Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions tests/unit/storages/test_request_queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
from __future__ import annotations

import asyncio
from datetime import timedelta
from typing import TYPE_CHECKING

import pytest
Expand Down Expand Up @@ -421,6 +422,36 @@ async def test_is_empty(rq: RequestQueue) -> None:
assert await rq.is_empty() is True


@pytest.mark.parametrize(
('wait_for_all'),
[
pytest.param(True, id='wait for all'),
pytest.param(False, id='don`t wait for all'),
],
)
async def test_add_requests_wait_for_all(rq: RequestQueue, *, wait_for_all: bool) -> None:
"""Test adding requests with wait_for_all_requests_to_be_added option."""
urls = [f'https://example.com/{i}' for i in range(15)]

# Add requests without waiting
await rq.add_requests(
urls,
batch_size=5,
wait_for_all_requests_to_be_added=wait_for_all,
wait_time_between_batches=timedelta(milliseconds=100),
)

if not wait_for_all:
# Immediately after adding, the total count may be less than 15 due to background processing
assert await rq.get_total_count() <= 15

# Wait a 250 milliseconds for background tasks to complete
await asyncio.sleep(0.25)

# Verify all requests were added
assert await rq.get_total_count() == 15


async def test_is_finished(rq: RequestQueue) -> None:
"""Test checking if a request queue is finished."""
# Initially the queue should be finished (empty and no background tasks)
Expand Down