From 65ea2fc2d6bb0834538be1a55aa706c3fe9bd5c3 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 27 Jun 2025 14:58:35 +0200 Subject: [PATCH 1/8] feat: Persist RequestList state --- src/crawlee/_utils/recoverable_state.py | 43 ++++++-- src/crawlee/request_loaders/_request_list.py | 105 ++++++++++++++++--- uv.lock | 6 +- 3 files changed, 127 insertions(+), 27 deletions(-) diff --git a/src/crawlee/_utils/recoverable_state.py b/src/crawlee/_utils/recoverable_state.py index 2cfdcd9ec7..6f2d6003b5 100644 --- a/src/crawlee/_utils/recoverable_state.py +++ b/src/crawlee/_utils/recoverable_state.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Generic, TypeVar +from typing import TYPE_CHECKING, Generic, Literal, TypeVar from pydantic import BaseModel @@ -34,7 +34,7 @@ def __init__( *, default_state: TStateModel, persist_state_key: str, - persistence_enabled: bool = False, + persistence_enabled: Literal[True, False, 'explicit_only'] = False, persist_state_kvs_name: str | None = None, persist_state_kvs_id: str | None = None, logger: logging.Logger, @@ -71,7 +71,7 @@ async def initialize(self) -> TStateModel: Returns: The loaded state model """ - if not self._persistence_enabled: + if self._persistence_enabled is False: self._state = self._default_state.model_copy(deep=True) return self.current_value @@ -81,8 +81,9 @@ async def initialize(self) -> TStateModel: await self._load_saved_state() - event_manager = service_locator.get_event_manager() - event_manager.on(event=Event.PERSIST_STATE, listener=self.persist_state) + if self._persistence_enabled is True: + event_manager = service_locator.get_event_manager() + event_manager.on(event=Event.PERSIST_STATE, listener=self.persist_state) return self.current_value @@ -95,9 +96,10 @@ async def teardown(self) -> None: if not self._persistence_enabled: return - event_manager = service_locator.get_event_manager() - event_manager.off(event=Event.PERSIST_STATE, listener=self.persist_state) - await self.persist_state() + if self._persistence_enabled is True: + event_manager = service_locator.get_event_manager() + event_manager.off(event=Event.PERSIST_STATE, listener=self.persist_state) + await self.persist_state() @property def current_value(self) -> TStateModel: @@ -107,6 +109,23 @@ def current_value(self) -> TStateModel: return self._state + @property + def is_initialized(self) -> bool: + """Check if the state has already been initialized.""" + return self._state is not None + + async def has_persisted_state(self) -> bool: + """Check if there is any persisted state in the key-value store.""" + if not self._persistence_enabled: + return False + + if self._key_value_store is None: + raise RuntimeError('Recoverable state has not yet been initialized') + + return ( + await self._key_value_store.get_value(self._persist_state_key) is not None + ) # TODO do not fetch the whole record + async def reset(self) -> None: """Reset the state to the default values and clear any persisted state. @@ -130,17 +149,21 @@ async def persist_state(self, event_data: EventPersistStateData | None = None) - Args: event_data: Optional data associated with a PERSIST_STATE event """ - self._log.debug(f'Persisting state of the Statistics (event_data={event_data}).') + self._log.debug( + f'Persisting RecoverableState (model={self._default_state.__class__.__name__}, event_data={event_data}).' + ) if self._key_value_store is None or self._state is None: raise RuntimeError('Recoverable state has not yet been initialized') - if self._persistence_enabled: + if self._persistence_enabled is True or self._persistence_enabled == 'explicit_only': await self._key_value_store.set_value( self._persist_state_key, self._state.model_dump(mode='json', by_alias=True), 'application/json', ) + else: + self._log.debug('Persistence is not enabled - not doing anything') async def _load_saved_state(self) -> None: if self._key_value_store is None: diff --git a/src/crawlee/request_loaders/_request_list.py b/src/crawlee/request_loaders/_request_list.py index 5964b106d0..167454e9dc 100644 --- a/src/crawlee/request_loaders/_request_list.py +++ b/src/crawlee/request_loaders/_request_list.py @@ -1,46 +1,83 @@ from __future__ import annotations import asyncio +import contextlib from collections.abc import AsyncIterable, AsyncIterator, Iterable -from typing import TYPE_CHECKING +from logging import getLogger +from typing import Annotated +from pydantic import BaseModel, ConfigDict, Field from typing_extensions import override +from crawlee._request import Request from crawlee._utils.docs import docs_group from crawlee.request_loaders._request_loader import RequestLoader -if TYPE_CHECKING: - from crawlee._request import Request +logger = getLogger(__name__) + + +class RequestListState(BaseModel): + model_config = ConfigDict(populate_by_name=True) + + next_index: Annotated[int, Field(alias='nextIndex')] = 0 + next_unique_key: Annotated[str | None, Field(alias='nextUniqueKey')] = None + in_progress: Annotated[set[str], Field(alias='inProgress')] = set() + + +class RequestListData(BaseModel): + requests: Annotated[Iterable[Request], Field()] @docs_group('Classes') class RequestList(RequestLoader): - """Represents a (potentially very large) list of URLs to crawl. - - Disclaimer: The `RequestList` class is in its early version and is not fully implemented. It is currently - intended mainly for testing purposes and small-scale projects. The current implementation is only in-memory - storage and is very limited. It will be (re)implemented in the future. For more details, see the GitHub issue: - https://github.com/apify/crawlee-python/issues/99. For production usage we recommend to use the `RequestQueue`. - """ + """Represents a (potentially very large) list of URLs to crawl.""" def __init__( self, requests: Iterable[str | Request] | AsyncIterable[str | Request] | None = None, name: str | None = None, + persist_state_key: str | None = None, + persist_requests_key: str | None = None, ) -> None: """Initialize a new instance. Args: requests: The request objects (or their string representations) to be added to the provider. name: A name of the request list. + persist_state_key: A key for persisting the progress information of the RequestList. + If you do not pass a key but pass a `name`, a key will be derived using the name. + Otherwise, state will not be persisted. + persist_requests_key: A key for persisting the request data loaded from the `requests` iterator. + If specified, the request data will be stored in the KeyValueStore to make sure that they don't change + over time. This is useful if the `requests` iterator pulls the data dynamically. """ + from crawlee._utils.recoverable_state import RecoverableState + self._name = name self._handled_count = 0 self._assumed_total_count = 0 - self._in_progress = set[str]() self._next: Request | None = None + if persist_state_key is None and name is not None: + persist_state_key = f'SDK_REQUEST_LIST_STATE-{name}' + + self._state = RecoverableState( + default_state=RequestListState(), + persistence_enabled=bool(persist_state_key), + persist_state_key=persist_state_key or '', + logger=logger, + ) + + self._persist_request_data = bool(persist_requests_key) + + self._requests_data = RecoverableState( + default_state=RequestListData(requests=[]), + persistence_enabled='explicit_only' if self._persist_request_data else False, + persist_state_key=persist_requests_key or '', + logger=logger, + ) + if isinstance(requests, AsyncIterable): self._requests = requests.__aiter__() elif requests is None: @@ -50,6 +87,38 @@ def __init__( self._requests_lock: asyncio.Lock | None = None + async def _get_state(self) -> RequestListState: + if self._state.is_initialized: + return self._state.current_value + + await self._state.initialize() + await self._requests_data.initialize() + + if self._persist_request_data: + if self._requests_lock is None: + self._requests_lock = asyncio.Lock() + + async with self._requests_lock: + if not await self._requests_data.has_persisted_state(): + self._requests_data.current_value.requests = [ + request if isinstance(request, Request) else Request.from_url(request) + async for request in self._requests + ] + await self._requests_data.persist_state() + + self._requests = self._iterate_in_threadpool(self._requests_data.current_value.requests) + + for _ in range(0, self._state.current_value.next_index): + with contextlib.suppress(StopAsyncIteration): + await self._requests.__anext__() + + if (unique_key_to_check := self._state.current_value.next_unique_key) is not None: + await self._ensure_next_request() + if self._next is None or self._next.unique_key != unique_key_to_check: + raise RuntimeError() + + return self._state.current_value + @property def name(self) -> str | None: return self._name @@ -65,7 +134,8 @@ async def is_empty(self) -> bool: @override async def is_finished(self) -> bool: - return len(self._in_progress) == 0 and await self.is_empty() + state = await self._get_state() + return len(state.in_progress) == 0 and await self.is_empty() @override async def fetch_next_request(self) -> Request | None: @@ -74,7 +144,8 @@ async def fetch_next_request(self) -> Request | None: if self._next is None: return None - self._in_progress.add(self._next.id) + state = await self._get_state() + state.in_progress.add(self._next.id) self._assumed_total_count += 1 next_request = self._next @@ -85,13 +156,16 @@ async def fetch_next_request(self) -> Request | None: @override async def mark_request_as_handled(self, request: Request) -> None: self._handled_count += 1 - self._in_progress.remove(request.id) + state = await self._get_state() + state.in_progress.remove(request.id) @override async def get_handled_count(self) -> int: return self._handled_count async def _ensure_next_request(self) -> None: + state = await self._get_state() + if self._requests_lock is None: self._requests_lock = asyncio.Lock() @@ -101,6 +175,9 @@ async def _ensure_next_request(self) -> None: self._next = self._transform_request(await self._requests.__anext__()) except StopAsyncIteration: self._next = None + else: + state.next_index += 1 + state.next_unique_key = self._next.unique_key async def _iterate_in_threadpool(self, iterable: Iterable[str | Request]) -> AsyncIterator[str | Request]: """Inspired by a function of the same name from encode/starlette.""" diff --git a/uv.lock b/uv.lock index 4e2074521c..46c3412b27 100644 --- a/uv.lock +++ b/uv.lock @@ -627,7 +627,7 @@ toml = [ [[package]] name = "crawlee" -version = "0.6.11" +version = "0.6.12" source = { editable = "." } dependencies = [ { name = "apify-fingerprint-datapoints" }, @@ -949,7 +949,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -1194,7 +1194,7 @@ name = "importlib-metadata" version = "8.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } wheels = [ From fa701f81e5d03001ccd02342abb36894ef04fed7 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Thu, 17 Jul 2025 13:08:54 +0200 Subject: [PATCH 2/8] Use record_exists --- src/crawlee/_utils/recoverable_state.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/crawlee/_utils/recoverable_state.py b/src/crawlee/_utils/recoverable_state.py index 2893c22f9c..b1dd79a342 100644 --- a/src/crawlee/_utils/recoverable_state.py +++ b/src/crawlee/_utils/recoverable_state.py @@ -131,9 +131,7 @@ async def has_persisted_state(self) -> bool: if self._key_value_store is None: raise RuntimeError('Recoverable state has not yet been initialized') - return ( - await self._key_value_store.get_value(self._persist_state_key) is not None - ) # TODO do not fetch the whole record + return await self._key_value_store.record_exists(self._persist_state_key) async def reset(self) -> None: """Reset the state to the default values and clear any persisted state. From 8a1c9287b3ca634fd711e48320ded2a3e77acf03 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 21 Jul 2025 20:24:01 +0200 Subject: [PATCH 3/8] Add tests and correctly implement request persistence --- src/crawlee/request_loaders/_request_list.py | 60 +++++--- .../unit/request_loaders/test_request_list.py | 130 ++++++++++++++++++ 2 files changed, 169 insertions(+), 21 deletions(-) diff --git a/src/crawlee/request_loaders/_request_list.py b/src/crawlee/request_loaders/_request_list.py index 6cce1a3725..67f717cc35 100644 --- a/src/crawlee/request_loaders/_request_list.py +++ b/src/crawlee/request_loaders/_request_list.py @@ -2,7 +2,7 @@ import asyncio import contextlib -from collections.abc import AsyncIterable, AsyncIterator, Iterable +from collections.abc import AsyncGenerator, AsyncIterable, AsyncIterator, Iterable from logging import getLogger from typing import Annotated @@ -25,7 +25,7 @@ class RequestListState(BaseModel): class RequestListData(BaseModel): - requests: Annotated[Iterable[Request], Field()] + requests: Annotated[list[Request], Field()] @docs_group('Classes') @@ -51,13 +51,13 @@ def __init__( If specified, the request data will be stored in the KeyValueStore to make sure that they don't change over time. This is useful if the `requests` iterator pulls the data dynamically. """ - from crawlee._utils.recoverable_state import RecoverableState + from crawlee._utils.recoverable_state import RecoverableState # noqa: PLC0415 self._name = name self._handled_count = 0 self._assumed_total_count = 0 - self._next: Request | None = None + self._next: tuple[Request | None, Request | None] = (None, None) if persist_state_key is None and name is not None: persist_state_key = f'SDK_REQUEST_LIST_STATE-{name}' @@ -108,14 +108,20 @@ async def _get_state(self) -> RequestListState: self._requests = self._iterate_in_threadpool(self._requests_data.current_value.requests) - for _ in range(0, self._state.current_value.next_index): + for _ in range(self._state.current_value.next_index): with contextlib.suppress(StopAsyncIteration): await self._requests.__anext__() if (unique_key_to_check := self._state.current_value.next_unique_key) is not None: await self._ensure_next_request() - if self._next is None or self._next.unique_key != unique_key_to_check: - raise RuntimeError() + + next_unique_key = self._next[0].unique_key if self._next[0] is not None else None + if next_unique_key != unique_key_to_check: + raise RuntimeError( + f'Mismatch at index { + self._state.current_value.next_index + } in persisted requests - Expected unique key `{unique_key_to_check}`, got `{next_unique_key}`' + ) return self._state.current_value @@ -134,7 +140,7 @@ async def get_total_count(self) -> int: @override async def is_empty(self) -> bool: await self._ensure_next_request() - return self._next is None + return self._next[0] is None @override async def is_finished(self) -> bool: @@ -145,15 +151,19 @@ async def is_finished(self) -> bool: async def fetch_next_request(self) -> Request | None: await self._ensure_next_request() - if self._next is None: + if self._next[0] is None: return None state = await self._get_state() - state.in_progress.add(self._next.id) + state.in_progress.add(self._next[0].id) self._assumed_total_count += 1 - next_request = self._next - self._next = None + next_request = self._next[0] + if next_request is not None: + state.next_index += 1 + + self._next = (self._next[1], None) + await self._ensure_next_request() return next_request @@ -169,15 +179,23 @@ async def _ensure_next_request(self) -> None: if self._requests_lock is None: self._requests_lock = asyncio.Lock() - try: - async with self._requests_lock: - if self._next is None: - self._next = self._transform_request(await self._requests.__anext__()) - except StopAsyncIteration: - self._next = None - else: - state.next_index += 1 - state.next_unique_key = self._next.unique_key + async with self._requests_lock: + if None in self._next: + if self._next[0] is None: + to_enqueue = [item async for item in self._dequeue_requests(2)] + self._next = (to_enqueue[0], to_enqueue[1]) + else: + to_enqueue = [item async for item in self._dequeue_requests(1)] + self._next = (self._next[0], to_enqueue[0]) + + state.next_unique_key = self._next[1].unique_key if self._next[1] is not None else None + + async def _dequeue_requests(self, count: int) -> AsyncGenerator[Request | None]: + for _ in range(count): + try: + yield self._transform_request(await self._requests.__anext__()) + except StopAsyncIteration: # noqa: PERF203 + yield None async def _iterate_in_threadpool(self, iterable: Iterable[str | Request]) -> AsyncIterator[str | Request]: """Inspired by a function of the same name from encode/starlette.""" diff --git a/tests/unit/request_loaders/test_request_list.py b/tests/unit/request_loaders/test_request_list.py index e3ded91b7f..e137c572a6 100644 --- a/tests/unit/request_loaders/test_request_list.py +++ b/tests/unit/request_loaders/test_request_list.py @@ -1,6 +1,7 @@ from collections.abc import AsyncGenerator from crawlee.request_loaders._request_list import RequestList +from crawlee.storages import KeyValueStore async def test_sync_traversal() -> None: @@ -56,3 +57,132 @@ async def test_is_empty_does_not_depend_on_fetch_next_request() -> None: assert await request_list.is_empty() assert await request_list.is_finished() + + +async def test_persist_requests_key_with_sync_iterable() -> None: + """Test that persist_requests_key persists request data from a sync iterable.""" + persist_key = 'test_requests_persist_sync' + urls = ['https://a.placeholder.com', 'https://b.placeholder.com', 'https://c.placeholder.com'] + + # Create a request list with persistence enabled + request_list = RequestList(urls, persist_requests_key=persist_key) + + # Fetch one request to trigger initialization + first_request = await request_list.fetch_next_request() + assert first_request is not None + assert first_request.url == 'https://a.placeholder.com' + + # Check that the requests were persisted + kvs = await KeyValueStore.open() + persisted_data = await kvs.get_value(persist_key) + assert persisted_data is not None + + +async def test_persist_requests_key_with_empty_iterator() -> None: + """Test behavior when persist_requests_key is provided but the iterator is empty.""" + persist_key = 'test_empty_iterator' + + # Create request list with empty iterator + request_list = RequestList([], persist_requests_key=persist_key) + + # Should be empty immediately + assert await request_list.is_empty() + assert await request_list.is_finished() + + # Check that empty requests were persisted + kvs = await KeyValueStore.open() + persisted_data = await kvs.get_value(persist_key) + assert persisted_data is not None + + +async def test_requests_restoration_without_state() -> None: + """Test that persisted request data is properly restored on subsequent RequestList creation.""" + persist_requests_key = 'test_requests_restoration' + urls = ['https://restore1.placeholder.com', 'https://restore2.placeholder.com'] + + # Create first request list and process one request + request_list_1 = RequestList(urls, persist_requests_key=persist_requests_key) + first_request = await request_list_1.fetch_next_request() + assert first_request is not None + assert first_request.url == 'https://restore1.placeholder.com' + await request_list_1.mark_request_as_handled(first_request) + + # Create second request list with same persist key (simulating restart) + # Since we don't have state persistence, it will start from the beginning of the persisted data + spy = iter(['1', '2', '3']) + request_list_2 = RequestList(spy, persist_requests_key=persist_requests_key) + + # Should be able to fetch requests from persisted data, but starts from beginning + first_request_again = await request_list_2.fetch_next_request() + assert first_request_again is not None + assert first_request_again.url == 'https://restore1.placeholder.com' + await request_list_2.mark_request_as_handled(first_request_again) + + # Make sure that the second instance did not consume the input iterator + assert len(list(spy)) == 3 + + +async def test_requests_and_state_restoration() -> None: + """Test that persisted request data and processing state is properly restored on subsequent RequestList creation.""" + persist_requests_key = 'test_requests_restoration' + persist_state_key = 'test_state_restoration' + urls = ['https://restore1.placeholder.com', 'https://restore2.placeholder.com'] + + # Create first request list and process one request + request_list_1 = RequestList( + urls, + persist_requests_key=persist_requests_key, + persist_state_key=persist_state_key, + ) + + first_request = await request_list_1.fetch_next_request() + assert first_request is not None + assert first_request.url == 'https://restore1.placeholder.com' + await request_list_1.mark_request_as_handled(first_request) + await request_list_1._state.persist_state() + + # Create second request list with same persist key (simulating restart) + spy = iter(['1', '2', '3']) + request_list_2 = RequestList( + spy, + persist_requests_key=persist_requests_key, + persist_state_key=persist_state_key, + ) + + # Should be able to fetch requests from persisted data and continue where the previous instance left off + next_request = await request_list_2.fetch_next_request() + assert next_request is not None + assert next_request.url == 'https://restore2.placeholder.com' + await request_list_2.mark_request_as_handled(next_request) + + # Make sure that the second instance did not consume the input iterator + assert len(list(spy)) == 3 + + +async def test_persist_requests_key_only_persists_once() -> None: + """Test that requests are only persisted once, even with multiple RequestList instances.""" + persist_key = 'test_requests_once' + urls = ['https://once1.placeholder.com', 'https://once2.placeholder.com'] + + # Create first request list + request_list_1 = RequestList(urls, persist_requests_key=persist_key) + await request_list_1.fetch_next_request() # Trigger persistence + + # Get initial persisted data + kvs = await KeyValueStore.open() + initial_data = await kvs.get_value(persist_key) + assert initial_data is not None + + # Create second request list with different data + different_urls = ['https://different.placeholder.com'] + request_list_2 = RequestList(different_urls, persist_requests_key=persist_key) + await request_list_2.fetch_next_request() # Should use persisted data, not new data + + # Verify the persisted data hasn't changed + current_data = await kvs.get_value(persist_key) + assert current_data == initial_data + + # The request should come from the original persisted data, not the new iterator + fetched_request = await request_list_2.fetch_next_request() + assert fetched_request is not None + assert fetched_request.url == 'https://once2.placeholder.com' # From original data From f601ce3b3ef530dc458fab71350cfbd3d01e6290 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 21 Jul 2025 20:25:22 +0200 Subject: [PATCH 4/8] Fix type mismatch --- src/crawlee/statistics/_statistics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crawlee/statistics/_statistics.py b/src/crawlee/statistics/_statistics.py index 02c41a7b3e..b1f5e0bf2f 100644 --- a/src/crawlee/statistics/_statistics.py +++ b/src/crawlee/statistics/_statistics.py @@ -67,7 +67,7 @@ class Statistics(Generic[TStatisticsState]): def __init__( self, *, - persistence_enabled: bool = False, + persistence_enabled: bool | Literal['explicit_only'] = False, persist_state_kvs_name: str | None = None, persist_state_key: str | None = None, log_message: str = 'Statistics', From c825b185c54c1c9431595d51a6daa4e51961e783 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 21 Jul 2025 20:30:33 +0200 Subject: [PATCH 5/8] More comments --- src/crawlee/_utils/recoverable_state.py | 9 +++++---- src/crawlee/request_loaders/_request_list.py | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/crawlee/_utils/recoverable_state.py b/src/crawlee/_utils/recoverable_state.py index b1dd79a342..fc115eb403 100644 --- a/src/crawlee/_utils/recoverable_state.py +++ b/src/crawlee/_utils/recoverable_state.py @@ -43,13 +43,14 @@ def __init__( Args: default_state: The default state model instance to use when no persisted state is found. - A deep copy is made each time the state is used. + A deep copy is made each time the state is used. persist_state_key: The key under which the state is stored in the KeyValueStore - persistence_enabled: Flag to enable or disable state persistence + persistence_enabled: Flag to enable or disable state persistence. Use 'explicit_only' if you want to be able + to save the state manually, but without any automatic persistence. persist_state_kvs_name: The name of the KeyValueStore to use for persistence. - If neither a name nor and id are supplied, the default store will be used. + If neither a name nor and id are supplied, the default store will be used. persist_state_kvs_id: The identifier of the KeyValueStore to use for persistence. - If neither a name nor and id are supplied, the default store will be used. + If neither a name nor and id are supplied, the default store will be used. logger: A logger instance for logging operations related to state persistence """ self._default_state = default_state diff --git a/src/crawlee/request_loaders/_request_list.py b/src/crawlee/request_loaders/_request_list.py index 67f717cc35..521c43f033 100644 --- a/src/crawlee/request_loaders/_request_list.py +++ b/src/crawlee/request_loaders/_request_list.py @@ -73,6 +73,7 @@ def __init__( self._requests_data = RecoverableState( default_state=RequestListData(requests=[]), + # With request data persistence enabled, a snapshot of the requests will be done on initialization persistence_enabled='explicit_only' if self._persist_request_data else False, persist_state_key=persist_requests_key or '', logger=logger, From 3a7312d78a450b40d9ae536b435377d86f3fe0ce Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 21 Jul 2025 20:33:03 +0200 Subject: [PATCH 6/8] Huh --- src/crawlee/request_loaders/_request_list.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crawlee/request_loaders/_request_list.py b/src/crawlee/request_loaders/_request_list.py index 521c43f033..86405cda16 100644 --- a/src/crawlee/request_loaders/_request_list.py +++ b/src/crawlee/request_loaders/_request_list.py @@ -119,9 +119,9 @@ async def _get_state(self) -> RequestListState: next_unique_key = self._next[0].unique_key if self._next[0] is not None else None if next_unique_key != unique_key_to_check: raise RuntimeError( - f'Mismatch at index { + f"""Mismatch at index { self._state.current_value.next_index - } in persisted requests - Expected unique key `{unique_key_to_check}`, got `{next_unique_key}`' + } in persisted requests - Expected unique key `{unique_key_to_check}`, got `{next_unique_key}`""" ) return self._state.current_value From 68c4c31d4145b0080f3ed9c5e6f15bf4536630cf Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 23 Jul 2025 11:05:02 +0200 Subject: [PATCH 7/8] More fixes, more comments, more tests --- src/crawlee/request_loaders/_request_list.py | 30 ++++++---- .../unit/request_loaders/test_request_list.py | 56 ++++++++++++++++++- 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/src/crawlee/request_loaders/_request_list.py b/src/crawlee/request_loaders/_request_list.py index 86405cda16..3be8b22c82 100644 --- a/src/crawlee/request_loaders/_request_list.py +++ b/src/crawlee/request_loaders/_request_list.py @@ -89,16 +89,20 @@ def __init__( self._requests_lock: asyncio.Lock | None = None async def _get_state(self) -> RequestListState: + # If state is already initialized, we are done if self._state.is_initialized: return self._state.current_value + # Initialize recoverable state await self._state.initialize() await self._requests_data.initialize() - if self._persist_request_data: - if self._requests_lock is None: - self._requests_lock = asyncio.Lock() + # Initialize lock if necessary + if self._requests_lock is None: + self._requests_lock = asyncio.Lock() + # If the RequestList is configured to persist request data, ensure that a copy of request data is used + if self._persist_request_data: async with self._requests_lock: if not await self._requests_data.has_persisted_state(): self._requests_data.current_value.requests = [ @@ -107,12 +111,17 @@ async def _get_state(self) -> RequestListState: ] await self._requests_data.persist_state() - self._requests = self._iterate_in_threadpool(self._requests_data.current_value.requests) - - for _ in range(self._state.current_value.next_index): - with contextlib.suppress(StopAsyncIteration): - await self._requests.__anext__() + self._requests = self._iterate_in_threadpool( + self._requests_data.current_value.requests[self._state.current_value.next_index :] + ) + # If not using persistent request data, advance the request iterator + else: + async with self._requests_lock: + for _ in range(self._state.current_value.next_index): + with contextlib.suppress(StopAsyncIteration): + await self._requests.__anext__() + # Check consistency of the stored state and the request iterator if (unique_key_to_check := self._state.current_value.next_unique_key) is not None: await self._ensure_next_request() @@ -162,6 +171,7 @@ async def fetch_next_request(self) -> Request | None: next_request = self._next[0] if next_request is not None: state.next_index += 1 + state.next_unique_key = self._next[1].unique_key if self._next[1] is not None else None self._next = (self._next[1], None) await self._ensure_next_request() @@ -175,8 +185,6 @@ async def mark_request_as_handled(self, request: Request) -> None: state.in_progress.remove(request.id) async def _ensure_next_request(self) -> None: - state = await self._get_state() - if self._requests_lock is None: self._requests_lock = asyncio.Lock() @@ -189,8 +197,6 @@ async def _ensure_next_request(self) -> None: to_enqueue = [item async for item in self._dequeue_requests(1)] self._next = (self._next[0], to_enqueue[0]) - state.next_unique_key = self._next[1].unique_key if self._next[1] is not None else None - async def _dequeue_requests(self, count: int) -> AsyncGenerator[Request | None]: for _ in range(count): try: diff --git a/tests/unit/request_loaders/test_request_list.py b/tests/unit/request_loaders/test_request_list.py index e137c572a6..bf7c094e8c 100644 --- a/tests/unit/request_loaders/test_request_list.py +++ b/tests/unit/request_loaders/test_request_list.py @@ -122,11 +122,60 @@ async def test_requests_restoration_without_state() -> None: assert len(list(spy)) == 3 +async def test_state_restoration() -> None: + """Test that persisted processing state is properly restored on subsequent RequestList creation.""" + persist_state_key = 'test_state_restoration' + urls = [ + 'https://restore1.placeholder.com', + 'https://restore2.placeholder.com', + 'https://restore3.placeholder.com', + 'https://restore4.placeholder.com', + ] + + # Create first request list and process one request + request_list_1 = RequestList( + urls, + persist_state_key=persist_state_key, + ) + + first_request = await request_list_1.fetch_next_request() + assert first_request is not None + assert first_request.url == 'https://restore1.placeholder.com' + await request_list_1.mark_request_as_handled(first_request) + await request_list_1._state.persist_state() + + # Create second request list with same persist key (simulating restart) + request_list_2 = RequestList( + urls, + persist_state_key=persist_state_key, + ) + + # Should be able to continue where the previous instance left off + next_request = await request_list_2.fetch_next_request() + assert next_request is not None + assert next_request.url == 'https://restore2.placeholder.com' + await request_list_2.mark_request_as_handled(next_request) + + next_request = await request_list_2.fetch_next_request() + assert next_request is not None + assert next_request.url == 'https://restore3.placeholder.com' + await request_list_2.mark_request_as_handled(next_request) + + next_request = await request_list_2.fetch_next_request() + assert next_request is not None + assert next_request.url == 'https://restore4.placeholder.com' + await request_list_2.mark_request_as_handled(next_request) + + async def test_requests_and_state_restoration() -> None: """Test that persisted request data and processing state is properly restored on subsequent RequestList creation.""" persist_requests_key = 'test_requests_restoration' persist_state_key = 'test_state_restoration' - urls = ['https://restore1.placeholder.com', 'https://restore2.placeholder.com'] + urls = [ + 'https://restore1.placeholder.com', + 'https://restore2.placeholder.com', + 'https://restore3.placeholder.com', + ] # Create first request list and process one request request_list_1 = RequestList( @@ -155,6 +204,11 @@ async def test_requests_and_state_restoration() -> None: assert next_request.url == 'https://restore2.placeholder.com' await request_list_2.mark_request_as_handled(next_request) + next_request = await request_list_2.fetch_next_request() + assert next_request is not None + assert next_request.url == 'https://restore3.placeholder.com' + await request_list_2.mark_request_as_handled(next_request) + # Make sure that the second instance did not consume the input iterator assert len(list(spy)) == 3 From 7d1c36a35559815c8b7d91f84d12453f509c6c03 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 23 Jul 2025 11:34:08 +0200 Subject: [PATCH 8/8] Turns out _get_state is important --- src/crawlee/request_loaders/_request_list.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/crawlee/request_loaders/_request_list.py b/src/crawlee/request_loaders/_request_list.py index 3be8b22c82..f8da9434c2 100644 --- a/src/crawlee/request_loaders/_request_list.py +++ b/src/crawlee/request_loaders/_request_list.py @@ -159,6 +159,7 @@ async def is_finished(self) -> bool: @override async def fetch_next_request(self) -> Request | None: + await self._get_state() await self._ensure_next_request() if self._next[0] is None: @@ -185,6 +186,8 @@ async def mark_request_as_handled(self, request: Request) -> None: state.in_progress.remove(request.id) async def _ensure_next_request(self) -> None: + await self._get_state() + if self._requests_lock is None: self._requests_lock = asyncio.Lock()