From 38b55318f686d72f68a0fd768013f0637c5d6f47 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 13 Mar 2025 23:30:34 +0000 Subject: [PATCH 01/15] basic implementation for bind request to session --- src/crawlee/_request.py | 20 +++++++++++++ src/crawlee/crawlers/_basic/_basic_crawler.py | 30 ++++++++++++++++++- src/crawlee/errors.py | 6 ++++ uv.lock | 2 +- 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/src/crawlee/_request.py b/src/crawlee/_request.py index 544de2b69c..85d44d535a 100644 --- a/src/crawlee/_request.py +++ b/src/crawlee/_request.py @@ -75,6 +75,9 @@ class UserData(BaseModel, MutableMapping[str, JsonSerializable]): label: Annotated[str | None, Field()] = None """Label used for request routing.""" + session_id: Annotated[str | None, Field()] = None + """Bind a request to a specific Session""" + def __getitem__(self, key: str) -> JsonSerializable: return self.__pydantic_extra__[key] @@ -84,6 +87,13 @@ def __setitem__(self, key: str, value: JsonSerializable) -> None: raise ValueError('`label` must be str or None') self.label = value + + if key == 'session_id': + if value is not None and not isinstance(value, str): + raise ValueError('`session_id` must be str or None') + + self.session_id = value + self.__pydantic_extra__[key] = value def __delitem__(self, key: str) -> None: @@ -227,6 +237,7 @@ def from_url( headers: HttpHeaders | dict[str, str] | None = None, payload: HttpPayload | str | None = None, label: str | None = None, + session_id: str | None = None, unique_key: str | None = None, id: str | None = None, keep_url_fragment: bool = False, @@ -248,6 +259,7 @@ def from_url( payload: The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests. label: A custom label to differentiate between request types. This is stored in `user_data`, and it is used for request routing (different requests go to different handlers). + session_id: TODO unique_key: A unique key identifying the request. If not provided, it is automatically computed based on the URL and other parameters. Requests with the same `unique_key` are treated as identical. id: A unique identifier for the request. If not provided, it is automatically generated from the @@ -296,6 +308,9 @@ def from_url( if label is not None: request.user_data['label'] = label + if session_id is not None: + request.user_data['session_id'] = session_id + return request def get_query_param_from_url(self, param: str, *, default: str | None = None) -> str | None: @@ -308,6 +323,11 @@ def label(self) -> str | None: """A string used to differentiate between arbitrary request types.""" return cast(UserData, self.user_data).label + @property + def session_id(self) -> str | None: + """A string used to differentiate between arbitrary request types.""" + return cast(UserData, self.user_data).session_id + @property def crawlee_data(self) -> CrawleeRequestData: """Crawlee-specific configuration stored in the `user_data`.""" diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index a6eae3c677..3f4709b4e3 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -40,6 +40,7 @@ ContextPipelineInterruptedError, HttpClientStatusCodeError, HttpStatusCodeError, + RequestCollisionError, RequestHandlerError, SessionError, UserDefinedErrorHandlerError, @@ -439,6 +440,20 @@ async def _get_session(self) -> Session | None: logger=self._logger, ) + async def _get_session_by_id(self, session_id: str | None) -> Session | None: + """If session pool is being used, try to take a session by id from it.""" + if not self._use_session_pool or not session_id: + return None + + return await wait_for( + partial(self._session_pool.get_session_by_id, session_id), + timeout=self._internal_timeout, + timeout_message='Fetching a session from the pool timed out after ' + f'{self._internal_timeout.total_seconds()} seconds', + max_retries=3, + logger=self._logger, + ) + async def _get_proxy_info(self, request: Request, session: Session | None) -> ProxyInfo | None: """Retrieve a new ProxyInfo object based on crawler configuration and the current request and session.""" if not self._proxy_configuration: @@ -1043,7 +1058,10 @@ async def __run_task_function(self) -> None: if request is None: return - session = await self._get_session() + if request.session_id: + session = await self._get_session_by_id(request.session_id) + else: + session = await self._get_session() proxy_info = await self._get_proxy_info(request, session) result = RequestHandlerRunResult(key_value_store_getter=self.get_key_value_store) @@ -1066,6 +1084,8 @@ async def __run_task_function(self) -> None: try: request.state = RequestState.REQUEST_HANDLER + self._raise_context_request_colission(context) + try: await self._run_request_handler(context=context) except asyncio.TimeoutError as e: @@ -1088,6 +1108,10 @@ async def __run_task_function(self) -> None: self._statistics.record_request_processing_finish(statistics_id) + except RequestCollisionError as request_error: + context.request.no_retry = True + await self._handle_request_error(context, request_error) + except RequestHandlerError as primary_error: primary_error = cast( RequestHandlerError[TCrawlingContext], primary_error @@ -1204,3 +1228,7 @@ def _raise_for_session_blocked_status_code(self, session: Session | None, status ignore_http_error_status_codes=self._ignore_http_error_status_codes, ): raise SessionError(f'Assuming the session is blocked based on HTTP status code {status_code}') + + def _raise_context_request_colission(self, context: BasicCrawlingContext) -> None: + if self._use_session_pool and context.request.session_id and not context.session: + raise RequestCollisionError('The `Session` binded to the `Request` is no longer available in `SessionPool`') diff --git a/src/crawlee/errors.py b/src/crawlee/errors.py index 07f7848bdb..217ce31091 100644 --- a/src/crawlee/errors.py +++ b/src/crawlee/errors.py @@ -14,6 +14,7 @@ 'HttpClientStatusCodeError', 'HttpStatusCodeError', 'ProxyError', + 'RequestCollisionError', 'RequestHandlerError', 'ServiceConflictError', 'SessionError', @@ -106,3 +107,8 @@ def __init__(self, wrapped_exception: Exception, crawling_context: BasicCrawling @docs_group('Errors') class ContextPipelineInterruptedError(Exception): """May be thrown in the initialization phase of a middleware to signal that the request should not be processed.""" + + +@docs_group('Errors') +class RequestCollisionError(Exception): + """Raised when a request cannot be processed due to a conflict with required resources.""" diff --git a/uv.lock b/uv.lock index c83aa5c7f1..a2937f9b95 100644 --- a/uv.lock +++ b/uv.lock @@ -587,7 +587,7 @@ toml = [ [[package]] name = "crawlee" -version = "0.6.4" +version = "0.6.5" source = { editable = "." } dependencies = [ { name = "apify-fingerprint-datapoints" }, From 08a7d01d0d944c7a00595454e2953c01fc490fdd Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 13 Mar 2025 23:56:23 +0000 Subject: [PATCH 02/15] update docs --- src/crawlee/_request.py | 9 ++++++--- src/crawlee/crawlers/_basic/_basic_crawler.py | 19 +++++++++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/src/crawlee/_request.py b/src/crawlee/_request.py index 85d44d535a..1b1b2cacb2 100644 --- a/src/crawlee/_request.py +++ b/src/crawlee/_request.py @@ -76,7 +76,7 @@ class UserData(BaseModel, MutableMapping[str, JsonSerializable]): """Label used for request routing.""" session_id: Annotated[str | None, Field()] = None - """Bind a request to a specific Session""" + """ID of a session to which the request is bound.""" def __getitem__(self, key: str) -> JsonSerializable: return self.__pydantic_extra__[key] @@ -129,6 +129,7 @@ class RequestOptions(TypedDict): headers: NotRequired[HttpHeaders | dict[str, str] | None] payload: NotRequired[HttpPayload | str | None] label: NotRequired[str | None] + session_id: NotRequired[str | None] unique_key: NotRequired[str | None] id: NotRequired[str | None] keep_url_fragment: NotRequired[bool] @@ -259,7 +260,9 @@ def from_url( payload: The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests. label: A custom label to differentiate between request types. This is stored in `user_data`, and it is used for request routing (different requests go to different handlers). - session_id: TODO + session_id: ID of a specific Session to which the request will be strictly bound. + If the session becomes unavailable when the request is processed, a RequestCollisionError will be + raised. unique_key: A unique key identifying the request. If not provided, it is automatically computed based on the URL and other parameters. Requests with the same `unique_key` are treated as identical. id: A unique identifier for the request. If not provided, it is automatically generated from the @@ -325,7 +328,7 @@ def label(self) -> str | None: @property def session_id(self) -> str | None: - """A string used to differentiate between arbitrary request types.""" + """A string used to identify the bound session.""" return cast(UserData, self.user_data).session_id @property diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 3f4709b4e3..6d5c5c932b 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -1084,7 +1084,7 @@ async def __run_task_function(self) -> None: try: request.state = RequestState.REQUEST_HANDLER - self._raise_context_request_colission(context) + self._raise_request_collision(context.request, context.session) try: await self._run_request_handler(context=context) @@ -1229,6 +1229,17 @@ def _raise_for_session_blocked_status_code(self, session: Session | None, status ): raise SessionError(f'Assuming the session is blocked based on HTTP status code {status_code}') - def _raise_context_request_colission(self, context: BasicCrawlingContext) -> None: - if self._use_session_pool and context.request.session_id and not context.session: - raise RequestCollisionError('The `Session` binded to the `Request` is no longer available in `SessionPool`') + def _raise_request_collision(self, request: Request, session: Session | None) -> None: + """Raise an exception if a request cannot access required resources. + + Args: + request: The Request that might require specific resources (like a session). + session: The Session object that was retrieved for the request, or None if not available. + + Raises: + RequestCollisionError: If the Session referenced by the request is not available. + """ + if self._use_session_pool and request.session_id and not session: + raise RequestCollisionError( + f'The Session (id: {request.session_id}) bound to the Request is no longer available in SessionPool' + ) From a67d3b3b42d2a1d59923810fe6f17dabeea3e98b Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Fri, 14 Mar 2025 01:57:26 +0000 Subject: [PATCH 03/15] add tests --- .../crawlers/_basic/test_basic_crawler.py | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 65e58d9079..489f70f88e 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -22,7 +22,7 @@ from crawlee._types import BasicCrawlingContext, EnqueueLinksKwargs, HttpHeaders from crawlee.configuration import Configuration from crawlee.crawlers import BasicCrawler -from crawlee.errors import SessionError, UserDefinedErrorHandlerError +from crawlee.errors import RequestCollisionError, SessionError, UserDefinedErrorHandlerError from crawlee.events._local_event_manager import LocalEventManager from crawlee.request_loaders import RequestList, RequestManagerTandem from crawlee.sessions import SessionPool @@ -38,6 +38,7 @@ from yarl import URL from crawlee._types import JsonSerializable + from crawlee.sessions._session import Session async def test_processes_requests_from_explicit_queue() -> None: @@ -1194,3 +1195,52 @@ async def handler(context: BasicCrawlingContext) -> None: assert sessions[1] != sessions[0] else: assert sessions[1] == sessions[0] + + +async def test_bound_session_to_request() -> None: + async with SessionPool() as session_pool: + check_session: Session = await session_pool.get_session() + used_sessions = list[str]() + crawler = BasicCrawler(session_pool=session_pool) + + @crawler.router.default_handler + async def handler(context: BasicCrawlingContext) -> None: + if context.session: + used_sessions.append(context.session.id) + + requests = [ + Request.from_url('http://a.com/', session_id=check_session.id, always_enqueue=True) for _ in range(10) + ] + + await crawler.run(requests) + + assert len(used_sessions) == 10 + assert all(session_id == check_session.id for session_id in used_sessions) + + +async def test_error_bound_session_to_request() -> None: + crawler = BasicCrawler(request_handler=AsyncMock()) + + requests = [Request.from_url('http://a.com/', session_id='1', always_enqueue=True) for _ in range(10)] + + stats = await crawler.run(requests) + + assert stats.requests_total == 10 + assert stats.requests_failed == 10 + assert stats.retry_histogram == [10] + + +async def test_handle_error_bound_session_to_request() -> None: + error_handler_mock = AsyncMock() + crawler = BasicCrawler(request_handler=AsyncMock()) + + @crawler.failed_request_handler + async def error_req_hook(context: BasicCrawlingContext, error: Exception) -> None: + if isinstance(error, RequestCollisionError): + await error_handler_mock(context, error) + + requests = [Request.from_url('http://a.com/', session_id='1')] + + await crawler.run(requests) + + assert error_handler_mock.call_count == 1 From 039f12f53893b05f875149954e73d63a3b241f7b Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Fri, 14 Mar 2025 12:20:54 +0000 Subject: [PATCH 04/15] move session_id to crawlee_data --- src/crawlee/_request.py | 16 +++++----------- tests/unit/crawlers/_basic/test_basic_crawler.py | 2 +- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/src/crawlee/_request.py b/src/crawlee/_request.py index 1b1b2cacb2..3ea39dc253 100644 --- a/src/crawlee/_request.py +++ b/src/crawlee/_request.py @@ -58,6 +58,9 @@ class CrawleeRequestData(BaseModel): crawl_depth: Annotated[int, Field(alias='crawlDepth')] = 0 """The depth of the request in the crawl tree.""" + session_id: Annotated[str | None, Field()] = None + """ID of a session to which the request is bound.""" + class UserData(BaseModel, MutableMapping[str, JsonSerializable]): """Represents the `user_data` part of a Request. @@ -75,9 +78,6 @@ class UserData(BaseModel, MutableMapping[str, JsonSerializable]): label: Annotated[str | None, Field()] = None """Label used for request routing.""" - session_id: Annotated[str | None, Field()] = None - """ID of a session to which the request is bound.""" - def __getitem__(self, key: str) -> JsonSerializable: return self.__pydantic_extra__[key] @@ -88,12 +88,6 @@ def __setitem__(self, key: str, value: JsonSerializable) -> None: self.label = value - if key == 'session_id': - if value is not None and not isinstance(value, str): - raise ValueError('`session_id` must be str or None') - - self.session_id = value - self.__pydantic_extra__[key] = value def __delitem__(self, key: str) -> None: @@ -312,7 +306,7 @@ def from_url( request.user_data['label'] = label if session_id is not None: - request.user_data['session_id'] = session_id + request.crawlee_data.session_id = session_id return request @@ -329,7 +323,7 @@ def label(self) -> str | None: @property def session_id(self) -> str | None: """A string used to identify the bound session.""" - return cast(UserData, self.user_data).session_id + return self.crawlee_data.session_id @property def crawlee_data(self) -> CrawleeRequestData: diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 489f70f88e..ab1f6506c9 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -1215,7 +1215,7 @@ async def handler(context: BasicCrawlingContext) -> None: await crawler.run(requests) assert len(used_sessions) == 10 - assert all(session_id == check_session.id for session_id in used_sessions) + assert set(used_sessions) == {check_session.id} async def test_error_bound_session_to_request() -> None: From 7fb5fde69cfe7e0b194796fc13e11d9715e5bdf8 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 20 Mar 2025 01:21:14 +0000 Subject: [PATCH 05/15] add docs and examples --- .../session_management/multi_sessions_http.py | 86 +++++++++++++++++++ .../session_management/one_session_http.py | 56 ++++++++++++ docs/guides/session_management.mdx | 24 ++++++ 3 files changed, 166 insertions(+) create mode 100644 docs/guides/code_examples/session_management/multi_sessions_http.py create mode 100644 docs/guides/code_examples/session_management/one_session_http.py diff --git a/docs/guides/code_examples/session_management/multi_sessions_http.py b/docs/guides/code_examples/session_management/multi_sessions_http.py new file mode 100644 index 0000000000..013d211f0b --- /dev/null +++ b/docs/guides/code_examples/session_management/multi_sessions_http.py @@ -0,0 +1,86 @@ +import asyncio +from datetime import timedelta +from typing import Callable + +from crawlee import ConcurrencySettings, Request +from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext +from crawlee.errors import RequestCollisionError +from crawlee.sessions import Session, SessionPool + + +# Define a function for creating sessions with simple logic for unique `id` generation. +# This is necessary if you need to specify a particular session for the first request, +# for example during authentication +def create_session_function() -> Callable[[], Session]: + counter = 0 + + def create_session() -> Session: + nonlocal counter + counter += 1 + return Session( + id=str(counter), + max_usage_count=999_999, + max_age=timedelta(hours=999_999), + max_error_score=100, + blocked_status_codes=[403], + ) + + return create_session + + +async def main() -> None: + crawler = HttpCrawler( + # Adjust request limits according to your pool size + concurrency_settings=ConcurrencySettings(max_tasks_per_minute=500), + # Requests are bound to specific sessions, no rotation needed + max_session_rotations=0, + session_pool=SessionPool( + max_pool_size=10, create_session_function=create_session_function() + ), + ) + + @crawler.router.default_handler + async def basic_handler(context: HttpCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url}') + + # Initialize the session and bind the next request to this session if needed + @crawler.router.handler(label='session_init') + async def session_init(context: HttpCrawlingContext) -> None: + next_requests = [] + if context.session: + context.log.info(f'Init session {context.session.id}') + next_request = Request.from_url( + 'https://placeholder.dev', session_id=context.session.id + ) + next_requests.append(next_request) + + await context.add_requests(next_requests) + + # Handle errors when a session is blocked and no longer available in the pool + # when attempting to execute requests bound to it + @crawler.failed_request_handler + async def error_processing(context: BasicCrawlingContext, error: Exception) -> None: + if isinstance(error, RequestCollisionError) and context.session: + context.log.error( + f'Request {context.request.url} failed, because the bound ' + 'session is unavailable' + ) + + # Create a pool of requests bound to their respective sessions + # Use `always_enqueue=True` if session initialization happens on a non-unique address, + # such as the site's main page + init_requests = [ + Request.from_url( + 'https://example.org/', + label='session_init', + session_id=str(session_id), + always_enqueue=True, + ) + for session_id in range(1, 11) + ] + + await crawler.run(init_requests) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/session_management/one_session_http.py b/docs/guides/code_examples/session_management/one_session_http.py new file mode 100644 index 0000000000..28cec44b63 --- /dev/null +++ b/docs/guides/code_examples/session_management/one_session_http.py @@ -0,0 +1,56 @@ +import asyncio +from datetime import timedelta + +from crawlee import ConcurrencySettings, Request +from crawlee.crawlers import BasicCrawlingContext, HttpCrawler, HttpCrawlingContext +from crawlee.errors import SessionError +from crawlee.sessions import SessionPool + + +async def main() -> None: + crawler = HttpCrawler( + # Limit requests per minute to reduce the chance of being blocked + concurrency_settings=ConcurrencySettings(max_tasks_per_minute=50), + # Disable session rotation + max_session_rotations=0, + session_pool=SessionPool( + # Only one session in the pool + max_pool_size=1, + create_session_settings={ + # High value for session usage limit + 'max_usage_count': 999_999, + # High value for session lifetime + 'max_age': timedelta(hours=999_999), + # High score allows the session to encounter more errors + # before crawlee decides the session is blocked + # Make sure you know how to handle these errors + 'max_error_score': 100, + # 403 status usually indicates you're already blocked + 'blocked_status_codes': [403], + }, + ), + ) + + # Basic request handling logic + @crawler.router.default_handler + async def basic_handler(context: HttpCrawlingContext) -> None: + context.log.info(f'Processing {context.request.url}') + + # Handler for session initialization (authentication, initial cookies, etc.) + @crawler.router.handler(label='session_init') + async def session_init(context: HttpCrawlingContext) -> None: + if context.session: + context.log.info(f'Init session {context.session.id}') + + # Monitor if our session gets blocked and explicitly stop the crawler + @crawler.error_handler + async def error_processing(context: BasicCrawlingContext, error: Exception) -> None: + if isinstance(error, SessionError) and context.session: + context.log.info(f'Session {context.session.id} blocked') + crawler.stop() + + await crawler.run([Request.from_url('https://example.org/', label='session_init')]) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdx index f172761204..4bbbb90640 100644 --- a/docs/guides/session_management.mdx +++ b/docs/guides/session_management.mdx @@ -15,6 +15,8 @@ import BeautifulSoupSource from '!!raw-loader!./code_examples/session_management import ParselSource from '!!raw-loader!./code_examples/session_management/session_management_parsel.py'; import PlaywrightSource from '!!raw-loader!./code_examples/session_management/session_management_playwright.py'; import StandaloneSource from '!!raw-loader!./code_examples/session_management/session_management_standalone.py'; +import OneSession from '!!raw-loader!./code_examples/session_management/one_session_http.py'; +import MultiSessions from '!!raw-loader!./code_examples/session_management/multi_sessions_http.py'; The `SessionPool` class provides a robust way to manage the rotation of proxy IP addresses, cookies, and other custom settings in Crawlee. Its primary advantage is the ability to filter out blocked or non-functional proxies, ensuring that your scraper avoids retrying requests through known problematic proxies. @@ -68,3 +70,25 @@ Now, let's explore examples of how to use the `S These examples demonstrate the basics of configuring and using the `SessionPool`. Please, bear in mind that `SessionPool` requires some time to establish a stable pool of working IPs. During the initial setup, you may encounter errors as the pool identifies and filters out blocked or non-functional IPs. This stabilization period is expected and will improve over time. + +## Single Session Configuration + +In some cases, you need fine-grained control over session usage. For example, when working with websites requiring authentication or initialization of certain parameters like cookies. + +When working with a site that requires authentication, we typically don't want multiple sessions with different browser fingerprints or client parameters accessing the site. In this case, we need to configure the `SessionPool` appropriately: + + + {OneSession} + + +## Binding Requests to Specific Sessions + +In the previous example, there's one obvious limitation - you're restricted to only one session. + +In some cases, we need to achieve the same behavior but using multiple sessions in parallel, such as authenticating with different profiles or using different proxies. + +To do this, use the `session_id` parameter for the `Request` object to bind a request to a specific session: + + + {MultiSessions} + From 300cf4f8f8fbcd2c91216334b6bf2122572bd5d1 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 24 Mar 2025 13:26:23 +0100 Subject: [PATCH 06/15] Polishment --- docs/guides/session_management.mdx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/guides/session_management.mdx b/docs/guides/session_management.mdx index 4bbbb90640..20b4dfd1b2 100644 --- a/docs/guides/session_management.mdx +++ b/docs/guides/session_management.mdx @@ -71,9 +71,9 @@ These examples demonstrate the basics of configuring and using the `SessionPool` requires some time to establish a stable pool of working IPs. During the initial setup, you may encounter errors as the pool identifies and filters out blocked or non-functional IPs. This stabilization period is expected and will improve over time. -## Single Session Configuration +## Configuring a single session -In some cases, you need fine-grained control over session usage. For example, when working with websites requiring authentication or initialization of certain parameters like cookies. +In some cases, you need full control over session usage. For example, when working with websites requiring authentication or initialization of certain parameters like cookies. When working with a site that requires authentication, we typically don't want multiple sessions with different browser fingerprints or client parameters accessing the site. In this case, we need to configure the `SessionPool` appropriately: @@ -81,7 +81,7 @@ When working with a site that requires authentication, we typically don't want m {OneSession} -## Binding Requests to Specific Sessions +## Binding requests to specific sessions In the previous example, there's one obvious limitation - you're restricted to only one session. From 603782990a625a06f082aebe528a0fbc746a3127 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 24 Mar 2025 13:30:25 +0100 Subject: [PATCH 07/15] Update src/crawlee/_request.py --- src/crawlee/_request.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crawlee/_request.py b/src/crawlee/_request.py index 05154dd049..63a1eff24c 100644 --- a/src/crawlee/_request.py +++ b/src/crawlee/_request.py @@ -254,8 +254,8 @@ def from_url( payload: The data to be sent as the request body. Typically used with 'POST' or 'PUT' requests. label: A custom label to differentiate between request types. This is stored in `user_data`, and it is used for request routing (different requests go to different handlers). - session_id: ID of a specific Session to which the request will be strictly bound. - If the session becomes unavailable when the request is processed, a RequestCollisionError will be + session_id: ID of a specific `Session` to which the request will be strictly bound. + If the `Session` becomes unavailable when the request is processed, a `RequestCollisionError` will be raised. unique_key: A unique key identifying the request. If not provided, it is automatically computed based on the URL and other parameters. Requests with the same `unique_key` are treated as identical. From 5e94912fbdf1a972e296a21b15da259b440d20f5 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 24 Mar 2025 13:30:31 +0100 Subject: [PATCH 08/15] Update src/crawlee/_request.py --- src/crawlee/_request.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crawlee/_request.py b/src/crawlee/_request.py index 63a1eff24c..0659c4e407 100644 --- a/src/crawlee/_request.py +++ b/src/crawlee/_request.py @@ -322,7 +322,7 @@ def label(self) -> str | None: @property def session_id(self) -> str | None: - """A string used to identify the bound session.""" + """The ID of the bound session, if there is any.""" return self.crawlee_data.session_id @property From 25a21a0cb8093cade7ec0a15d0684505ffd7e64c Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 24 Mar 2025 13:30:39 +0100 Subject: [PATCH 09/15] Update src/crawlee/crawlers/_basic/_basic_crawler.py --- src/crawlee/crawlers/_basic/_basic_crawler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 544fa6e32f..abe15ff33b 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -1233,11 +1233,11 @@ def _raise_request_collision(self, request: Request, session: Session | None) -> """Raise an exception if a request cannot access required resources. Args: - request: The Request that might require specific resources (like a session). - session: The Session object that was retrieved for the request, or None if not available. + request: The `Request` that might require specific resources (like a session). + session: The `Session` that was retrieved for the request, or `None` if not available. Raises: - RequestCollisionError: If the Session referenced by the request is not available. + RequestCollisionError: If the `Session` referenced by the `Request` is not available. """ if self._use_session_pool and request.session_id and not session: raise RequestCollisionError( From 14b897491e251ed2498a34bd6b01f0c8e6cd31e9 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 27 Mar 2025 13:17:42 +0000 Subject: [PATCH 10/15] add session_id in extended_unique_key --- .../session_management/multi_sessions_http.py | 2 +- src/crawlee/_request.py | 7 ++-- src/crawlee/_utils/requests.py | 5 ++- .../crawlers/_basic/test_basic_crawler.py | 37 +++++++++++++++++-- 4 files changed, 43 insertions(+), 8 deletions(-) diff --git a/docs/guides/code_examples/session_management/multi_sessions_http.py b/docs/guides/code_examples/session_management/multi_sessions_http.py index 013d211f0b..399ab6f442 100644 --- a/docs/guides/code_examples/session_management/multi_sessions_http.py +++ b/docs/guides/code_examples/session_management/multi_sessions_http.py @@ -74,7 +74,7 @@ async def error_processing(context: BasicCrawlingContext, error: Exception) -> N 'https://example.org/', label='session_init', session_id=str(session_id), - always_enqueue=True, + use_extended_unique_key=True, ) for session_id in range(1, 11) ] diff --git a/src/crawlee/_request.py b/src/crawlee/_request.py index 0659c4e407..adb43949ea 100644 --- a/src/crawlee/_request.py +++ b/src/crawlee/_request.py @@ -255,7 +255,7 @@ def from_url( label: A custom label to differentiate between request types. This is stored in `user_data`, and it is used for request routing (different requests go to different handlers). session_id: ID of a specific `Session` to which the request will be strictly bound. - If the `Session` becomes unavailable when the request is processed, a `RequestCollisionError` will be + If the session becomes unavailable when the request is processed, a `RequestCollisionError` will be raised. unique_key: A unique key identifying the request. If not provided, it is automatically computed based on the URL and other parameters. Requests with the same `unique_key` are treated as identical. @@ -263,8 +263,8 @@ def from_url( `unique_key`. keep_url_fragment: Determines whether the URL fragment (e.g., `#section`) should be included in the `unique_key` computation. This is only relevant when `unique_key` is not provided. - use_extended_unique_key: Determines whether to include the HTTP method and payload in the `unique_key` - computation. This is only relevant when `unique_key` is not provided. + use_extended_unique_key: Determines whether to include the HTTP method, ID Session and payload in the + `unique_key` computation. This is only relevant when `unique_key` is not provided. always_enqueue: If set to `True`, the request will be enqueued even if it is already present in the queue. Using this is not allowed when a custom `unique_key` is also provided and will result in a `ValueError`. **kwargs: Additional request properties. @@ -283,6 +283,7 @@ def from_url( method=method, headers=headers, payload=payload, + session_id=session_id, keep_url_fragment=keep_url_fragment, use_extended_unique_key=use_extended_unique_key, ) diff --git a/src/crawlee/_utils/requests.py b/src/crawlee/_utils/requests.py index aed4664765..3723ad4e8b 100644 --- a/src/crawlee/_utils/requests.py +++ b/src/crawlee/_utils/requests.py @@ -78,6 +78,7 @@ def compute_unique_key( method: HttpMethod = 'GET', headers: HttpHeaders | None = None, payload: HttpPayload | None = None, + session_id: str | None = None, *, keep_url_fragment: bool = False, use_extended_unique_key: bool = False, @@ -96,6 +97,7 @@ def compute_unique_key( payload: The data to be sent as the request body. keep_url_fragment: A flag indicating whether to keep the URL fragment. use_extended_unique_key: A flag indicating whether to include a hashed payload in the key. + session_id: The ID of a specific `Session` to which the request will be strictly bound Returns: A string representing the unique key for the request. @@ -114,9 +116,10 @@ def compute_unique_key( if use_extended_unique_key: payload_hash = _get_payload_hash(payload) headers_hash = _get_headers_hash(headers) + normilizead_session = '' if session_id is None else session_id.lower() # Return the extended unique key. Use pipe as a separator of the different parts of the unique key. - return f'{normalized_method}|{headers_hash}|{payload_hash}|{normalized_url}' + return f'{normalized_method}|{headers_hash}|{payload_hash}|{normilizead_session}|{normalized_url}' # Log information if there is a non-GET request with a payload. if normalized_method != 'GET' and payload: diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 2cc59601ab..f8f3b26e56 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -25,19 +25,18 @@ from crawlee.errors import RequestCollisionError, SessionError, UserDefinedErrorHandlerError from crawlee.events._local_event_manager import LocalEventManager from crawlee.request_loaders import RequestList, RequestManagerTandem -from crawlee.sessions import SessionPool +from crawlee.sessions import Session, SessionPool from crawlee.statistics import FinalStatistics from crawlee.storage_clients import MemoryStorageClient from crawlee.storages import Dataset, KeyValueStore, RequestQueue if TYPE_CHECKING: - from collections.abc import Sequence + from collections.abc import Callable, Sequence import respx from yarl import URL from crawlee._types import JsonSerializable - from crawlee.sessions._session import Session from crawlee.storage_clients._memory import DatasetClient @@ -1218,6 +1217,38 @@ async def handler(context: BasicCrawlingContext) -> None: assert set(used_sessions) == {check_session.id} +async def test_bound_sessions_to_same_request() -> None: + # Use a custom function to avoid errors due to random Session retrieval + def create_session_function() -> Callable[[], Session]: + counter = -1 + + def create_session() -> Session: + nonlocal counter + counter += 1 + return Session(id=str(counter)) + + return create_session + + check_sessions = [str(session_id) for session_id in range(10)] + used_sessions = list[str]() + crawler = BasicCrawler(session_pool=SessionPool(create_session_function=create_session_function())) + + @crawler.router.default_handler + async def handler(context: BasicCrawlingContext) -> None: + if context.session: + used_sessions.append(context.session.id) + + requests = [ + Request.from_url('http://a.com/', session_id=str(session_id), use_extended_unique_key=True) + for session_id in range(10) + ] + + await crawler.run(requests) + + assert len(used_sessions) == 10 + assert set(used_sessions) == set(check_sessions) + + async def test_error_bound_session_to_request() -> None: crawler = BasicCrawler(request_handler=AsyncMock()) From b5fc9a47a0c5e292e200311b6363c7531d61e7c5 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 27 Mar 2025 13:37:30 +0000 Subject: [PATCH 11/15] remove empty part from extended if session_id empty --- src/crawlee/_utils/requests.py | 5 ++++- tests/unit/_utils/test_requests.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/crawlee/_utils/requests.py b/src/crawlee/_utils/requests.py index 9d9a490938..1af01f352a 100644 --- a/src/crawlee/_utils/requests.py +++ b/src/crawlee/_utils/requests.py @@ -119,7 +119,10 @@ def compute_unique_key( normilizead_session = '' if session_id is None else session_id.lower() # Return the extended unique key. Use pipe as a separator of the different parts of the unique key. - return f'{normalized_method}|{headers_hash}|{payload_hash}|{normilizead_session}|{normalized_url}' + extended_part = f'{normalized_method}|{headers_hash}|{payload_hash}' + if normilizead_session: + extended_part = f'{extended_part}|{normilizead_session}' + return f'{extended_part}|{normalized_url}' # Log information if there is a non-GET request with a payload. if normalized_method != 'GET' and payload: diff --git a/tests/unit/_utils/test_requests.py b/tests/unit/_utils/test_requests.py index 4092b29e05..348ae95948 100644 --- a/tests/unit/_utils/test_requests.py +++ b/tests/unit/_utils/test_requests.py @@ -133,6 +133,7 @@ def test_compute_unique_key_complex() -> None: method='POST', headers=headers, payload=payload, + session_id='test_session', use_extended_unique_key=False, ) assert uk == url @@ -142,9 +143,10 @@ def test_compute_unique_key_complex() -> None: method='POST', headers=headers, payload=payload, + session_id='test_session', use_extended_unique_key=True, ) - assert extended_uk == 'POST|4e1a2cf6|9724c1e2|https://crawlee.dev' + assert extended_uk == 'POST|4e1a2cf6|9724c1e2|test_session|https://crawlee.dev' def test_compute_unique_key_post_with_none_payload() -> None: From 1a8cd228297cf8e2c8d81dd0441a2d883aa01f24 Mon Sep 17 00:00:00 2001 From: Max Bohomolov <34358312+Mantisus@users.noreply.github.com> Date: Mon, 31 Mar 2025 15:14:45 +0300 Subject: [PATCH 12/15] Update docs/guides/code_examples/session_management/multi_sessions_http.py Co-authored-by: Jan Buchar --- .../session_management/multi_sessions_http.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/guides/code_examples/session_management/multi_sessions_http.py b/docs/guides/code_examples/session_management/multi_sessions_http.py index 399ab6f442..aba4d5e9d0 100644 --- a/docs/guides/code_examples/session_management/multi_sessions_http.py +++ b/docs/guides/code_examples/session_management/multi_sessions_http.py @@ -1,5 +1,6 @@ import asyncio from datetime import timedelta +from itertools import count from typing import Callable from crawlee import ConcurrencySettings, Request @@ -12,13 +13,11 @@ # This is necessary if you need to specify a particular session for the first request, # for example during authentication def create_session_function() -> Callable[[], Session]: - counter = 0 + counter = count() def create_session() -> Session: - nonlocal counter - counter += 1 return Session( - id=str(counter), + id=str(next(counter)), max_usage_count=999_999, max_age=timedelta(hours=999_999), max_error_score=100, From b5d1dc54f6280647af381054b40425d96908c737 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Mon, 31 Mar 2025 12:33:09 +0000 Subject: [PATCH 13/15] fix typo --- src/crawlee/_utils/requests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crawlee/_utils/requests.py b/src/crawlee/_utils/requests.py index 1af01f352a..cea93376d0 100644 --- a/src/crawlee/_utils/requests.py +++ b/src/crawlee/_utils/requests.py @@ -116,12 +116,12 @@ def compute_unique_key( if use_extended_unique_key: payload_hash = _get_payload_hash(payload) headers_hash = _get_headers_hash(headers) - normilizead_session = '' if session_id is None else session_id.lower() + normilized_session = '' if session_id is None else session_id.lower() # Return the extended unique key. Use pipe as a separator of the different parts of the unique key. extended_part = f'{normalized_method}|{headers_hash}|{payload_hash}' - if normilizead_session: - extended_part = f'{extended_part}|{normilizead_session}' + if normilized_session: + extended_part = f'{extended_part}|{normilized_session}' return f'{extended_part}|{normalized_url}' # Log information if there is a non-GET request with a payload. From 9557ac536b322ee02d87cbcb6036afb175f32860 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Mon, 31 Mar 2025 12:59:37 +0000 Subject: [PATCH 14/15] rename `_raise_request_collision` to `_check_request_collision` --- src/crawlee/crawlers/_basic/_basic_crawler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index 8ae5fd16ce..8c76da798e 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -1106,7 +1106,7 @@ async def __run_task_function(self) -> None: try: request.state = RequestState.REQUEST_HANDLER - self._raise_request_collision(context.request, context.session) + self._check_request_collision(context.request, context.session) try: await self._run_request_handler(context=context) @@ -1251,7 +1251,7 @@ def _raise_for_session_blocked_status_code(self, session: Session | None, status ): raise SessionError(f'Assuming the session is blocked based on HTTP status code {status_code}') - def _raise_request_collision(self, request: Request, session: Session | None) -> None: + def _check_request_collision(self, request: Request, session: Session | None) -> None: """Raise an exception if a request cannot access required resources. Args: From 448f9e6d3778719e2903835012d88dd5e1063312 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Mon, 31 Mar 2025 13:24:31 +0000 Subject: [PATCH 15/15] fix typo --- src/crawlee/_utils/requests.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crawlee/_utils/requests.py b/src/crawlee/_utils/requests.py index cea93376d0..a3b152a51d 100644 --- a/src/crawlee/_utils/requests.py +++ b/src/crawlee/_utils/requests.py @@ -116,12 +116,12 @@ def compute_unique_key( if use_extended_unique_key: payload_hash = _get_payload_hash(payload) headers_hash = _get_headers_hash(headers) - normilized_session = '' if session_id is None else session_id.lower() + normalized_session = '' if session_id is None else session_id.lower() # Return the extended unique key. Use pipe as a separator of the different parts of the unique key. extended_part = f'{normalized_method}|{headers_hash}|{payload_hash}' - if normilized_session: - extended_part = f'{extended_part}|{normilized_session}' + if normalized_session: + extended_part = f'{extended_part}|{normalized_session}' return f'{extended_part}|{normalized_url}' # Log information if there is a non-GET request with a payload.