diff --git a/docs/upgrading/upgrading_to_v0x.md b/docs/upgrading/upgrading_to_v0x.md index a0a4789af7..eb51ee0925 100644 --- a/docs/upgrading/upgrading_to_v0x.md +++ b/docs/upgrading/upgrading_to_v0x.md @@ -13,6 +13,10 @@ This section summarizes the breaking changes between v0.5.x and v0.6.0. - Removed `HttpCrawlerOptions` - which contained options from `BasicCrawlerOptions` and unique options `additional_http_error_status_codes` and `ignore_http_error_status_codes`. Both of the unique options were added to `BasicCrawlerOptions` instead. +### SessionCookies + +- Replaces the `dict` used for cookie storage in `Session.cookies` with a new `SessionCookies` class. `SessionCookies` uses `CookieJar`, which enables support for multiple domains. + ### PlaywrightCrawler and PlaywrightBrowserPlugin - `PlaywrightCrawler` now use a persistent browser context instead of the standard browser context. diff --git a/src/crawlee/crawlers/_playwright/_playwright_crawler.py b/src/crawlee/crawlers/_playwright/_playwright_crawler.py index c16a536c8f..d5c7e7e203 100644 --- a/src/crawlee/crawlers/_playwright/_playwright_crawler.py +++ b/src/crawlee/crawlers/_playwright/_playwright_crawler.py @@ -6,7 +6,6 @@ from pydantic import ValidationError from typing_extensions import NotRequired, TypedDict, TypeVar -from yarl import URL from crawlee import EnqueueStrategy, RequestTransformAction from crawlee._request import Request, RequestOptions @@ -16,6 +15,7 @@ from crawlee.browsers import BrowserPool from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline from crawlee.errors import SessionError +from crawlee.sessions._cookies import PlaywrightCookieParam from crawlee.statistics import StatisticsState from ._playwright_crawling_context import PlaywrightCrawlingContext @@ -208,7 +208,8 @@ async def _navigate( """ async with context.page: if context.session: - await self._set_cookies(context.page, context.request.url, context.session.cookies) + session_cookies = context.session.cookies.get_cookies_as_playwright_format() + await self._update_cookies(context.page, session_cookies) if context.request.headers: await context.page.set_extra_http_headers(context.request.headers.model_dump()) @@ -222,8 +223,8 @@ async def _navigate( context.request.loaded_url = context.page.url if context.session: - cookies = await self._get_cookies(context.page) - context.session.cookies.update(cookies) + pw_cookies = await self._get_cookies(context.page) + context.session.cookies.set_cookies_from_playwright_format(pw_cookies) async def enqueue_links( *, @@ -335,17 +336,14 @@ def pre_navigation_hook(self, hook: Callable[[PlaywrightPreNavCrawlingContext], """ self._pre_navigation_hooks.append(hook) - async def _get_cookies(self, page: Page) -> dict[str, str]: + async def _get_cookies(self, page: Page) -> list[PlaywrightCookieParam]: """Get the cookies from the page.""" cookies = await page.context.cookies() - return {cookie['name']: cookie['value'] for cookie in cookies if cookie.get('name') and cookie.get('value')} + return [PlaywrightCookieParam(**cookie) for cookie in cookies] - async def _set_cookies(self, page: Page, url: str, cookies: dict[str, str]) -> None: - """Set the cookies to the page.""" - parsed_url = URL(url) - await page.context.add_cookies( - [{'name': name, 'value': value, 'domain': parsed_url.host, 'path': '/'} for name, value in cookies.items()] - ) + async def _update_cookies(self, page: Page, cookies: list[PlaywrightCookieParam]) -> None: + """Update the cookies in the page context.""" + await page.context.add_cookies([{**cookie} for cookie in cookies]) class _PlaywrightCrawlerAdditionalOptions(TypedDict): diff --git a/src/crawlee/http_clients/_curl_impersonate.py b/src/crawlee/http_clients/_curl_impersonate.py index a2c5a9462d..f75c8ebdd8 100644 --- a/src/crawlee/http_clients/_curl_impersonate.py +++ b/src/crawlee/http_clients/_curl_impersonate.py @@ -5,7 +5,8 @@ from curl_cffi import CurlInfo from curl_cffi.const import CurlHttpVersion from curl_cffi.requests import AsyncSession -from curl_cffi.requests.cookies import Cookies, CurlMorsel +from curl_cffi.requests.cookies import Cookies as CurlCookies +from curl_cffi.requests.cookies import CurlMorsel from curl_cffi.requests.exceptions import ProxyError as CurlProxyError from curl_cffi.requests.exceptions import RequestException as CurlRequestError from curl_cffi.requests.impersonate import DEFAULT_CHROME as CURL_DEFAULT_CHROME @@ -19,6 +20,7 @@ if TYPE_CHECKING: from collections.abc import Iterable + from http.cookiejar import Cookie from curl_cffi import Curl from curl_cffi.requests import Request as CurlRequest @@ -31,7 +33,7 @@ from crawlee.statistics import Statistics -class _EmptyCookies(Cookies): +class _EmptyCookies(CurlCookies): @override def get_cookies_for_curl(self, request: CurlRequest) -> list[CurlMorsel]: return [] @@ -147,7 +149,7 @@ async def crawl( method=request.method.upper(), # type: ignore[arg-type] # curl-cffi requires uppercase method headers=request.headers, data=request.payload, - cookies=session.cookies if session else None, + cookies=session.cookies.jar if session else None, allow_redirects=True, ) except CurlRequestError as exc: @@ -166,7 +168,7 @@ async def crawl( if self._persist_cookies_per_session and session and response.curl: response_cookies = self._get_cookies(response.curl) - session.cookies.update(response_cookies) + session.cookies.store_cookies(response_cookies) request.loaded_url = response.url @@ -197,7 +199,7 @@ async def send_request( method=method.upper(), # type: ignore[arg-type] # curl-cffi requires uppercase method headers=dict(headers) if headers else None, data=payload, - cookies=session.cookies if session else None, + cookies=session.cookies.jar if session else None, allow_redirects=True, ) except CurlRequestError as exc: @@ -213,7 +215,7 @@ async def send_request( if self._persist_cookies_per_session and session and response.curl: response_cookies = self._get_cookies(response.curl) - session.cookies.update(response_cookies) + session.cookies.store_cookies(response_cookies) return _CurlImpersonateResponse(response) @@ -253,9 +255,10 @@ def _is_proxy_error(error: CurlRequestError) -> bool: return False @staticmethod - def _get_cookies(curl: Curl) -> dict[str, str]: - cookies = {} + def _get_cookies(curl: Curl) -> list[Cookie]: + cookies: list[Cookie] = [] for curl_cookie in curl.getinfo(CurlInfo.COOKIELIST): # type: ignore[union-attr] curl_morsel = CurlMorsel.from_curl_format(curl_cookie) # type: ignore[arg-type] - cookies[curl_morsel.name] = curl_morsel.value + cookie = curl_morsel.to_cookiejar_cookie() + cookies.append(cookie) return cookies diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index 2f5941c032..24b4d69383 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -62,9 +62,7 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response: response.request = request if session := cast(Session, request.extensions.get('crawlee_session')): - response_cookies = httpx.Cookies() - response_cookies.extract_cookies(response) - session.cookies.update(response_cookies) + session.cookies.store_cookies(list(response.cookies.jar)) if 'Set-Cookie' in response.headers: del response.headers['Set-Cookie'] @@ -160,7 +158,7 @@ async def crawl( method=request.method, headers=headers, content=request.payload, - cookies=session.cookies if session else None, + cookies=session.cookies.jar if session else None, extensions={'crawlee_session': session if self._persist_cookies_per_session else None}, ) diff --git a/src/crawlee/sessions/__init__.py b/src/crawlee/sessions/__init__.py index af4db03a97..8852122cb4 100644 --- a/src/crawlee/sessions/__init__.py +++ b/src/crawlee/sessions/__init__.py @@ -1,4 +1,5 @@ +from ._cookies import CookieParam, SessionCookies from ._session import Session from ._session_pool import SessionPool -__all__ = ['Session', 'SessionPool'] +__all__ = ['CookieParam', 'Session', 'SessionCookies', 'SessionPool'] diff --git a/src/crawlee/sessions/_cookies.py b/src/crawlee/sessions/_cookies.py new file mode 100644 index 0000000000..33bf63d155 --- /dev/null +++ b/src/crawlee/sessions/_cookies.py @@ -0,0 +1,268 @@ +from __future__ import annotations + +from copy import deepcopy +from http.cookiejar import Cookie, CookieJar +from typing import TYPE_CHECKING, Any, Literal + +from typing_extensions import NotRequired, Required, TypedDict + +from crawlee._utils.docs import docs_group + +if TYPE_CHECKING: + from collections.abc import Iterator + + +@docs_group('Data structures') +class CookieParam(TypedDict, total=False): + """Dictionary representation of cookies for `SessionCookies.set` method.""" + + name: Required[str] + """Cookie name.""" + + value: Required[str] + """Cookie value.""" + + domain: NotRequired[str] + """Domain for which the cookie is set.""" + + path: NotRequired[str] + """Path on the specified domain for which the cookie is set.""" + + secure: NotRequired[bool] + """Set the `Secure` flag for the cookie.""" + + http_only: NotRequired[bool] + """Set the `HttpOnly` flag for the cookie.""" + + expires: NotRequired[int] + """Expiration date for the cookie, None for a session cookie.""" + + same_site: NotRequired[Literal['Lax', 'None', 'Strict']] + """Set the `SameSite` attribute for the cookie.""" + + +class PlaywrightCookieParam(TypedDict, total=False): + """Cookie parameters in Playwright format with camelCase naming.""" + + name: NotRequired[str] + value: NotRequired[str] + domain: NotRequired[str] + path: NotRequired[str] + secure: NotRequired[bool] + httpOnly: NotRequired[bool] + expires: NotRequired[float] + sameSite: NotRequired[Literal['Lax', 'None', 'Strict']] + + +@docs_group('Data structures') +class SessionCookies: + """Storage cookies for session with browser-compatible serialization and deserialization.""" + + def __init__(self, cookies: SessionCookies | CookieJar | dict[str, str] | list[CookieParam] | None = None) -> None: + if isinstance(cookies, CookieJar): + self._jar = cookies + return + + self._jar = CookieJar() + + if isinstance(cookies, dict): + for key, value in cookies.items(): + self.set(key, value) + + elif isinstance(cookies, list): + for item in cookies: + self.set(**item) + + elif isinstance(cookies, SessionCookies): + for cookie in cookies.jar: + self.jar.set_cookie(cookie) + + @property + def jar(self) -> CookieJar: + """The cookie jar instance.""" + return self._jar + + def set( + self, + name: str, + value: str, + *, + domain: str = '', + path: str = '/', + expires: int | None = None, + http_only: bool = False, + secure: bool = False, + same_site: Literal['Lax', 'None', 'Strict'] | None = None, + ) -> None: + """Create and store a cookie with modern browser attributes. + + Args: + name: Cookie name. + value: Cookie value. + domain: Cookie domain. + path: Cookie path. + expires: Cookie expiration timestamp. + http_only: Whether cookie is HTTP-only. + secure: Whether cookie requires secure context. + same_site: SameSite cookie attribute value. + """ + cookie = Cookie( + version=0, + name=name, + value=value, + port=None, + port_specified=False, + domain=domain, + domain_specified=bool(domain), + domain_initial_dot=domain.startswith('.'), + path=path, + path_specified=bool(path), + secure=secure, + expires=expires, + discard=True, + comment=None, + comment_url=None, + rest={'HttpOnly': ''} if http_only else {}, + rfc2109=False, + ) + + if same_site: + cookie.set_nonstandard_attr('SameSite', same_site) + + self.jar.set_cookie(cookie) + + def _convert_cookie_to_dict(self, cookie: Cookie) -> CookieParam: + """Convert `http.cookiejar.Cookie` to dictionary format. + + Args: + cookie: Cookie object to convert. + """ + cookie_dict = CookieParam( + name=cookie.name, + value=cookie.value if cookie.value else '', + domain=cookie.domain, + path=cookie.path, + secure=cookie.secure, + http_only=cookie.has_nonstandard_attr('HttpOnly'), + ) + + if cookie.expires: + cookie_dict['expires'] = cookie.expires + + if (same_site := cookie.get_nonstandard_attr('SameSite')) and same_site in {'Lax', 'None', 'Strict'}: + cookie_dict['same_site'] = same_site # type: ignore[typeddict-item] + + return cookie_dict + + def _to_playwright(self, cookie_dict: CookieParam) -> PlaywrightCookieParam: + """Convert internal cookie to Playwright format.""" + result: dict = dict(cookie_dict) + + if 'http_only' in result: + result['httpOnly'] = result.pop('http_only') + if 'same_site' in result: + result['sameSite'] = result.pop('same_site') + if 'expires' in result: + result['expires'] = float(result['expires']) + + return PlaywrightCookieParam(**result) + + def _from_playwright(self, cookie_dict: PlaywrightCookieParam) -> CookieParam: + """Convert Playwright cookie to internal format.""" + result: dict = dict(cookie_dict) + + if 'httpOnly' in result: + result['http_only'] = result.pop('httpOnly') + if 'sameSite' in result: + result['same_site'] = result.pop('sameSite') + if 'expires' in result: + expires = int(result['expires']) + result['expires'] = None if expires == -1 else expires + + return CookieParam(name=result.pop('name', ''), value=result.pop('value', ''), **result) + + def get_cookies_as_dicts(self) -> list[CookieParam]: + """Convert cookies to a list with `CookieParam` dicts.""" + return [self._convert_cookie_to_dict(cookie) for cookie in self.jar] + + def store_cookie(self, cookie: Cookie) -> None: + """Store a Cookie object in the session cookie jar. + + Args: + cookie: The Cookie object to store in the jar. + """ + self.jar.set_cookie(cookie) + + def store_cookies(self, cookies: list[Cookie]) -> None: + """Store multiple cookie objects in the session cookie jar. + + Args: + cookies: A list of cookie objects to store in the jar. + """ + for cookie in cookies: + self.store_cookie(cookie) + self._jar.clear_expired_cookies() + + def set_cookies(self, cookie_dicts: list[CookieParam]) -> None: + """Create and store cookies from their dictionary representations. + + Args: + cookie_dicts: List of dictionaries where each dict represents cookie parameters. + """ + for cookie_dict in cookie_dicts: + self.set(**cookie_dict) + self._jar.clear_expired_cookies() + + def get_cookies_as_playwright_format(self) -> list[PlaywrightCookieParam]: + """Get cookies in playwright format.""" + return [self._to_playwright(cookie) for cookie in self.get_cookies_as_dicts()] + + def set_cookies_from_playwright_format(self, pw_cookies: list[PlaywrightCookieParam]) -> None: + """Set cookies from playwright format.""" + for pw_cookie in pw_cookies: + cookie_param = self._from_playwright(pw_cookie) + self.set(**cookie_param) + self._jar.clear_expired_cookies() + + def __deepcopy__(self, memo: dict[int, Any] | None) -> SessionCookies: + # This is necessary because `CookieJar` use `RLock`, which prevents `deepcopy`. + cookie_dicts = self.get_cookies_as_dicts() + return self.__class__(deepcopy(cookie_dicts, memo)) + + def __len__(self) -> int: + return len(self._jar) + + def __setitem__(self, name: str, value: str) -> None: + self.set(name, value) + + def __getitem__(self, name: str) -> str | None: + for cookie in self._jar: + if cookie.name == name: + return cookie.value + raise KeyError(f"Cookie '{name}' not found") + + def __iter__(self) -> Iterator[CookieParam]: + return (self._convert_cookie_to_dict(cookie) for cookie in self._jar) + + def __repr__(self) -> str: + cookies_str: str = ', '.join( + [f'' for cookie in self._jar] + ) + return f'' + + def __bool__(self) -> bool: + for _ in self._jar: + return True + return False + + def __eq__(self, other: object) -> bool: + if not isinstance(other, SessionCookies): + return NotImplemented + + if len(self) != len(other): + return False + + self_keys = {(cookie.name, cookie.value, cookie.domain, cookie.path) for cookie in self._jar} + other_keys = {(cookie.name, cookie.value, cookie.domain, cookie.path) for cookie in other.jar} + + return self_keys == other_keys diff --git a/src/crawlee/sessions/_models.py b/src/crawlee/sessions/_models.py index b4efdb4542..1e28ddd2ce 100644 --- a/src/crawlee/sessions/_models.py +++ b/src/crawlee/sessions/_models.py @@ -5,6 +5,8 @@ from pydantic import BaseModel, ConfigDict, Field +from ._cookies import CookieParam + class SessionModel(BaseModel): """Model for a Session object.""" @@ -20,7 +22,7 @@ class SessionModel(BaseModel): usage_count: Annotated[int, Field(alias='usageCount')] max_usage_count: Annotated[int, Field(alias='maxUsageCount')] error_score: Annotated[float, Field(alias='errorScore')] - cookies: Annotated[dict, Field(alias='cookies')] + cookies: Annotated[list[CookieParam], Field(alias='cookies')] blocked_status_codes: Annotated[list[int], Field(alias='blockedStatusCodes')] diff --git a/src/crawlee/sessions/_session.py b/src/crawlee/sessions/_session.py index 8acb5500b1..295ef46b74 100644 --- a/src/crawlee/sessions/_session.py +++ b/src/crawlee/sessions/_session.py @@ -4,12 +4,16 @@ from datetime import datetime, timedelta, timezone from logging import getLogger -from typing import ClassVar, Literal, overload +from typing import TYPE_CHECKING, ClassVar, Literal, overload from crawlee._utils.crypto import crypto_random_object_id from crawlee._utils.docs import docs_group +from crawlee.sessions._cookies import CookieParam, SessionCookies from crawlee.sessions._models import SessionModel +if TYPE_CHECKING: + from http.cookiejar import CookieJar + logger = getLogger(__name__) @@ -38,7 +42,7 @@ def __init__( usage_count: int = 0, max_usage_count: int = 50, error_score: float = 0.0, - cookies: dict[str, str] | None = None, + cookies: SessionCookies | CookieJar | dict[str, str] | list[CookieParam] | None = None, blocked_status_codes: list | None = None, ) -> None: """A default constructor. @@ -65,13 +69,14 @@ def __init__( self._usage_count = usage_count self._max_usage_count = max_usage_count self._error_score = error_score - self._cookies = cookies or {} + self._cookies = SessionCookies(cookies) or SessionCookies() self._blocked_status_codes = set(blocked_status_codes or self._DEFAULT_BLOCKED_STATUS_CODES) @classmethod def from_model(cls, model: SessionModel) -> Session: """Create a new instance from a `SessionModel`.""" - return cls(**model.model_dump()) + cookies = SessionCookies(model.cookies) + return cls(**model.model_dump(exclude={'cookies'}), cookies=cookies) def __repr__(self) -> str: """Get a string representation.""" @@ -94,7 +99,7 @@ def user_data(self) -> dict: return self._user_data @property - def cookies(self) -> dict[str, str]: + def cookies(self) -> SessionCookies: """Get the cookies.""" return self._cookies @@ -151,7 +156,7 @@ def get_state(self, *, as_dict: bool = False) -> SessionModel | dict: usage_count=self._usage_count, max_usage_count=self._max_usage_count, error_score=self._error_score, - cookies=self._cookies, + cookies=self._cookies.get_cookies_as_dicts(), blocked_status_codes=self._blocked_status_codes, ) if as_dict: diff --git a/tests/unit/crawlers/_http/test_http_crawler.py b/tests/unit/crawlers/_http/test_http_crawler.py index 96c38e0603..55307b67a3 100644 --- a/tests/unit/crawlers/_http/test_http_crawler.py +++ b/tests/unit/crawlers/_http/test_http_crawler.py @@ -99,6 +99,17 @@ async def server() -> AsyncGenerator[respx.MockRouter, None]: """, ) + mock.get('/set-cookie', name='set_cookie_endpoint').return_value = Response( + 200, + headers=[ + ('set-cookie', 'basic=1; Path=/; HttpOnly; SameSite=Lax'), + ('set-cookie', 'withpath=2; Path=/html; SameSite=None'), + ('set-cookie', 'strict=3; Path=/; SameSite=Strict'), + ('set-cookie', 'secure=4; Path=/; HttpOnly; Secure; SameSite=Strict'), + ('set-cookie', 'short=5; Path=/;'), + ('set-cookie', 'domain=6; Path=/; Domain=.test.io;'), + ], + ) mock.get('/simple_set_cookies', name='simple_set_cookies').return_value = Response( 200, headers={'set-cookie': 'a=1; Path=/'} @@ -230,7 +241,11 @@ async def handler(context: HttpCrawlingContext) -> None: session = await session_pool.get_session_by_id(session_ids.pop()) assert session is not None - assert session.cookies == {'a': '1', 'b': '2', 'c': '3'} + assert {cookie['name']: cookie['value'] for cookie in session.cookies.get_cookies_as_playwright_format()} == { + 'a': '1', + 'b': '2', + 'c': '3', + } async def test_do_not_retry_on_client_errors(crawler: HttpCrawler, server: respx.MockRouter) -> None: @@ -448,7 +463,9 @@ async def handler(context: HttpCrawlingContext) -> None: if context.request.unique_key not in {'1', '2'}: return - sessions_cookies[context.session.id] = context.session.cookies + sessions_cookies[context.session.id] = { + cookie['name']: cookie['value'] for cookie in context.session.cookies.get_cookies_as_dicts() + } response_data = json.loads(context.http_response.read()) response_cookies[context.session.id] = response_data.get('cookies') @@ -484,69 +501,131 @@ async def handler(context: HttpCrawlingContext) -> None: assert sessions_cookies[clean_session_id] == response_cookies[clean_session_id] == {} -async def test_isolation_cookies_httpx(server: respx.MockRouter) -> None: - """Test isolation cookies for Session with httpx and respx""" - http_client = HttpxHttpClient() - sessions_ids: list[str] = [] - sessions_cookies: dict[str, dict[str, str]] = {} - response_cookies: dict[str, dict[str, str]] = {} +async def test_store_complex_cookies(server: respx.MockRouter) -> None: + visit = Mock() + track_session_usage = Mock() + async with SessionPool(max_pool_size=1) as session_pool: + crawler = HttpCrawler(session_pool=session_pool) - crawler = HttpCrawler( - session_pool=SessionPool(max_pool_size=1), - http_client=http_client, - concurrency_settings=ConcurrencySettings(max_concurrency=1), - ) + @crawler.router.default_handler + async def handler(context: HttpCrawlingContext) -> None: + visit(context.request.url) + track_session_usage(context.session.id if context.session else None) - @crawler.router.default_handler - async def handler(context: HttpCrawlingContext) -> None: - if not context.session: - return + await crawler.run(['https://test.io/set-cookie']) - sessions_ids.append(context.session.id) + assert server['set_cookie_endpoint'].called - if context.request.unique_key not in {'1', '2'}: - return + visited = {call[0][0] for call in visit.call_args_list} + assert len(visited) == 1 - sessions_cookies[context.session.id] = context.session.cookies - response_data = json.loads(context.http_response.read()) - print(response_data) - response_cookies[context.session.id] = response_data.get('cookies') + session_ids = {call[0][0] for call in track_session_usage.call_args_list} + assert len(session_ids) == 1 - if context.request.user_data.get('retire_session'): - context.session.retire() + session = await session_pool.get_session_by_id(session_ids.pop()) + assert session is not None - await crawler.run( - [ - # The first request sets the cookie in the session - 'https://test.io/simple_set_cookies', - # With the second request, we check the cookies in the session and set retire - Request.from_url('https://test.io/get_cookies', unique_key='1', user_data={'retire_session': True}), - # The third request is made with a new session to make sure it does not use another session's cookies - Request.from_url('https://test.io/get_cookies', unique_key='2'), - ] - ) + session_cookies_dict = {cookie['name']: cookie for cookie in session.cookies.get_cookies_as_playwright_format()} + + assert len(session_cookies_dict) == 6 + + # cookie string: 'basic=1; Path=/; HttpOnly; SameSite=Lax' + assert session_cookies_dict['basic'] == { + 'name': 'basic', + 'value': '1', + 'domain': 'test.io', + 'path': '/', + 'secure': False, + 'httpOnly': True, + 'sameSite': 'Lax', + } + + # cookie string: 'withpath=2; Path=/html; SameSite=None' + assert session_cookies_dict['withpath'] == { + 'name': 'withpath', + 'value': '2', + 'domain': 'test.io', + 'path': '/html', + 'secure': False, + 'httpOnly': False, + 'sameSite': 'None', + } + + # cookie string: 'strict=3; Path=/; SameSite=Strict' + assert session_cookies_dict['strict'] == { + 'name': 'strict', + 'value': '3', + 'domain': 'test.io', + 'path': '/', + 'secure': False, + 'httpOnly': False, + 'sameSite': 'Strict', + } + + # cookie string: 'secure=4; Path=/; HttpOnly; Secure; SameSite=Strict' + assert session_cookies_dict['secure'] == { + 'name': 'secure', + 'value': '4', + 'domain': 'test.io', + 'path': '/', + 'secure': True, + 'httpOnly': True, + 'sameSite': 'Strict', + } + + # cookie string: 'short=5; Path=/;' + assert session_cookies_dict['short'] == { + 'name': 'short', + 'value': '5', + 'domain': 'test.io', + 'path': '/', + 'secure': False, + 'httpOnly': False, + } + + assert session_cookies_dict['domain'] == { + 'name': 'domain', + 'value': '6', + 'domain': '.test.io', + 'path': '/', + 'secure': False, + 'httpOnly': False, + } + + +async def test_store_multidomain_cookies(server: respx.MockRouter) -> None: + visit = Mock() + track_session_usage = Mock() + with respx.mock(base_url='https://notest.io') as another_server: + another_server.get('/').return_value = Response(200, headers=[('set-cookie', 'basic=1; Path=/;')]) + async with SessionPool(max_pool_size=1) as session_pool: + crawler = HttpCrawler(session_pool=session_pool) - assert server['simple_set_cookies'].called - assert server['get_cookies'].called + @crawler.router.default_handler + async def handler(context: HttpCrawlingContext) -> None: + visit(context.request.url) + track_session_usage(context.session.id if context.session else None) - assert len(sessions_cookies) == 2 - assert len(response_cookies) == 2 + await crawler.run(['https://test.io/set-cookie', 'https://notest.io/']) - assert sessions_ids[0] == sessions_ids[1] + assert server['set_cookie_endpoint'].called - cookie_session_id = sessions_ids[0] - clean_session_id = sessions_ids[2] + visited = {call[0][0] for call in visit.call_args_list} + assert len(visited) == 2 - assert cookie_session_id != clean_session_id + session_ids = {call[0][0] for call in track_session_usage.call_args_list} + assert len(session_ids) == 1 - # The initiated cookies must match in both the response and the session store - assert sessions_cookies[cookie_session_id] == {'a': '1'} - assert response_cookies[cookie_session_id] == 'a=1' + session = await session_pool.get_session_by_id(session_ids.pop()) + assert session is not None - # For a clean session, the cookie should not be in the session store or in the response - # This way we can be sure that no cookies are being leaked through the http client - assert sessions_cookies[clean_session_id] == {} - assert response_cookies[clean_session_id] == '' + session_cookies_dict = {cookie['domain'] for cookie in session.cookies.get_cookies_as_playwright_format()} + + assert len(session_cookies_dict) == 3 + + assert 'test.io' in session_cookies_dict + assert '.test.io' in session_cookies_dict + assert 'notest.io' in session_cookies_dict def test_default_logger() -> None: diff --git a/tests/unit/crawlers/_playwright/test_playwright_crawler.py b/tests/unit/crawlers/_playwright/test_playwright_crawler.py index 4f7213da33..2765e1c7bd 100644 --- a/tests/unit/crawlers/_playwright/test_playwright_crawler.py +++ b/tests/unit/crawlers/_playwright/test_playwright_crawler.py @@ -317,7 +317,9 @@ async def handler(context: PlaywrightCrawlingContext) -> None: if context.request.unique_key not in {'1', '2'}: return - sessions_cookies[context.session.id] = context.session.cookies + sessions_cookies[context.session.id] = { + cookie['name']: cookie['value'] for cookie in context.session.cookies.get_cookies_as_dicts() + } response_data = json.loads(await context.response.text()) response_cookies[context.session.id] = response_data.get('cookies') diff --git a/tests/unit/sessions/test_cookies.py b/tests/unit/sessions/test_cookies.py new file mode 100644 index 0000000000..0c2042688d --- /dev/null +++ b/tests/unit/sessions/test_cookies.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +import pytest + +from crawlee.sessions._cookies import CookieParam, PlaywrightCookieParam, SessionCookies + + +@pytest.fixture +def cookie_dict() -> CookieParam: + return CookieParam( + { + 'name': 'test_cookie', + 'value': 'test_value', + 'domain': 'example.com', + 'path': '/test', + 'expires': 1735689600, + 'http_only': True, + 'secure': True, + 'same_site': 'Strict', + } + ) + + +@pytest.fixture +def session_cookies(cookie_dict: CookieParam) -> SessionCookies: + session_cookies = SessionCookies() + session_cookies.set(**cookie_dict) + return session_cookies + + +def test_set_basic_cookie() -> None: + """Test setting a basic cookie with minimal attributes.""" + session_cookies = SessionCookies() + session_cookies.set('test', 'value') + cookies = list(session_cookies.jar) + + assert len(cookies) == 1 + cookie = cookies[0] + assert cookie.name == 'test' + assert cookie.value == 'value' + assert cookie.path == '/' + assert not cookie.secure + assert not cookie.has_nonstandard_attr('httpOnpy') + + +def test_set_cookie_with_all_attributes(session_cookies: SessionCookies, cookie_dict: CookieParam) -> None: + """Test setting a cookie with all available attributes.""" + cookies = list(session_cookies.jar) + + assert len(cookies) == 1 + cookie = cookies[0] + + assert cookie.name == cookie_dict.get('name') + assert cookie.value == cookie_dict.get('value') + assert cookie.path == cookie_dict.get('path') + assert cookie.domain == cookie_dict.get('domain') + assert cookie.expires == cookie_dict.get('expires') + assert cookie.has_nonstandard_attr('HttpOnly') + assert cookie.secure + assert cookie.get_nonstandard_attr('SameSite') == 'Strict' + + +def test_convert_cookie_to_dict(session_cookies: SessionCookies, cookie_dict: CookieParam) -> None: + """Test converting Cookie object to dictionary representation.""" + cookies = list(session_cookies.jar) + + assert len(cookies) == 1 + cookie = cookies[0] + + converted_cookie_dict = session_cookies._convert_cookie_to_dict(cookie) + assert converted_cookie_dict == cookie_dict + + +def test_convert_dict_format(session_cookies: SessionCookies) -> None: + """Test normalizing cookie attributes between internal and browser formats.""" + internal_format = CookieParam({'name': 'test', 'value': 'value', 'http_only': True, 'same_site': 'Lax'}) + + # Test internal to browser format + browser_format = session_cookies._to_playwright(internal_format) + assert 'httpOnly' in browser_format + assert 'sameSite' in browser_format + assert 'http_only' not in browser_format + assert 'same_site' not in browser_format + + # Test browser to internal format + browser_format = PlaywrightCookieParam({'name': 'test', 'value': 'value', 'httpOnly': True, 'sameSite': 'Lax'}) + internal_format = session_cookies._from_playwright(browser_format) + assert 'http_only' in internal_format + assert 'same_site' in internal_format + assert 'httpOnly' not in internal_format + assert 'sameSite' not in internal_format + + +def test_get_cookies_as_browser_format(session_cookies: SessionCookies, cookie_dict: CookieParam) -> None: + """Test getting cookies in browser-compatible format.""" + browser_cookies = session_cookies.get_cookies_as_playwright_format() + + assert len(browser_cookies) == 1 + cookie = browser_cookies[0] + assert 'httpOnly' in cookie + assert 'sameSite' in cookie + assert cookie['httpOnly'] == cookie_dict.get('http_only') + assert cookie['sameSite'] == cookie_dict.get('same_site') + + +def test_get_cookies_as_dicts(session_cookies: SessionCookies, cookie_dict: CookieParam) -> None: + """Test get list of dictionary from a SessionCookies.""" + test_session_cookies = session_cookies.get_cookies_as_dicts() + + assert [cookie_dict] == test_session_cookies + + +def test_store_cookie(session_cookies: SessionCookies) -> None: + """Test storing a Cookie object directly.""" + test_session_cookies = SessionCookies() + cookies = list(session_cookies.jar) + test_session_cookies.store_cookie(cookies[0]) + + assert test_session_cookies == session_cookies diff --git a/tests/unit/sessions/test_models.py b/tests/unit/sessions/test_models.py index b954da7433..450777f226 100644 --- a/tests/unit/sessions/test_models.py +++ b/tests/unit/sessions/test_models.py @@ -4,6 +4,7 @@ import pytest +from crawlee.sessions._cookies import CookieParam from crawlee.sessions._models import SessionModel, SessionPoolModel SESSION_CREATED_AT = datetime.now(timezone.utc) @@ -22,7 +23,7 @@ def session_direct() -> SessionModel: usage_count=0, max_usage_count=10, error_score=0.0, - cookies={'cookie_key': 'cookie_value'}, + cookies=[CookieParam({'name': 'cookie_key', 'value': 'cookie_value'})], blocked_status_codes=[401, 403, 429], ) @@ -40,7 +41,7 @@ def session_args_camel() -> dict: 'usageCount': 0, 'maxUsageCount': 10, 'errorScore': 0.0, - 'cookies': {'cookie_key': 'cookie_value'}, + 'cookies': [CookieParam({'name': 'cookie_key', 'value': 'cookie_value'})], 'blockedStatusCodes': [401, 403, 429], } @@ -58,7 +59,7 @@ def session_args_snake() -> dict: 'usage_count': 0, 'max_usage_count': 10, 'error_score': 0.0, - 'cookies': {'cookie_key': 'cookie_value'}, + 'cookies': [CookieParam({'name': 'cookie_key', 'value': 'cookie_value'})], 'blocked_status_codes': [401, 403, 429], } diff --git a/tests/unit/sessions/test_session.py b/tests/unit/sessions/test_session.py index 8be69e47f6..98af9c06a5 100644 --- a/tests/unit/sessions/test_session.py +++ b/tests/unit/sessions/test_session.py @@ -4,6 +4,7 @@ import pytest +from crawlee.sessions._cookies import SessionCookies from crawlee.sessions._session import Session @@ -28,7 +29,7 @@ def test_session_init(session: Session) -> None: """Verify that the session initializes correctly with the expected properties.""" assert session.id == 'test_session' assert session.user_data == {'user_key': 'user_value'} - assert session.cookies == {'cookie_key': 'cookie_value'} + assert session.cookies == SessionCookies({'cookie_key': 'cookie_value'}) assert session.expires_at >= datetime.now(timezone.utc) assert not session.is_blocked assert not session.is_expired