Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/upgrading/upgrading_to_v0x.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@ This section summarizes the breaking changes between v0.5.x and v0.6.0.

- Removed `HttpCrawlerOptions` - which contained options from `BasicCrawlerOptions` and unique options `additional_http_error_status_codes` and `ignore_http_error_status_codes`. Both of the unique options were added to `BasicCrawlerOptions` instead.

### SessionCookies

- Replaces the `dict` used for cookie storage in `Session.cookies` with a new `SessionCookies` class. `SessionCookies` uses `CookieJar`, which enables support for multiple domains.

### PlaywrightCrawler and PlaywrightBrowserPlugin

- `PlaywrightCrawler` now use a persistent browser context instead of the standard browser context.
Expand Down
22 changes: 10 additions & 12 deletions src/crawlee/crawlers/_playwright/_playwright_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@

from pydantic import ValidationError
from typing_extensions import NotRequired, TypedDict, TypeVar
from yarl import URL

from crawlee import EnqueueStrategy, RequestTransformAction
from crawlee._request import Request, RequestOptions
Expand All @@ -16,6 +15,7 @@
from crawlee.browsers import BrowserPool
from crawlee.crawlers._basic import BasicCrawler, BasicCrawlerOptions, ContextPipeline
from crawlee.errors import SessionError
from crawlee.sessions._cookies import PlaywrightCookieParam
from crawlee.statistics import StatisticsState

from ._playwright_crawling_context import PlaywrightCrawlingContext
Expand Down Expand Up @@ -208,7 +208,8 @@ async def _navigate(
"""
async with context.page:
if context.session:
await self._set_cookies(context.page, context.request.url, context.session.cookies)
session_cookies = context.session.cookies.get_cookies_as_playwright_format()
await self._update_cookies(context.page, session_cookies)

if context.request.headers:
await context.page.set_extra_http_headers(context.request.headers.model_dump())
Expand All @@ -222,8 +223,8 @@ async def _navigate(
context.request.loaded_url = context.page.url

if context.session:
cookies = await self._get_cookies(context.page)
context.session.cookies.update(cookies)
pw_cookies = await self._get_cookies(context.page)
context.session.cookies.set_cookies_from_playwright_format(pw_cookies)

async def enqueue_links(
*,
Expand Down Expand Up @@ -335,17 +336,14 @@ def pre_navigation_hook(self, hook: Callable[[PlaywrightPreNavCrawlingContext],
"""
self._pre_navigation_hooks.append(hook)

async def _get_cookies(self, page: Page) -> dict[str, str]:
async def _get_cookies(self, page: Page) -> list[PlaywrightCookieParam]:
"""Get the cookies from the page."""
cookies = await page.context.cookies()
return {cookie['name']: cookie['value'] for cookie in cookies if cookie.get('name') and cookie.get('value')}
return [PlaywrightCookieParam(**cookie) for cookie in cookies]

async def _set_cookies(self, page: Page, url: str, cookies: dict[str, str]) -> None:
"""Set the cookies to the page."""
parsed_url = URL(url)
await page.context.add_cookies(
[{'name': name, 'value': value, 'domain': parsed_url.host, 'path': '/'} for name, value in cookies.items()]
)
async def _update_cookies(self, page: Page, cookies: list[PlaywrightCookieParam]) -> None:
"""Update the cookies in the page context."""
await page.context.add_cookies([{**cookie} for cookie in cookies])


class _PlaywrightCrawlerAdditionalOptions(TypedDict):
Expand Down
21 changes: 12 additions & 9 deletions src/crawlee/http_clients/_curl_impersonate.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
from curl_cffi import CurlInfo
from curl_cffi.const import CurlHttpVersion
from curl_cffi.requests import AsyncSession
from curl_cffi.requests.cookies import Cookies, CurlMorsel
from curl_cffi.requests.cookies import Cookies as CurlCookies
from curl_cffi.requests.cookies import CurlMorsel
from curl_cffi.requests.exceptions import ProxyError as CurlProxyError
from curl_cffi.requests.exceptions import RequestException as CurlRequestError
from curl_cffi.requests.impersonate import DEFAULT_CHROME as CURL_DEFAULT_CHROME
Expand All @@ -19,6 +20,7 @@

if TYPE_CHECKING:
from collections.abc import Iterable
from http.cookiejar import Cookie

from curl_cffi import Curl
from curl_cffi.requests import Request as CurlRequest
Expand All @@ -31,7 +33,7 @@
from crawlee.statistics import Statistics


class _EmptyCookies(Cookies):
class _EmptyCookies(CurlCookies):
@override
def get_cookies_for_curl(self, request: CurlRequest) -> list[CurlMorsel]:
return []
Expand Down Expand Up @@ -147,7 +149,7 @@ async def crawl(
method=request.method.upper(), # type: ignore[arg-type] # curl-cffi requires uppercase method
headers=request.headers,
data=request.payload,
cookies=session.cookies if session else None,
cookies=session.cookies.jar if session else None,
allow_redirects=True,
)
except CurlRequestError as exc:
Expand All @@ -166,7 +168,7 @@ async def crawl(

if self._persist_cookies_per_session and session and response.curl:
response_cookies = self._get_cookies(response.curl)
session.cookies.update(response_cookies)
session.cookies.store_cookies(response_cookies)

request.loaded_url = response.url

Expand Down Expand Up @@ -197,7 +199,7 @@ async def send_request(
method=method.upper(), # type: ignore[arg-type] # curl-cffi requires uppercase method
headers=dict(headers) if headers else None,
data=payload,
cookies=session.cookies if session else None,
cookies=session.cookies.jar if session else None,
allow_redirects=True,
)
except CurlRequestError as exc:
Expand All @@ -213,7 +215,7 @@ async def send_request(

if self._persist_cookies_per_session and session and response.curl:
response_cookies = self._get_cookies(response.curl)
session.cookies.update(response_cookies)
session.cookies.store_cookies(response_cookies)

return _CurlImpersonateResponse(response)

Expand Down Expand Up @@ -253,9 +255,10 @@ def _is_proxy_error(error: CurlRequestError) -> bool:
return False

@staticmethod
def _get_cookies(curl: Curl) -> dict[str, str]:
cookies = {}
def _get_cookies(curl: Curl) -> list[Cookie]:
cookies: list[Cookie] = []
for curl_cookie in curl.getinfo(CurlInfo.COOKIELIST): # type: ignore[union-attr]
curl_morsel = CurlMorsel.from_curl_format(curl_cookie) # type: ignore[arg-type]
cookies[curl_morsel.name] = curl_morsel.value
cookie = curl_morsel.to_cookiejar_cookie()
cookies.append(cookie)
return cookies
6 changes: 2 additions & 4 deletions src/crawlee/http_clients/_httpx.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,7 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
response.request = request

if session := cast(Session, request.extensions.get('crawlee_session')):
response_cookies = httpx.Cookies()
response_cookies.extract_cookies(response)
session.cookies.update(response_cookies)
session.cookies.store_cookies(list(response.cookies.jar))

if 'Set-Cookie' in response.headers:
del response.headers['Set-Cookie']
Expand Down Expand Up @@ -160,7 +158,7 @@ async def crawl(
method=request.method,
headers=headers,
content=request.payload,
cookies=session.cookies if session else None,
cookies=session.cookies.jar if session else None,
extensions={'crawlee_session': session if self._persist_cookies_per_session else None},
)

Expand Down
3 changes: 2 additions & 1 deletion src/crawlee/sessions/__init__.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from ._cookies import CookieParam, SessionCookies
from ._session import Session
from ._session_pool import SessionPool

__all__ = ['Session', 'SessionPool']
__all__ = ['CookieParam', 'Session', 'SessionCookies', 'SessionPool']
Loading