Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/crawlee/crawlers/_playwright/_playwright_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -232,10 +232,6 @@ async def _navigate(
# Set the loaded URL to the actual URL after redirection.
context.request.loaded_url = context.page.url

if context.session:
pw_cookies = await self._get_cookies(context.page)
context.session.cookies.set_cookies_from_playwright_format(pw_cookies)

extract_links = self._create_extract_links_function(context)

error = yield PlaywrightCrawlingContext(
Expand All @@ -256,6 +252,10 @@ async def _navigate(
block_requests=partial(block_requests, page=context.page),
)

if context.session:
pw_cookies = await self._get_cookies(context.page)
context.session.cookies.set_cookies_from_playwright_format(pw_cookies)

# Collect data in case of errors, before the page object is closed.
if error:
await self.statistics.error_tracker.add(error=error, context=context, early=True)
Expand Down
45 changes: 40 additions & 5 deletions tests/unit/crawlers/_playwright/test_playwright_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from crawlee.fingerprint_suite._browserforge_adapter import get_available_header_values
from crawlee.fingerprint_suite._consts import BROWSER_TYPE_HEADER_KEYWORD
from crawlee.proxy_configuration import ProxyConfiguration
from crawlee.sessions import SessionPool
from crawlee.sessions import Session, SessionPool
from crawlee.statistics import Statistics
from crawlee.statistics._error_snapshotter import ErrorSnapshotter
from tests.unit.server_endpoints import GENERIC_RESPONSE, HELLO_WORLD
Expand Down Expand Up @@ -304,6 +304,7 @@ async def some_hook(context: PlaywrightPreNavCrawlingContext) -> None:
)
async def test_isolation_cookies(*, use_incognito_pages: bool, server_url: URL) -> None:
sessions_ids: list[str] = []
sessions: dict[str, Session] = {}
sessions_cookies: dict[str, dict[str, str]] = {}
response_cookies: dict[str, dict[str, str]] = {}

Expand All @@ -319,13 +320,11 @@ async def handler(context: PlaywrightCrawlingContext) -> None:
return

sessions_ids.append(context.session.id)
sessions[context.session.id] = context.session

if context.request.unique_key not in {'1', '2'}:
return

sessions_cookies[context.session.id] = {
cookie['name']: cookie['value'] for cookie in context.session.cookies.get_cookies_as_dicts()
}
response_data = json.loads(await context.response.text())
response_cookies[context.session.id] = response_data.get('cookies')

Expand All @@ -343,11 +342,20 @@ async def handler(context: PlaywrightCrawlingContext) -> None:
]
)

assert len(sessions_cookies) == 2
assert len(response_cookies) == 2
assert len(sessions) == 2

assert sessions_ids[0] == sessions_ids[1]

sessions_cookies = {
sessions_id: {
cookie['name']: cookie['value'] for cookie in sessions[sessions_id].cookies.get_cookies_as_dicts()
}
for sessions_id in sessions_ids
}

assert len(sessions_cookies) == 2

cookie_session_id = sessions_ids[0]
clean_session_id = sessions_ids[2]

Expand All @@ -372,6 +380,33 @@ async def handler(context: PlaywrightCrawlingContext) -> None:
assert sessions_cookies[clean_session_id] == response_cookies[clean_session_id] == {'a': '1'}


async def test_save_cookies_after_handler_processing(server_url: URL) -> None:
"""Test that cookies are saved correctly."""
async with SessionPool(max_pool_size=1) as session_pool:
crawler = PlaywrightCrawler(session_pool=session_pool)

session_ids = []

@crawler.router.default_handler
async def request_handler(context: PlaywrightCrawlingContext) -> None:
# Simulate cookies installed from an external source in the browser
await context.page.context.add_cookies([{'name': 'check', 'value': 'test', 'url': str(server_url)}])

if context.session:
session_ids.append(context.session.id)

await crawler.run([str(server_url)])

assert len(session_ids) == 1

check_session = await session_pool.get_session()

assert check_session.id == session_ids[0]
session_cookies = {cookie['name']: cookie['value'] for cookie in check_session.cookies.get_cookies_as_dicts()}

assert session_cookies == {'check': 'test'}


async def test_custom_fingerprint_uses_generator_options(server_url: URL) -> None:
min_width = 300
max_width = 600
Expand Down