Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/crawlee/browsers/_browser_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,9 +112,9 @@ def with_default_plugin(
browser_options: Keyword arguments to pass to the browser launch method. These options are provided
directly to Playwright's `browser_type.launch` method. For more details, refer to the Playwright
documentation: https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch.
page_options: Keyword arguments to pass to the new page method. These options are provided directly to
Playwright's `browser_context.new_page` method. For more details, refer to the Playwright documentation:
https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page.
page_options: Keyword arguments to pass to the page object is set at the playwright context level.
These options are provided directly to Playwright's `browser.new_context` method. For more details,
refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
headless: Whether to run the browser in headless mode.
kwargs: Additional arguments for default constructor.
"""
Expand Down
20 changes: 9 additions & 11 deletions src/crawlee/browsers/_playwright_browser_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,13 +98,12 @@ async def new_page(
proxy_info: ProxyInfo | None = None,
) -> Page:
if not self._browser_context:
self._browser_context = await self._create_browser_context(proxy_info)
self._browser_context = await self._create_browser_context(page_options, proxy_info)

if not self.has_free_capacity:
raise ValueError('Cannot open more pages in this browser.')

page_options = dict(page_options) if page_options else {}
page = await self._browser_context.new_page(**page_options)
page = await self._browser_context.new_page()
Comment thread
Pijukatel marked this conversation as resolved.
Resolved

# Handle page close event
page.on(event='close', f=self._on_page_close)
Expand All @@ -130,17 +129,20 @@ def _on_page_close(self, page: Page) -> None:
"""Handle actions after a page is closed."""
self._pages.remove(page)

async def _create_browser_context(self, proxy_info: ProxyInfo | None = None) -> BrowserContext:
async def _create_browser_context(
self, page_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None
) -> BrowserContext:
"""Create a new browser context with the specified proxy settings."""
if self._header_generator:
common_headers = self._header_generator.get_common_headers()
sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type)
user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type)
extra_http_headers = dict(common_headers | sec_ch_ua_headers | user_agent_header)
user_agent = user_agent_header.get('User-Agent')
Comment thread
Pijukatel marked this conversation as resolved.
Resolved

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was here to replace the default "headless"-something user agent. Why is it removed? Does it mean we're now using the default (headless) user agent?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should have a test to double-check the default UA is not leaking, it burned as several times in the past in the JS version.

I think we have it tested on other places too, found this one now

https://github.com/apify/crawlee/blob/fdd895e94e848c7e5545a2e239f24130d341db61/test/core/crawlers/playwright_crawler.test.ts#L88

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

else:
extra_http_headers = None
user_agent = None

page_options = dict(page_options) if page_options else {}
page_options['extra_http_headers'] = page_options.get('extra_http_headers', extra_http_headers)

proxy = (
ProxySettings(
Expand All @@ -152,8 +154,4 @@ async def _create_browser_context(self, proxy_info: ProxyInfo | None = None) ->
else None
)

return await self._browser.new_context(
user_agent=user_agent,
extra_http_headers=extra_http_headers,
proxy=proxy,
)
return await self._browser.new_context(proxy=proxy, **page_options)
6 changes: 3 additions & 3 deletions src/crawlee/browsers/_playwright_browser_plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,9 @@ def __init__(
browser_options: Keyword arguments to pass to the browser launch method. These options are provided
directly to Playwright's `browser_type.launch` method. For more details, refer to the Playwright
documentation: https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch.
page_options: Keyword arguments to pass to the new page method. These options are provided directly to
Playwright's `browser_context.new_page` method. For more details, refer to the Playwright documentation:
https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page.
page_options: Keyword arguments to pass to the page object is set at the playwright context level.
These options are provided directly to Playwright's `browser.new_context` method. For more details,
refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context.
max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance.
Once reached, a new browser instance will be launched to handle the excess.
"""
Expand Down
9 changes: 9 additions & 0 deletions tests/unit/browsers/test_browser_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,3 +151,12 @@ async def test_methods_raise_error_when_not_active() -> None:

async with browser_pool:
assert browser_pool.active is True


async def test_with_plugin_contains_page_options(httpbin: URL) -> None:
plugin = PlaywrightBrowserPlugin(page_options={'user_agent': 'My Best User-Agent'})
async with BrowserPool(plugins=[plugin]) as browser_pool:
test_page = await browser_pool.new_page()
await test_page.page.goto(str(httpbin / 'user-agent'))
assert 'My Best User-Agent' in await test_page.page.content()
await test_page.page.close()