From 4d055335c8b942ae65e98cc57d9d3e8db80b7bc8 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Mon, 9 Dec 2024 22:52:10 +0000 Subject: [PATCH 1/2] fix: fix `page_options` for `PlaywrightBrowserPlugin` --- src/crawlee/browsers/_browser_pool.py | 6 +++--- .../_playwright_browser_controller.py | 20 +++++++++---------- .../browsers/_playwright_browser_plugin.py | 6 +++--- tests/unit/browsers/test_browser_pool.py | 9 +++++++++ 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/crawlee/browsers/_browser_pool.py b/src/crawlee/browsers/_browser_pool.py index 07908286f5..0cf8561ac6 100644 --- a/src/crawlee/browsers/_browser_pool.py +++ b/src/crawlee/browsers/_browser_pool.py @@ -112,9 +112,9 @@ def with_default_plugin( browser_options: Keyword arguments to pass to the browser launch method. These options are provided directly to Playwright's `browser_type.launch` method. For more details, refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch. - page_options: Keyword arguments to pass to the new page method. These options are provided directly to - Playwright's `browser_context.new_page` method. For more details, refer to the Playwright documentation: - https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page. + page_options: Keyword arguments to pass to the page object is set at the playwright context level. + These options are provided directly to Playwright's `browser.new_context` method. For more details, + refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context. headless: Whether to run the browser in headless mode. kwargs: Additional arguments for default constructor. """ diff --git a/src/crawlee/browsers/_playwright_browser_controller.py b/src/crawlee/browsers/_playwright_browser_controller.py index f0a91deb51..9e9a83a90c 100644 --- a/src/crawlee/browsers/_playwright_browser_controller.py +++ b/src/crawlee/browsers/_playwright_browser_controller.py @@ -98,13 +98,12 @@ async def new_page( proxy_info: ProxyInfo | None = None, ) -> Page: if not self._browser_context: - self._browser_context = await self._create_browser_context(proxy_info) + self._browser_context = await self._create_browser_context(page_options, proxy_info) if not self.has_free_capacity: raise ValueError('Cannot open more pages in this browser.') - page_options = dict(page_options) if page_options else {} - page = await self._browser_context.new_page(**page_options) + page = await self._browser_context.new_page() # Handle page close event page.on(event='close', f=self._on_page_close) @@ -130,17 +129,20 @@ def _on_page_close(self, page: Page) -> None: """Handle actions after a page is closed.""" self._pages.remove(page) - async def _create_browser_context(self, proxy_info: ProxyInfo | None = None) -> BrowserContext: + async def _create_browser_context( + self, page_options: Mapping[str, Any] | None = None, proxy_info: ProxyInfo | None = None + ) -> BrowserContext: """Create a new browser context with the specified proxy settings.""" if self._header_generator: common_headers = self._header_generator.get_common_headers() sec_ch_ua_headers = self._header_generator.get_sec_ch_ua_headers(browser_type=self.browser_type) user_agent_header = self._header_generator.get_user_agent_header(browser_type=self.browser_type) extra_http_headers = dict(common_headers | sec_ch_ua_headers | user_agent_header) - user_agent = user_agent_header.get('User-Agent') else: extra_http_headers = None - user_agent = None + + page_options = dict(page_options) if page_options else {} + page_options['extra_http_headers'] = page_options.get('extra_http_headers', extra_http_headers) proxy = ( ProxySettings( @@ -152,8 +154,4 @@ async def _create_browser_context(self, proxy_info: ProxyInfo | None = None) -> else None ) - return await self._browser.new_context( - user_agent=user_agent, - extra_http_headers=extra_http_headers, - proxy=proxy, - ) + return await self._browser.new_context(proxy=proxy, **page_options) diff --git a/src/crawlee/browsers/_playwright_browser_plugin.py b/src/crawlee/browsers/_playwright_browser_plugin.py index cd5cda314a..c00dc8455e 100644 --- a/src/crawlee/browsers/_playwright_browser_plugin.py +++ b/src/crawlee/browsers/_playwright_browser_plugin.py @@ -46,9 +46,9 @@ def __init__( browser_options: Keyword arguments to pass to the browser launch method. These options are provided directly to Playwright's `browser_type.launch` method. For more details, refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browsertype#browser-type-launch. - page_options: Keyword arguments to pass to the new page method. These options are provided directly to - Playwright's `browser_context.new_page` method. For more details, refer to the Playwright documentation: - https://playwright.dev/python/docs/api/class-browsercontext#browser-context-new-page. + page_options: Keyword arguments to pass to the page object is set at the playwright context level. + These options are provided directly to Playwright's `browser.new_context` method. For more details, + refer to the Playwright documentation: https://playwright.dev/python/docs/api/class-browser#browser-new-context. max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance. Once reached, a new browser instance will be launched to handle the excess. """ diff --git a/tests/unit/browsers/test_browser_pool.py b/tests/unit/browsers/test_browser_pool.py index ca26454d3d..1256651206 100644 --- a/tests/unit/browsers/test_browser_pool.py +++ b/tests/unit/browsers/test_browser_pool.py @@ -151,3 +151,12 @@ async def test_methods_raise_error_when_not_active() -> None: async with browser_pool: assert browser_pool.active is True + + +async def test_with_plugin_contains_page_options(httpbin: URL) -> None: + plugin = PlaywrightBrowserPlugin(page_options={'user_agent': 'My Best Uset-Agent'}) + async with BrowserPool(plugins=[plugin]) as browser_pool: + test_page = await browser_pool.new_page() + await test_page.page.goto(str(httpbin / 'user-agent')) + assert 'My Best Uset-Agent' in await test_page.page.content() + await test_page.page.close() From 3039c1fb01e42e719eb536207cb9652d13ae0c26 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Tue, 10 Dec 2024 16:55:44 +0000 Subject: [PATCH 2/2] fix typo --- tests/unit/browsers/test_browser_pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/browsers/test_browser_pool.py b/tests/unit/browsers/test_browser_pool.py index 1256651206..d9966dddd1 100644 --- a/tests/unit/browsers/test_browser_pool.py +++ b/tests/unit/browsers/test_browser_pool.py @@ -154,9 +154,9 @@ async def test_methods_raise_error_when_not_active() -> None: async def test_with_plugin_contains_page_options(httpbin: URL) -> None: - plugin = PlaywrightBrowserPlugin(page_options={'user_agent': 'My Best Uset-Agent'}) + plugin = PlaywrightBrowserPlugin(page_options={'user_agent': 'My Best User-Agent'}) async with BrowserPool(plugins=[plugin]) as browser_pool: test_page = await browser_pool.new_page() await test_page.page.goto(str(httpbin / 'user-agent')) - assert 'My Best Uset-Agent' in await test_page.page.content() + assert 'My Best User-Agent' in await test_page.page.content() await test_page.page.close()