diff --git a/CHANGELOG.md b/CHANGELOG.md index 14079c3053..0150b5a1fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Features +- Integrate proxies into PlaywrightCrawler. - Expose `crawler.log` to public. ### Bug fixes diff --git a/src/crawlee/browsers/base_browser_controller.py b/src/crawlee/browsers/base_browser_controller.py index 7632d85964..cd84bf8cf7 100644 --- a/src/crawlee/browsers/base_browser_controller.py +++ b/src/crawlee/browsers/base_browser_controller.py @@ -11,6 +11,8 @@ from playwright.async_api import Page + from crawlee.proxy_configuration import ProxyInfo + class BaseBrowserController(ABC): """An abstract class for managing browser instance and their pages.""" @@ -49,11 +51,16 @@ def is_browser_connected(self) -> bool: """Return if the browser is closed.""" @abstractmethod - async def new_page(self, page_options: Mapping[str, Any] | None = None) -> Page: + async def new_page( + self, + page_options: Mapping[str, Any] | None = None, + proxy_info: ProxyInfo | None = None, + ) -> Page: """Create a new page with the given context options. Args: page_options: Options to configure the new page. + proxy_info: The proxy configuration to use for the new page. Returns: Page: The newly created page. diff --git a/src/crawlee/browsers/browser_pool.py b/src/crawlee/browsers/browser_pool.py index a1bebedabc..b45ab61226 100644 --- a/src/crawlee/browsers/browser_pool.py +++ b/src/crawlee/browsers/browser_pool.py @@ -21,6 +21,7 @@ from types import TracebackType from crawlee.browsers.base_browser_plugin import BaseBrowserPlugin + from crawlee.proxy_configuration import ProxyInfo logger = getLogger(__name__) @@ -180,6 +181,7 @@ async def new_page( *, page_id: str | None = None, browser_plugin: BaseBrowserPlugin | None = None, + proxy_info: ProxyInfo | None = None, ) -> CrawleePage: """Opens a new page in a browser using the specified or a random browser plugin. @@ -187,6 +189,7 @@ async def new_page( page_id: The ID to assign to the new page. If not provided, a random ID is generated. browser_plugin: browser_plugin: The browser plugin to use for creating the new page. If not provided, the next plugin in the rotation is used. + proxy_info: The proxy configuration to use for the new page. Returns: The newly created browser page. @@ -200,7 +203,7 @@ async def new_page( page_id = page_id or crypto_random_object_id(self._GENERATED_PAGE_ID_LENGTH) plugin = browser_plugin or next(self._plugins_cycle) - return await self._get_new_page(page_id, plugin) + return await self._get_new_page(page_id, plugin, proxy_info) async def new_page_with_each_plugin(self) -> Sequence[CrawleePage]: """Create a new page with each browser plugin in the pool. @@ -219,6 +222,7 @@ async def _get_new_page( self, page_id: str, plugin: BaseBrowserPlugin, + proxy_info: ProxyInfo | None, ) -> CrawleePage: """Internal method to initialize a new page in a browser using the specified plugin.""" timeout = self._operation_timeout.total_seconds() @@ -227,7 +231,10 @@ async def _get_new_page( try: if not browser: browser = await asyncio.wait_for(self._launch_new_browser(plugin), timeout) - page = await asyncio.wait_for(browser.new_page(page_options=plugin.page_options), timeout) + page = await asyncio.wait_for( + browser.new_page(page_options=plugin.page_options, proxy_info=proxy_info), + timeout, + ) except asyncio.TimeoutError as exc: raise TimeoutError(f'Creating a new page with plugin {plugin} timed out.') from exc except RuntimeError as exc: diff --git a/src/crawlee/browsers/playwright_browser_controller.py b/src/crawlee/browsers/playwright_browser_controller.py index b61fb8c547..d20c3fee33 100644 --- a/src/crawlee/browsers/playwright_browser_controller.py +++ b/src/crawlee/browsers/playwright_browser_controller.py @@ -15,6 +15,8 @@ from playwright.async_api import Browser + from crawlee.proxy_configuration import ProxyInfo + class PlaywrightBrowserController(BaseBrowserController): """Controller for managing Playwright browser instances and their pages. @@ -69,8 +71,20 @@ def is_browser_connected(self) -> bool: return self._browser.is_connected() @override - async def new_page(self, page_options: Mapping[str, Any] | None = None) -> Page: - page_options = page_options or {} + async def new_page( + self, + page_options: Mapping[str, Any] | None = None, + proxy_info: ProxyInfo | None = None, + ) -> Page: + page_options = dict(page_options) if page_options else {} + + # If "proxy_info" is provided and no proxy is already set in "page_options", configure the proxy. + if proxy_info and 'proxy' not in page_options: + page_options['proxy'] = { + 'server': f'{proxy_info.scheme}://{proxy_info.hostname}:{proxy_info.port}', + 'username': proxy_info.username, + 'password': proxy_info.password, + } if not self.has_free_capacity: raise ValueError('Cannot open more pages in this browser.') diff --git a/src/crawlee/playwright_crawler/playwright_crawler.py b/src/crawlee/playwright_crawler/playwright_crawler.py index 1b847be4b7..466e77525a 100644 --- a/src/crawlee/playwright_crawler/playwright_crawler.py +++ b/src/crawlee/playwright_crawler/playwright_crawler.py @@ -57,14 +57,11 @@ def __init__( super().__init__(**kwargs) - async def _page_goto( - self, - context: BasicCrawlingContext, - ) -> AsyncGenerator[PlaywrightCrawlingContext, None]: + async def _page_goto(self, context: BasicCrawlingContext) -> AsyncGenerator[PlaywrightCrawlingContext, None]: if self._browser_pool is None: raise ValueError('Browser pool is not initialized.') - crawlee_page = await self._browser_pool.new_page() + crawlee_page = await self._browser_pool.new_page(proxy_info=context.proxy_info) await crawlee_page.page.goto(context.request.url) context.request.loaded_url = crawlee_page.page.url diff --git a/src/crawlee/proxy_configuration.py b/src/crawlee/proxy_configuration.py index 8599db509c..da9582f845 100644 --- a/src/crawlee/proxy_configuration.py +++ b/src/crawlee/proxy_configuration.py @@ -29,6 +29,9 @@ class ProxyInfo: url: str """The URL of the proxy.""" + scheme: str + """The scheme of the proxy.""" + hostname: str """The hostname of the proxy.""" @@ -157,6 +160,7 @@ async def new_proxy_info( info = ProxyInfo( url=str(url), + scheme=url.scheme, hostname=url.host, port=cast(int, url.port), username=url.username, diff --git a/tests/unit/httpx_client/test_httpx_client.py b/tests/unit/httpx_client/test_httpx_client.py index 67253668be..8d2f4be138 100644 --- a/tests/unit/httpx_client/test_httpx_client.py +++ b/tests/unit/httpx_client/test_httpx_client.py @@ -18,6 +18,7 @@ async def proxy_info(unused_tcp_port: int) -> ProxyInfo: return ProxyInfo( url=f'http://{username}:{password}@127.0.0.1:{unused_tcp_port}', + scheme='http', hostname='127.0.0.1', port=unused_tcp_port, username=username,