Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

### Features

- Integrate proxies into PlaywrightCrawler.
- Expose `crawler.log` to public.

### Bug fixes
Expand Down
9 changes: 8 additions & 1 deletion src/crawlee/browsers/base_browser_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@

from playwright.async_api import Page

from crawlee.proxy_configuration import ProxyInfo


class BaseBrowserController(ABC):
"""An abstract class for managing browser instance and their pages."""
Expand Down Expand Up @@ -49,11 +51,16 @@ def is_browser_connected(self) -> bool:
"""Return if the browser is closed."""

@abstractmethod
async def new_page(self, page_options: Mapping[str, Any] | None = None) -> Page:
async def new_page(
self,
page_options: Mapping[str, Any] | None = None,
proxy_info: ProxyInfo | None = None,
) -> Page:
"""Create a new page with the given context options.

Args:
page_options: Options to configure the new page.
proxy_info: The proxy configuration to use for the new page.

Returns:
Page: The newly created page.
Expand Down
11 changes: 9 additions & 2 deletions src/crawlee/browsers/browser_pool.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
from types import TracebackType

from crawlee.browsers.base_browser_plugin import BaseBrowserPlugin
from crawlee.proxy_configuration import ProxyInfo

logger = getLogger(__name__)

Expand Down Expand Up @@ -180,13 +181,15 @@ async def new_page(
*,
page_id: str | None = None,
browser_plugin: BaseBrowserPlugin | None = None,
proxy_info: ProxyInfo | None = None,
) -> CrawleePage:
"""Opens a new page in a browser using the specified or a random browser plugin.

Args:
page_id: The ID to assign to the new page. If not provided, a random ID is generated.
browser_plugin: browser_plugin: The browser plugin to use for creating the new page.
If not provided, the next plugin in the rotation is used.
proxy_info: The proxy configuration to use for the new page.

Returns:
The newly created browser page.
Expand All @@ -200,7 +203,7 @@ async def new_page(
page_id = page_id or crypto_random_object_id(self._GENERATED_PAGE_ID_LENGTH)
plugin = browser_plugin or next(self._plugins_cycle)

return await self._get_new_page(page_id, plugin)
return await self._get_new_page(page_id, plugin, proxy_info)

async def new_page_with_each_plugin(self) -> Sequence[CrawleePage]:
"""Create a new page with each browser plugin in the pool.
Expand All @@ -219,6 +222,7 @@ async def _get_new_page(
self,
page_id: str,
plugin: BaseBrowserPlugin,
proxy_info: ProxyInfo | None,
) -> CrawleePage:
"""Internal method to initialize a new page in a browser using the specified plugin."""
timeout = self._operation_timeout.total_seconds()
Expand All @@ -227,7 +231,10 @@ async def _get_new_page(
try:
if not browser:
browser = await asyncio.wait_for(self._launch_new_browser(plugin), timeout)
page = await asyncio.wait_for(browser.new_page(page_options=plugin.page_options), timeout)
page = await asyncio.wait_for(
browser.new_page(page_options=plugin.page_options, proxy_info=proxy_info),
timeout,
)
except asyncio.TimeoutError as exc:
raise TimeoutError(f'Creating a new page with plugin {plugin} timed out.') from exc
except RuntimeError as exc:
Expand Down
18 changes: 16 additions & 2 deletions src/crawlee/browsers/playwright_browser_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@

from playwright.async_api import Browser

from crawlee.proxy_configuration import ProxyInfo


class PlaywrightBrowserController(BaseBrowserController):
"""Controller for managing Playwright browser instances and their pages.
Expand Down Expand Up @@ -69,8 +71,20 @@ def is_browser_connected(self) -> bool:
return self._browser.is_connected()

@override
async def new_page(self, page_options: Mapping[str, Any] | None = None) -> Page:
page_options = page_options or {}
async def new_page(
self,
page_options: Mapping[str, Any] | None = None,
proxy_info: ProxyInfo | None = None,
) -> Page:
page_options = dict(page_options) if page_options else {}

# If "proxy_info" is provided and no proxy is already set in "page_options", configure the proxy.
if proxy_info and 'proxy' not in page_options:
page_options['proxy'] = {
'server': f'{proxy_info.scheme}://{proxy_info.hostname}:{proxy_info.port}',
Comment thread
vdusek marked this conversation as resolved.
Resolved
'username': proxy_info.username,
'password': proxy_info.password,
}

if not self.has_free_capacity:
raise ValueError('Cannot open more pages in this browser.')
Expand Down
7 changes: 2 additions & 5 deletions src/crawlee/playwright_crawler/playwright_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,14 +57,11 @@ def __init__(

super().__init__(**kwargs)

async def _page_goto(
self,
context: BasicCrawlingContext,
) -> AsyncGenerator[PlaywrightCrawlingContext, None]:
async def _page_goto(self, context: BasicCrawlingContext) -> AsyncGenerator[PlaywrightCrawlingContext, None]:
if self._browser_pool is None:
raise ValueError('Browser pool is not initialized.')

crawlee_page = await self._browser_pool.new_page()
crawlee_page = await self._browser_pool.new_page(proxy_info=context.proxy_info)
await crawlee_page.page.goto(context.request.url)
context.request.loaded_url = crawlee_page.page.url

Expand Down
4 changes: 4 additions & 0 deletions src/crawlee/proxy_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ class ProxyInfo:
url: str
"""The URL of the proxy."""

scheme: str
"""The scheme of the proxy."""

hostname: str
"""The hostname of the proxy."""

Expand Down Expand Up @@ -157,6 +160,7 @@ async def new_proxy_info(

info = ProxyInfo(
url=str(url),
scheme=url.scheme,
hostname=url.host,
port=cast(int, url.port),
username=url.username,
Expand Down
1 change: 1 addition & 0 deletions tests/unit/httpx_client/test_httpx_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ async def proxy_info(unused_tcp_port: int) -> ProxyInfo:

return ProxyInfo(
url=f'http://{username}:{password}@127.0.0.1:{unused_tcp_port}',
scheme='http',
hostname='127.0.0.1',
port=unused_tcp_port,
username=username,
Expand Down