From 173cd94e6091aaa026f30781e3c7f17a55a6ec88 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 31 May 2024 17:28:10 +0200 Subject: [PATCH 01/10] feat: browser rotation and better browser abstraction Closes: #131 --- .../browsers/base_browser_controller.py | 79 +++++++++++++ src/crawlee/browsers/base_browser_plugin.py | 34 ++++-- src/crawlee/browsers/browser_pool.py | 92 +++++++++++++-- .../browsers/playwright_browser_controller.py | 109 ++++++++++++++++++ .../browsers/playwright_browser_plugin.py | 63 ++++++---- tests/unit/browsers/test_browser_pool.py | 57 ++++++--- .../test_playwright_browser_controller.py | 109 ++++++++++++++++++ .../test_playwright_browser_plugin.py | 57 +++++++++ tests/unit/browsers/test_playwright_plugin.py | 50 -------- 9 files changed, 545 insertions(+), 105 deletions(-) create mode 100644 src/crawlee/browsers/base_browser_controller.py create mode 100644 src/crawlee/browsers/playwright_browser_controller.py create mode 100644 tests/unit/browsers/test_playwright_browser_controller.py create mode 100644 tests/unit/browsers/test_playwright_browser_plugin.py delete mode 100644 tests/unit/browsers/test_playwright_plugin.py diff --git a/src/crawlee/browsers/base_browser_controller.py b/src/crawlee/browsers/base_browser_controller.py new file mode 100644 index 0000000000..9b7a494ad1 --- /dev/null +++ b/src/crawlee/browsers/base_browser_controller.py @@ -0,0 +1,79 @@ +# Inspiration: https://github.com/apify/crawlee/blob/v3.10.1/packages/browser-pool/src/abstract-classes/browser-controller.ts + +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Mapping + from datetime import datetime, timedelta + + from playwright.async_api import Page + + +class BaseBrowserController(ABC): + """An abstract class for managing browser instance and their pages.""" + + AUTOMATION_LIBRARY: str | None = None + """The name of the automation library that the controller is using.""" + + @property + @abstractmethod + def pages(self) -> list[Page]: + """Return the list of opened pages.""" + + @property + @abstractmethod + def num_of_pages(self) -> int: + """Return the number of currently open pages.""" + + @property + @abstractmethod + def num_of_total_pages(self) -> int: + """Return the total number of pages opened since the browser was launched.""" + + @property + @abstractmethod + def last_page_opened_at(self) -> datetime: + """Return the time when the last page was opened.""" + + @property + @abstractmethod + def idle_time(self) -> timedelta: + """Return the idle time of the browser controller.""" + + @property + @abstractmethod + def has_free_capacity(self) -> bool: + """Return if the browser has free capacity to open a new page.""" + + @property + @abstractmethod + def is_browser_connected(self) -> bool: + """Return if the browser is closed.""" + + @abstractmethod + async def new_page(self, page_options: Mapping | None = None) -> Page: + """Create a new page with the given context options. + + Args: + page_options: Options to configure the new page. + + Returns: + Page: The newly created page. + + Raises: + ValueError: If the browser has reached the maximum number of open pages. + """ + + @abstractmethod + async def close(self, *, force: bool = False) -> None: + """Close the browser. + + Args: + force: Whether to force close all open pages before closing the browser. + + Raises: + ValueError: If there are still open pages when trying to close the browser. + """ diff --git a/src/crawlee/browsers/base_browser_plugin.py b/src/crawlee/browsers/base_browser_plugin.py index 1f2162b07a..b28b995c02 100644 --- a/src/crawlee/browsers/base_browser_plugin.py +++ b/src/crawlee/browsers/base_browser_plugin.py @@ -1,4 +1,4 @@ -# Inspiration: https://github.com/apify/crawlee/blob/v3.10.0/packages/browser-pool/src/abstract-classes/browser-plugin.ts +# Inspiration: https://github.com/apify/crawlee/blob/v3.10.1/packages/browser-pool/src/abstract-classes/browser-plugin.ts from __future__ import annotations @@ -6,9 +6,10 @@ from typing import TYPE_CHECKING, Literal if TYPE_CHECKING: + from collections.abc import Mapping from types import TracebackType - from playwright.async_api import Browser, Page + from crawlee.browsers.base_browser_controller import BaseBrowserController class BaseBrowserPlugin(ABC): @@ -18,16 +19,29 @@ class BaseBrowserPlugin(ABC): providing a unified interface for interacting with browsers. """ - @property - @abstractmethod - def browser(self) -> Browser | None: - """Return the browser instance.""" + AUTOMATION_LIBRARY: str | None = None + """The name of the automation library that the plugin is managing.""" @property @abstractmethod def browser_type(self) -> Literal['chromium', 'firefox', 'webkit']: """Return the browser type name.""" + @property + @abstractmethod + def browser_options(self) -> Mapping: + """Return the options for a new browser.""" + + @property + @abstractmethod + def page_options(self) -> Mapping: + """Return the options for a new page.""" + + @property + @abstractmethod + def max_open_pages_per_browser(self) -> int: + """Return the maximum number of pages that can be opened in a single browser.""" + @abstractmethod async def __aenter__(self) -> BaseBrowserPlugin: """Enter the context manager and initialize the browser plugin.""" @@ -42,5 +56,9 @@ async def __aexit__( """Exit the context manager and close the browser plugin.""" @abstractmethod - async def new_page(self) -> Page: - """Get a new page in a browser.""" + async def new_browser(self) -> BaseBrowserController: + """Create a new browser instance. + + Returns: + A new browser instance wrapped in a controller. + """ diff --git a/src/crawlee/browsers/browser_pool.py b/src/crawlee/browsers/browser_pool.py index 35a1932bae..50fff8b033 100644 --- a/src/crawlee/browsers/browser_pool.py +++ b/src/crawlee/browsers/browser_pool.py @@ -11,6 +11,8 @@ from weakref import WeakValueDictionary from crawlee._utils.crypto import crypto_random_object_id +from crawlee._utils.recurring_task import RecurringTask +from crawlee.browsers.base_browser_controller import BaseBrowserController from crawlee.browsers.playwright_browser_plugin import PlaywrightBrowserPlugin from crawlee.browsers.types import CrawleePage @@ -30,6 +32,8 @@ class BrowserPool: and handling the overall lifecycle of these resources. It provides flexible configuration via constructor options, which include various hooks that allow for the insertion of custom behavior at different stages of the browser and page lifecycles. + + The browsers in the pool can be in one of three states: active, inactive, or closed. """ _GENERATED_PAGE_ID_LENGTH = 8 @@ -40,6 +44,9 @@ def __init__( plugins: Sequence[BaseBrowserPlugin] | None = None, *, operation_timeout: timedelta = timedelta(seconds=15), + browser_inactive_threshold: timedelta = timedelta(seconds=3), + identify_inactive_browsers_interval: timedelta = timedelta(seconds=10), + close_inactive_browsers_interval: timedelta = timedelta(seconds=30), ) -> None: """Create a new instance. @@ -49,9 +56,29 @@ def __init__( operation_timeout: Operations of the underlying automation libraries, such as launching a browser or opening a new page, can sometimes get stuck. To prevent `BrowserPool` from becoming unresponsive, we add a timeout to these operations. + browser_inactive_threshold: The period of inactivity after which a browser is considered as inactive. + identify_inactive_browsers_interval: The period of inactivity after which a browser is considered + as retired. + close_inactive_browsers_interval: The interval at which the pool checks for inactive browsers + and closes them. The browser is considered as inactive if it has no active pages and has been idle + for the specified period. """ self._plugins = plugins or [PlaywrightBrowserPlugin()] self._operation_timeout = operation_timeout + self._browser_inactive_threshold = browser_inactive_threshold + + self._identify_inactive_browsers_task = RecurringTask( + self._identify_inactive_browsers, + identify_inactive_browsers_interval, + ) + + self._close_inactive_browsers_task = RecurringTask( + self._close_inactive_browsers, + close_inactive_browsers_interval, + ) + + self._active_browsers = list[BaseBrowserController]() + self._inactive_browsers = list[BaseBrowserController]() self._pages = WeakValueDictionary[str, CrawleePage]() # Track the pages in the pool self._plugins_cycle = itertools.cycle(self._plugins) # Cycle through the plugins @@ -64,7 +91,7 @@ def with_default_plugin( browser_type: Literal['chromium', 'firefox', 'webkit'] | None = None, **kwargs: Any, ) -> BrowserPool: - """Create a new instance with a single `PlaywrightBrowserPlugin` configured with the provided options. + """Create a new instance with a single `BaseBrowserPlugin` configured with the provided options. Args: headless: Whether to run the browser in headless mode. @@ -87,6 +114,16 @@ def plugins(self) -> Sequence[BaseBrowserPlugin]: """Return the browser plugins.""" return self._plugins + @property + def active_browsers(self) -> Sequence[BaseBrowserController]: + """Return the active browsers in the pool.""" + return self._active_browsers + + @property + def inactive_browsers(self) -> Sequence[BaseBrowserController]: + """Return the inactive browsers in the pool.""" + return self._inactive_browsers + @property def pages(self) -> Mapping[str, CrawleePage]: """Return the pages in the pool.""" @@ -113,6 +150,9 @@ async def __aexit__( ) -> None: """Exit the context manager and close all browser plugins.""" logger.debug('Closing browser pool.') + for browser in self._active_browsers + self._inactive_browsers: + await browser.close(force=True) + for plugin in self._plugins: await plugin.__aexit__(exc_type, exc_value, exc_traceback) @@ -141,7 +181,7 @@ async def new_page( page_id = page_id or crypto_random_object_id(self._GENERATED_PAGE_ID_LENGTH) plugin = browser_plugin or next(self._plugins_cycle) - return await self._initialize_page(page_id, plugin) + return await self._get_new_page(page_id, plugin) async def new_page_with_each_plugin(self) -> Sequence[CrawleePage]: """Create a new page with each browser plugin in the pool. @@ -150,31 +190,61 @@ async def new_page_with_each_plugin(self) -> Sequence[CrawleePage]: or website analysis. Each page is created using a different browser plugin, allowing you to interact with various browser types concurrently. - Args: - page_options: Options to configure the new pages. These options will be applied to all pages created. - If not provided, an empty dictionary is used. - Returns: A list of newly created pages, one for each plugin in the pool. """ pages_coroutines = [self.new_page(browser_plugin=plugin) for plugin in self._plugins] return await asyncio.gather(*pages_coroutines) - async def _initialize_page( + async def _get_new_page( self, page_id: str, plugin: BaseBrowserPlugin, ) -> CrawleePage: """Internal method to initialize a new page in a browser using the specified plugin.""" timeout = self._operation_timeout.total_seconds() + browser = self._pick_browser_with_free_capacity(plugin) try: - raw_page = await asyncio.wait_for(plugin.new_page(), timeout) + if not browser: + browser = await asyncio.wait_for(self._launch_new_browser(plugin), timeout) + page = await asyncio.wait_for(browser.new_page(page_options=plugin.page_options), timeout) except asyncio.TimeoutError as exc: raise TimeoutError(f'Creating a new page with plugin {plugin} timed out.') from exc except RuntimeError as exc: raise RuntimeError('Browser pool is not initialized.') from exc - page = CrawleePage(id=page_id, page=raw_page, browser_type=plugin.browser_type) - self._pages[page_id] = page - return page + crawlee_page = CrawleePage(id=page_id, page=page, browser_type=plugin.browser_type) + self._pages[page_id] = crawlee_page + return crawlee_page + + def _pick_browser_with_free_capacity( + self, + browser_plugin: BaseBrowserPlugin, + ) -> BaseBrowserController | None: + """Pick a browser with free capacity that matches the specified plugin.""" + for browser in self._active_browsers: + if browser.has_free_capacity and browser.AUTOMATION_LIBRARY == browser_plugin.AUTOMATION_LIBRARY: + return browser + + return None + + async def _launch_new_browser(self, plugin: BaseBrowserPlugin) -> BaseBrowserController: + """Launch a new browser instance using the specified plugin.""" + browser = await plugin.new_browser() + self._active_browsers.append(browser) + return browser + + def _identify_inactive_browsers(self) -> None: + """Identify inactive browsers and move them to the inactive list if their idle time exceeds the threshold.""" + for browser in self._active_browsers: + if browser.idle_time >= self._browser_inactive_threshold: + self._active_browsers.remove(browser) + self._inactive_browsers.append(browser) + + async def _close_inactive_browsers(self) -> None: + """Close the browsers that have no active pages and have been idle for a certain period.""" + for browser in self._inactive_browsers: + if not browser.pages: + await browser.close() + self._inactive_browsers.remove(browser) diff --git a/src/crawlee/browsers/playwright_browser_controller.py b/src/crawlee/browsers/playwright_browser_controller.py new file mode 100644 index 0000000000..784ab09c5b --- /dev/null +++ b/src/crawlee/browsers/playwright_browser_controller.py @@ -0,0 +1,109 @@ +# Inspiration: https://github.com/apify/crawlee/blob/v3.10.0/packages/browser-pool/src/playwright/playwright-controller.ts + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from typing import TYPE_CHECKING + +from playwright.async_api import Page +from typing_extensions import override + +from crawlee.browsers.base_browser_controller import BaseBrowserController + +if TYPE_CHECKING: + from collections.abc import Mapping + + from playwright.async_api import Browser + + +class PlaywrightBrowserController(BaseBrowserController): + """Controller for managing Playwright browser instances and their pages. + + This class provides methods to manage pages within a browser instance, ensuring that the number + of open pages does not exceed the specified limit and tracking the state of the pages. + """ + + AUTOMATION_LIBRARY = 'playwright' + + def __init__(self, browser: Browser, *, max_open_pages_per_browser: int = 20) -> None: + """Create a new instance. + + Args: + browser: The browser instance to control. + max_open_pages_per_browser: The maximum number of pages that can be open at the same time. + """ + self._browser = browser + self._max_open_pages_per_browser = max_open_pages_per_browser + + self._pages = list[Page]() + self._num_of_total_pages = 0 + self._last_page_opened_at = datetime.now(timezone.utc) + + @property + @override + def pages(self) -> list[Page]: + return self._pages + + @property + @override + def num_of_pages(self) -> int: + return len(self._pages) + + @property + @override + def num_of_total_pages(self) -> int: + return self._num_of_total_pages + + @property + @override + def last_page_opened_at(self) -> datetime: + return self._last_page_opened_at + + @property + @override + def idle_time(self) -> timedelta: + return datetime.now(timezone.utc) - self._last_page_opened_at + + @property + @override + def has_free_capacity(self) -> bool: + return self.num_of_pages < self._max_open_pages_per_browser + + @property + @override + def is_browser_connected(self) -> bool: + return self._browser.is_connected() + + @override + async def new_page(self, page_options: Mapping | None = None) -> Page: + page_options = page_options or {} + + if not self.has_free_capacity: + raise ValueError('Cannot open more pages in this browser.') + + page = await self._browser.new_page(**page_options) + + # Handle page close event + page.on(event='close', f=self._on_page_close) + + # Update internal state + self._pages.append(page) + self._num_of_total_pages += 1 + self._last_page_opened_at = datetime.now(timezone.utc) + + return page + + @override + async def close(self, *, force: bool = False) -> None: + if force: + for page in self._pages: + await page.close() + + if self.num_of_pages > 0: + raise ValueError('Cannot close the browser while there are open pages.') + + await self._browser.close() + + def _on_page_close(self, page: Page) -> None: + """Handle actions after a page is closed.""" + self._pages.remove(page) diff --git a/src/crawlee/browsers/playwright_browser_plugin.py b/src/crawlee/browsers/playwright_browser_plugin.py index 020474d832..2855b88e00 100644 --- a/src/crawlee/browsers/playwright_browser_plugin.py +++ b/src/crawlee/browsers/playwright_browser_plugin.py @@ -9,18 +9,22 @@ from typing_extensions import override from crawlee.browsers.base_browser_plugin import BaseBrowserPlugin +from crawlee.browsers.playwright_browser_controller import PlaywrightBrowserController if TYPE_CHECKING: from collections.abc import Mapping from types import TracebackType - from playwright.async_api import Browser, Page - logger = getLogger(__name__) class PlaywrightBrowserPlugin(BaseBrowserPlugin): - """A plugin for managing Playwright browser instances.""" + """A plugin for managing Playwright automation library. + + It should work as a factory for creating new browser instances. + """ + + AUTOMATION_LIBRARY = 'playwright' def __init__( self, @@ -28,6 +32,7 @@ def __init__( browser_type: Literal['chromium', 'firefox', 'webkit'] = 'chromium', browser_options: Mapping | None = None, page_options: Mapping | None = None, + max_open_pages_per_browser: int = 20, ) -> None: """Create a new instance. @@ -35,39 +40,41 @@ def __init__( browser_type: The type of the browser to launch. browser_options: Options to configure the browser instance. page_options: Options to configure a new page instance. + max_open_pages_per_browser: The maximum number of pages that can be opened in a single browser instance. + Once reached, a new browser instance will be launched to handle the excess. """ self._browser_type = browser_type self._browser_options = browser_options or {} self._page_options = page_options or {} + self._max_open_pages_per_browser = max_open_pages_per_browser self._playwright_context_manager = async_playwright() self._playwright: Playwright | None = None - self._browser: Browser | None = None @property @override - def browser(self) -> Browser | None: - return self._browser + def browser_type(self) -> Literal['chromium', 'firefox', 'webkit']: + return self._browser_type @property @override - def browser_type(self) -> Literal['chromium', 'firefox', 'webkit']: - return self._browser_type + def browser_options(self) -> Mapping: + return self._browser_options + + @property + @override + def page_options(self) -> Mapping: + return self._page_options + + @property + @override + def max_open_pages_per_browser(self) -> int: + return self._max_open_pages_per_browser @override async def __aenter__(self) -> PlaywrightBrowserPlugin: logger.debug('Initializing Playwright browser plugin.') self._playwright = await self._playwright_context_manager.__aenter__() - - if self._browser_type == 'chromium': - self._browser = await self._playwright.chromium.launch(**self._browser_options) - elif self._browser_type == 'firefox': - self._browser = await self._playwright.firefox.launch(**self._browser_options) - elif self._browser_type == 'webkit': - self._browser = await self._playwright.webkit.launch(**self._browser_options) - else: - raise ValueError(f'Invalid browser type: {self._browser_type}') - return self @override @@ -78,13 +85,23 @@ async def __aexit__( exc_traceback: TracebackType | None, ) -> None: logger.debug('Closing Playwright browser plugin.') - if self._browser: - await self._browser.close() await self._playwright_context_manager.__aexit__(exc_type, exc_value, exc_traceback) @override - async def new_page(self) -> Page: - if not self._browser: + async def new_browser(self) -> PlaywrightBrowserController: + if not self._playwright: raise RuntimeError('Playwright browser plugin is not initialized.') - return await self._browser.new_page(**self._page_options) + if self._browser_type == 'chromium': + browser = await self._playwright.chromium.launch(**self._browser_options) + elif self._browser_type == 'firefox': + browser = await self._playwright.firefox.launch(**self._browser_options) + elif self._browser_type == 'webkit': + browser = await self._playwright.webkit.launch(**self._browser_options) + else: + raise ValueError(f'Invalid browser type: {self._browser_type}') + + return PlaywrightBrowserController( + browser, + max_open_pages_per_browser=self._max_open_pages_per_browser, + ) diff --git a/tests/unit/browsers/test_browser_pool.py b/tests/unit/browsers/test_browser_pool.py index 74d035c918..0bc0fbc6d3 100644 --- a/tests/unit/browsers/test_browser_pool.py +++ b/tests/unit/browsers/test_browser_pool.py @@ -6,12 +6,8 @@ from crawlee.browsers.playwright_browser_plugin import PlaywrightBrowserPlugin -async def test_new_page_single_plugin(httpbin: str) -> None: - plugin = PlaywrightBrowserPlugin(browser_type='chromium') - - async with BrowserPool([plugin]) as browser_pool: - assert browser_pool.plugins == [plugin] - +async def test_default_plugin_new_page_creation(httpbin: str) -> None: + async with BrowserPool() as browser_pool: page_1 = await browser_pool.new_page() await page_1.page.goto(f'{httpbin}/get') assert page_1.browser_type == 'chromium' @@ -24,8 +20,11 @@ async def test_new_page_single_plugin(httpbin: str) -> None: assert page_2.page.url == f'{httpbin}/status/200' assert ' None: +async def test_multiple_plugins_new_page_creation(httpbin: str) -> None: plugin_chromium = PlaywrightBrowserPlugin(browser_type='chromium') plugin_firefox = PlaywrightBrowserPlugin(browser_type='firefox') @@ -50,6 +49,10 @@ async def test_new_page_multiple_plugins(httpbin: str) -> None: assert page_3.page.url == f'{httpbin}/user-agent' assert ' None: plugin_chromium = PlaywrightBrowserPlugin(browser_type='chromium') @@ -71,6 +74,39 @@ async def test_new_page_with_each_plugin(httpbin: str) -> None: assert pages[1].page.url == f'{httpbin}/headers' assert ' None: + async with BrowserPool.with_default_plugin(headless=True, browser_type='firefox') as browser_pool: + assert len(browser_pool.plugins) == 1 + assert isinstance(browser_pool.plugins[0], PlaywrightBrowserPlugin) + + page = await browser_pool.new_page() + assert page.browser_type == 'firefox' + + await page.page.goto(f'{httpbin}/get') + assert page.page.url == f'{httpbin}/get' + assert ' None: + async with BrowserPool() as browser_pool: + page_1 = await browser_pool.new_page() + with pytest.raises(ValueError, match='Page with ID: .* already exists.'): + await browser_pool.new_page(page_id=page_1.id) + + +async def test_new_page_with_invalid_plugin() -> None: + plugin_1 = PlaywrightBrowserPlugin(browser_type='chromium') + plugin_2 = PlaywrightBrowserPlugin(browser_type='firefox') + async with BrowserPool([plugin_1]) as browser_pool: + with pytest.raises(ValueError, match='Provided browser_plugin is not one of the plugins used by BrowserPool.'): + await browser_pool.new_page(browser_plugin=plugin_2) + async def test_resource_management(httpbin: str) -> None: playwright_plugin = PlaywrightBrowserPlugin(browser_type='chromium') @@ -81,14 +117,9 @@ async def test_resource_management(httpbin: str) -> None: assert page.page.url == f'{httpbin}/get' assert ' None: plugin = PlaywrightBrowserPlugin() diff --git a/tests/unit/browsers/test_playwright_browser_controller.py b/tests/unit/browsers/test_playwright_browser_controller.py new file mode 100644 index 0000000000..53d5274a18 --- /dev/null +++ b/tests/unit/browsers/test_playwright_browser_controller.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone +from typing import AsyncGenerator + +import pytest +from playwright.async_api import Browser, Playwright, async_playwright + +from crawlee.browsers.playwright_browser_controller import PlaywrightBrowserController + +# pytestmark = pytest.mark.only() + + +@pytest.fixture() +async def playwright() -> AsyncGenerator[Playwright, None]: + async with async_playwright() as playwright: + yield playwright + + +@pytest.fixture() +async def browser(playwright: Playwright) -> AsyncGenerator[Browser, None]: + browser = await playwright.chromium.launch() + yield browser + await browser.close() + + +@pytest.fixture() +async def controller(browser: Browser) -> AsyncGenerator[PlaywrightBrowserController, None]: + controller = PlaywrightBrowserController(browser, max_open_pages_per_browser=2) + yield controller + await controller.close() + + +async def test_initial_state(browser: Browser) -> None: + controller = PlaywrightBrowserController(browser) + + # Test initial state + assert controller.pages == [] + assert controller.num_of_pages == 0 + assert controller.num_of_total_pages == 0 + assert isinstance(controller.last_page_opened_at, datetime) + assert controller.idle_time < timedelta(seconds=1) + assert controller.has_free_capacity + + +async def test_open_and_close_page(controller: PlaywrightBrowserController, httpbin: str) -> None: + page = await controller.new_page() + await page.goto(f'{httpbin}') + + assert page in controller.pages + assert controller.num_of_pages == 1 + assert controller.num_of_total_pages == 1 + assert controller.last_page_opened_at <= datetime.now(timezone.utc) + + await page.close() + + assert page not in controller.pages + assert controller.num_of_pages == 0 + assert controller.num_of_total_pages == 1 + + +async def test_max_open_pages_limit(controller: PlaywrightBrowserController) -> None: + page1 = await controller.new_page() + assert controller.num_of_pages == 1 + + page2 = await controller.new_page() + assert controller.num_of_pages == 2 + + with pytest.raises(ValueError, match='Cannot open more pages in this browser.'): + await controller.new_page() + + assert controller.num_of_pages == 2 + + await page1.close() + assert controller.num_of_pages == 1 + + page3 = await controller.new_page() + assert controller.num_of_pages == 2 + + await page2.close() + await page3.close() + + assert controller.pages == [] + assert controller.num_of_pages == 0 + assert controller.num_of_total_pages == 3 + + +async def test_idle_time(controller: PlaywrightBrowserController) -> None: + idle_time_before = controller.idle_time + await asyncio.sleep(1) # Simulate waiting + idle_time_after = controller.idle_time + assert idle_time_after > idle_time_before + + +async def test_close_browser_with_open_pages(browser: Browser) -> None: + controller = PlaywrightBrowserController(browser, max_open_pages_per_browser=2) + _ = await controller.new_page() + + with pytest.raises(ValueError, match='Cannot close the browser while there are open pages.'): + await controller.close() + + assert controller.num_of_pages == 1 + assert controller.is_browser_connected + + await controller.close(force=True) + + assert controller.num_of_pages == 0 + assert not controller.is_browser_connected diff --git a/tests/unit/browsers/test_playwright_browser_plugin.py b/tests/unit/browsers/test_playwright_browser_plugin.py new file mode 100644 index 0000000000..c55d6466f8 --- /dev/null +++ b/tests/unit/browsers/test_playwright_browser_plugin.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from typing import AsyncGenerator + +import pytest + +from crawlee.browsers.playwright_browser_plugin import PlaywrightBrowserPlugin + +# pytestmark = pytest.mark.only() + + +@pytest.fixture() +async def plugin() -> AsyncGenerator[PlaywrightBrowserPlugin, None]: + async with PlaywrightBrowserPlugin() as plugin: + yield plugin + + +async def test_initial_state() -> None: + plugin = PlaywrightBrowserPlugin( + browser_type='chromium', + browser_options={'headless': False}, + page_options={'viewport': {'width': 1920, 'height': 1080}}, + max_open_pages_per_browser=10, + ) + + # Test initial state + assert plugin.browser_type == 'chromium' + assert plugin.browser_options == {'headless': False} + assert plugin.page_options == {'viewport': {'width': 1920, 'height': 1080}} + assert plugin.max_open_pages_per_browser == 10 + + +async def test_new_browser(plugin: PlaywrightBrowserPlugin, httpbin: str) -> None: + browser_controller = await plugin.new_browser() + + assert browser_controller.is_browser_connected + + page = await browser_controller.new_page() + await page.goto(f'{httpbin}') + + await page.close() + await browser_controller.close() + + assert not browser_controller.is_browser_connected + + +async def test_multiple_new_browsers(plugin: PlaywrightBrowserPlugin) -> None: + browser_controller_1 = await plugin.new_browser() + browser_controller_2 = await plugin.new_browser() + + assert browser_controller_1 is not browser_controller_2 + + +async def test_new_browser_without_initialization() -> None: + plugin = PlaywrightBrowserPlugin() + with pytest.raises(RuntimeError): + await plugin.new_browser() diff --git a/tests/unit/browsers/test_playwright_plugin.py b/tests/unit/browsers/test_playwright_plugin.py deleted file mode 100644 index 3bfa014832..0000000000 --- a/tests/unit/browsers/test_playwright_plugin.py +++ /dev/null @@ -1,50 +0,0 @@ -from __future__ import annotations - -import pytest - -from crawlee.browsers.playwright_browser_plugin import PlaywrightBrowserPlugin - - -async def test_new_page(httpbin: str) -> None: - async with PlaywrightBrowserPlugin() as plugin: - # Get a new page with default options - page_1 = await plugin.new_page() - await page_1.goto(f'{httpbin}/get') - assert page_1.url == f'{httpbin}/get' - assert ' None: - async with PlaywrightBrowserPlugin() as plugin: - assert plugin.browser is not None - # Browser should be connected - assert plugin.browser.is_connected() is True - - page = await plugin.new_page() - await page.goto(f'{httpbin}/get') - assert page.url == f'{httpbin}/get' - assert ' None: - plugin = PlaywrightBrowserPlugin() - with pytest.raises(RuntimeError, match='Playwright browser plugin is not initialized.'): - await plugin.new_page() From faa57f48b42a236a63968d8be24e62fedae8e961 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 6 Jun 2024 20:27:57 +0200 Subject: [PATCH 02/10] rm commented out code --- tests/unit/browsers/test_playwright_browser_controller.py | 2 -- tests/unit/browsers/test_playwright_browser_plugin.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/tests/unit/browsers/test_playwright_browser_controller.py b/tests/unit/browsers/test_playwright_browser_controller.py index 53d5274a18..9a2f6995e2 100644 --- a/tests/unit/browsers/test_playwright_browser_controller.py +++ b/tests/unit/browsers/test_playwright_browser_controller.py @@ -9,8 +9,6 @@ from crawlee.browsers.playwright_browser_controller import PlaywrightBrowserController -# pytestmark = pytest.mark.only() - @pytest.fixture() async def playwright() -> AsyncGenerator[Playwright, None]: diff --git a/tests/unit/browsers/test_playwright_browser_plugin.py b/tests/unit/browsers/test_playwright_browser_plugin.py index c55d6466f8..bbd481354f 100644 --- a/tests/unit/browsers/test_playwright_browser_plugin.py +++ b/tests/unit/browsers/test_playwright_browser_plugin.py @@ -6,8 +6,6 @@ from crawlee.browsers.playwright_browser_plugin import PlaywrightBrowserPlugin -# pytestmark = pytest.mark.only() - @pytest.fixture() async def plugin() -> AsyncGenerator[PlaywrightBrowserPlugin, None]: From c49fdc98d1bdb7b223ae39026ff4d8c99a5a0a88 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 6 Jun 2024 20:29:49 +0200 Subject: [PATCH 03/10] unify comments --- src/crawlee/browsers/browser_pool.py | 2 +- src/crawlee/browsers/playwright_browser_controller.py | 2 +- src/crawlee/browsers/playwright_browser_plugin.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crawlee/browsers/browser_pool.py b/src/crawlee/browsers/browser_pool.py index 50fff8b033..3be9b557bb 100644 --- a/src/crawlee/browsers/browser_pool.py +++ b/src/crawlee/browsers/browser_pool.py @@ -1,4 +1,4 @@ -# Inspiration: https://github.com/apify/crawlee/tree/v3.10.0/packages/browser-pool/ +# Inspiration: https://github.com/apify/crawlee/tree/v3.10.1/packages/browser-pool/ from __future__ import annotations diff --git a/src/crawlee/browsers/playwright_browser_controller.py b/src/crawlee/browsers/playwright_browser_controller.py index 784ab09c5b..01c1b5f032 100644 --- a/src/crawlee/browsers/playwright_browser_controller.py +++ b/src/crawlee/browsers/playwright_browser_controller.py @@ -1,4 +1,4 @@ -# Inspiration: https://github.com/apify/crawlee/blob/v3.10.0/packages/browser-pool/src/playwright/playwright-controller.ts +# Inspiration: https://github.com/apify/crawlee/blob/v3.10.1/packages/browser-pool/src/playwright/playwright-controller.ts from __future__ import annotations diff --git a/src/crawlee/browsers/playwright_browser_plugin.py b/src/crawlee/browsers/playwright_browser_plugin.py index 2855b88e00..c27e4271a3 100644 --- a/src/crawlee/browsers/playwright_browser_plugin.py +++ b/src/crawlee/browsers/playwright_browser_plugin.py @@ -1,4 +1,4 @@ -# Inspiration: https://github.com/apify/crawlee/blob/v3.10.0/packages/browser-pool/src/playwright/playwright-plugin.ts +# Inspiration: https://github.com/apify/crawlee/blob/v3.10.1/packages/browser-pool/src/playwright/playwright-plugin.ts from __future__ import annotations From 486a9d3bd5678e758b09e3d8185dd22eb244a103 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 6 Jun 2024 20:32:09 +0200 Subject: [PATCH 04/10] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f362931c3..99c1bdbae2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ ### Adds - Add explicit error messages for missing package extras during import +- Better browser abstraction: + - `BrowserController` - Wraps a single browser instance and maintains its state. + - `BrowserPlugin` - Manages the browser automation framework, and basically acts as a factory for controllers. +- Browser rotation with a maximum number of pages opened per browser. ## [0.0.4](../../releases/tag/v0.0.4) - 2024-05-30 From 8d4bc90b247724863793b69a0a4b93d1f2e66cdc Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 7 Jun 2024 09:00:08 +0200 Subject: [PATCH 05/10] starting tasks --- src/crawlee/browsers/browser_pool.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/crawlee/browsers/browser_pool.py b/src/crawlee/browsers/browser_pool.py index 3be9b557bb..5aeb2e1f63 100644 --- a/src/crawlee/browsers/browser_pool.py +++ b/src/crawlee/browsers/browser_pool.py @@ -132,6 +132,8 @@ def pages(self) -> Mapping[str, CrawleePage]: async def __aenter__(self) -> BrowserPool: """Enter the context manager and initialize all browser plugins.""" logger.debug('Initializing browser pool.') + self._identify_inactive_browsers_task.start() + self._close_inactive_browsers_task.start() timeout = self._operation_timeout.total_seconds() try: From 4eacb063581ee8f8aaac6a713af3a545e6a9ae2c Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 7 Jun 2024 13:26:06 +0200 Subject: [PATCH 06/10] updates acc to review --- .../browsers/base_browser_controller.py | 8 ++--- src/crawlee/browsers/browser_pool.py | 11 +++++-- .../browsers/playwright_browser_controller.py | 14 ++++----- tests/unit/browsers/test_browser_pool.py | 2 +- .../test_playwright_browser_controller.py | 30 +++++++++---------- 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/src/crawlee/browsers/base_browser_controller.py b/src/crawlee/browsers/base_browser_controller.py index 9b7a494ad1..3617824bd8 100644 --- a/src/crawlee/browsers/base_browser_controller.py +++ b/src/crawlee/browsers/base_browser_controller.py @@ -25,13 +25,13 @@ def pages(self) -> list[Page]: @property @abstractmethod - def num_of_pages(self) -> int: - """Return the number of currently open pages.""" + def pages_count(self) -> int: + """Returns the number of currently open pages.""" @property @abstractmethod - def num_of_total_pages(self) -> int: - """Return the total number of pages opened since the browser was launched.""" + def total_pages_count(self) -> int: + """Returns the total number of pages opened since the browser was launched.""" @property @abstractmethod diff --git a/src/crawlee/browsers/browser_pool.py b/src/crawlee/browsers/browser_pool.py index 5aeb2e1f63..f28db3f416 100644 --- a/src/crawlee/browsers/browser_pool.py +++ b/src/crawlee/browsers/browser_pool.py @@ -44,8 +44,8 @@ def __init__( plugins: Sequence[BaseBrowserPlugin] | None = None, *, operation_timeout: timedelta = timedelta(seconds=15), - browser_inactive_threshold: timedelta = timedelta(seconds=3), - identify_inactive_browsers_interval: timedelta = timedelta(seconds=10), + browser_inactive_threshold: timedelta = timedelta(seconds=10), + identify_inactive_browsers_interval: timedelta = timedelta(seconds=20), close_inactive_browsers_interval: timedelta = timedelta(seconds=30), ) -> None: """Create a new instance. @@ -132,8 +132,11 @@ def pages(self) -> Mapping[str, CrawleePage]: async def __aenter__(self) -> BrowserPool: """Enter the context manager and initialize all browser plugins.""" logger.debug('Initializing browser pool.') + + # Start the recurring tasks for identifying and closing inactive browsers self._identify_inactive_browsers_task.start() self._close_inactive_browsers_task.start() + timeout = self._operation_timeout.total_seconds() try: @@ -152,6 +155,10 @@ async def __aexit__( ) -> None: """Exit the context manager and close all browser plugins.""" logger.debug('Closing browser pool.') + + await self._identify_inactive_browsers_task.stop() + await self._close_inactive_browsers_task.stop() + for browser in self._active_browsers + self._inactive_browsers: await browser.close(force=True) diff --git a/src/crawlee/browsers/playwright_browser_controller.py b/src/crawlee/browsers/playwright_browser_controller.py index 01c1b5f032..1aac9f28e5 100644 --- a/src/crawlee/browsers/playwright_browser_controller.py +++ b/src/crawlee/browsers/playwright_browser_controller.py @@ -36,7 +36,7 @@ def __init__(self, browser: Browser, *, max_open_pages_per_browser: int = 20) -> self._max_open_pages_per_browser = max_open_pages_per_browser self._pages = list[Page]() - self._num_of_total_pages = 0 + self._total_pages_count = 0 self._last_page_opened_at = datetime.now(timezone.utc) @property @@ -46,13 +46,13 @@ def pages(self) -> list[Page]: @property @override - def num_of_pages(self) -> int: + def pages_count(self) -> int: return len(self._pages) @property @override - def num_of_total_pages(self) -> int: - return self._num_of_total_pages + def total_pages_count(self) -> int: + return self._total_pages_count @property @override @@ -67,7 +67,7 @@ def idle_time(self) -> timedelta: @property @override def has_free_capacity(self) -> bool: - return self.num_of_pages < self._max_open_pages_per_browser + return self.pages_count < self._max_open_pages_per_browser @property @override @@ -88,7 +88,7 @@ async def new_page(self, page_options: Mapping | None = None) -> Page: # Update internal state self._pages.append(page) - self._num_of_total_pages += 1 + self._total_pages_count += 1 self._last_page_opened_at = datetime.now(timezone.utc) return page @@ -99,7 +99,7 @@ async def close(self, *, force: bool = False) -> None: for page in self._pages: await page.close() - if self.num_of_pages > 0: + if self.pages_count > 0: raise ValueError('Cannot close the browser while there are open pages.') await self._browser.close() diff --git a/tests/unit/browsers/test_browser_pool.py b/tests/unit/browsers/test_browser_pool.py index 0bc0fbc6d3..9c72d8afba 100644 --- a/tests/unit/browsers/test_browser_pool.py +++ b/tests/unit/browsers/test_browser_pool.py @@ -117,7 +117,7 @@ async def test_resource_management(httpbin: str) -> None: assert page.page.url == f'{httpbin}/get' assert ' None: # Test initial state assert controller.pages == [] - assert controller.num_of_pages == 0 - assert controller.num_of_total_pages == 0 + assert controller.pages_count == 0 + assert controller.total_pages_count == 0 assert isinstance(controller.last_page_opened_at, datetime) assert controller.idle_time < timedelta(seconds=1) assert controller.has_free_capacity @@ -47,41 +47,41 @@ async def test_open_and_close_page(controller: PlaywrightBrowserController, http await page.goto(f'{httpbin}') assert page in controller.pages - assert controller.num_of_pages == 1 - assert controller.num_of_total_pages == 1 + assert controller.pages_count == 1 + assert controller.total_pages_count == 1 assert controller.last_page_opened_at <= datetime.now(timezone.utc) await page.close() assert page not in controller.pages - assert controller.num_of_pages == 0 - assert controller.num_of_total_pages == 1 + assert controller.pages_count == 0 + assert controller.total_pages_count == 1 async def test_max_open_pages_limit(controller: PlaywrightBrowserController) -> None: page1 = await controller.new_page() - assert controller.num_of_pages == 1 + assert controller.pages_count == 1 page2 = await controller.new_page() - assert controller.num_of_pages == 2 + assert controller.pages_count == 2 with pytest.raises(ValueError, match='Cannot open more pages in this browser.'): await controller.new_page() - assert controller.num_of_pages == 2 + assert controller.pages_count == 2 await page1.close() - assert controller.num_of_pages == 1 + assert controller.pages_count == 1 page3 = await controller.new_page() - assert controller.num_of_pages == 2 + assert controller.pages_count == 2 await page2.close() await page3.close() assert controller.pages == [] - assert controller.num_of_pages == 0 - assert controller.num_of_total_pages == 3 + assert controller.pages_count == 0 + assert controller.total_pages_count == 3 async def test_idle_time(controller: PlaywrightBrowserController) -> None: @@ -98,10 +98,10 @@ async def test_close_browser_with_open_pages(browser: Browser) -> None: with pytest.raises(ValueError, match='Cannot close the browser while there are open pages.'): await controller.close() - assert controller.num_of_pages == 1 + assert controller.pages_count == 1 assert controller.is_browser_connected await controller.close(force=True) - assert controller.num_of_pages == 0 + assert controller.pages_count == 0 assert not controller.is_browser_connected From 72f5d1edce92ec5d56c82d0c7e3a3bd3e31003d2 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 7 Jun 2024 14:12:50 +0200 Subject: [PATCH 07/10] use mapping[str, any] explicitly --- src/crawlee/browsers/base_browser_controller.py | 4 ++-- src/crawlee/browsers/base_browser_plugin.py | 6 +++--- src/crawlee/browsers/playwright_browser_controller.py | 4 ++-- src/crawlee/browsers/playwright_browser_plugin.py | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/crawlee/browsers/base_browser_controller.py b/src/crawlee/browsers/base_browser_controller.py index 3617824bd8..aec7bb4428 100644 --- a/src/crawlee/browsers/base_browser_controller.py +++ b/src/crawlee/browsers/base_browser_controller.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from collections.abc import Mapping @@ -54,7 +54,7 @@ def is_browser_connected(self) -> bool: """Return if the browser is closed.""" @abstractmethod - async def new_page(self, page_options: Mapping | None = None) -> Page: + async def new_page(self, page_options: Mapping[str, Any] | None = None) -> Page: """Create a new page with the given context options. Args: diff --git a/src/crawlee/browsers/base_browser_plugin.py b/src/crawlee/browsers/base_browser_plugin.py index b28b995c02..d8ddf10dc0 100644 --- a/src/crawlee/browsers/base_browser_plugin.py +++ b/src/crawlee/browsers/base_browser_plugin.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal if TYPE_CHECKING: from collections.abc import Mapping @@ -29,12 +29,12 @@ def browser_type(self) -> Literal['chromium', 'firefox', 'webkit']: @property @abstractmethod - def browser_options(self) -> Mapping: + def browser_options(self) -> Mapping[str, Any]: """Return the options for a new browser.""" @property @abstractmethod - def page_options(self) -> Mapping: + def page_options(self) -> Mapping[str, Any]: """Return the options for a new page.""" @property diff --git a/src/crawlee/browsers/playwright_browser_controller.py b/src/crawlee/browsers/playwright_browser_controller.py index 1aac9f28e5..004516cbf2 100644 --- a/src/crawlee/browsers/playwright_browser_controller.py +++ b/src/crawlee/browsers/playwright_browser_controller.py @@ -3,7 +3,7 @@ from __future__ import annotations from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from playwright.async_api import Page from typing_extensions import override @@ -75,7 +75,7 @@ def is_browser_connected(self) -> bool: return self._browser.is_connected() @override - async def new_page(self, page_options: Mapping | None = None) -> Page: + async def new_page(self, page_options: Mapping[str, Any] | None = None) -> Page: page_options = page_options or {} if not self.has_free_capacity: diff --git a/src/crawlee/browsers/playwright_browser_plugin.py b/src/crawlee/browsers/playwright_browser_plugin.py index c27e4271a3..66176b26b7 100644 --- a/src/crawlee/browsers/playwright_browser_plugin.py +++ b/src/crawlee/browsers/playwright_browser_plugin.py @@ -3,7 +3,7 @@ from __future__ import annotations from logging import getLogger -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal from playwright.async_api import Playwright, async_playwright from typing_extensions import override @@ -30,8 +30,8 @@ def __init__( self, *, browser_type: Literal['chromium', 'firefox', 'webkit'] = 'chromium', - browser_options: Mapping | None = None, - page_options: Mapping | None = None, + browser_options: Mapping[str, Any] | None = None, + page_options: Mapping[str, Any] | None = None, max_open_pages_per_browser: int = 20, ) -> None: """Create a new instance. @@ -58,12 +58,12 @@ def browser_type(self) -> Literal['chromium', 'firefox', 'webkit']: @property @override - def browser_options(self) -> Mapping: + def browser_options(self) -> Mapping[str, Any]: return self._browser_options @property @override - def page_options(self) -> Mapping: + def page_options(self) -> Mapping[str, Any]: return self._page_options @property From bc8430268267d386225ae237ba1fd01ce6d50259 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 7 Jun 2024 14:15:30 +0200 Subject: [PATCH 08/10] count instead of num_of --- src/crawlee/autoscaling/snapshotter.py | 4 ++-- src/crawlee/autoscaling/types.py | 10 +++++----- tests/unit/autoscaling/test_system_status.py | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/crawlee/autoscaling/snapshotter.py b/src/crawlee/autoscaling/snapshotter.py index e139814584..4d626e5124 100644 --- a/src/crawlee/autoscaling/snapshotter.py +++ b/src/crawlee/autoscaling/snapshotter.py @@ -269,8 +269,8 @@ def _snapshot_client(self) -> None: # Attribute `self._client_rate_limit_error_retry_count` will be used here. # https://github.com/apify/crawlee-py/issues/60 - num_of_errors = 0 - snapshot = ClientSnapshot(num_of_errors=num_of_errors, max_num_of_errors=self._max_client_errors) + error_count = 0 + snapshot = ClientSnapshot(error_count=error_count, max_error_count=self._max_client_errors) snapshots = cast(list[Snapshot], self._client_snapshots) self._prune_snapshots(snapshots, snapshot.created_at) diff --git a/src/crawlee/autoscaling/types.py b/src/crawlee/autoscaling/types.py index a02df33d87..91a3bb0e1c 100644 --- a/src/crawlee/autoscaling/types.py +++ b/src/crawlee/autoscaling/types.py @@ -141,19 +141,19 @@ class ClientSnapshot: """Snapshot of the state of the client. Args: - num_of_errors: The number of errors (HTTP 429) that occurred. - max_num_of_errors: The maximum number of errors that is considered acceptable. + error_count: The number of errors (HTTP 429) that occurred. + max_error_count: The maximum number of errors that is considered acceptable. created_at: The time at which the measurement was taken. """ - num_of_errors: int - max_num_of_errors: int + error_count: int + max_error_count: int created_at: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) @property def is_overloaded(self) -> bool: """Returns whether the client is considered as overloaded.""" - return self.num_of_errors > self.max_num_of_errors + return self.error_count > self.max_error_count Snapshot = Union[MemorySnapshot, CpuSnapshot, EventLoopSnapshot, ClientSnapshot] diff --git a/tests/unit/autoscaling/test_system_status.py b/tests/unit/autoscaling/test_system_status.py index 9b0b17b1d8..63d0a5d28c 100644 --- a/tests/unit/autoscaling/test_system_status.py +++ b/tests/unit/autoscaling/test_system_status.py @@ -140,10 +140,10 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None: # Add client snapshots system_status._snapshotter._client_snapshots = [ - ClientSnapshot(num_of_errors=1, max_num_of_errors=2, created_at=now - timedelta(minutes=3)), - ClientSnapshot(num_of_errors=1, max_num_of_errors=2, created_at=now - timedelta(minutes=2)), - ClientSnapshot(num_of_errors=2, max_num_of_errors=2, created_at=now - timedelta(minutes=1)), - ClientSnapshot(num_of_errors=0, max_num_of_errors=2, created_at=now), + ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=3)), + ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=2)), + ClientSnapshot(error_count=2, max_error_count=2, created_at=now - timedelta(minutes=1)), + ClientSnapshot(error_count=0, max_error_count=2, created_at=now), ] # Test current system info From 727e44c535e4b5e32cd82623c3ba242d9b2763be Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 7 Jun 2024 14:22:48 +0200 Subject: [PATCH 09/10] total pages on browser pool --- src/crawlee/browsers/base_browser_controller.py | 5 ----- src/crawlee/browsers/browser_pool.py | 7 +++++++ src/crawlee/browsers/playwright_browser_controller.py | 7 ------- tests/unit/browsers/test_browser_pool.py | 8 ++++++++ tests/unit/browsers/test_playwright_browser_controller.py | 4 ---- 5 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/crawlee/browsers/base_browser_controller.py b/src/crawlee/browsers/base_browser_controller.py index aec7bb4428..7632d85964 100644 --- a/src/crawlee/browsers/base_browser_controller.py +++ b/src/crawlee/browsers/base_browser_controller.py @@ -28,11 +28,6 @@ def pages(self) -> list[Page]: def pages_count(self) -> int: """Returns the number of currently open pages.""" - @property - @abstractmethod - def total_pages_count(self) -> int: - """Returns the total number of pages opened since the browser was launched.""" - @property @abstractmethod def last_page_opened_at(self) -> datetime: diff --git a/src/crawlee/browsers/browser_pool.py b/src/crawlee/browsers/browser_pool.py index f28db3f416..1f68c81b37 100644 --- a/src/crawlee/browsers/browser_pool.py +++ b/src/crawlee/browsers/browser_pool.py @@ -80,6 +80,7 @@ def __init__( self._active_browsers = list[BaseBrowserController]() self._inactive_browsers = list[BaseBrowserController]() + self._total_pages_count = 0 self._pages = WeakValueDictionary[str, CrawleePage]() # Track the pages in the pool self._plugins_cycle = itertools.cycle(self._plugins) # Cycle through the plugins @@ -129,6 +130,11 @@ def pages(self) -> Mapping[str, CrawleePage]: """Return the pages in the pool.""" return self._pages + @property + def total_pages_count(self) -> int: + """Returns the total number of pages opened since the browser pool was launched.""" + return self._total_pages_count + async def __aenter__(self) -> BrowserPool: """Enter the context manager and initialize all browser plugins.""" logger.debug('Initializing browser pool.') @@ -225,6 +231,7 @@ async def _get_new_page( crawlee_page = CrawleePage(id=page_id, page=page, browser_type=plugin.browser_type) self._pages[page_id] = crawlee_page + self._total_pages_count += 1 return crawlee_page def _pick_browser_with_free_capacity( diff --git a/src/crawlee/browsers/playwright_browser_controller.py b/src/crawlee/browsers/playwright_browser_controller.py index 004516cbf2..b61fb8c547 100644 --- a/src/crawlee/browsers/playwright_browser_controller.py +++ b/src/crawlee/browsers/playwright_browser_controller.py @@ -36,7 +36,6 @@ def __init__(self, browser: Browser, *, max_open_pages_per_browser: int = 20) -> self._max_open_pages_per_browser = max_open_pages_per_browser self._pages = list[Page]() - self._total_pages_count = 0 self._last_page_opened_at = datetime.now(timezone.utc) @property @@ -49,11 +48,6 @@ def pages(self) -> list[Page]: def pages_count(self) -> int: return len(self._pages) - @property - @override - def total_pages_count(self) -> int: - return self._total_pages_count - @property @override def last_page_opened_at(self) -> datetime: @@ -88,7 +82,6 @@ async def new_page(self, page_options: Mapping[str, Any] | None = None) -> Page: # Update internal state self._pages.append(page) - self._total_pages_count += 1 self._last_page_opened_at = datetime.now(timezone.utc) return page diff --git a/tests/unit/browsers/test_browser_pool.py b/tests/unit/browsers/test_browser_pool.py index 9c72d8afba..82ec6d9b8e 100644 --- a/tests/unit/browsers/test_browser_pool.py +++ b/tests/unit/browsers/test_browser_pool.py @@ -13,12 +13,14 @@ async def test_default_plugin_new_page_creation(httpbin: str) -> None: assert page_1.browser_type == 'chromium' assert page_1.page.url == f'{httpbin}/get' assert ' None: await page_2.page.close() await page_3.page.close() + assert browser_pool.total_pages_count == 3 + async def test_new_page_with_each_plugin(httpbin: str) -> None: plugin_chromium = PlaywrightBrowserPlugin(browser_type='chromium') @@ -77,6 +81,8 @@ async def test_new_page_with_each_plugin(httpbin: str) -> None: for page in pages: await page.page.close() + assert browser_pool.total_pages_count == 2 + async def test_with_default_plugin_constructor(httpbin: str) -> None: async with BrowserPool.with_default_plugin(headless=True, browser_type='firefox') as browser_pool: @@ -91,6 +97,7 @@ async def test_with_default_plugin_constructor(httpbin: str) -> None: assert ' None: @@ -116,6 +123,7 @@ async def test_resource_management(httpbin: str) -> None: await page.page.goto(f'{httpbin}/get') assert page.page.url == f'{httpbin}/get' assert ' None: # Test initial state assert controller.pages == [] assert controller.pages_count == 0 - assert controller.total_pages_count == 0 assert isinstance(controller.last_page_opened_at, datetime) assert controller.idle_time < timedelta(seconds=1) assert controller.has_free_capacity @@ -48,14 +47,12 @@ async def test_open_and_close_page(controller: PlaywrightBrowserController, http assert page in controller.pages assert controller.pages_count == 1 - assert controller.total_pages_count == 1 assert controller.last_page_opened_at <= datetime.now(timezone.utc) await page.close() assert page not in controller.pages assert controller.pages_count == 0 - assert controller.total_pages_count == 1 async def test_max_open_pages_limit(controller: PlaywrightBrowserController) -> None: @@ -81,7 +78,6 @@ async def test_max_open_pages_limit(controller: PlaywrightBrowserController) -> assert controller.pages == [] assert controller.pages_count == 0 - assert controller.total_pages_count == 3 async def test_idle_time(controller: PlaywrightBrowserController) -> None: From 68e239eb46d75f2f955df4f43440fbbcb5380848 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 7 Jun 2024 18:11:48 +0200 Subject: [PATCH 10/10] add explaining comments to attrs --- src/crawlee/browsers/browser_pool.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/crawlee/browsers/browser_pool.py b/src/crawlee/browsers/browser_pool.py index 1f68c81b37..a1bebedabc 100644 --- a/src/crawlee/browsers/browser_pool.py +++ b/src/crawlee/browsers/browser_pool.py @@ -67,6 +67,13 @@ def __init__( self._operation_timeout = operation_timeout self._browser_inactive_threshold = browser_inactive_threshold + self._active_browsers = list[BaseBrowserController]() + """A list of browsers currently active and being used to open pages.""" + + self._inactive_browsers = list[BaseBrowserController]() + """A list of browsers currently inactive and not being used to open new pages, + but may still contain open pages.""" + self._identify_inactive_browsers_task = RecurringTask( self._identify_inactive_browsers, identify_inactive_browsers_interval, @@ -77,9 +84,6 @@ def __init__( close_inactive_browsers_interval, ) - self._active_browsers = list[BaseBrowserController]() - self._inactive_browsers = list[BaseBrowserController]() - self._total_pages_count = 0 self._pages = WeakValueDictionary[str, CrawleePage]() # Track the pages in the pool self._plugins_cycle = itertools.cycle(self._plugins) # Cycle through the plugins