diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index eebd86fb7a..1ec780ff0f 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -155,8 +155,8 @@ def __init__( self._context_pipeline = (_context_pipeline or ContextPipeline()).compose(self._check_url_after_redirects) - self._error_handler: ErrorHandler[TCrawlingContext] | None = None - self._failed_request_handler: FailedRequestHandler[TCrawlingContext] | None = None + self._error_handler: ErrorHandler[TCrawlingContext | BasicCrawlingContext] | None = None + self._failed_request_handler: FailedRequestHandler[TCrawlingContext | BasicCrawlingContext] | None = None self._max_request_retries = max_request_retries self._max_requests_per_crawl = max_requests_per_crawl @@ -309,13 +309,15 @@ async def get_key_value_store( """Return the key-value store with the given ID or name. If none is provided, return the default KVS.""" return await KeyValueStore.open(id=id, name=name) - def error_handler(self, handler: ErrorHandler[TCrawlingContext]) -> ErrorHandler[TCrawlingContext]: + def error_handler( + self, handler: ErrorHandler[TCrawlingContext | BasicCrawlingContext] + ) -> ErrorHandler[TCrawlingContext]: """Decorator for configuring an error handler (called after a request handler error and before retrying).""" self._error_handler = handler return handler def failed_request_handler( - self, handler: FailedRequestHandler[TCrawlingContext] + self, handler: FailedRequestHandler[TCrawlingContext | BasicCrawlingContext] ) -> FailedRequestHandler[TCrawlingContext]: """Decorator for configuring a failed request handler (called after max retries are reached).""" self._failed_request_handler = handler @@ -575,7 +577,9 @@ def _check_url_patterns( # The URL does not match any `include` pattern - reject it return False - async def _handle_request_error(self, crawling_context: TCrawlingContext, error: Exception) -> None: + async def _handle_request_retries( + self, crawling_context: TCrawlingContext | BasicCrawlingContext, error: Exception + ) -> None: request_provider = await self.get_request_provider() request = crawling_context.request @@ -605,7 +609,39 @@ async def _handle_request_error(self, crawling_context: TCrawlingContext, error: await self._handle_failed_request(crawling_context, error) self._statistics.record_request_processing_failure(request.id or request.unique_key) - async def _handle_failed_request(self, crawling_context: TCrawlingContext, error: Exception) -> None: + async def _handle_request_error( + self, crawling_context: TCrawlingContext | BasicCrawlingContext, error: Exception + ) -> None: + try: + crawling_context.request.state = RequestState.ERROR_HANDLER + + await wait_for( + partial(self._handle_request_retries, crawling_context, error), + timeout=self._internal_timeout, + timeout_message='Handling request failure timed out after ' + f'{self._internal_timeout.total_seconds()} seconds', + logger=self._logger, + ) + + crawling_context.request.state = RequestState.DONE + except UserDefinedErrorHandlerError: + crawling_context.request.state = RequestState.ERROR + raise + except Exception as secondary_error: + self._logger.exception( + 'An exception occurred during handling of failed request. This places the crawler ' + 'and its underlying storages into an unknown state and crawling will be terminated.', + exc_info=secondary_error, + ) + crawling_context.request.state = RequestState.ERROR + raise + + if crawling_context.session: + crawling_context.session.mark_bad() + + async def _handle_failed_request( + self, crawling_context: TCrawlingContext | BasicCrawlingContext, error: Exception + ) -> None: self._logger.exception('Request failed and reached maximum retries', exc_info=error) self._statistics.error_tracker.add(error) @@ -753,34 +789,11 @@ async def __run_task_function(self) -> None: RequestHandlerError[TCrawlingContext], primary_error ) # valid thanks to ContextPipeline - try: - request.state = RequestState.ERROR_HANDLER - - await wait_for( - partial( - self._handle_request_error, primary_error.crawling_context, primary_error.wrapped_exception - ), - timeout=self._internal_timeout, - timeout_message='Handling request failure timed out after ' - f'{self._internal_timeout.total_seconds()} seconds', - logger=self._logger, - ) - - request.state = RequestState.DONE - except UserDefinedErrorHandlerError: - request.state = RequestState.ERROR - raise - except Exception as secondary_error: - self._logger.exception( - 'An exception occurred during handling of failed request. This places the crawler ' - 'and its underlying storages into an unknown state and crawling will be terminated.', - exc_info=secondary_error, - ) - request.state = RequestState.ERROR - raise - - if crawling_context.session: - crawling_context.session.mark_bad() + self._logger.debug( + 'An exception occurred in the user-defined request handler', + exc_info=primary_error.wrapped_exception, + ) + await self._handle_request_error(primary_error.crawling_context, primary_error.wrapped_exception) except SessionError as session_error: if not crawling_context.session: raise RuntimeError('SessionError raised in a crawling context without a session') from session_error @@ -822,30 +835,11 @@ async def __run_task_function(self) -> None: max_retries=3, ) except ContextPipelineInitializationError as initialization_error: - if self._should_retry_request(crawling_context, initialization_error): - self._logger.debug( - 'An exception occurred during the initialization of crawling context, a retry is in order', - exc_info=initialization_error, - ) - - request = crawling_context.request - request.retry_count += 1 - request.state = RequestState.DONE - await request_provider.reclaim_request(request) - else: - self._logger.exception('Request failed and reached maximum retries', exc_info=initialization_error) - - await wait_for( - lambda: request_provider.mark_request_as_handled(crawling_context.request), - timeout=self._internal_timeout, - timeout_message='Marking request as handled timed out after ' - f'{self._internal_timeout.total_seconds()} seconds', - logger=self._logger, - max_retries=3, - ) - - if crawling_context.session: - crawling_context.session.mark_bad() + self._logger.debug( + 'An exception occurred during the initialization of crawling context', + exc_info=initialization_error, + ) + await self._handle_request_error(crawling_context, initialization_error) except Exception as internal_error: self._logger.exception( 'An exception occurred during handling of a request. This places the crawler ' diff --git a/src/crawlee/playwright_crawler/playwright_crawler.py b/src/crawlee/playwright_crawler/playwright_crawler.py index d69a6bff51..d7cc80e0f5 100644 --- a/src/crawlee/playwright_crawler/playwright_crawler.py +++ b/src/crawlee/playwright_crawler/playwright_crawler.py @@ -97,59 +97,60 @@ async def _make_http_request( if self._browser_pool is None: raise ValueError('Browser pool is not initialized.') - # Create a new browser page, navigate to the URL and get response. + # Create a new browser page crawlee_page = await self._browser_pool.new_page(proxy_info=context.proxy_info) - response = await crawlee_page.page.goto(context.request.url) - - if response is None: - raise SessionError(f'Failed to load the URL: {context.request.url}') - - # Set the loaded URL to the actual URL after redirection. - context.request.loaded_url = crawlee_page.page.url - - async def enqueue_links( - *, - selector: str = 'a', - label: str | None = None, - user_data: dict | None = None, - **kwargs: Unpack[AddRequestsKwargs], - ) -> None: - """The `PlaywrightCrawler` implementation of the `EnqueueLinksFunction` function.""" - kwargs.setdefault('strategy', EnqueueStrategy.SAME_HOSTNAME) - - requests = list[BaseRequestData]() - user_data = user_data or {} - - elements = await crawlee_page.page.query_selector_all(selector) - - for element in elements: - href = await element.get_attribute('href') - - if href: - link_user_data = user_data.copy() - - if label is not None: - link_user_data.setdefault('label', label) - - request = BaseRequestData.from_url(href, user_data=link_user_data) - requests.append(request) - - await context.add_requests(requests, **kwargs) - - yield PlaywrightCrawlingContext( - request=context.request, - session=context.session, - add_requests=context.add_requests, - send_request=context.send_request, - push_data=context.push_data, - proxy_info=context.proxy_info, - log=context.log, - page=crawlee_page.page, - response=response, - enqueue_links=enqueue_links, - ) - await crawlee_page.page.close() + async with crawlee_page.page: + # Navigate to the URL and get response. + response = await crawlee_page.page.goto(context.request.url) + + if response is None: + raise SessionError(f'Failed to load the URL: {context.request.url}') + + # Set the loaded URL to the actual URL after redirection. + context.request.loaded_url = crawlee_page.page.url + + async def enqueue_links( + *, + selector: str = 'a', + label: str | None = None, + user_data: dict | None = None, + **kwargs: Unpack[AddRequestsKwargs], + ) -> None: + """The `PlaywrightCrawler` implementation of the `EnqueueLinksFunction` function.""" + kwargs.setdefault('strategy', EnqueueStrategy.SAME_HOSTNAME) + + requests = list[BaseRequestData]() + user_data = user_data or {} + + elements = await crawlee_page.page.query_selector_all(selector) + + for element in elements: + href = await element.get_attribute('href') + + if href: + link_user_data = user_data.copy() + + if label is not None: + link_user_data.setdefault('label', label) + + request = BaseRequestData.from_url(href, user_data=link_user_data) + requests.append(request) + + await context.add_requests(requests, **kwargs) + + yield PlaywrightCrawlingContext( + request=context.request, + session=context.session, + add_requests=context.add_requests, + send_request=context.send_request, + push_data=context.push_data, + proxy_info=context.proxy_info, + log=context.log, + page=crawlee_page.page, + response=response, + enqueue_links=enqueue_links, + ) async def _handle_blocked_request( self, diff --git a/tests/unit/playwright_crawler/test_playwright_crawler.py b/tests/unit/playwright_crawler/test_playwright_crawler.py index f8c64c64eb..aec88776dc 100644 --- a/tests/unit/playwright_crawler/test_playwright_crawler.py +++ b/tests/unit/playwright_crawler/test_playwright_crawler.py @@ -9,6 +9,7 @@ from crawlee import Glob from crawlee.playwright_crawler import PlaywrightCrawler +from crawlee.storages import RequestList if TYPE_CHECKING: from crawlee.playwright_crawler import PlaywrightCrawlingContext @@ -50,3 +51,23 @@ async def request_handler(context: PlaywrightCrawlingContext) -> None: assert len(visited) >= 10 assert all(url.startswith('https://crawlee.dev/docs/examples') for url in visited) + + +async def test_nonexistent_url_invokes_error_handler() -> None: + crawler = PlaywrightCrawler( + max_request_retries=4, request_provider=RequestList(['https://this-does-not-exist-22343434.com']) + ) + + error_handler = mock.AsyncMock(return_value=None) + crawler.error_handler(error_handler) + + failed_handler = mock.AsyncMock(return_value=None) + crawler.failed_request_handler(failed_handler) + + @crawler.router.default_handler + async def request_handler(_context: PlaywrightCrawlingContext) -> None: + pass + + await crawler.run() + assert error_handler.call_count == 3 + assert failed_handler.call_count == 1