diff --git a/Makefile b/Makefile index ea715fb5ab..96cbeb49b6 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ clean: install-dev: python3 -m pip install --upgrade pip poetry - poetry install + poetry install --all-extras poetry run pre-commit install lint: diff --git a/pyproject.toml b/pyproject.toml index b28d73be0a..1095c75c92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,10 +47,13 @@ keywords = [ python = "^3.9" aiofiles = "^23.2.1" aioshutil = "^1.3" +beautifulsoup4 = { version = "^4.12.3", optional = true } colorama = "^0.4.6" docutils = "^0.21.0" eval-type-backport = "^0.1.3" +html5lib = { version = "^1.1", optional = true } httpx = "^0.27.0" +lxml = { version = "^5.2.1", optional = true } more_itertools = "^10.2.0" psutil = "^5.9.8" pydantic = "^2.6.3" @@ -76,10 +79,14 @@ respx = "~0.21.0" ruff = "~0.3.0" twine = "~5.0.0" types-aiofiles = "^23.2.0.20240106" +types-beautifulsoup4 = "^4.12.0.20240229" types-colorama = "~0.4.15.20240106" types-psutil = "~5.9.5.20240205" types-python-dateutil = "^2.9.0.20240316" +[tool.poetry.extras] +beautifulsoup = ["beautifulsoup4", "lxml", "html5lib"] + [tool.ruff] line-length = 120 diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index 9bb28104c6..adb4d59b72 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -12,7 +12,11 @@ from crawlee.autoscaling import AutoscaledPool, ConcurrencySettings from crawlee.autoscaling.snapshotter import Snapshotter from crawlee.autoscaling.system_status import SystemStatus -from crawlee.basic_crawler.context_pipeline import ContextPipeline, RequestHandlerError +from crawlee.basic_crawler.context_pipeline import ( + ContextPipeline, + ContextPipelineInitializationError, + RequestHandlerError, +) from crawlee.basic_crawler.router import Router from crawlee.basic_crawler.types import BasicCrawlingContext, FinalStatistics from crawlee.configuration import Configuration @@ -162,16 +166,17 @@ async def run(self, requests: list[str | BaseRequestData] | None = None) -> Fina return FinalStatistics() - async def _handle_request_error(self, crawling_context: TCrawlingContext, error: Exception) -> None: + def _should_retry_request(self, crawling_context: BasicCrawlingContext) -> bool: max_request_retries = crawling_context.request.max_retries if max_request_retries is None: max_request_retries = self._max_request_retries - should_retry_request = ( + return ( not crawling_context.request.no_retry and (crawling_context.request.retry_count + 1) < max_request_retries ) - if should_retry_request: + async def _handle_request_error(self, crawling_context: TCrawlingContext, error: Exception) -> None: + if self._should_retry_request(crawling_context): request = crawling_context.request request.retry_count += 1 @@ -224,7 +229,7 @@ async def __run_task_function(self) -> None: return # TODO: fetch session from the session pool - # https://github.com/apify/crawlee-py/issues/78 + # https://github.com/apify/crawlee-py/issues/110 crawling_context = BasicCrawlingContext(request=request) @@ -240,7 +245,7 @@ async def __run_task_function(self) -> None: ) await wait_for( - lambda: self._request_provider.mark_request_as_handled(request), + lambda: self._request_provider.mark_request_as_handled(crawling_context.request), timeout=self._internal_timeout, timeout_message='Marking request as handled timed out after ' f'{self._internal_timeout.total_seconds()} seconds', @@ -279,6 +284,28 @@ async def __run_task_function(self) -> None: ) request.state = RequestState.ERROR raise + except ContextPipelineInitializationError as initialization_error: + if self._should_retry_request(crawling_context): + logger.debug( + 'An exception occured during the initialization of crawling context, a retry is in order', + exc_info=initialization_error, + ) + + request = crawling_context.request + request.retry_count += 1 + request.state = RequestState.DONE + await self._request_provider.reclaim_request(request) + else: + logger.exception('Request failed and reached maximum retries', exc_info=initialization_error) + + await wait_for( + lambda: self._request_provider.mark_request_as_handled(crawling_context.request), + timeout=self._internal_timeout, + timeout_message='Marking request as handled timed out after ' + f'{self._internal_timeout.total_seconds()} seconds', + logger=logger, + max_retries=3, + ) except Exception as internal_error: logger.exception( 'An exception occurred during handling of a request. This places the crawler ' diff --git a/src/crawlee/basic_crawler/context_pipeline.py b/src/crawlee/basic_crawler/context_pipeline.py index cc746b5c7e..39e4a9077e 100644 --- a/src/crawlee/basic_crawler/context_pipeline.py +++ b/src/crawlee/basic_crawler/context_pipeline.py @@ -18,6 +18,28 @@ def __init__(self, wrapped_exception: Exception, crawling_context: TCrawlingCont self.crawling_context = crawling_context +class ContextPipelineInitializationError(Exception): + """Wraps an exception thrown in the initialization step of a context pipeline middleware. + + We may not have the complete context at this point, so only `BasicCrawlingContext` is provided. + """ + + def __init__(self, wrapped_exception: Exception, crawling_context: BasicCrawlingContext) -> None: + self.wrapped_exception = wrapped_exception + self.crawling_context = crawling_context + + +class ContextPipelineFinalizationError(Exception): + """Wraps an exception thrown in the finalization step of a context pipeline middleware. + + We may not have the complete context at this point, so only `BasicCrawlingContext` is provided. + """ + + def __init__(self, wrapped_exception: Exception, crawling_context: BasicCrawlingContext) -> None: + self.wrapped_exception = wrapped_exception + self.crawling_context = crawling_context + + class ContextPipeline(Generic[TCrawlingContext]): """Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities. @@ -55,29 +77,34 @@ async def __call__( chain = list(self._middleware_chain()) cleanup_stack = list[AsyncGenerator]() - for member in reversed(chain): - if member._middleware: # noqa: SLF001 - middleware_instance = member._middleware(crawling_context) # noqa: SLF001 - try: - result = await middleware_instance.__anext__() - except StopAsyncIteration as e: - raise RuntimeError('The middleware did not yield') from e - - crawling_context = result - cleanup_stack.append(middleware_instance) - try: - await final_context_consumer(cast(TCrawlingContext, crawling_context)) - except Exception as e: - raise RequestHandlerError(e, crawling_context) from e + for member in reversed(chain): + if member._middleware: # noqa: SLF001 + middleware_instance = member._middleware(crawling_context) # noqa: SLF001 + try: + result = await middleware_instance.__anext__() + except StopAsyncIteration as e: + raise RuntimeError('The middleware did not yield') from e + except Exception as e: + raise ContextPipelineInitializationError(e, crawling_context) from e + + crawling_context = result + cleanup_stack.append(middleware_instance) - for middleware_instance in reversed(cleanup_stack): try: - result = await middleware_instance.__anext__() - except StopAsyncIteration: # noqa: PERF203 - pass - else: - raise RuntimeError('The middleware yielded more than once') + await final_context_consumer(cast(TCrawlingContext, crawling_context)) + except Exception as e: + raise RequestHandlerError(e, crawling_context) from e + finally: + for middleware_instance in reversed(cleanup_stack): + try: + result = await middleware_instance.__anext__() + except StopAsyncIteration: # noqa: PERF203 + pass + except Exception as e: + raise ContextPipelineFinalizationError(e, crawling_context) from e + else: + raise RuntimeError('The middleware yielded more than once') def compose( self, diff --git a/src/crawlee/beautifulsoup_crawler/__init__.py b/src/crawlee/beautifulsoup_crawler/__init__.py new file mode 100644 index 0000000000..ec511fe9ce --- /dev/null +++ b/src/crawlee/beautifulsoup_crawler/__init__.py @@ -0,0 +1 @@ +from .beautifulsoup_crawler import BeautifulSoupCrawler, BeautifulSoupCrawlingContext diff --git a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py new file mode 100644 index 0000000000..ef2e08ad16 --- /dev/null +++ b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import asyncio +from datetime import timedelta +from typing import TYPE_CHECKING, AsyncGenerator, Awaitable, Callable, Iterable, Literal + +import httpx + +from crawlee.basic_crawler.basic_crawler import BasicCrawler +from crawlee.basic_crawler.context_pipeline import ContextPipeline +from crawlee.beautifulsoup_crawler.types import BeautifulSoupCrawlingContext +from crawlee.http_crawler.http_crawler import make_http_request +from crawlee.http_crawler.types import HttpCrawlingContext + +if TYPE_CHECKING: + from datetime import timedelta + + from crawlee.autoscaling.autoscaled_pool import ConcurrencySettings + from crawlee.basic_crawler.types import BasicCrawlingContext + from crawlee.configuration import Configuration + from crawlee.storages.request_provider import RequestProvider + + +class BeautifulSoupCrawler(BasicCrawler[BeautifulSoupCrawlingContext]): + """A crawler that fetches the request URL using `httpx` and parses the result with `BeautifulSoup`.""" + + def __init__( + self, + *, + parser: Literal['html.parser', 'lxml', 'xml', 'html5lib'] = 'lxml', + request_provider: RequestProvider, + router: Callable[[BeautifulSoupCrawlingContext], Awaitable[None]] | None = None, + concurrency_settings: ConcurrencySettings | None = None, + configuration: Configuration | None = None, + request_handler_timeout: timedelta | None = None, + additional_http_error_status_codes: Iterable[int] = (), + ignore_http_error_status_codes: Iterable[int] = (), + ) -> None: + """Initialize the BeautifulSoupCrawler. + + Args: + parser: The type of parser that should be used by BeautifulSoup + + request_provider: Provides requests to be processed + + router: A callable to which request handling is delegated + + concurrency_settings: Allows fine-tuning concurrency levels + + configuration: Crawler configuration + + request_handler_timeout: How long is a single request handler allowed to run + + additional_http_error_status_codes: HTTP status codes that should be considered errors (and trigger a retry) + + ignore_http_error_status_codes: HTTP status codes that are normally considered errors but we want to treat + them as successful + """ + context_pipeline = ContextPipeline().compose(self._make_http_request).compose(self._parse_http_response) + + self._client = httpx.AsyncClient() + self._parser = parser + + self._additional_http_error_status_codes = set(additional_http_error_status_codes) + self._ignore_http_error_status_codes = set(ignore_http_error_status_codes) + + basic_crawler_kwargs = {} + + if request_handler_timeout is not None: + basic_crawler_kwargs['request_handler_timeout'] = request_handler_timeout + + super().__init__( + request_provider=request_provider, + router=router, + concurrency_settings=concurrency_settings, + configuration=configuration, + _context_pipeline=context_pipeline, + **basic_crawler_kwargs, # type: ignore + ) + + async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenerator[HttpCrawlingContext, None]: + result = await make_http_request( + self._client, + context.request, + additional_http_error_status_codes=self._additional_http_error_status_codes, + ignore_http_error_status_codes=self._ignore_http_error_status_codes, + ) + + yield HttpCrawlingContext(request=context.request, http_response=result.http_response) + + async def _parse_http_response( + self, context: HttpCrawlingContext + ) -> AsyncGenerator[BeautifulSoupCrawlingContext, None]: + from bs4 import BeautifulSoup + + soup = await asyncio.to_thread(lambda: BeautifulSoup(context.http_response.read(), self._parser)) + yield BeautifulSoupCrawlingContext(request=context.request, http_response=context.http_response, soup=soup) diff --git a/src/crawlee/beautifulsoup_crawler/types.py b/src/crawlee/beautifulsoup_crawler/types.py new file mode 100644 index 0000000000..4bcc560540 --- /dev/null +++ b/src/crawlee/beautifulsoup_crawler/types.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from crawlee.basic_crawler.types import BasicCrawlingContext +from crawlee.http_crawler.types import HttpCrawlResult + +if TYPE_CHECKING: + from bs4 import BeautifulSoup + + +@dataclass(frozen=True) +class BeautifulSoupCrawlingContext(HttpCrawlResult, BasicCrawlingContext): + """Crawling context used by BeautifulSoupCrawler.""" + soup: BeautifulSoup diff --git a/src/crawlee/http_crawler/http_crawler.py b/src/crawlee/http_crawler/http_crawler.py index 4fecf432a0..2c8c1a5610 100644 --- a/src/crawlee/http_crawler/http_crawler.py +++ b/src/crawlee/http_crawler/http_crawler.py @@ -1,12 +1,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, AsyncGenerator, Awaitable, Callable +from typing import TYPE_CHECKING, AsyncGenerator, Awaitable, Callable, Iterable import httpx from crawlee.basic_crawler.basic_crawler import BasicCrawler from crawlee.basic_crawler.context_pipeline import ContextPipeline -from crawlee.http_crawler.types import HttpCrawlingContext +from crawlee.http_crawler.types import HttpCrawlingContext, HttpCrawlResult if TYPE_CHECKING: from datetime import timedelta @@ -14,6 +14,7 @@ from crawlee.autoscaling.autoscaled_pool import ConcurrencySettings from crawlee.basic_crawler.types import BasicCrawlingContext from crawlee.configuration import Configuration + from crawlee.request import Request from crawlee.storages.request_provider import RequestProvider @@ -28,15 +29,26 @@ def __init__( concurrency_settings: ConcurrencySettings | None = None, configuration: Configuration | None = None, request_handler_timeout: timedelta | None = None, + additional_http_error_status_codes: Iterable[int] = (), + ignore_http_error_status_codes: Iterable[int] = (), ) -> None: """Initialize the HttpCrawler. Args: request_provider: Provides requests to be processed + router: A callable to which request handling is delegated + concurrency_settings: Allows fine-tuning concurrency levels + configuration: Crawler configuration + request_handler_timeout: How long is a single request handler allowed to run + + additional_http_error_status_codes: HTTP status codes that should be considered errors (and trigger a retry) + + ignore_http_error_status_codes: HTTP status codes that are normally considered errors but we want to treat + them as successful """ context_pipeline = ContextPipeline().compose(self._make_http_request) self._client = httpx.AsyncClient() @@ -46,6 +58,9 @@ def __init__( if request_handler_timeout is not None: basic_crawler_kwargs['request_handler_timeout'] = request_handler_timeout + self._additional_http_error_status_codes = set(additional_http_error_status_codes) + self._ignore_http_error_status_codes = set(ignore_http_error_status_codes) + super().__init__( router=router, _context_pipeline=context_pipeline, @@ -58,7 +73,43 @@ def __init__( async def _make_http_request( self, crawling_context: BasicCrawlingContext ) -> AsyncGenerator[HttpCrawlingContext, None]: - response = await self._client.request(crawling_context.request.method, crawling_context.request.url) - response.raise_for_status() + result = await make_http_request( + self._client, + crawling_context.request, + additional_http_error_status_codes=self._additional_http_error_status_codes, + ignore_http_error_status_codes=self._ignore_http_error_status_codes, + ) + + yield HttpCrawlingContext(request=crawling_context.request, http_response=result.http_response) + + +async def make_http_request( + client: httpx.AsyncClient, + request: Request, + *, + additional_http_error_status_codes: set[int] | None = None, + ignore_http_error_status_codes: set[int] | None = None, +) -> HttpCrawlResult: + """Perform a request using `httpx`.""" + response = await client.request(request.method, request.url, follow_redirects=True) + + exclude_error = ( + ignore_http_error_status_codes is not None and response.status_code in ignore_http_error_status_codes + ) + include_error = additional_http_error_status_codes and response.status_code in additional_http_error_status_codes + + if (response.is_server_error and not exclude_error) or include_error: + if include_error: + raise httpx.HTTPStatusError( + f'Status code {response.status_code} (user-configured to be an error) returned', + request=response.request, + response=response, + ) + + raise httpx.HTTPStatusError( + f'Status code {response.status_code} returned', request=response.request, response=response + ) + + request.loaded_url = str(response.url) - yield HttpCrawlingContext(request=crawling_context.request, http_response=response) + return HttpCrawlResult(http_response=response) diff --git a/src/crawlee/http_crawler/types.py b/src/crawlee/http_crawler/types.py index 52c32c8048..6b32c48e53 100644 --- a/src/crawlee/http_crawler/types.py +++ b/src/crawlee/http_crawler/types.py @@ -10,7 +10,12 @@ @dataclass(frozen=True) -class HttpCrawlingContext(BasicCrawlingContext): - """HTTP crawling context.""" +class HttpCrawlResult: + """Result of loading a page with raw HTTP.""" http_response: Response + + +@dataclass(frozen=True) +class HttpCrawlingContext(BasicCrawlingContext, HttpCrawlResult): + """HTTP crawling context.""" diff --git a/tests/unit/basic_crawler/test_context_pipeline.py b/tests/unit/basic_crawler/test_context_pipeline.py index 19741f156f..67bde6a086 100644 --- a/tests/unit/basic_crawler/test_context_pipeline.py +++ b/tests/unit/basic_crawler/test_context_pipeline.py @@ -6,7 +6,12 @@ import pytest -from crawlee.basic_crawler.context_pipeline import ContextPipeline, RequestHandlerError +from crawlee.basic_crawler.context_pipeline import ( + ContextPipeline, + ContextPipelineFinalizationError, + ContextPipelineInitializationError, + RequestHandlerError, +) from crawlee.basic_crawler.types import BasicCrawlingContext from crawlee.request import Request @@ -71,3 +76,47 @@ async def test_wraps_consumer_errors() -> None: with pytest.raises(RequestHandlerError): await pipeline(context, consumer) + + +async def test_handles_exceptions_in_middleware_initialization() -> None: + consumer = AsyncMock() + cleanup = AsyncMock() + + async def step_1(context: BasicCrawlingContext) -> AsyncGenerator[BasicCrawlingContext, None]: + yield context + await cleanup() + + async def step_2(context: BasicCrawlingContext) -> AsyncGenerator[BasicCrawlingContext, None]: + raise RuntimeError('Crash during middleware initialization') + yield context # type: ignore[unreachable] + + pipeline = ContextPipeline().compose(step_1).compose(step_2) + context = BasicCrawlingContext(request=Request.from_url(url='aaa')) + + with pytest.raises(ContextPipelineInitializationError): + await pipeline(context, consumer) + + assert not consumer.called + assert cleanup.called + + +async def test_handles_exceptions_in_middleware_finalization() -> None: + consumer = AsyncMock() + cleanup = AsyncMock() + + async def step_1(context: BasicCrawlingContext) -> AsyncGenerator[BasicCrawlingContext, None]: + yield context + await cleanup() + + async def step_2(context: BasicCrawlingContext) -> AsyncGenerator[BasicCrawlingContext, None]: + yield context + raise RuntimeError('Crash during middleware finalization') + + pipeline = ContextPipeline().compose(step_1).compose(step_2) + context = BasicCrawlingContext(request=Request.from_url(url='aaa')) + + with pytest.raises(ContextPipelineFinalizationError): + await pipeline(context, consumer) + + assert consumer.called + assert not cleanup.called diff --git a/tests/unit/beautifulsoup_crawler/test_beautifulsoup_crawler.py b/tests/unit/beautifulsoup_crawler/test_beautifulsoup_crawler.py new file mode 100644 index 0000000000..1bc511c3ce --- /dev/null +++ b/tests/unit/beautifulsoup_crawler/test_beautifulsoup_crawler.py @@ -0,0 +1,47 @@ +from typing import AsyncGenerator +from unittest import mock + +import pytest +import respx +from httpx import Response + +from crawlee.beautifulsoup_crawler.beautifulsoup_crawler import BeautifulSoupCrawler +from crawlee.beautifulsoup_crawler.types import BeautifulSoupCrawlingContext +from crawlee.storages.request_list import RequestList + + +@pytest.fixture() +async def server() -> AsyncGenerator[respx.MockRouter, None]: + with respx.mock(base_url='https://test.io', assert_all_called=False) as mock: + mock.get('/html', name='html_endpoint').return_value = Response( + 200, + text=""" +
+