Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ clean:

install-dev:
python3 -m pip install --upgrade pip poetry
poetry install
poetry install --all-extras
poetry run pre-commit install

lint:
Expand Down
7 changes: 7 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,13 @@ keywords = [
python = "^3.9"
aiofiles = "^23.2.1"
aioshutil = "^1.3"
beautifulsoup4 = { version = "^4.12.3", optional = true }
colorama = "^0.4.6"
docutils = "^0.21.0"
eval-type-backport = "^0.1.3"
html5lib = { version = "^1.1", optional = true }
httpx = "^0.27.0"
lxml = { version = "^5.2.1", optional = true }
more_itertools = "^10.2.0"
psutil = "^5.9.8"
pydantic = "^2.6.3"
Expand All @@ -76,10 +79,14 @@ respx = "~0.21.0"
ruff = "~0.3.0"
twine = "~5.0.0"
types-aiofiles = "^23.2.0.20240106"
types-beautifulsoup4 = "^4.12.0.20240229"
types-colorama = "~0.4.15.20240106"
types-psutil = "~5.9.5.20240205"
types-python-dateutil = "^2.9.0.20240316"

[tool.poetry.extras]
beautifulsoup = ["beautifulsoup4", "lxml", "html5lib"]

[tool.ruff]
line-length = 120

Expand Down
39 changes: 33 additions & 6 deletions src/crawlee/basic_crawler/basic_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,11 @@
from crawlee.autoscaling import AutoscaledPool, ConcurrencySettings
from crawlee.autoscaling.snapshotter import Snapshotter
from crawlee.autoscaling.system_status import SystemStatus
from crawlee.basic_crawler.context_pipeline import ContextPipeline, RequestHandlerError
from crawlee.basic_crawler.context_pipeline import (
ContextPipeline,
ContextPipelineInitializationError,
RequestHandlerError,
)
from crawlee.basic_crawler.router import Router
from crawlee.basic_crawler.types import BasicCrawlingContext, FinalStatistics
from crawlee.configuration import Configuration
Expand Down Expand Up @@ -162,16 +166,17 @@ async def run(self, requests: list[str | BaseRequestData] | None = None) -> Fina

return FinalStatistics()

async def _handle_request_error(self, crawling_context: TCrawlingContext, error: Exception) -> None:
def _should_retry_request(self, crawling_context: BasicCrawlingContext) -> bool:
max_request_retries = crawling_context.request.max_retries
if max_request_retries is None:
max_request_retries = self._max_request_retries

should_retry_request = (
return (
not crawling_context.request.no_retry and (crawling_context.request.retry_count + 1) < max_request_retries
)

if should_retry_request:
async def _handle_request_error(self, crawling_context: TCrawlingContext, error: Exception) -> None:
if self._should_retry_request(crawling_context):
request = crawling_context.request
request.retry_count += 1

Expand Down Expand Up @@ -224,7 +229,7 @@ async def __run_task_function(self) -> None:
return

# TODO: fetch session from the session pool
# https://github.com/apify/crawlee-py/issues/78
# https://github.com/apify/crawlee-py/issues/110

crawling_context = BasicCrawlingContext(request=request)

Expand All @@ -240,7 +245,7 @@ async def __run_task_function(self) -> None:
)

await wait_for(
lambda: self._request_provider.mark_request_as_handled(request),
lambda: self._request_provider.mark_request_as_handled(crawling_context.request),
timeout=self._internal_timeout,
timeout_message='Marking request as handled timed out after '
f'{self._internal_timeout.total_seconds()} seconds',
Expand Down Expand Up @@ -279,6 +284,28 @@ async def __run_task_function(self) -> None:
)
request.state = RequestState.ERROR
raise
except ContextPipelineInitializationError as initialization_error:
Comment thread
vdusek marked this conversation as resolved.
Resolved
if self._should_retry_request(crawling_context):
logger.debug(
'An exception occured during the initialization of crawling context, a retry is in order',
exc_info=initialization_error,
)

request = crawling_context.request
request.retry_count += 1
request.state = RequestState.DONE
await self._request_provider.reclaim_request(request)
else:
logger.exception('Request failed and reached maximum retries', exc_info=initialization_error)

await wait_for(
lambda: self._request_provider.mark_request_as_handled(crawling_context.request),
timeout=self._internal_timeout,
timeout_message='Marking request as handled timed out after '
f'{self._internal_timeout.total_seconds()} seconds',
logger=logger,
max_retries=3,
)
except Exception as internal_error:
logger.exception(
'An exception occurred during handling of a request. This places the crawler '
Expand Down
67 changes: 47 additions & 20 deletions src/crawlee/basic_crawler/context_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,28 @@ def __init__(self, wrapped_exception: Exception, crawling_context: TCrawlingCont
self.crawling_context = crawling_context


class ContextPipelineInitializationError(Exception):
"""Wraps an exception thrown in the initialization step of a context pipeline middleware.

We may not have the complete context at this point, so only `BasicCrawlingContext` is provided.
"""

def __init__(self, wrapped_exception: Exception, crawling_context: BasicCrawlingContext) -> None:
self.wrapped_exception = wrapped_exception
self.crawling_context = crawling_context


class ContextPipelineFinalizationError(Exception):
"""Wraps an exception thrown in the finalization step of a context pipeline middleware.

We may not have the complete context at this point, so only `BasicCrawlingContext` is provided.
"""

def __init__(self, wrapped_exception: Exception, crawling_context: BasicCrawlingContext) -> None:
self.wrapped_exception = wrapped_exception
self.crawling_context = crawling_context


class ContextPipeline(Generic[TCrawlingContext]):
"""Encapsulates the logic of gradually enhancing the crawling context with additional information and utilities.

Expand Down Expand Up @@ -55,29 +77,34 @@ async def __call__(
chain = list(self._middleware_chain())
cleanup_stack = list[AsyncGenerator]()

for member in reversed(chain):
if member._middleware: # noqa: SLF001
middleware_instance = member._middleware(crawling_context) # noqa: SLF001
try:
result = await middleware_instance.__anext__()
except StopAsyncIteration as e:
raise RuntimeError('The middleware did not yield') from e

crawling_context = result
cleanup_stack.append(middleware_instance)

try:
await final_context_consumer(cast(TCrawlingContext, crawling_context))
except Exception as e:
raise RequestHandlerError(e, crawling_context) from e
for member in reversed(chain):
if member._middleware: # noqa: SLF001
middleware_instance = member._middleware(crawling_context) # noqa: SLF001
try:
result = await middleware_instance.__anext__()
except StopAsyncIteration as e:
raise RuntimeError('The middleware did not yield') from e
except Exception as e:
raise ContextPipelineInitializationError(e, crawling_context) from e

crawling_context = result
cleanup_stack.append(middleware_instance)

for middleware_instance in reversed(cleanup_stack):
try:
result = await middleware_instance.__anext__()
except StopAsyncIteration: # noqa: PERF203
pass
else:
raise RuntimeError('The middleware yielded more than once')
await final_context_consumer(cast(TCrawlingContext, crawling_context))
except Exception as e:
raise RequestHandlerError(e, crawling_context) from e
finally:
for middleware_instance in reversed(cleanup_stack):
try:
result = await middleware_instance.__anext__()
except StopAsyncIteration: # noqa: PERF203
pass
except Exception as e:
raise ContextPipelineFinalizationError(e, crawling_context) from e
else:
raise RuntimeError('The middleware yielded more than once')

def compose(
self,
Expand Down
1 change: 1 addition & 0 deletions src/crawlee/beautifulsoup_crawler/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
from .beautifulsoup_crawler import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
97 changes: 97 additions & 0 deletions src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
from __future__ import annotations

import asyncio
from datetime import timedelta
from typing import TYPE_CHECKING, AsyncGenerator, Awaitable, Callable, Iterable, Literal

import httpx

from crawlee.basic_crawler.basic_crawler import BasicCrawler
from crawlee.basic_crawler.context_pipeline import ContextPipeline
from crawlee.beautifulsoup_crawler.types import BeautifulSoupCrawlingContext
from crawlee.http_crawler.http_crawler import make_http_request
from crawlee.http_crawler.types import HttpCrawlingContext

if TYPE_CHECKING:
from datetime import timedelta

from crawlee.autoscaling.autoscaled_pool import ConcurrencySettings
from crawlee.basic_crawler.types import BasicCrawlingContext
from crawlee.configuration import Configuration
from crawlee.storages.request_provider import RequestProvider


class BeautifulSoupCrawler(BasicCrawler[BeautifulSoupCrawlingContext]):
"""A crawler that fetches the request URL using `httpx` and parses the result with `BeautifulSoup`."""

def __init__(
self,
*,
parser: Literal['html.parser', 'lxml', 'xml', 'html5lib'] = 'lxml',
request_provider: RequestProvider,
router: Callable[[BeautifulSoupCrawlingContext], Awaitable[None]] | None = None,
concurrency_settings: ConcurrencySettings | None = None,
configuration: Configuration | None = None,
request_handler_timeout: timedelta | None = None,
additional_http_error_status_codes: Iterable[int] = (),
ignore_http_error_status_codes: Iterable[int] = (),
) -> None:
"""Initialize the BeautifulSoupCrawler.

Args:
parser: The type of parser that should be used by BeautifulSoup

request_provider: Provides requests to be processed

router: A callable to which request handling is delegated

concurrency_settings: Allows fine-tuning concurrency levels

configuration: Crawler configuration

request_handler_timeout: How long is a single request handler allowed to run

additional_http_error_status_codes: HTTP status codes that should be considered errors (and trigger a retry)

ignore_http_error_status_codes: HTTP status codes that are normally considered errors but we want to treat
them as successful
"""
context_pipeline = ContextPipeline().compose(self._make_http_request).compose(self._parse_http_response)

self._client = httpx.AsyncClient()
self._parser = parser

self._additional_http_error_status_codes = set(additional_http_error_status_codes)
self._ignore_http_error_status_codes = set(ignore_http_error_status_codes)

basic_crawler_kwargs = {}

if request_handler_timeout is not None:
basic_crawler_kwargs['request_handler_timeout'] = request_handler_timeout

super().__init__(
request_provider=request_provider,
router=router,
concurrency_settings=concurrency_settings,
configuration=configuration,
_context_pipeline=context_pipeline,
**basic_crawler_kwargs, # type: ignore
)

async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenerator[HttpCrawlingContext, None]:
result = await make_http_request(
self._client,
context.request,
additional_http_error_status_codes=self._additional_http_error_status_codes,
ignore_http_error_status_codes=self._ignore_http_error_status_codes,
)

yield HttpCrawlingContext(request=context.request, http_response=result.http_response)

async def _parse_http_response(
self, context: HttpCrawlingContext
) -> AsyncGenerator[BeautifulSoupCrawlingContext, None]:
from bs4 import BeautifulSoup

soup = await asyncio.to_thread(lambda: BeautifulSoup(context.http_response.read(), self._parser))
yield BeautifulSoupCrawlingContext(request=context.request, http_response=context.http_response, soup=soup)
16 changes: 16 additions & 0 deletions src/crawlee/beautifulsoup_crawler/types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
from __future__ import annotations

from dataclasses import dataclass
from typing import TYPE_CHECKING

from crawlee.basic_crawler.types import BasicCrawlingContext
from crawlee.http_crawler.types import HttpCrawlResult

if TYPE_CHECKING:
from bs4 import BeautifulSoup


@dataclass(frozen=True)
class BeautifulSoupCrawlingContext(HttpCrawlResult, BasicCrawlingContext):
"""Crawling context used by BeautifulSoupCrawler."""
soup: BeautifulSoup
Loading