From a5b6b208f78c56834718cb465fe4225b208449d4 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Thu, 18 Apr 2024 15:20:56 +0200 Subject: [PATCH 01/20] Add type definitions for add_requests and enqueue_links --- src/crawlee/basic_crawler/types.py | 44 +++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/crawlee/basic_crawler/types.py b/src/crawlee/basic_crawler/types.py index 33695ad3e0..e545312e34 100644 --- a/src/crawlee/basic_crawler/types.py +++ b/src/crawlee/basic_crawler/types.py @@ -1,14 +1,55 @@ # ruff: noqa: TCH003 from __future__ import annotations +import re from dataclasses import dataclass -from typing import TYPE_CHECKING, Coroutine, Protocol +from enum import Enum +from typing import TYPE_CHECKING, Any, Coroutine, Protocol + +from typing_extensions import NotRequired, TypedDict, Unpack if TYPE_CHECKING: from crawlee.http_clients.base_http_client import HttpResponse from crawlee.request import Request +class EnqueueStrategy(str, Enum): + """Strategy for deciding which links should be followed and which ones should be ignored.""" + + ALL = 'all' + SAME_DOMAIN = 'same-domain' + SAME_HOSTNAME = 'same-hostname' + SAME_ORIGIN = 'same-origin' + + +class AddRequestsFunctionKwargs(TypedDict): + """Keyword arguments type for AddRequestsFunction.""" + + label: NotRequired[str] + user_data: NotRequired[dict[str, Any]] + limit: NotRequired[int] + base_url: NotRequired[str] + strategy: NotRequired[EnqueueStrategy] + include: NotRequired[list[re.Pattern]] + exclude: NotRequired[list[re.Pattern]] + + +class AddRequestsFunction(Protocol): + """Type of a function for adding URLs to the request queue with optional filtering.""" + + def __call__( # noqa: D102 + self, urls: list[str], **kwargs: Unpack[AddRequestsFunctionKwargs] + ) -> Coroutine[None, None, None]: ... + + +class EnqueueLinksFunction(Protocol): + """Type of a function for enqueueing links based on a selector.""" + + def __call__( # noqa: D102 + self, *, selector: str, **kwargs: Unpack[AddRequestsFunctionKwargs] + ) -> Coroutine[None, None, None]: ... + + class SendRequestFunction(Protocol): """Type of a function for performing an HTTP request.""" @@ -23,6 +64,7 @@ class BasicCrawlingContext: request: Request send_request: SendRequestFunction + add_requests: AddRequestsFunction @dataclass(frozen=True) From 46b3761cbd2129b30b2a7264e9cdd7df8c707542 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Thu, 18 Apr 2024 15:40:11 +0200 Subject: [PATCH 02/20] Add glob support --- src/crawlee/basic_crawler/types.py | 5 ++- src/crawlee/globs.py | 69 ++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 src/crawlee/globs.py diff --git a/src/crawlee/basic_crawler/types.py b/src/crawlee/basic_crawler/types.py index e545312e34..66854c14ba 100644 --- a/src/crawlee/basic_crawler/types.py +++ b/src/crawlee/basic_crawler/types.py @@ -9,6 +9,7 @@ from typing_extensions import NotRequired, TypedDict, Unpack if TYPE_CHECKING: + from crawlee.globs import Glob from crawlee.http_clients.base_http_client import HttpResponse from crawlee.request import Request @@ -30,8 +31,8 @@ class AddRequestsFunctionKwargs(TypedDict): limit: NotRequired[int] base_url: NotRequired[str] strategy: NotRequired[EnqueueStrategy] - include: NotRequired[list[re.Pattern]] - exclude: NotRequired[list[re.Pattern]] + include: NotRequired[list[re.Pattern | Glob]] + exclude: NotRequired[list[re.Pattern | Glob]] class AddRequestsFunction(Protocol): diff --git a/src/crawlee/globs.py b/src/crawlee/globs.py new file mode 100644 index 0000000000..bf840ae90a --- /dev/null +++ b/src/crawlee/globs.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import fnmatch +import os +import re +from typing import Sequence + + +class Glob: + def __init__(self, glob: str) -> None: + self.glob = glob + self.regexp = re.compile(_translate(self.glob)) + + +def _translate( + pat: str, *, recursive: bool = False, include_hidden: bool = False, seps: Sequence[str] | None = None +) -> str: + """Translate a pathname with shell wildcards to a regular expression. + + If `recursive` is true, the pattern segment '**' will match any number of + path segments. + + If `include_hidden` is true, wildcards can match path segments beginning + with a dot ('.'). + + If a sequence of separator characters is given to `seps`, they will be + used to split the pattern into segments and match path separators. If not + given, os.path.sep and os.path.altsep (where available) are used. + + HACK: This function is copied from CPython stdlib source. It will be released in Python 3.13 as `glob.translate` + """ + if not seps: + seps = (os.path.sep, os.path.altsep) if os.path.altsep else os.path.sep + + escaped_seps = ''.join(map(re.escape, seps)) + any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps + not_sep = f'[^{escaped_seps}]' + if include_hidden: + one_last_segment = f'{not_sep}+' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:.+{any_sep})?' + any_last_segments = '.*' + else: + one_last_segment = f'[^{escaped_seps}.]{not_sep}*' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:{one_segment})*' + any_last_segments = f'{any_segments}(?:{one_last_segment})?' + + results = [] + parts = re.split(any_sep, pat) + last_part_idx = len(parts) - 1 + for idx, part in enumerate(parts): + if part == '*': + results.append(one_segment if idx < last_part_idx else one_last_segment) + elif recursive and part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + else: + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) # type: ignore # noqa: SLF001 + if idx < last_part_idx: + results.append(any_sep) + res = ''.join(results) + return rf'(?s:{res})\Z' From f3ee56673249bc51ecf0d2c33c47805f01d6b043 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 19 Apr 2024 14:43:26 +0200 Subject: [PATCH 03/20] Rough implementation of add_requests --- src/crawlee/basic_crawler/basic_crawler.py | 65 ++++++++++++++----- src/crawlee/basic_crawler/types.py | 34 ++++++++-- .../beautifulsoup_crawler.py | 13 +++- src/crawlee/http_crawler/http_crawler.py | 3 +- src/crawlee/storages/request_list.py | 11 ++-- src/crawlee/storages/request_provider.py | 8 +-- src/crawlee/storages/request_queue.py | 6 +- .../unit/basic_crawler/test_basic_crawler.py | 24 ++++++- .../basic_crawler/test_context_pipeline.py | 28 ++++++-- tests/unit/basic_crawler/test_router.py | 6 +- 10 files changed, 150 insertions(+), 48 deletions(-) diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index f32aff777e..be82bdf0a1 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -19,17 +19,22 @@ RequestHandlerError, ) from crawlee.basic_crawler.router import Router -from crawlee.basic_crawler.types import BasicCrawlingContext, FinalStatistics, SendRequestFunction +from crawlee.basic_crawler.types import ( + BasicCrawlingContext, + FinalStatistics, + RequestHandlerRunResult, + SendRequestFunction, +) from crawlee.configuration import Configuration from crawlee.events.local_event_manager import LocalEventManager from crawlee.http_clients.httpx_client import HttpxClient from crawlee.request import BaseRequestData, Request, RequestState +from crawlee.storages.request_queue import RequestQueue if TYPE_CHECKING: from crawlee.http_clients.base_http_client import BaseHttpClient, HttpResponse from crawlee.storages.request_provider import RequestProvider - TCrawlingContext = TypeVar('TCrawlingContext', bound=BasicCrawlingContext, default=BasicCrawlingContext) ErrorHandler = Callable[[TCrawlingContext, Exception], Awaitable[Union[Request, None]]] FailedRequestHandler = Callable[[TCrawlingContext, Exception], Awaitable[None]] @@ -55,10 +60,8 @@ class BasicCrawler(Generic[TCrawlingContext]): def __init__( self, *, - request_provider: RequestProvider, + request_provider: RequestProvider | None = None, router: Callable[[TCrawlingContext], Awaitable[None]] | None = None, - # TODO: make request_provider optional (and instantiate based on configuration if None is supplied) - # https://github.com/apify/crawlee-py/issues/83 http_client: BaseHttpClient | None = None, concurrency_settings: ConcurrencySettings | None = None, max_request_retries: int = 3, @@ -132,6 +135,13 @@ def router(self, router: Router[TCrawlingContext]) -> None: self._router = router + async def get_request_provider(self) -> RequestProvider: + """Return the configured request provider. If none is configured, open and return the default request queue.""" + if not self._request_provider: + self._request_provider = await RequestQueue.open() + + return self._request_provider + def error_handler(self, handler: ErrorHandler[TCrawlingContext]) -> ErrorHandler[TCrawlingContext]: """Decorator for configuring an error handler (called after a request handler error and before retrying).""" self._error_handler = handler @@ -153,7 +163,7 @@ async def add_requests( wait_time_between_batches: timedelta = timedelta(0), ) -> None: """Add requests to the underlying queue.""" - await self._request_provider.add_requests_batched( + await (await self.get_request_provider()).add_requests_batched( [ request if isinstance(request, BaseRequestData) else BaseRequestData.from_url(url=request) for request in requests @@ -183,6 +193,8 @@ def _should_retry_request(self, crawling_context: BasicCrawlingContext) -> bool: ) async def _handle_request_error(self, crawling_context: TCrawlingContext, error: Exception) -> None: + request_provider = await self.get_request_provider() + if self._should_retry_request(crawling_context): request = crawling_context.request request.retry_count += 1 @@ -196,10 +208,10 @@ async def _handle_request_error(self, crawling_context: TCrawlingContext, error: if new_request is not None: request = new_request - await self._request_provider.reclaim_request(request) + await request_provider.reclaim_request(request) else: await wait_for( - lambda: self._request_provider.mark_request_as_handled(crawling_context.request), + lambda: request_provider.mark_request_as_handled(crawling_context.request), timeout=self._internal_timeout, timeout_message='Marking request as handled timed out after ' f'{self._internal_timeout.total_seconds()} seconds', @@ -223,15 +235,30 @@ async def send_request(url: str, *, method: str = 'get', headers: dict[str, str] return send_request + async def _commit_request_handler_result(self, result: RequestHandlerRunResult) -> None: + request_provider = await self.get_request_provider() + + for call in result.add_requests_calls: + # TODO: handle strategy, limit, include/exclude, etc. + await request_provider.add_requests_batched( + requests=[ + request if isinstance(request, BaseRequestData) else BaseRequestData.from_url(request) + for request in call['requests'] + ], + wait_for_all_requests_to_be_added=False, + ) + async def __is_finished_function(self) -> bool: - return await self._request_provider.is_finished() + return await (await self.get_request_provider()).is_finished() async def __is_task_ready_function(self) -> bool: - return not await self._request_provider.is_empty() + return not await (await self.get_request_provider()).is_empty() async def __run_task_function(self) -> None: + request_provider = await self.get_request_provider() + request = await wait_for( - lambda: self._request_provider.fetch_next_request(), + lambda: request_provider.fetch_next_request(), timeout=self._internal_timeout, timeout_message=f'Fetching next request failed after {self._internal_timeout.total_seconds()} seconds', logger=logger, @@ -244,7 +271,13 @@ async def __run_task_function(self) -> None: # TODO: fetch session from the session pool # https://github.com/apify/crawlee-py/issues/110 - crawling_context = BasicCrawlingContext(request=request, send_request=self._prepare_send_request_function()) + result = RequestHandlerRunResult() + + crawling_context = BasicCrawlingContext( + request=request, + send_request=self._prepare_send_request_function(), + add_requests=result.add_requests, + ) try: request.state = RequestState.REQUEST_HANDLER @@ -257,8 +290,10 @@ async def __run_task_function(self) -> None: logger=logger, ) + await self._commit_request_handler_result(result) + await wait_for( - lambda: self._request_provider.mark_request_as_handled(crawling_context.request), + lambda: request_provider.mark_request_as_handled(crawling_context.request), timeout=self._internal_timeout, timeout_message='Marking request as handled timed out after ' f'{self._internal_timeout.total_seconds()} seconds', @@ -307,12 +342,12 @@ async def __run_task_function(self) -> None: request = crawling_context.request request.retry_count += 1 request.state = RequestState.DONE - await self._request_provider.reclaim_request(request) + await request_provider.reclaim_request(request) else: logger.exception('Request failed and reached maximum retries', exc_info=initialization_error) await wait_for( - lambda: self._request_provider.mark_request_as_handled(crawling_context.request), + lambda: request_provider.mark_request_as_handled(crawling_context.request), timeout=self._internal_timeout, timeout_message='Marking request as handled timed out after ' f'{self._internal_timeout.total_seconds()} seconds', diff --git a/src/crawlee/basic_crawler/types.py b/src/crawlee/basic_crawler/types.py index 66854c14ba..948d1fe5c0 100644 --- a/src/crawlee/basic_crawler/types.py +++ b/src/crawlee/basic_crawler/types.py @@ -2,7 +2,7 @@ from __future__ import annotations import re -from dataclasses import dataclass +from dataclasses import dataclass, field from enum import Enum from typing import TYPE_CHECKING, Any, Coroutine, Protocol @@ -11,7 +11,7 @@ if TYPE_CHECKING: from crawlee.globs import Glob from crawlee.http_clients.base_http_client import HttpResponse - from crawlee.request import Request + from crawlee.request import BaseRequestData, Request class EnqueueStrategy(str, Enum): @@ -26,8 +26,6 @@ class EnqueueStrategy(str, Enum): class AddRequestsFunctionKwargs(TypedDict): """Keyword arguments type for AddRequestsFunction.""" - label: NotRequired[str] - user_data: NotRequired[dict[str, Any]] limit: NotRequired[int] base_url: NotRequired[str] strategy: NotRequired[EnqueueStrategy] @@ -39,7 +37,7 @@ class AddRequestsFunction(Protocol): """Type of a function for adding URLs to the request queue with optional filtering.""" def __call__( # noqa: D102 - self, urls: list[str], **kwargs: Unpack[AddRequestsFunctionKwargs] + self, requests: list[str | BaseRequestData], **kwargs: Unpack[AddRequestsFunctionKwargs] ) -> Coroutine[None, None, None]: ... @@ -47,7 +45,12 @@ class EnqueueLinksFunction(Protocol): """Type of a function for enqueueing links based on a selector.""" def __call__( # noqa: D102 - self, *, selector: str, **kwargs: Unpack[AddRequestsFunctionKwargs] + self, + *, + selector: str, + label: str | None = None, + user_data: dict[str, Any] | None = None, + **kwargs: Unpack[AddRequestsFunctionKwargs], ) -> Coroutine[None, None, None]: ... @@ -71,3 +74,22 @@ class BasicCrawlingContext: @dataclass(frozen=True) class FinalStatistics: """Statistics about a crawler run.""" + + +class AddRequestsFunctionCall(AddRequestsFunctionKwargs): + """Record of a call to `add_requests1.""" + + requests: list[str | BaseRequestData] + + +@dataclass() +class RequestHandlerRunResult: + """Record of calls to storage-related context helpers.""" + + add_requests_calls: list[AddRequestsFunctionCall] = field(default_factory=list) + + async def add_requests( + self, requests: list[str | BaseRequestData], **kwargs: Unpack[AddRequestsFunctionKwargs] + ) -> None: + """Track a call to the `add_requests` context helper.""" + self.add_requests_calls.append(AddRequestsFunctionCall(requests=requests, **kwargs)) diff --git a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py index 4fc400aeb2..ba2a23047a 100644 --- a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py +++ b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py @@ -28,7 +28,7 @@ def __init__( self, *, parser: Literal['html.parser', 'lxml', 'xml', 'html5lib'] = 'lxml', - request_provider: RequestProvider, + request_provider: RequestProvider | None = None, router: Callable[[BeautifulSoupCrawlingContext], Awaitable[None]] | None = None, concurrency_settings: ConcurrencySettings | None = None, configuration: Configuration | None = None, @@ -83,7 +83,10 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera result = await self._http_client.crawl(context.request) yield HttpCrawlingContext( - request=context.request, send_request=context.send_request, http_response=result.http_response + request=context.request, + send_request=context.send_request, + add_requests=context.add_requests, + http_response=result.http_response, ) async def _parse_http_response( @@ -93,5 +96,9 @@ async def _parse_http_response( soup = await asyncio.to_thread(lambda: BeautifulSoup(context.http_response.read(), self._parser)) yield BeautifulSoupCrawlingContext( - request=context.request, send_request=context.send_request, http_response=context.http_response, soup=soup + request=context.request, + send_request=context.send_request, + add_requests=context.add_requests, + http_response=context.http_response, + soup=soup, ) diff --git a/src/crawlee/http_crawler/http_crawler.py b/src/crawlee/http_crawler/http_crawler.py index 324c8e43b3..057b967589 100644 --- a/src/crawlee/http_crawler/http_crawler.py +++ b/src/crawlee/http_crawler/http_crawler.py @@ -22,7 +22,7 @@ class HttpCrawler(BasicCrawler[HttpCrawlingContext]): def __init__( self, *, - request_provider: RequestProvider, + request_provider: RequestProvider | None = None, router: Callable[[HttpCrawlingContext], Awaitable[None]] | None = None, concurrency_settings: ConcurrencySettings | None = None, configuration: Configuration | None = None, @@ -76,5 +76,6 @@ async def _make_http_request( yield HttpCrawlingContext( request=crawling_context.request, send_request=crawling_context.send_request, + add_requests=crawling_context.add_requests, http_response=result.http_response, ) diff --git a/src/crawlee/storages/request_list.py b/src/crawlee/storages/request_list.py index da4115fd0f..814e756f45 100644 --- a/src/crawlee/storages/request_list.py +++ b/src/crawlee/storages/request_list.py @@ -1,16 +1,13 @@ from __future__ import annotations from collections import deque -from typing import TYPE_CHECKING +from datetime import timedelta from typing_extensions import override from crawlee.request import BaseRequestData, Request from crawlee.storages.request_provider import RequestProvider -if TYPE_CHECKING: - from datetime import timedelta - class RequestList(RequestProvider): """Represents a (potentially very large) list of URLs to crawl.""" @@ -81,9 +78,9 @@ async def add_requests_batched( self, requests: list[BaseRequestData | Request], *, - batch_size: int, - wait_for_all_requests_to_be_added: bool, - wait_time_between_batches: timedelta, + batch_size: int = 1000, + wait_for_all_requests_to_be_added: bool = False, + wait_time_between_batches: timedelta = timedelta(seconds=1), ) -> None: self._sources.extend( request if isinstance(request, Request) else Request.from_base_request_data(request) for request in requests diff --git a/src/crawlee/storages/request_provider.py b/src/crawlee/storages/request_provider.py index 92e55070c7..c31a22a59a 100644 --- a/src/crawlee/storages/request_provider.py +++ b/src/crawlee/storages/request_provider.py @@ -1,10 +1,10 @@ from __future__ import annotations from abc import ABC, abstractmethod +from datetime import timedelta from typing import TYPE_CHECKING if TYPE_CHECKING: - from datetime import timedelta from crawlee.request import BaseRequestData, Request from crawlee.storages.models import RequestQueueOperationInfo @@ -58,9 +58,9 @@ async def add_requests_batched( self, requests: list[BaseRequestData | Request], *, - batch_size: int, - wait_for_all_requests_to_be_added: bool, - wait_time_between_batches: timedelta, + batch_size: int = 1000, + wait_for_all_requests_to_be_added: bool = False, + wait_time_between_batches: timedelta = timedelta(seconds=1), ) -> None: """Adds requests to the queue in batches. diff --git a/src/crawlee/storages/request_queue.py b/src/crawlee/storages/request_queue.py index 027cc6d20a..96e077996a 100644 --- a/src/crawlee/storages/request_queue.py +++ b/src/crawlee/storages/request_queue.py @@ -208,9 +208,9 @@ async def add_requests_batched( self, requests: list[BaseRequestData | Request], *, - batch_size: int, - wait_for_all_requests_to_be_added: bool, - wait_time_between_batches: timedelta, + batch_size: int = 1000, + wait_for_all_requests_to_be_added: bool = False, + wait_time_between_batches: timedelta = timedelta(seconds=1), ) -> None: for request in requests: if isinstance(request, Request): diff --git a/tests/unit/basic_crawler/test_basic_crawler.py b/tests/unit/basic_crawler/test_basic_crawler.py index 6c776c12f8..9a5863a80d 100644 --- a/tests/unit/basic_crawler/test_basic_crawler.py +++ b/tests/unit/basic_crawler/test_basic_crawler.py @@ -3,13 +3,14 @@ import json from typing import TYPE_CHECKING, Any +from unittest.mock import Mock import pytest from httpx import Headers, Response from crawlee.basic_crawler.basic_crawler import BasicCrawler, UserDefinedErrorHandlerError from crawlee.basic_crawler.types import BasicCrawlingContext -from crawlee.request import Request +from crawlee.request import BaseRequestData, Request from crawlee.storages import RequestList if TYPE_CHECKING: @@ -265,3 +266,24 @@ async def handler(context: BasicCrawlingContext) -> None: assert response_headers is not None assert response_headers.get('content-type').endswith('/json') + + +async def test_add_requests_works() -> None: + visit = Mock() + + crawler = BasicCrawler( + request_provider=RequestList(['http://a.com']), + max_request_retries=3, + ) + + @crawler.router.default_handler + async def handler(context: BasicCrawlingContext) -> None: + visit(context.request.url) + + if context.request.url == 'http://a.com': + await context.add_requests([BaseRequestData.from_url('http://b.com'), 'http://c.com']) + + await crawler.run() + + visited = {call[0][0] for call in visit.call_args_list} + assert visited == {'http://a.com', 'http://b.com', 'http://c.com'} diff --git a/tests/unit/basic_crawler/test_context_pipeline.py b/tests/unit/basic_crawler/test_context_pipeline.py index 10809492fa..42bc3f61a7 100644 --- a/tests/unit/basic_crawler/test_context_pipeline.py +++ b/tests/unit/basic_crawler/test_context_pipeline.py @@ -30,7 +30,9 @@ async def test_calls_consumer_without_middleware() -> None: consumer = AsyncMock() pipeline = ContextPipeline() - context = BasicCrawlingContext(request=Request.from_url(url='aaa'), send_request=AsyncMock()) + context = BasicCrawlingContext( + request=Request.from_url(url='aaa'), send_request=AsyncMock(), add_requests=AsyncMock() + ) await pipeline(context, consumer) @@ -46,17 +48,23 @@ async def consumer(context: MoreEnhancedCrawlingContext) -> None: async def middleware_a(context: BasicCrawlingContext) -> AsyncGenerator[EnhancedCrawlingContext, None]: events.append('middleware_a_in') - yield EnhancedCrawlingContext(request=context.request, foo='foo', send_request=AsyncMock()) + yield EnhancedCrawlingContext( + request=context.request, foo='foo', send_request=AsyncMock(), add_requests=AsyncMock() + ) events.append('middleware_a_out') async def middleware_b(context: EnhancedCrawlingContext) -> AsyncGenerator[MoreEnhancedCrawlingContext, None]: events.append('middleware_b_in') - yield MoreEnhancedCrawlingContext(request=context.request, foo=context.foo, bar=4, send_request=AsyncMock()) + yield MoreEnhancedCrawlingContext( + request=context.request, foo=context.foo, bar=4, send_request=AsyncMock(), add_requests=AsyncMock() + ) events.append('middleware_b_out') pipeline = ContextPipeline[BasicCrawlingContext]().compose(middleware_a).compose(middleware_b) - context = BasicCrawlingContext(request=Request.from_url(url='aaa'), send_request=AsyncMock()) + context = BasicCrawlingContext( + request=Request.from_url(url='aaa'), send_request=AsyncMock(), add_requests=AsyncMock() + ) await pipeline(context, consumer) assert events == [ @@ -72,7 +80,9 @@ async def test_wraps_consumer_errors() -> None: consumer = AsyncMock(side_effect=RuntimeError('Arbitrary crash for testing purposes')) pipeline = ContextPipeline() - context = BasicCrawlingContext(request=Request.from_url(url='aaa'), send_request=AsyncMock()) + context = BasicCrawlingContext( + request=Request.from_url(url='aaa'), send_request=AsyncMock(), add_requests=AsyncMock() + ) with pytest.raises(RequestHandlerError): await pipeline(context, consumer) @@ -91,7 +101,9 @@ async def step_2(context: BasicCrawlingContext) -> AsyncGenerator[BasicCrawlingC yield context # type: ignore[unreachable] pipeline = ContextPipeline().compose(step_1).compose(step_2) - context = BasicCrawlingContext(request=Request.from_url(url='aaa'), send_request=AsyncMock()) + context = BasicCrawlingContext( + request=Request.from_url(url='aaa'), send_request=AsyncMock(), add_requests=AsyncMock() + ) with pytest.raises(ContextPipelineInitializationError): await pipeline(context, consumer) @@ -113,7 +125,9 @@ async def step_2(context: BasicCrawlingContext) -> AsyncGenerator[BasicCrawlingC raise RuntimeError('Crash during middleware finalization') pipeline = ContextPipeline().compose(step_1).compose(step_2) - context = BasicCrawlingContext(request=Request.from_url(url='aaa'), send_request=AsyncMock()) + context = BasicCrawlingContext( + request=Request.from_url(url='aaa'), send_request=AsyncMock(), add_requests=AsyncMock() + ) with pytest.raises(ContextPipelineFinalizationError): await pipeline(context, consumer) diff --git a/tests/unit/basic_crawler/test_router.py b/tests/unit/basic_crawler/test_router.py index f09d4c6880..e910343343 100644 --- a/tests/unit/basic_crawler/test_router.py +++ b/tests/unit/basic_crawler/test_router.py @@ -11,7 +11,11 @@ class MockContext(BasicCrawlingContext): def __init__(self, *, label: str | None) -> None: - super().__init__(request=Request.from_url(url='42', user_data={'label': label}), send_request=AsyncMock()) + super().__init__( + request=Request.from_url(url='42', user_data={'label': label}), + send_request=AsyncMock(), + add_requests=AsyncMock(), + ) async def test_router_no_handlers() -> None: From 70629f112d79869a669f074d399acb62dd3c17e9 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 19 Apr 2024 16:30:34 +0200 Subject: [PATCH 04/20] Initial implementation of enqueue_links for BS crawler --- src/crawlee/basic_crawler/basic_crawler.py | 25 +++++--- src/crawlee/basic_crawler/types.py | 10 ++-- .../beautifulsoup_crawler.py | 25 +++++++- src/crawlee/beautifulsoup_crawler/types.py | 3 +- .../test_beautifulsoup_crawler.py | 59 ++++++++++++++++++- 5 files changed, 103 insertions(+), 19 deletions(-) diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index be82bdf0a1..8853507582 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -235,16 +235,27 @@ async def send_request(url: str, *, method: str = 'get', headers: dict[str, str] return send_request - async def _commit_request_handler_result(self, result: RequestHandlerRunResult) -> None: + async def _commit_request_handler_result( + self, context: BasicCrawlingContext, result: RequestHandlerRunResult + ) -> None: request_provider = await self.get_request_provider() + origin = httpx.URL(context.request.loaded_url or context.request.url) for call in result.add_requests_calls: - # TODO: handle strategy, limit, include/exclude, etc. + requests = list[BaseRequestData]() + + for request in call['requests']: + # TODO: handle strategy, limit, include/exclude, etc. + request_model = request if isinstance(request, BaseRequestData) else BaseRequestData.from_url(request) + destination = httpx.URL(request_model.url) + if destination.is_relative_url: + base_url = httpx.URL(call.get('base_url', origin)) + request_model.url = str(base_url.join(destination)) + + requests.append(request_model) + await request_provider.add_requests_batched( - requests=[ - request if isinstance(request, BaseRequestData) else BaseRequestData.from_url(request) - for request in call['requests'] - ], + requests=requests, wait_for_all_requests_to_be_added=False, ) @@ -290,7 +301,7 @@ async def __run_task_function(self) -> None: logger=logger, ) - await self._commit_request_handler_result(result) + await self._commit_request_handler_result(crawling_context, result) await wait_for( lambda: request_provider.mark_request_as_handled(crawling_context.request), diff --git a/src/crawlee/basic_crawler/types.py b/src/crawlee/basic_crawler/types.py index 948d1fe5c0..eecdb37c51 100644 --- a/src/crawlee/basic_crawler/types.py +++ b/src/crawlee/basic_crawler/types.py @@ -4,7 +4,7 @@ import re from dataclasses import dataclass, field from enum import Enum -from typing import TYPE_CHECKING, Any, Coroutine, Protocol +from typing import TYPE_CHECKING, Any, Coroutine, Protocol, Sequence from typing_extensions import NotRequired, TypedDict, Unpack @@ -37,7 +37,7 @@ class AddRequestsFunction(Protocol): """Type of a function for adding URLs to the request queue with optional filtering.""" def __call__( # noqa: D102 - self, requests: list[str | BaseRequestData], **kwargs: Unpack[AddRequestsFunctionKwargs] + self, requests: Sequence[str | BaseRequestData], **kwargs: Unpack[AddRequestsFunctionKwargs] ) -> Coroutine[None, None, None]: ... @@ -47,7 +47,7 @@ class EnqueueLinksFunction(Protocol): def __call__( # noqa: D102 self, *, - selector: str, + selector: str = 'a', label: str | None = None, user_data: dict[str, Any] | None = None, **kwargs: Unpack[AddRequestsFunctionKwargs], @@ -79,7 +79,7 @@ class FinalStatistics: class AddRequestsFunctionCall(AddRequestsFunctionKwargs): """Record of a call to `add_requests1.""" - requests: list[str | BaseRequestData] + requests: Sequence[str | BaseRequestData] @dataclass() @@ -89,7 +89,7 @@ class RequestHandlerRunResult: add_requests_calls: list[AddRequestsFunctionCall] = field(default_factory=list) async def add_requests( - self, requests: list[str | BaseRequestData], **kwargs: Unpack[AddRequestsFunctionKwargs] + self, requests: Sequence[str | BaseRequestData], **kwargs: Unpack[AddRequestsFunctionKwargs] ) -> None: """Track a call to the `add_requests` context helper.""" self.add_requests_calls.append(AddRequestsFunctionCall(requests=requests, **kwargs)) diff --git a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py index ba2a23047a..5eb85c83c0 100644 --- a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py +++ b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py @@ -2,21 +2,23 @@ import asyncio from datetime import timedelta -from typing import TYPE_CHECKING, AsyncGenerator, Awaitable, Callable, Iterable, Literal +from typing import TYPE_CHECKING, Any, AsyncGenerator, Awaitable, Callable, Iterable, Literal import httpx +from typing_extensions import Unpack from crawlee.basic_crawler.basic_crawler import BasicCrawler from crawlee.basic_crawler.context_pipeline import ContextPipeline from crawlee.beautifulsoup_crawler.types import BeautifulSoupCrawlingContext from crawlee.http_clients.httpx_client import HttpxClient from crawlee.http_crawler.types import HttpCrawlingContext +from crawlee.request import BaseRequestData if TYPE_CHECKING: from datetime import timedelta from crawlee.autoscaling.autoscaled_pool import ConcurrencySettings - from crawlee.basic_crawler.types import BasicCrawlingContext + from crawlee.basic_crawler.types import AddRequestsFunctionKwargs, BasicCrawlingContext from crawlee.configuration import Configuration from crawlee.storages.request_provider import RequestProvider @@ -92,13 +94,30 @@ async def _make_http_request(self, context: BasicCrawlingContext) -> AsyncGenera async def _parse_http_response( self, context: HttpCrawlingContext ) -> AsyncGenerator[BeautifulSoupCrawlingContext, None]: - from bs4 import BeautifulSoup + from bs4 import BeautifulSoup, Tag soup = await asyncio.to_thread(lambda: BeautifulSoup(context.http_response.read(), self._parser)) + + async def enqueue_links( + *, + selector: str = 'a', + label: str | None = None, + user_data: dict[str, Any] | None = None, + **kwargs: Unpack[AddRequestsFunctionKwargs], + ) -> None: + requests = list[BaseRequestData]() + for link in soup.find_all(selector): + link: Tag + if (href := link.attrs.get('href')) is not None: + requests.append(BaseRequestData.from_url(href)) + + await context.add_requests(requests, **kwargs) + yield BeautifulSoupCrawlingContext( request=context.request, send_request=context.send_request, add_requests=context.add_requests, + enqueue_links=enqueue_links, http_response=context.http_response, soup=soup, ) diff --git a/src/crawlee/beautifulsoup_crawler/types.py b/src/crawlee/beautifulsoup_crawler/types.py index e0ff48049e..707e8027d0 100644 --- a/src/crawlee/beautifulsoup_crawler/types.py +++ b/src/crawlee/beautifulsoup_crawler/types.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from crawlee.basic_crawler.types import BasicCrawlingContext +from crawlee.basic_crawler.types import BasicCrawlingContext, EnqueueLinksFunction from crawlee.http_crawler.types import HttpCrawlingResult if TYPE_CHECKING: @@ -15,3 +15,4 @@ class BeautifulSoupCrawlingContext(HttpCrawlingResult, BasicCrawlingContext): """Crawling context used by BeautifulSoupCrawler.""" soup: BeautifulSoup + enqueue_links: EnqueueLinksFunction diff --git a/tests/unit/beautifulsoup_crawler/test_beautifulsoup_crawler.py b/tests/unit/beautifulsoup_crawler/test_beautifulsoup_crawler.py index 1bc511c3ce..5405310fcb 100644 --- a/tests/unit/beautifulsoup_crawler/test_beautifulsoup_crawler.py +++ b/tests/unit/beautifulsoup_crawler/test_beautifulsoup_crawler.py @@ -13,7 +13,7 @@ @pytest.fixture() async def server() -> AsyncGenerator[respx.MockRouter, None]: with respx.mock(base_url='https://test.io', assert_all_called=False) as mock: - mock.get('/html', name='html_endpoint').return_value = Response( + mock.get('/', name='index_endpoint').return_value = Response( 200, text=""" @@ -26,11 +26,40 @@ async def server() -> AsyncGenerator[respx.MockRouter, None]: """, ) + mock.get('/asdf', name='secondary_index_endpoint').return_value = Response( + 200, + text=""" + + Hello + + + Link 3 + Link 4 + + """, + ) + + generic_response = Response( + 200, + text=""" + + Hello + + + Insightful content + + """, + ) + + mock.get('/hjkl').return_value = generic_response + mock.get('/qwer').return_value = generic_response + mock.get('/uiop').return_value = generic_response + yield mock async def test_basic(server: respx.MockRouter) -> None: - crawler = BeautifulSoupCrawler(request_provider=RequestList(['https://test.io/html'])) + crawler = BeautifulSoupCrawler(request_provider=RequestList(['https://test.io/'])) handler = mock.AsyncMock() @crawler.router.default_handler @@ -40,8 +69,32 @@ async def request_handler(context: BeautifulSoupCrawlingContext) -> None: await crawler.run() - assert server['html_endpoint'].called + assert server['index_endpoint'].called assert handler.called # The handler should find two links assert len(handler.call_args[0][0]) == 2 + + +async def test_enqueue_links(server: respx.MockRouter) -> None: + crawler = BeautifulSoupCrawler(request_provider=RequestList(['https://test.io/'])) + visit = mock.Mock() + + @crawler.router.default_handler + async def request_handler(context: BeautifulSoupCrawlingContext) -> None: + visit(context.request.url) + await context.enqueue_links() + + await crawler.run() + + assert server['index_endpoint'].called + assert server['secondary_index_endpoint'].called + + visited = {call[0][0] for call in visit.call_args_list} + assert visited == { + 'https://test.io/', + 'https://test.io/asdf', + 'https://test.io/hjkl', + 'https://test.io/qwer', + 'https://test.io/uiop', + } From 46e43753e717f03538653cfe6ea330066fd3e13f Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 19 Apr 2024 16:48:46 +0200 Subject: [PATCH 05/20] Fix loop variable init --- src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py index 5eb85c83c0..d02e8c42a6 100644 --- a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py +++ b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py @@ -106,8 +106,9 @@ async def enqueue_links( **kwargs: Unpack[AddRequestsFunctionKwargs], ) -> None: requests = list[BaseRequestData]() + + link: Tag for link in soup.find_all(selector): - link: Tag if (href := link.attrs.get('href')) is not None: requests.append(BaseRequestData.from_url(href)) From a02a1ed76d9aa05f7ab4a04269bddb5cc205ed9c Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 19 Apr 2024 17:00:22 +0200 Subject: [PATCH 06/20] Handle label and user_data in enqueue_links --- .../beautifulsoup_crawler/beautifulsoup_crawler.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py index d02e8c42a6..399c718cbe 100644 --- a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py +++ b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py @@ -109,8 +109,17 @@ async def enqueue_links( link: Tag for link in soup.find_all(selector): + link_user_data = user_data + + if user_data is not None or label is not None: + if link_user_data is None: + link_user_data = {} + + if label is not None: + link_user_data.setdefault('label', label) + if (href := link.attrs.get('href')) is not None: - requests.append(BaseRequestData.from_url(href)) + requests.append(BaseRequestData.from_url(href, user_data=link_user_data)) await context.add_requests(requests, **kwargs) From 731dbc0cfa8f1a0d75fea147099df4ea3e864496 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 19 Apr 2024 17:05:27 +0200 Subject: [PATCH 07/20] Missing doc --- src/crawlee/globs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/crawlee/globs.py b/src/crawlee/globs.py index bf840ae90a..7a25eee835 100644 --- a/src/crawlee/globs.py +++ b/src/crawlee/globs.py @@ -7,6 +7,8 @@ class Glob: + """Wraps a glob pattern (supports the `*`, `**`, `?` wildcards).""" + def __init__(self, glob: str) -> None: self.glob = glob self.regexp = re.compile(_translate(self.glob)) From c411757f28ed78464650e41a9ad03961dc1db3ac Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 22 Apr 2024 14:45:32 +0200 Subject: [PATCH 08/20] Handle EnqueueStrategy --- pyproject.toml | 7 ++- src/crawlee/basic_crawler/basic_crawler.py | 61 +++++++++++++++++-- src/crawlee/basic_crawler/context_pipeline.py | 8 +++ src/crawlee/basic_crawler/types.py | 15 +---- src/crawlee/enqueue_strategy.py | 10 +++ src/crawlee/request.py | 45 +++++++++++--- .../unit/basic_crawler/test_basic_crawler.py | 49 ++++++++++++++- 7 files changed, 169 insertions(+), 26 deletions(-) create mode 100644 src/crawlee/enqueue_strategy.py diff --git a/pyproject.toml b/pyproject.toml index 3a8619e6cf..fb3f0ae6fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ pyee = "^11.1.0" python-dateutil = "^2.9.0" sortedcollections = "^2.1.0" typing-extensions = "^4.1.0" +tldextract = "^5.1.2" [tool.poetry.group.dev.dependencies] build = "~1.2.0" @@ -190,4 +191,8 @@ ignore_missing_imports = true ignore_missing_imports = true [tool.coverage.report] -exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"] +exclude_lines = [ + "pragma: no cover", + "if TYPE_CHECKING:", + "assert_never()" +] diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index 8853507582..1d5d26750d 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -1,13 +1,15 @@ # Inspiration: https://github.com/apify/crawlee/blob/v3.7.3/packages/basic-crawler/src/internals/basic-crawler.ts from __future__ import annotations +import tempfile from datetime import timedelta from functools import partial from logging import getLogger -from typing import TYPE_CHECKING, Awaitable, Callable, Generic, Union, cast +from typing import TYPE_CHECKING, AsyncGenerator, Awaitable, Callable, Generic, Union, cast import httpx -from typing_extensions import TypeVar +from tldextract import TLDExtract +from typing_extensions import TypeVar, assert_never from crawlee._utils.wait import wait_for from crawlee.autoscaling import AutoscaledPool, ConcurrencySettings @@ -16,6 +18,7 @@ from crawlee.basic_crawler.context_pipeline import ( ContextPipeline, ContextPipelineInitializationError, + ContextPipelineInterruptedError, RequestHandlerError, ) from crawlee.basic_crawler.router import Router @@ -26,6 +29,7 @@ SendRequestFunction, ) from crawlee.configuration import Configuration +from crawlee.enqueue_strategy import EnqueueStrategy from crawlee.events.local_event_manager import LocalEventManager from crawlee.http_clients.httpx_client import HttpxClient from crawlee.request import BaseRequestData, Request, RequestState @@ -92,7 +96,7 @@ def __init__( self._http_client = http_client or HttpxClient() - self._context_pipeline = _context_pipeline or ContextPipeline() + self._context_pipeline = (_context_pipeline or ContextPipeline()).compose(self._check_url_after_redirects) self._error_handler: ErrorHandler[TCrawlingContext] | None = None self._failed_request_handler: FailedRequestHandler[TCrawlingContext] | None = None @@ -109,6 +113,8 @@ def __init__( else max(2 * request_handler_timeout, timedelta(minutes=5)) ) + self._tld_extractor = TLDExtract(cache_dir=tempfile.TemporaryDirectory().name) + self._event_manager = LocalEventManager() # TODO: switch based on configuration # https://github.com/apify/crawlee-py/issues/83 self._snapshotter = Snapshotter(self._event_manager) @@ -192,6 +198,39 @@ def _should_retry_request(self, crawling_context: BasicCrawlingContext) -> bool: not crawling_context.request.no_retry and (crawling_context.request.retry_count + 1) < max_request_retries ) + async def _check_url_after_redirects( + self, crawling_context: TCrawlingContext + ) -> AsyncGenerator[TCrawlingContext, None]: + if crawling_context.request.loaded_url is not None and not self._check_enqueue_strategy( + httpx.URL(crawling_context.request.url), + httpx.URL(crawling_context.request.loaded_url), + crawling_context.request.enqueue_strategy, + ): + raise ContextPipelineInterruptedError( + f'Skipping URL {crawling_context.request.loaded_url} (redirected from {crawling_context.request.url})' + ) + + yield crawling_context + + def _check_enqueue_strategy( + self, target_url: httpx.URL, origin_url: httpx.URL, strategy: EnqueueStrategy | None + ) -> bool: + if strategy == EnqueueStrategy.SAME_HOSTNAME: + return target_url.host == origin_url.host + + if strategy == EnqueueStrategy.SAME_DOMAIN: + origin_domain = self._tld_extractor.extract_str(origin_url.host).domain + target_domain = self._tld_extractor.extract_str(target_url.host).domain + return origin_domain == target_domain + + if strategy == EnqueueStrategy.SAME_ORIGIN: + return target_url.host == origin_url.host and target_url.scheme == origin_url.scheme + + if strategy is None or strategy == EnqueueStrategy.ALL: + return True + + assert_never() + async def _handle_request_error(self, crawling_context: TCrawlingContext, error: Exception) -> None: request_provider = await self.get_request_provider() @@ -245,14 +284,15 @@ async def _commit_request_handler_result( requests = list[BaseRequestData]() for request in call['requests']: - # TODO: handle strategy, limit, include/exclude, etc. + # TODO: handle deduplication, limit, include/exclude, etc. request_model = request if isinstance(request, BaseRequestData) else BaseRequestData.from_url(request) destination = httpx.URL(request_model.url) if destination.is_relative_url: base_url = httpx.URL(call.get('base_url', origin)) request_model.url = str(base_url.join(destination)) - requests.append(request_model) + if self._check_enqueue_strategy(destination, origin, call.get('strategy')): + requests.append(request_model) await request_provider.add_requests_batched( requests=requests, @@ -343,6 +383,17 @@ async def __run_task_function(self) -> None: ) request.state = RequestState.ERROR raise + except ContextPipelineInterruptedError as interruped_error: + logger.debug('The context pipeline was interrupted', exc_info=interruped_error) + + await wait_for( + lambda: request_provider.mark_request_as_handled(crawling_context.request), + timeout=self._internal_timeout, + timeout_message='Marking request as handled timed out after ' + f'{self._internal_timeout.total_seconds()} seconds', + logger=logger, + max_retries=3, + ) except ContextPipelineInitializationError as initialization_error: if self._should_retry_request(crawling_context): logger.debug( diff --git a/src/crawlee/basic_crawler/context_pipeline.py b/src/crawlee/basic_crawler/context_pipeline.py index 39e4a9077e..121e43a5f2 100644 --- a/src/crawlee/basic_crawler/context_pipeline.py +++ b/src/crawlee/basic_crawler/context_pipeline.py @@ -29,6 +29,10 @@ def __init__(self, wrapped_exception: Exception, crawling_context: BasicCrawling self.crawling_context = crawling_context +class ContextPipelineInterruptedError(Exception): + """May be thrown in the initialization phase of a middleware to signal that the request should not be processed.""" + + class ContextPipelineFinalizationError(Exception): """Wraps an exception thrown in the finalization step of a context pipeline middleware. @@ -85,6 +89,8 @@ async def __call__( result = await middleware_instance.__anext__() except StopAsyncIteration as e: raise RuntimeError('The middleware did not yield') from e + except ContextPipelineInterruptedError: + raise except Exception as e: raise ContextPipelineInitializationError(e, crawling_context) from e @@ -101,6 +107,8 @@ async def __call__( result = await middleware_instance.__anext__() except StopAsyncIteration: # noqa: PERF203 pass + except ContextPipelineInterruptedError as e: + raise RuntimeError('Invalid state - pipeline interrupted in the finalization step') from e except Exception as e: raise ContextPipelineFinalizationError(e, crawling_context) from e else: diff --git a/src/crawlee/basic_crawler/types.py b/src/crawlee/basic_crawler/types.py index eecdb37c51..5f8fbf99c0 100644 --- a/src/crawlee/basic_crawler/types.py +++ b/src/crawlee/basic_crawler/types.py @@ -3,32 +3,23 @@ import re from dataclasses import dataclass, field -from enum import Enum from typing import TYPE_CHECKING, Any, Coroutine, Protocol, Sequence from typing_extensions import NotRequired, TypedDict, Unpack if TYPE_CHECKING: + from crawlee.enqueue_strategy import EnqueueStrategy from crawlee.globs import Glob from crawlee.http_clients.base_http_client import HttpResponse from crawlee.request import BaseRequestData, Request -class EnqueueStrategy(str, Enum): - """Strategy for deciding which links should be followed and which ones should be ignored.""" - - ALL = 'all' - SAME_DOMAIN = 'same-domain' - SAME_HOSTNAME = 'same-hostname' - SAME_ORIGIN = 'same-origin' - - class AddRequestsFunctionKwargs(TypedDict): """Keyword arguments type for AddRequestsFunction.""" limit: NotRequired[int] base_url: NotRequired[str] - strategy: NotRequired[EnqueueStrategy] + strategy: NotRequired[EnqueueStrategy | None] include: NotRequired[list[re.Pattern | Glob]] exclude: NotRequired[list[re.Pattern | Glob]] @@ -77,7 +68,7 @@ class FinalStatistics: class AddRequestsFunctionCall(AddRequestsFunctionKwargs): - """Record of a call to `add_requests1.""" + """Record of a call to `add_requests`.""" requests: Sequence[str | BaseRequestData] diff --git a/src/crawlee/enqueue_strategy.py b/src/crawlee/enqueue_strategy.py new file mode 100644 index 0000000000..873edce86d --- /dev/null +++ b/src/crawlee/enqueue_strategy.py @@ -0,0 +1,10 @@ +from enum import Enum + + +class EnqueueStrategy(str, Enum): + """Strategy for deciding which links should be followed and which ones should be ignored.""" + + ALL = 'all' + SAME_DOMAIN = 'same-domain' + SAME_HOSTNAME = 'same-hostname' + SAME_ORIGIN = 'same-origin' diff --git a/src/crawlee/request.py b/src/crawlee/request.py index 38e46a7bf0..ca122d3deb 100644 --- a/src/crawlee/request.py +++ b/src/crawlee/request.py @@ -10,6 +10,7 @@ from typing_extensions import Self from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id +from crawlee.enqueue_strategy import EnqueueStrategy class BaseRequestData(BaseModel): @@ -56,12 +57,20 @@ def from_url( cls, url: str, *, + label: str | None = None, unique_key: str | None = None, **kwargs: Any, ) -> Self: """Create a new `RequestData` instance from a URL.""" unique_key = unique_key or compute_unique_key(url) - return cls(url=url, unique_key=unique_key, **kwargs) + result = cls(url=url, unique_key=unique_key, **kwargs) + + if label is not None: + if result.user_data is None: + result.user_data = {} + result.user_data['label'] = label + + return result class Request(BaseRequestData): @@ -80,6 +89,7 @@ def from_url( cls, url: str, *, + label: str | None = None, unique_key: str | None = None, id: str | None = None, **kwargs: Any, @@ -87,18 +97,21 @@ def from_url( """Create a new `RequestData` instance from a URL.""" unique_key = unique_key or compute_unique_key(url) id = id or unique_key_to_request_id(unique_key) - return cls(url=url, unique_key=unique_key, id=id, **kwargs) + + result = cls(url=url, unique_key=unique_key, id=id, **kwargs) + + if label is not None: + if result.user_data is None: + result.user_data = {} + result.user_data['label'] = label + + return result @classmethod def from_base_request_data(cls, base_request_data: BaseRequestData, *, id: str | None = None) -> Self: """Create a complete Request object based on a BaseRequestData instance.""" return cls(**base_request_data.model_dump(), id=id or unique_key_to_request_id(base_request_data.unique_key)) - @property - def crawlee_data(self) -> CrawleeRequestData: - """Crawlee-specific configuration stored in the user_data.""" - return CrawleeRequestData.model_validate(self.user_data.get('__crawlee', {}) if self.user_data else {}) - @property def label(self) -> str | None: """A string used to differentiate between arbitrary request types.""" @@ -106,6 +119,11 @@ def label(self) -> str | None: return str(self.user_data['label']) return None + @property + def crawlee_data(self) -> CrawleeRequestData: + """Crawlee-specific configuration stored in the user_data.""" + return CrawleeRequestData.model_validate(self.user_data.get('__crawlee', {}) if self.user_data else {}) + @property def state(self) -> RequestState | None: """Crawlee-specific request handling state.""" @@ -132,6 +150,19 @@ def max_retries(self, new_max_retries: int) -> None: self.user_data.setdefault('__crawlee', {}) self.user_data['__crawlee']['maxRetries'] = new_max_retries + @property + def enqueue_strategy(self) -> EnqueueStrategy | None: + """The strategy used when enqueueing the request.""" + return EnqueueStrategy(self.crawlee_data.enqueue_strategy) if self.crawlee_data.enqueue_strategy else None + + @enqueue_strategy.setter + def enqueue_strategy(self, new_enqueue_strategy: EnqueueStrategy) -> None: + if self.user_data is None: + self.user_data = {} + + self.user_data.setdefault('__crawlee', {}) + self.user_data['__crawlee']['enqueueStrategy'] = str(new_enqueue_strategy) + class RequestState(Enum): """Crawlee-specific request handling state.""" diff --git a/tests/unit/basic_crawler/test_basic_crawler.py b/tests/unit/basic_crawler/test_basic_crawler.py index 9a5863a80d..3f2b437772 100644 --- a/tests/unit/basic_crawler/test_basic_crawler.py +++ b/tests/unit/basic_crawler/test_basic_crawler.py @@ -2,7 +2,8 @@ from __future__ import annotations import json -from typing import TYPE_CHECKING, Any +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, Sequence from unittest.mock import Mock import pytest @@ -10,6 +11,7 @@ from crawlee.basic_crawler.basic_crawler import BasicCrawler, UserDefinedErrorHandlerError from crawlee.basic_crawler.types import BasicCrawlingContext +from crawlee.enqueue_strategy import EnqueueStrategy from crawlee.request import BaseRequestData, Request from crawlee.storages import RequestList @@ -287,3 +289,48 @@ async def handler(context: BasicCrawlingContext) -> None: visited = {call[0][0] for call in visit.call_args_list} assert visited == {'http://a.com', 'http://b.com', 'http://c.com'} + + +@dataclass +class EnqueueStrategyTestInput: + strategy: EnqueueStrategy | None + expected_urls: Sequence[str] + + URLS: Sequence[str] = ( + 'https://someplace.com/index.html', + 'http://someplace.com/index.html', + 'https://blog.someplace.com/index.html', + 'https://other.place.com/index.html', + ) + + +@pytest.mark.parametrize( + 'test_input', + argvalues=[ + EnqueueStrategyTestInput(strategy=None, expected_urls=EnqueueStrategyTestInput.URLS), + EnqueueStrategyTestInput(strategy=EnqueueStrategy.ALL, expected_urls=EnqueueStrategyTestInput.URLS), + EnqueueStrategyTestInput(strategy=EnqueueStrategy.SAME_DOMAIN, expected_urls=EnqueueStrategyTestInput.URLS[:3]), + EnqueueStrategyTestInput( + strategy=EnqueueStrategy.SAME_HOSTNAME, expected_urls=EnqueueStrategyTestInput.URLS[:2] + ), + EnqueueStrategyTestInput(strategy=EnqueueStrategy.SAME_ORIGIN, expected_urls=EnqueueStrategyTestInput.URLS[:1]), + ], +) +async def test_enqueue_strategy(test_input: EnqueueStrategyTestInput) -> None: + visit = Mock() + crawler = BasicCrawler(request_provider=RequestList(['https://someplace.com'])) + + @crawler.router.default_handler + async def default_handler(context: BasicCrawlingContext) -> None: + await context.add_requests( + [BaseRequestData.from_url(url, label='a') for url in test_input.URLS], strategy=test_input.strategy + ) + + @crawler.router.handler('a') + async def handler(context: BasicCrawlingContext) -> None: + visit(context.request.url) + + await crawler.run() + + visited = {call[0][0] for call in visit.call_args_list} + assert visited == set(test_input.expected_urls) From 7343b5cfc41ff9089d5eb68968d750400b4cd314 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 22 Apr 2024 15:23:16 +0200 Subject: [PATCH 09/20] Handle limit --- src/crawlee/basic_crawler/basic_crawler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index 1d5d26750d..a5949c1529 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -284,7 +284,10 @@ async def _commit_request_handler_result( requests = list[BaseRequestData]() for request in call['requests']: - # TODO: handle deduplication, limit, include/exclude, etc. + if (limit := call.get('limit')) is not None and len(requests) >= limit: + break + + # TODO: handle deduplication, include/exclude, etc. request_model = request if isinstance(request, BaseRequestData) else BaseRequestData.from_url(request) destination = httpx.URL(request_model.url) if destination.is_relative_url: From 6320f4e6b34f619c161d5bb37b24c9cbb1d3d81c Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 23 Apr 2024 15:08:13 +0200 Subject: [PATCH 10/20] Handle include/exclude globs --- src/crawlee/basic_crawler/basic_crawler.py | 36 ++++- src/crawlee/globs.py | 76 +++++++++- src/crawlee/storages/request_provider.py | 1 - .../unit/basic_crawler/test_basic_crawler.py | 141 +++++++++++++----- 4 files changed, 206 insertions(+), 48 deletions(-) diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index a5949c1529..6afe3a02d2 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -5,7 +5,7 @@ from datetime import timedelta from functools import partial from logging import getLogger -from typing import TYPE_CHECKING, AsyncGenerator, Awaitable, Callable, Generic, Union, cast +from typing import TYPE_CHECKING, AsyncGenerator, Awaitable, Callable, Generic, Sequence, Union, cast import httpx from tldextract import TLDExtract @@ -31,11 +31,14 @@ from crawlee.configuration import Configuration from crawlee.enqueue_strategy import EnqueueStrategy from crawlee.events.local_event_manager import LocalEventManager +from crawlee.globs import Glob from crawlee.http_clients.httpx_client import HttpxClient from crawlee.request import BaseRequestData, Request, RequestState from crawlee.storages.request_queue import RequestQueue if TYPE_CHECKING: + import re + from crawlee.http_clients.base_http_client import BaseHttpClient, HttpResponse from crawlee.storages.request_provider import RequestProvider @@ -231,6 +234,32 @@ def _check_enqueue_strategy( assert_never() + def _check_url_patterns( + self, + target_url: httpx.URL, + include: Sequence[re.Pattern | Glob] | None, + exclude: Sequence[re.Pattern | Glob] | None, + ) -> bool: + if exclude is not None: + for pattern in exclude: + if isinstance(pattern, Glob): + pattern = pattern.regexp # noqa: PLW2901 + + if pattern.match(str(target_url)) is not None: + return False + + if include is None: + return True + + for pattern in include: + if isinstance(pattern, Glob): + pattern = pattern.regexp # noqa: PLW2901 + + if pattern.match(str(target_url)) is not None: + return True + + return False + async def _handle_request_error(self, crawling_context: TCrawlingContext, error: Exception) -> None: request_provider = await self.get_request_provider() @@ -287,14 +316,15 @@ async def _commit_request_handler_result( if (limit := call.get('limit')) is not None and len(requests) >= limit: break - # TODO: handle deduplication, include/exclude, etc. request_model = request if isinstance(request, BaseRequestData) else BaseRequestData.from_url(request) destination = httpx.URL(request_model.url) if destination.is_relative_url: base_url = httpx.URL(call.get('base_url', origin)) request_model.url = str(base_url.join(destination)) - if self._check_enqueue_strategy(destination, origin, call.get('strategy')): + if self._check_enqueue_strategy(destination, origin, call.get('strategy')) and self._check_url_patterns( + destination, call.get('include', None), call.get('exclude', None) + ): requests.append(request_model) await request_provider.add_requests_batched( diff --git a/src/crawlee/globs.py b/src/crawlee/globs.py index 7a25eee835..e9c0697228 100644 --- a/src/crawlee/globs.py +++ b/src/crawlee/globs.py @@ -1,6 +1,5 @@ from __future__ import annotations -import fnmatch import os import re from typing import Sequence @@ -11,7 +10,7 @@ class Glob: def __init__(self, glob: str) -> None: self.glob = glob - self.regexp = re.compile(_translate(self.glob)) + self.regexp = re.compile(_translate(self.glob, recursive=True)) def _translate( @@ -64,8 +63,79 @@ def _translate( if part: if not include_hidden and part[0] in '*?': results.append(r'(?!\.)') - results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) # type: ignore # noqa: SLF001 + results.extend(_fnmatch_translate(part, f'{not_sep}*', not_sep)) if idx < last_part_idx: results.append(any_sep) res = ''.join(results) return rf'(?s:{res})\Z' + + +def _fnmatch_translate(pat: str, star: str, question_mark: str) -> list[str]: # noqa: PLR0912 + """Copy of fnmatch._translate from Python 3.13.""" + res = list[str]() + add = res.append + i, n = 0, len(pat) + while i < n: + c = pat[i] + i = i + 1 + if c == '*': + # compress consecutive `*` into one + if (not res) or res[-1] is not star: + add(star) + elif c == '?': + add(question_mark) + elif c == '[': + j = i + if j < n and pat[j] == '!': + j = j + 1 + if j < n and pat[j] == ']': + j = j + 1 + while j < n and pat[j] != ']': + j = j + 1 + if j >= n: + add('\\[') + else: + stuff = pat[i:j] + if '-' not in stuff: + stuff = stuff.replace('\\', r'\\') + else: + chunks = [] + k = i + 2 if pat[i] == '!' else i + 1 + while True: + k = pat.find('-', k, j) + if k < 0: + break + chunks.append(pat[i:k]) + i = k + 1 + k = k + 3 + chunk = pat[i:j] + if chunk: + chunks.append(chunk) + else: + chunks[-1] += '-' + # Remove empty ranges -- invalid in RE. + for k in range(len(chunks) - 1, 0, -1): + if chunks[k - 1][-1] > chunks[k][0]: + chunks[k - 1] = chunks[k - 1][:-1] + chunks[k][1:] + del chunks[k] + # Escape backslashes and hyphens for set difference (--). + # Hyphens that create ranges shouldn't be escaped. + stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') for s in chunks) + # Escape set operations (&&, ~~ and ||). + stuff = re.sub(r'([&~|])', r'\\\1', stuff) + i = j + 1 + if not stuff: + # Empty range: never match. + add('(?!)') + elif stuff == '!': + # Negated empty range: match any character. + add('.') + else: + if stuff[0] == '!': + stuff = '^' + stuff[1:] + elif stuff[0] in ('^', '['): + stuff = '\\' + stuff + add(f'[{stuff}]') + else: + add(re.escape(c)) + return res diff --git a/src/crawlee/storages/request_provider.py b/src/crawlee/storages/request_provider.py index c31a22a59a..762f562bef 100644 --- a/src/crawlee/storages/request_provider.py +++ b/src/crawlee/storages/request_provider.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from crawlee.request import BaseRequestData, Request from crawlee.storages.models import RequestQueueOperationInfo diff --git a/tests/unit/basic_crawler/test_basic_crawler.py b/tests/unit/basic_crawler/test_basic_crawler.py index 3f2b437772..ccf6633b12 100644 --- a/tests/unit/basic_crawler/test_basic_crawler.py +++ b/tests/unit/basic_crawler/test_basic_crawler.py @@ -10,8 +10,9 @@ from httpx import Headers, Response from crawlee.basic_crawler.basic_crawler import BasicCrawler, UserDefinedErrorHandlerError -from crawlee.basic_crawler.types import BasicCrawlingContext +from crawlee.basic_crawler.types import AddRequestsFunctionKwargs, BasicCrawlingContext from crawlee.enqueue_strategy import EnqueueStrategy +from crawlee.globs import Glob from crawlee.request import BaseRequestData, Request from crawlee.storages import RequestList @@ -270,63 +271,121 @@ async def handler(context: BasicCrawlingContext) -> None: assert response_headers.get('content-type').endswith('/json') -async def test_add_requests_works() -> None: - visit = Mock() - - crawler = BasicCrawler( - request_provider=RequestList(['http://a.com']), - max_request_retries=3, - ) - - @crawler.router.default_handler - async def handler(context: BasicCrawlingContext) -> None: - visit(context.request.url) - - if context.request.url == 'http://a.com': - await context.add_requests([BaseRequestData.from_url('http://b.com'), 'http://c.com']) - - await crawler.run() - - visited = {call[0][0] for call in visit.call_args_list} - assert visited == {'http://a.com', 'http://b.com', 'http://c.com'} - - @dataclass -class EnqueueStrategyTestInput: - strategy: EnqueueStrategy | None +class AddRequestsTestInput: + start_url: str + requests: Sequence[str | BaseRequestData] expected_urls: Sequence[str] + kwargs: AddRequestsFunctionKwargs - URLS: Sequence[str] = ( - 'https://someplace.com/index.html', - 'http://someplace.com/index.html', - 'https://blog.someplace.com/index.html', - 'https://other.place.com/index.html', - ) + +STRATEGY_TEST_URLS = ( + 'https://someplace.com/index.html', + 'http://someplace.com/index.html', + 'https://blog.someplace.com/index.html', + 'https://other.place.com/index.html', +) + +INCLUDE_TEST_URLS = ( + 'https://someplace.com', + 'https://someplace.com/blog/category/cats', + 'https://someplace.com/blog/category/boots', + 'https://someplace.com/blog/archive/index.html', + 'https://someplace.com/blog/archive/cats', +) @pytest.mark.parametrize( 'test_input', argvalues=[ - EnqueueStrategyTestInput(strategy=None, expected_urls=EnqueueStrategyTestInput.URLS), - EnqueueStrategyTestInput(strategy=EnqueueStrategy.ALL, expected_urls=EnqueueStrategyTestInput.URLS), - EnqueueStrategyTestInput(strategy=EnqueueStrategy.SAME_DOMAIN, expected_urls=EnqueueStrategyTestInput.URLS[:3]), - EnqueueStrategyTestInput( - strategy=EnqueueStrategy.SAME_HOSTNAME, expected_urls=EnqueueStrategyTestInput.URLS[:2] + # Basic use case + AddRequestsTestInput( + start_url='https://a.com', + requests=[ + 'https://a.com', + BaseRequestData.from_url('http://b.com'), + 'http://c.com', + ], + kwargs={}, + expected_urls=['https://a.com', 'http://b.com', 'http://c.com'], + ), + # Enqueue strategy + AddRequestsTestInput( + start_url=STRATEGY_TEST_URLS[0], + requests=STRATEGY_TEST_URLS, + kwargs=AddRequestsFunctionKwargs(strategy=None), + expected_urls=STRATEGY_TEST_URLS, + ), + AddRequestsTestInput( + start_url=STRATEGY_TEST_URLS[0], + requests=STRATEGY_TEST_URLS, + kwargs=AddRequestsFunctionKwargs(strategy=EnqueueStrategy.ALL), + expected_urls=STRATEGY_TEST_URLS, + ), + AddRequestsTestInput( + start_url=STRATEGY_TEST_URLS[0], + requests=STRATEGY_TEST_URLS, + kwargs=AddRequestsFunctionKwargs(strategy=EnqueueStrategy.SAME_DOMAIN), + expected_urls=STRATEGY_TEST_URLS[:3], ), - EnqueueStrategyTestInput(strategy=EnqueueStrategy.SAME_ORIGIN, expected_urls=EnqueueStrategyTestInput.URLS[:1]), + AddRequestsTestInput( + start_url=STRATEGY_TEST_URLS[0], + requests=STRATEGY_TEST_URLS, + kwargs=AddRequestsFunctionKwargs(strategy=EnqueueStrategy.SAME_HOSTNAME), + expected_urls=STRATEGY_TEST_URLS[:2], + ), + AddRequestsTestInput( + start_url=STRATEGY_TEST_URLS[0], + requests=STRATEGY_TEST_URLS, + kwargs=AddRequestsFunctionKwargs(strategy=EnqueueStrategy.SAME_ORIGIN), + expected_urls=STRATEGY_TEST_URLS[:1], + ), + # Include/exclude + AddRequestsTestInput( + start_url=INCLUDE_TEST_URLS[0], + requests=INCLUDE_TEST_URLS, + kwargs=AddRequestsFunctionKwargs(include=[Glob('https://someplace.com/**/cats')]), + expected_urls=[INCLUDE_TEST_URLS[1], INCLUDE_TEST_URLS[4]], + ), + AddRequestsTestInput( + start_url=INCLUDE_TEST_URLS[0], + requests=INCLUDE_TEST_URLS, + kwargs=AddRequestsFunctionKwargs(exclude=[Glob('https://someplace.com/**/cats')]), + expected_urls=[INCLUDE_TEST_URLS[0], INCLUDE_TEST_URLS[2], INCLUDE_TEST_URLS[3]], + ), + AddRequestsTestInput( + start_url=INCLUDE_TEST_URLS[0], + requests=INCLUDE_TEST_URLS, + kwargs=AddRequestsFunctionKwargs( + include=[Glob('https://someplace.com/**/cats')], exclude=[Glob('https://**/archive/**')] + ), + expected_urls=[INCLUDE_TEST_URLS[1]], + ), + ], + ids=[ + 'basic', + 'enqueue_strategy_1', + 'enqueue_strategy_2', + 'enqueue_strategy_3', + 'enqueue_strategy_4', + 'enqueue_strategy_5', + 'include_exclude_1', + 'include_exclude_2', + 'include_exclude_3', ], ) -async def test_enqueue_strategy(test_input: EnqueueStrategyTestInput) -> None: +async def test_enqueue_strategy(test_input: AddRequestsTestInput) -> None: visit = Mock() - crawler = BasicCrawler(request_provider=RequestList(['https://someplace.com'])) + crawler = BasicCrawler(request_provider=RequestList([Request.from_url('https://someplace.com', label='start')])) - @crawler.router.default_handler + @crawler.router.handler('start') async def default_handler(context: BasicCrawlingContext) -> None: await context.add_requests( - [BaseRequestData.from_url(url, label='a') for url in test_input.URLS], strategy=test_input.strategy + test_input.requests, + **test_input.kwargs, ) - @crawler.router.handler('a') + @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: visit(context.request.url) From 6cfb9e777653dad7e99373fffb50362b344ab764 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 24 Apr 2024 18:26:43 +0200 Subject: [PATCH 11/20] Apply suggestions from code review Co-authored-by: Vlada Dusek --- src/crawlee/basic_crawler/basic_crawler.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index 6afe3a02d2..8e1dc24821 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -172,7 +172,8 @@ async def add_requests( wait_time_between_batches: timedelta = timedelta(0), ) -> None: """Add requests to the underlying queue.""" - await (await self.get_request_provider()).add_requests_batched( + request_provider = await self.get_request_provider() + await request_provider.add_requests_batched( [ request if isinstance(request, BaseRequestData) else BaseRequestData.from_url(url=request) for request in requests @@ -333,10 +334,12 @@ async def _commit_request_handler_result( ) async def __is_finished_function(self) -> bool: - return await (await self.get_request_provider()).is_finished() + request_provider = await self.get_request_provider() + return await request_provider.is_finished() async def __is_task_ready_function(self) -> bool: - return not await (await self.get_request_provider()).is_empty() + request_provider = await self.get_request_provider() + return not await request_provider.is_empty() async def __run_task_function(self) -> None: request_provider = await self.get_request_provider() From 38d11b5ee0652738bb6f1e6a83831eea76b9a77c Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 24 Apr 2024 18:39:20 +0200 Subject: [PATCH 12/20] default value for user_data --- src/crawlee/request.py | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/src/crawlee/request.py b/src/crawlee/request.py index ca122d3deb..b486083b1b 100644 --- a/src/crawlee/request.py +++ b/src/crawlee/request.py @@ -39,7 +39,7 @@ class BaseRequestData(BaseModel): headers: Annotated[dict[str, str] | None, Field(default_factory=dict)] = None - user_data: Annotated[dict[str, Any] | None, Field(alias='userData')] = None + user_data: Annotated[dict[str, Any], Field(alias='userData', default_factory=dict)] """Custom user data assigned to the request. Use this to save any request related data to the request's scope, keeping them accessible on retries, failures etc. """ @@ -66,8 +66,6 @@ def from_url( result = cls(url=url, unique_key=unique_key, **kwargs) if label is not None: - if result.user_data is None: - result.user_data = {} result.user_data['label'] = label return result @@ -101,8 +99,6 @@ def from_url( result = cls(url=url, unique_key=unique_key, id=id, **kwargs) if label is not None: - if result.user_data is None: - result.user_data = {} result.user_data['label'] = label return result @@ -115,14 +111,14 @@ def from_base_request_data(cls, base_request_data: BaseRequestData, *, id: str | @property def label(self) -> str | None: """A string used to differentiate between arbitrary request types.""" - if self.user_data and 'label' in self.user_data: + if 'label' in self.user_data: return str(self.user_data['label']) return None @property def crawlee_data(self) -> CrawleeRequestData: """Crawlee-specific configuration stored in the user_data.""" - return CrawleeRequestData.model_validate(self.user_data.get('__crawlee', {}) if self.user_data else {}) + return CrawleeRequestData.model_validate(self.user_data.get('__crawlee', {})) @property def state(self) -> RequestState | None: @@ -131,9 +127,6 @@ def state(self) -> RequestState | None: @state.setter def state(self, new_state: RequestState) -> None: - if self.user_data is None: - self.user_data = {} - self.user_data.setdefault('__crawlee', {}) self.user_data['__crawlee']['state'] = new_state @@ -144,9 +137,6 @@ def max_retries(self) -> int | None: @max_retries.setter def max_retries(self, new_max_retries: int) -> None: - if self.user_data is None: - self.user_data = {} - self.user_data.setdefault('__crawlee', {}) self.user_data['__crawlee']['maxRetries'] = new_max_retries @@ -157,9 +147,6 @@ def enqueue_strategy(self) -> EnqueueStrategy | None: @enqueue_strategy.setter def enqueue_strategy(self, new_enqueue_strategy: EnqueueStrategy) -> None: - if self.user_data is None: - self.user_data = {} - self.user_data.setdefault('__crawlee', {}) self.user_data['__crawlee']['enqueueStrategy'] = str(new_enqueue_strategy) From 9fbe84a599b2ce5b6fea5ad7cd0d0e1f886af9eb Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 24 Apr 2024 18:48:30 +0200 Subject: [PATCH 13/20] _check_url_patterns comments --- src/crawlee/basic_crawler/basic_crawler.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index 8e1dc24821..3fc8ebdaca 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -241,17 +241,19 @@ def _check_url_patterns( include: Sequence[re.Pattern | Glob] | None, exclude: Sequence[re.Pattern | Glob] | None, ) -> bool: - if exclude is not None: - for pattern in exclude: - if isinstance(pattern, Glob): - pattern = pattern.regexp # noqa: PLW2901 + # If the URL matches any `exclude` pattern, reject it + for pattern in exclude or (): + if isinstance(pattern, Glob): + pattern = pattern.regexp # noqa: PLW2901 - if pattern.match(str(target_url)) is not None: - return False + if pattern.match(str(target_url)) is not None: + return False + # If there are no `include` patterns and the URL passed all `exclude` patterns, accept the URL if include is None: return True + # If the URL matches any `include` pattern, accept it for pattern in include: if isinstance(pattern, Glob): pattern = pattern.regexp # noqa: PLW2901 @@ -259,6 +261,7 @@ def _check_url_patterns( if pattern.match(str(target_url)) is not None: return True + # The URL does not match any `include` pattern - reject it return False async def _handle_request_error(self, crawling_context: TCrawlingContext, error: Exception) -> None: From 50ce6ba7b03122dda3b55dca04465318ee4c6fc3 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 29 Apr 2024 11:02:24 +0200 Subject: [PATCH 14/20] fix type error --- src/crawlee/resource_clients/request_queue_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/crawlee/resource_clients/request_queue_client.py b/src/crawlee/resource_clients/request_queue_client.py index dc21f8d431..8cbd02e974 100644 --- a/src/crawlee/resource_clients/request_queue_client.py +++ b/src/crawlee/resource_clients/request_queue_client.py @@ -549,6 +549,7 @@ async def _create_internal_request(self, request: Request, forefront: bool | Non retry_count=request.retry_count, order_no=order_no, json_=json_request, + user_data={}, ) def _calculate_order_no(self, request: Request, forefront: bool | None) -> Decimal | None: From 61f5381e8118beeabb5f4d736510fdf67be5d246 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 29 Apr 2024 11:05:43 +0200 Subject: [PATCH 15/20] add more comments --- src/crawlee/basic_crawler/basic_crawler.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index 3fc8ebdaca..821ab97fb6 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -205,6 +205,10 @@ def _should_retry_request(self, crawling_context: BasicCrawlingContext) -> bool: async def _check_url_after_redirects( self, crawling_context: TCrawlingContext ) -> AsyncGenerator[TCrawlingContext, None]: + """Invoked at the end of the context pipeline to make sure that the `loaded_url` still matches enqueue_strategy. + + This is done to filter out links that redirect outside of the crawled domain. + """ if crawling_context.request.loaded_url is not None and not self._check_enqueue_strategy( httpx.URL(crawling_context.request.url), httpx.URL(crawling_context.request.loaded_url), @@ -219,6 +223,7 @@ async def _check_url_after_redirects( def _check_enqueue_strategy( self, target_url: httpx.URL, origin_url: httpx.URL, strategy: EnqueueStrategy | None ) -> bool: + """Check if a URL matches the enqueue_strategy.""" if strategy == EnqueueStrategy.SAME_HOSTNAME: return target_url.host == origin_url.host @@ -241,6 +246,7 @@ def _check_url_patterns( include: Sequence[re.Pattern | Glob] | None, exclude: Sequence[re.Pattern | Glob] | None, ) -> bool: + """Check if a URL matches configured include/exclude patterns.""" # If the URL matches any `exclude` pattern, reject it for pattern in exclude or (): if isinstance(pattern, Glob): From d6a6c2354dee5fe7188cc40879054a9b9cba33ba Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 29 Apr 2024 11:08:16 +0200 Subject: [PATCH 16/20] move Glob under _utils --- src/crawlee/__init__.py | 1 + src/crawlee/{ => _utils}/globs.py | 0 src/crawlee/basic_crawler/basic_crawler.py | 2 +- src/crawlee/basic_crawler/types.py | 2 +- tests/unit/basic_crawler/test_basic_crawler.py | 2 +- 5 files changed, 4 insertions(+), 3 deletions(-) rename src/crawlee/{ => _utils}/globs.py (100%) diff --git a/src/crawlee/__init__.py b/src/crawlee/__init__.py index e69de29bb2..d1e26facfd 100644 --- a/src/crawlee/__init__.py +++ b/src/crawlee/__init__.py @@ -0,0 +1 @@ +from ._utils.globs import Glob diff --git a/src/crawlee/globs.py b/src/crawlee/_utils/globs.py similarity index 100% rename from src/crawlee/globs.py rename to src/crawlee/_utils/globs.py diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index 821ab97fb6..2baa0d5644 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -11,6 +11,7 @@ from tldextract import TLDExtract from typing_extensions import TypeVar, assert_never +from crawlee import Glob from crawlee._utils.wait import wait_for from crawlee.autoscaling import AutoscaledPool, ConcurrencySettings from crawlee.autoscaling.snapshotter import Snapshotter @@ -31,7 +32,6 @@ from crawlee.configuration import Configuration from crawlee.enqueue_strategy import EnqueueStrategy from crawlee.events.local_event_manager import LocalEventManager -from crawlee.globs import Glob from crawlee.http_clients.httpx_client import HttpxClient from crawlee.request import BaseRequestData, Request, RequestState from crawlee.storages.request_queue import RequestQueue diff --git a/src/crawlee/basic_crawler/types.py b/src/crawlee/basic_crawler/types.py index 5f8fbf99c0..fc7ad09a78 100644 --- a/src/crawlee/basic_crawler/types.py +++ b/src/crawlee/basic_crawler/types.py @@ -8,8 +8,8 @@ from typing_extensions import NotRequired, TypedDict, Unpack if TYPE_CHECKING: + from crawlee import Glob from crawlee.enqueue_strategy import EnqueueStrategy - from crawlee.globs import Glob from crawlee.http_clients.base_http_client import HttpResponse from crawlee.request import BaseRequestData, Request diff --git a/tests/unit/basic_crawler/test_basic_crawler.py b/tests/unit/basic_crawler/test_basic_crawler.py index ccf6633b12..9d50242e27 100644 --- a/tests/unit/basic_crawler/test_basic_crawler.py +++ b/tests/unit/basic_crawler/test_basic_crawler.py @@ -9,10 +9,10 @@ import pytest from httpx import Headers, Response +from crawlee import Glob from crawlee.basic_crawler.basic_crawler import BasicCrawler, UserDefinedErrorHandlerError from crawlee.basic_crawler.types import AddRequestsFunctionKwargs, BasicCrawlingContext from crawlee.enqueue_strategy import EnqueueStrategy -from crawlee.globs import Glob from crawlee.request import BaseRequestData, Request from crawlee.storages import RequestList From bbad23ef969ea03c628fa8736104de5f3d840786 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 29 Apr 2024 11:10:47 +0200 Subject: [PATCH 17/20] basic Glob test --- tests/unit/_utils/test_globs.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 tests/unit/_utils/test_globs.py diff --git a/tests/unit/_utils/test_globs.py b/tests/unit/_utils/test_globs.py new file mode 100644 index 0000000000..df61e902cb --- /dev/null +++ b/tests/unit/_utils/test_globs.py @@ -0,0 +1,15 @@ +from crawlee._utils.globs import Glob + + +def test_asterisk() -> None: + glob = Glob('foo/*') + assert glob.regexp.match('bar/') is None + assert glob.regexp.match('foo/bar') is not None + assert glob.regexp.match('foo/bar/baz') is None + + +def test_double_asteritsk() -> None: + glob = Glob('foo/**') + assert glob.regexp.match('bar/') is None + assert glob.regexp.match('foo/bar') is not None + assert glob.regexp.match('foo/bar/baz') is not None From d9fba92c57c42d25c5394c6bfe6e18eeae9bbfa7 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 29 Apr 2024 11:15:37 +0200 Subject: [PATCH 18/20] fix user_data handling --- .../beautifulsoup_crawler/beautifulsoup_crawler.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py index 399c718cbe..637cbfd9ca 100644 --- a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py +++ b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py @@ -106,17 +106,14 @@ async def enqueue_links( **kwargs: Unpack[AddRequestsFunctionKwargs], ) -> None: requests = list[BaseRequestData]() + user_data = user_data or {} link: Tag for link in soup.find_all(selector): link_user_data = user_data - if user_data is not None or label is not None: - if link_user_data is None: - link_user_data = {} - - if label is not None: - link_user_data.setdefault('label', label) + if label is not None: + link_user_data.setdefault('label', label) if (href := link.attrs.get('href')) is not None: requests.append(BaseRequestData.from_url(href, user_data=link_user_data)) From 5a7a8aff6fe1f8f70eaa471ad0267a29136fb3e4 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 29 Apr 2024 12:59:09 +0200 Subject: [PATCH 19/20] kw only args --- src/crawlee/basic_crawler/basic_crawler.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index 2baa0d5644..832844e919 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -210,9 +210,9 @@ async def _check_url_after_redirects( This is done to filter out links that redirect outside of the crawled domain. """ if crawling_context.request.loaded_url is not None and not self._check_enqueue_strategy( - httpx.URL(crawling_context.request.url), - httpx.URL(crawling_context.request.loaded_url), crawling_context.request.enqueue_strategy, + origin_url=httpx.URL(crawling_context.request.url), + target_url=httpx.URL(crawling_context.request.loaded_url), ): raise ContextPipelineInterruptedError( f'Skipping URL {crawling_context.request.loaded_url} (redirected from {crawling_context.request.url})' @@ -221,7 +221,7 @@ async def _check_url_after_redirects( yield crawling_context def _check_enqueue_strategy( - self, target_url: httpx.URL, origin_url: httpx.URL, strategy: EnqueueStrategy | None + self, strategy: EnqueueStrategy | None, *, target_url: httpx.URL, origin_url: httpx.URL ) -> bool: """Check if a URL matches the enqueue_strategy.""" if strategy == EnqueueStrategy.SAME_HOSTNAME: @@ -332,9 +332,9 @@ async def _commit_request_handler_result( base_url = httpx.URL(call.get('base_url', origin)) request_model.url = str(base_url.join(destination)) - if self._check_enqueue_strategy(destination, origin, call.get('strategy')) and self._check_url_patterns( - destination, call.get('include', None), call.get('exclude', None) - ): + if self._check_enqueue_strategy( + call.get('strategy'), target_url=destination, origin_url=origin + ) and self._check_url_patterns(destination, call.get('include', None), call.get('exclude', None)): requests.append(request_model) await request_provider.add_requests_batched( From 92f05effdb591c96cc554e64aeb9294a3bacfad7 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 30 Apr 2024 10:39:18 +0200 Subject: [PATCH 20/20] make strategy non-optional internally --- src/crawlee/basic_crawler/basic_crawler.py | 6 +++--- src/crawlee/basic_crawler/types.py | 2 +- .../beautifulsoup_crawler/beautifulsoup_crawler.py | 4 ++++ src/crawlee/request.py | 8 ++++++-- tests/unit/basic_crawler/test_basic_crawler.py | 2 +- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/crawlee/basic_crawler/basic_crawler.py b/src/crawlee/basic_crawler/basic_crawler.py index 832844e919..719f13b6b7 100644 --- a/src/crawlee/basic_crawler/basic_crawler.py +++ b/src/crawlee/basic_crawler/basic_crawler.py @@ -221,7 +221,7 @@ async def _check_url_after_redirects( yield crawling_context def _check_enqueue_strategy( - self, strategy: EnqueueStrategy | None, *, target_url: httpx.URL, origin_url: httpx.URL + self, strategy: EnqueueStrategy, *, target_url: httpx.URL, origin_url: httpx.URL ) -> bool: """Check if a URL matches the enqueue_strategy.""" if strategy == EnqueueStrategy.SAME_HOSTNAME: @@ -235,7 +235,7 @@ def _check_enqueue_strategy( if strategy == EnqueueStrategy.SAME_ORIGIN: return target_url.host == origin_url.host and target_url.scheme == origin_url.scheme - if strategy is None or strategy == EnqueueStrategy.ALL: + if strategy == EnqueueStrategy.ALL: return True assert_never() @@ -333,7 +333,7 @@ async def _commit_request_handler_result( request_model.url = str(base_url.join(destination)) if self._check_enqueue_strategy( - call.get('strategy'), target_url=destination, origin_url=origin + call.get('strategy', EnqueueStrategy.ALL), target_url=destination, origin_url=origin ) and self._check_url_patterns(destination, call.get('include', None), call.get('exclude', None)): requests.append(request_model) diff --git a/src/crawlee/basic_crawler/types.py b/src/crawlee/basic_crawler/types.py index fc7ad09a78..6b2dfd82a3 100644 --- a/src/crawlee/basic_crawler/types.py +++ b/src/crawlee/basic_crawler/types.py @@ -19,7 +19,7 @@ class AddRequestsFunctionKwargs(TypedDict): limit: NotRequired[int] base_url: NotRequired[str] - strategy: NotRequired[EnqueueStrategy | None] + strategy: NotRequired[EnqueueStrategy] include: NotRequired[list[re.Pattern | Glob]] exclude: NotRequired[list[re.Pattern | Glob]] diff --git a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py index 637cbfd9ca..3ddbcf6b64 100644 --- a/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py +++ b/src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py @@ -10,6 +10,7 @@ from crawlee.basic_crawler.basic_crawler import BasicCrawler from crawlee.basic_crawler.context_pipeline import ContextPipeline from crawlee.beautifulsoup_crawler.types import BeautifulSoupCrawlingContext +from crawlee.enqueue_strategy import EnqueueStrategy from crawlee.http_clients.httpx_client import HttpxClient from crawlee.http_crawler.types import HttpCrawlingContext from crawlee.request import BaseRequestData @@ -118,6 +119,9 @@ async def enqueue_links( if (href := link.attrs.get('href')) is not None: requests.append(BaseRequestData.from_url(href, user_data=link_user_data)) + uses_patterns = 'include' in kwargs or 'exclude' in kwargs + kwargs.setdefault('strategy', EnqueueStrategy.SAME_HOSTNAME if uses_patterns else EnqueueStrategy.ALL) + await context.add_requests(requests, **kwargs) yield BeautifulSoupCrawlingContext( diff --git a/src/crawlee/request.py b/src/crawlee/request.py index b486083b1b..ccf726f5b1 100644 --- a/src/crawlee/request.py +++ b/src/crawlee/request.py @@ -141,9 +141,13 @@ def max_retries(self, new_max_retries: int) -> None: self.user_data['__crawlee']['maxRetries'] = new_max_retries @property - def enqueue_strategy(self) -> EnqueueStrategy | None: + def enqueue_strategy(self) -> EnqueueStrategy: """The strategy used when enqueueing the request.""" - return EnqueueStrategy(self.crawlee_data.enqueue_strategy) if self.crawlee_data.enqueue_strategy else None + return ( + EnqueueStrategy(self.crawlee_data.enqueue_strategy) + if self.crawlee_data.enqueue_strategy + else EnqueueStrategy.ALL + ) @enqueue_strategy.setter def enqueue_strategy(self, new_enqueue_strategy: EnqueueStrategy) -> None: diff --git a/tests/unit/basic_crawler/test_basic_crawler.py b/tests/unit/basic_crawler/test_basic_crawler.py index 9d50242e27..ff9a541065 100644 --- a/tests/unit/basic_crawler/test_basic_crawler.py +++ b/tests/unit/basic_crawler/test_basic_crawler.py @@ -313,7 +313,7 @@ class AddRequestsTestInput: AddRequestsTestInput( start_url=STRATEGY_TEST_URLS[0], requests=STRATEGY_TEST_URLS, - kwargs=AddRequestsFunctionKwargs(strategy=None), + kwargs=AddRequestsFunctionKwargs(), expected_urls=STRATEGY_TEST_URLS, ), AddRequestsTestInput(