Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/crawlee/crawlers/_basic/_basic_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -670,6 +670,7 @@ async def _run_crawler(self) -> None:
self._snapshotter,
self._statistics,
self._session_pool if self._use_session_pool else None,
self._http_client,
*self._additional_context_managers,
)
if cm and getattr(cm, 'active', False) is False
Expand Down
22 changes: 21 additions & 1 deletion src/crawlee/crawlers/_playwright/_playwright_http_client.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from __future__ import annotations

import contextvars
from contextlib import asynccontextmanager
from contextlib import AbstractAsyncContextManager, asynccontextmanager
from typing import TYPE_CHECKING

from typing_extensions import override
Expand All @@ -12,6 +12,7 @@

if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from datetime import timedelta

from playwright.async_api import Page

Expand Down Expand Up @@ -48,6 +49,7 @@ class PlaywrightHttpClient(HttpClient):

def __init__(self) -> None:
"""Initialize a new instance."""
self._active = False

@override
async def crawl(
Expand Down Expand Up @@ -89,3 +91,21 @@ async def send_request(
)

return await PlaywrightHttpResponse.from_playwright_response(response, protocol='')

@override
def stream(
self,
url: str,
*,
method: HttpMethod = 'GET',
headers: HttpHeaders | dict[str, str] | None = None,
payload: HttpPayload | None = None,
session: Session | None = None,
proxy_info: ProxyInfo | None = None,
timeout: timedelta | None = None,
) -> AbstractAsyncContextManager[HttpResponse]:
raise NotImplementedError('The `stream` method should not be used for `PlaywrightHttpClient`')

async def cleanup(self) -> None:
# The `browser_page_context` is responsible for resource cleanup
return
7 changes: 7 additions & 0 deletions src/crawlee/crawlers/_playwright/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
from crawlee._utils.docs import docs_group

if TYPE_CHECKING:
from collections.abc import AsyncGenerator

from playwright.async_api import APIResponse, Response
from typing_extensions import Self

Expand Down Expand Up @@ -42,6 +44,11 @@ class PlaywrightHttpResponse:
def read(self) -> bytes:
return self._content

async def read_stream(self) -> AsyncGenerator[bytes, None]:
# Playwright does not support `streaming` responses.
# This is a workaround to make it compatible with `HttpResponse` protocol.
yield self._content

@classmethod
async def from_playwright_response(cls, response: Response | APIResponse, protocol: str) -> Self:
headers = HttpHeaders(response.headers)
Expand Down
102 changes: 101 additions & 1 deletion src/crawlee/http_clients/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@
from crawlee._utils.docs import docs_group

if TYPE_CHECKING:
from collections.abc import AsyncIterator
from contextlib import AbstractAsyncContextManager
from datetime import timedelta
from types import TracebackType

from crawlee import Request
from crawlee._types import HttpHeaders, HttpMethod, HttpPayload
from crawlee.proxy_configuration import ProxyInfo
Expand All @@ -31,7 +36,26 @@ def headers(self) -> HttpHeaders:
"""The HTTP headers received in the response."""

def read(self) -> bytes:
"""Read the content of the response body."""
"""Read the entire content of the response body.

This method loads the complete response body into memory at once. It should be used
for responses received from regular HTTP requests (via `send_request` or `crawl` methods).

Raises:
RuntimeError: If called on a response received from the `stream` method.
"""

def read_stream(self) -> AsyncIterator[bytes]:
"""Iterate over the content of the response body in chunks.

This method should be used for responses received from the `stream` method to process
large response bodies without loading them entirely into memory. It allows for efficient
processing of potentially large data by yielding chunks sequentially.

Raises:
RuntimeError: If the stream has already been consumed or if the response was not obtained from the `stream`
method.
"""


@dataclass(frozen=True)
Expand Down Expand Up @@ -64,6 +88,14 @@ def __init__(
"""
self._persist_cookies_per_session = persist_cookies_per_session

# Flag to indicate the context state.
self._active = False

@property
def active(self) -> bool:
"""Indicate whether the context is active."""
return self._active

@abstractmethod
async def crawl(
self,
Expand Down Expand Up @@ -119,3 +151,71 @@ async def send_request(
Returns:
The HTTP response received from the server.
"""

@abstractmethod
def stream(
self,
url: str,
*,
method: HttpMethod = 'GET',
headers: HttpHeaders | dict[str, str] | None = None,
payload: HttpPayload | None = None,
session: Session | None = None,
proxy_info: ProxyInfo | None = None,
timeout: timedelta | None = None,
) -> AbstractAsyncContextManager[HttpResponse]:
"""Stream an HTTP request via the client.

This method should be used for downloading potentially large data where you need to process
the response body in chunks rather than loading it entirely into memory.

Args:
url: The URL to send the request to.
method: The HTTP method to use.
headers: The headers to include in the request.
payload: The data to be sent as the request body.
session: The session associated with the request.
proxy_info: The information about the proxy to be used.
timeout: The maximum time to wait for establishing the connection.

Raises:
ProxyError: Raised if a proxy-related error occurs.

Returns:
An async context manager yielding the HTTP response with streaming capabilities.
"""

@abstractmethod
async def cleanup(self) -> None:
"""Clean up resources used by the client.

This method is called when the client is no longer needed and should be overridden
in subclasses to perform any necessary cleanup such as closing connections,
releasing file handles, or other resource deallocation.
"""

async def __aenter__(self) -> HttpClient:
"""Initialize the client when entering the context manager.

Raises:
RuntimeError: If the context manager is already active.
"""
if self._active:
raise RuntimeError(f'The {self.__class__.__name__} is already active.')

self._active = True
return self

async def __aexit__(
self, exc_type: BaseException | None, exc_value: BaseException | None, traceback: TracebackType | None
) -> None:
"""Deinitialize the client and clean up resources when exiting the context manager.

Raises:
RuntimeError: If the context manager is already active.
"""
if not self._active:
raise RuntimeError(f'The {self.__class__.__name__} is not active.')

await self.cleanup()
self._active = False
Comment on lines +197 to +221

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So we use the _active flag only for the detection of multiple aenter/aexit calls?

If so, I remember we had some discussion a while ago with either @janbuchar or @Pijukatel on how the context managers should behave in that case. Do you guys remember?

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for pointing that out. I added this flag to match the logic in basic_crawler - https://github.com/apify/crawlee-python/blob/master/src/crawlee/crawlers/_basic/_basic_crawler.py#L675

But I completely forgot about the public property.

Although you may have discussed some other details )

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We have 3 options. Re-entrant , re-usable, single-use:
https://docs.python.org/3/library/contextlib.html#single-use-reusable-and-reentrant-context-managers

Re-entrant does not make sense here in my opinion.

So then second question. Does it make sense and is it possible to re-use context that was already left before? If yes, then it is re-usable. (Based on the flags, this is the setup in your change. But is it really possible to enter closed context again and everything will work as expected?)

If not then it is single use and we should probably throw some error if someone tries to enter it again.
Based on the code in the cleanup, it might not be possible to re-use already closed client (unless __aenter__ is implemented in a away to make the client ready again).

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thank you for the detailed explanation.

Yes, HttpClient should be reuse. Fixed and added the tests

62 changes: 62 additions & 0 deletions src/crawlee/http_clients/_curl_impersonate.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from __future__ import annotations

from contextlib import asynccontextmanager
from typing import TYPE_CHECKING, Any, Optional

from curl_cffi import CurlInfo
Expand All @@ -19,6 +20,8 @@
from crawlee.http_clients import HttpClient, HttpCrawlingResult, HttpResponse

if TYPE_CHECKING:
from collections.abc import AsyncGenerator
from datetime import timedelta
from http.cookiejar import Cookie

from curl_cffi import Curl
Expand Down Expand Up @@ -83,8 +86,19 @@ def headers(self) -> HttpHeaders:
return HttpHeaders({key: value for key, value in self._response.headers.items() if value})

def read(self) -> bytes:
if self._response.astream_task:
raise RuntimeError('Use `read_stream` to read the body of the Response received from the `stream` method')
return self._response.content

async def read_stream(self) -> AsyncGenerator[bytes, None]:
if not self._response.astream_task or self._response.astream_task.done(): # type: ignore[attr-defined]
raise RuntimeError(
'Cannot read stream: either already consumed or Response not obtained from `stream` method'
)

async for chunk in self._response.aiter_content(): # type: ignore[no-untyped-call]
yield chunk


@docs_group('Classes')
class CurlImpersonateHttpClient(HttpClient):
Expand Down Expand Up @@ -198,6 +212,49 @@ async def send_request(

return _CurlImpersonateResponse(response)

@asynccontextmanager
@override
async def stream(
self,
url: str,
*,
method: HttpMethod = 'GET',
headers: HttpHeaders | dict[str, str] | None = None,
payload: HttpPayload | None = None,
session: Session | None = None,
proxy_info: ProxyInfo | None = None,
timeout: timedelta | None = None,
) -> AsyncGenerator[HttpResponse]:
if isinstance(headers, dict) or headers is None:
headers = HttpHeaders(headers or {})

proxy_url = proxy_info.url if proxy_info else None
client = self._get_client(proxy_url)

try:
response = await client.request(
url=url,
method=method.upper(), # type: ignore[arg-type] # curl-cffi requires uppercase method
headers=dict(headers) if headers else None,
data=payload,
cookies=session.cookies.jar if session else None,
stream=True,
timeout=timeout.total_seconds() if timeout else None,
)
except CurlRequestError as exc:
if self._is_proxy_error(exc):
raise ProxyError from exc
raise

if self._persist_cookies_per_session and session and response.curl:
response_cookies = self._get_cookies(response.curl)
session.cookies.store_cookies(response_cookies)

try:
yield _CurlImpersonateResponse(response)
finally:
await response.aclose()

def _get_client(self, proxy_url: str | None) -> AsyncSession:
"""Retrieve or create an asynchronous HTTP session for the given proxy URL.

Expand Down Expand Up @@ -245,3 +302,8 @@ def _get_cookies(curl: Curl) -> list[Cookie]:
cookie = curl_morsel.to_cookiejar_cookie()
cookies.append(cookie)
return cookies

async def cleanup(self) -> None:
for client in self._client_by_proxy_url.values():
await client.close()
self._client_by_proxy_url.clear()
Loading