diff --git a/src/crawlee/_utils/measure_time.py b/src/crawlee/_utils/measure_time.py new file mode 100644 index 0000000000..082ef9f869 --- /dev/null +++ b/src/crawlee/_utils/measure_time.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +import time +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Iterator + + +@dataclass +class TimerResult: + wall: float | None = None + cpu: float | None = None + + +@contextmanager +def measure_time() -> Iterator[TimerResult]: + """Measure the execution time (wall-clock and CPU) between the start and end of the with-block.""" + result = TimerResult() + before_wall = time.monotonic() + before_cpu = time.thread_time() + + try: + yield result + finally: + after_wall = time.monotonic() + after_cpu = time.thread_time() + result.wall = after_wall - before_wall + result.cpu = after_cpu - before_cpu diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py new file mode 100644 index 0000000000..ce6b094766 --- /dev/null +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -0,0 +1,298 @@ +# Inspiration: https://github.com/apify/crawlee/blob/v3.7.3/packages/core/src/autoscaling/autoscaled_pool.ts + +from __future__ import annotations + +import asyncio +import math +from contextlib import suppress +from datetime import timedelta +from logging import getLogger +from typing import TYPE_CHECKING, Awaitable, Callable + +from crawlee._utils.recurring_task import RecurringTask + +if TYPE_CHECKING: + from crawlee.autoscaling.system_status import SystemStatus + +logger = getLogger(__name__) + + +class AbortError(Exception): + """Raised when an AutoscaledPool run is aborted. Not for direct use.""" + + +class AutoscaledPool: + """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel. + + The pool only starts new tasks if there is enough free CPU and memory available. If an exception is thrown in + any of the tasks, it is propagated and the pool is stopped. + """ + + def __init__( + self: AutoscaledPool, + *, + system_status: SystemStatus, + run_task_function: Callable[..., Awaitable], + is_task_ready_function: Callable[..., bool], + is_finished_function: Callable[..., bool], + task_timeout: timedelta | None = None, + autoscale_interval: timedelta = timedelta(seconds=10), + logging_interval: timedelta = timedelta(minutes=1), + desired_concurrency: int | None = None, + desired_concurrency_ratio: float = 0.9, + min_concurrency: int = 1, + max_concurrency: int = 200, + scale_up_step_ratio: float = 0.05, + scale_down_step_ratio: float = 0.05, + max_tasks_per_minute: int | None = None, + ) -> None: + """Initialize the AutoscaledPool. + + Args: + system_status: Provides data about system utilization (load). + + run_task_function: A function that performs an asynchronous resource-intensive task. + + is_task_ready_function: A function that indicates whether `run_task_function` should be called. This + function is called every time there is free capacity for a new task and it should indicate whether + it should start a new task or not by resolving to either `True` or `False`. Besides its obvious use, + it is also useful for task throttling to save resources. is_finished_function: A function that is + called only when there are no tasks to be processed. If it resolves to `True` then the pool's run + finishes. Being called only when there are no tasks being processed means that as long as + `is_task_ready_function()` keeps resolving to `True`, `is_finished_function()` will never be called. + To abort a run, use the `abort` method. + + is_finished_function: A function that is called only when there are no tasks to be processed. If it resolves + to `true` then the pool's run finishes. Being called only when there are no tasks being processed means + that as long as `isTaskReadyFunction()` keeps resolving to `true`, `isFinishedFunction()` will never + be called. To abort a run, use the `abort` method. + + task_timeout: Timeout in which the `run_task_function` needs to finish. + + autoscale_interval: Defines how often the pool should attempt to adjust the desired concurrency based on + the latest system status. Setting it lower than 1 might have a severe impact on performance. We suggest + using a value from 5 to 20. + + logging_interval: Specifies a period in which the instance logs its state, in seconds. + + desired_concurrency: The desired number of tasks that should be running parallel on the start of the pool, + if there is a large enough supply of them. By default, it is `min_concurrency`. + + desired_concurrency_ratio: Minimum level of desired concurrency to reach before more scaling up is allowed. + + min_concurrency: The minimum number of tasks running in parallel. If you set this value too high with + respect to the available system memory and CPU, your code might run extremely slow or crash. + + max_concurrency: The maximum number of tasks running in parallel. + + scale_up_step_ratio: Defines the fractional amount of desired concurrency to be added with each scaling up. + + scale_down_step_ratio: Defines the amount of desired concurrency to be subtracted with each scaling down. + + max_tasks_per_minute: The maximum number of tasks per minute the pool can run. By default, this is set to + `Infinity`, but you can pass any positive, non-zero integer. + """ + self._system_status = system_status + + self._run_task_function = run_task_function + self._is_task_ready_function = is_task_ready_function + self._is_finished_function = is_finished_function + + self._task_timeout = task_timeout + + self._logging_interval = logging_interval + self._log_system_status_task = RecurringTask(self._log_system_status, logging_interval) + + self._autoscale_task = RecurringTask(self._autoscale, autoscale_interval) + + self._worker_tasks = list[asyncio.Task]() + """A list of worker tasks currently in progress""" + + self._worker_tasks_updated = asyncio.Event() + self._cleanup_done = asyncio.Event() + self._run_result: asyncio.Future = asyncio.Future() + + if desired_concurrency is not None and desired_concurrency < 1: + raise ValueError('desired_concurrency must be 1 or larger') + + if min_concurrency < 1: + raise ValueError('min_concurrency must be 1 or larger') + + if max_concurrency < min_concurrency: + raise ValueError('max_concurrency cannot be less than min_concurrency') + + if desired_concurrency_ratio < 0 or desired_concurrency_ratio > 1: + raise ValueError('desired_concurrency_ratio must be between 0 and 1 (non-inclusive)') + + self._desired_concurrency_ratio = desired_concurrency_ratio + self._desired_concurrency = desired_concurrency if desired_concurrency is not None else min_concurrency + self._max_concurrency = max_concurrency + self._min_concurrency = min_concurrency + + self._scale_up_step_ratio = scale_up_step_ratio + self._scale_down_step_ratio = scale_down_step_ratio + + self._max_tasks_per_minute = max_tasks_per_minute + self._is_paused = False + self._is_running = False + + async def run(self: AutoscaledPool) -> None: + """Start the autoscaled pool and return when all tasks are completed and `is_finished_function` returns True. + + If there is an exception in one of the tasks, it will be re-raised. + """ + if self._is_running: + raise RuntimeError('The pool is already running') + + self._is_running = True + logger.debug('Starting the pool') + + self._autoscale_task.start() + self._log_system_status_task.start() + + orchestrator = asyncio.create_task( + self._worker_task_orchestrator(), name='autoscaled pool worker task orchestrator' + ) + + try: + await self._run_result + except AbortError: + orchestrator.cancel() + for task in self._worker_tasks: + if not task.done(): + task.cancel() + finally: + with suppress(asyncio.CancelledError): + await self._autoscale_task.stop() + with suppress(asyncio.CancelledError): + await self._log_system_status_task.stop() + + if not orchestrator.done(): + orchestrator.cancel() + elif not orchestrator.cancelled() and orchestrator.exception() is not None: + logger.error('Exception in worker task orchestrator', exc_info=orchestrator.exception()) + + logger.info('Waiting for remaining tasks to finish') + + for task in self._worker_tasks: + if not task.done(): + with suppress(BaseException): + await task + + self._run_result = asyncio.Future() + self._cleanup_done.set() + self._is_running = False + + logger.debug('Pool cleanup finished') + + async def abort(self: AutoscaledPool) -> None: + """Interrupt the autoscaled pool and all the tasks in progress.""" + self._run_result.set_exception(AbortError()) + await self._cleanup_done.wait() + + def pause(self: AutoscaledPool) -> None: + """Pause the autoscaled pool so that it does not start new tasks.""" + self._is_paused = True + + def resume(self: AutoscaledPool) -> None: + """Resume a paused autoscaled pool so that it continues starting new tasks.""" + self._is_paused = False + + @property + def desired_concurrency(self: AutoscaledPool) -> int: + """The current desired concurrency, possibly updated by the pool according to system load.""" + return self._desired_concurrency + + @property + def current_concurrency(self: AutoscaledPool) -> int: + """The number of concurrent tasks in progress.""" + return len(self._worker_tasks) + + def _autoscale(self: AutoscaledPool) -> None: + """Inspect system load status and adjust desired concurrency if necessary. Do not call directly.""" + status = self._system_status.get_historical_status() + + min_current_concurrency = math.floor(self._desired_concurrency_ratio * self.current_concurrency) + should_scale_up = ( + status.is_system_idle + and self._desired_concurrency < self._max_concurrency + and self.current_concurrency >= min_current_concurrency + ) + + should_scale_down = not status.is_system_idle and self._desired_concurrency > self._min_concurrency + + if should_scale_up: + step = math.ceil(self._scale_up_step_ratio * self._desired_concurrency) + self._desired_concurrency = min(self._max_concurrency, self._desired_concurrency + step) + elif should_scale_down: + step = math.ceil(self._scale_down_step_ratio * self._desired_concurrency) + self._desired_concurrency = max(self._min_concurrency, self._desired_concurrency - step) + + def _log_system_status(self: AutoscaledPool) -> None: + system_status = self._system_status.get_historical_status() + + logger.info( + f'current_concurrency = {self.current_concurrency}; ' + f'desired_concurrency = {self.desired_concurrency}; ' + f'{system_status!s}' + ) + + async def _worker_task_orchestrator(self: AutoscaledPool) -> None: + """Launches worker tasks whenever there is free capacity and a task is ready. + + Exits when `is_finished_function` returns True. + """ + try: + while not self._is_finished_function() and not self._run_result.done(): + self._worker_tasks_updated.clear() + + current_status = self._system_status.get_current_status() + if not current_status.is_system_idle: + logger.debug('Not scheduling new tasks - system is overloaded') + elif self._is_paused: + logger.debug('Not scheduling new tasks - the autoscaled pool is paused') + elif self.current_concurrency >= self.desired_concurrency: + logger.debug('Not scheduling new tasks - already running at desired concurrency') + elif not self._is_task_ready_function(): + logger.debug('Not scheduling new task - no task is ready') + else: + logger.debug('Scheduling a new task') + worker_task = asyncio.create_task(self._worker_task(), name='autoscaled pool worker task') + worker_task.add_done_callback(lambda task: self._reap_worker_task(task)) + self._worker_tasks.append(worker_task) + + if self._max_tasks_per_minute is not None: + await asyncio.sleep(60 / self._max_tasks_per_minute) + + continue + + with suppress(asyncio.TimeoutError): + await asyncio.wait_for(self._worker_tasks_updated.wait(), timeout=0.5) + finally: + if not (self._run_result.done() and self._run_result.exception() is not None): + self._run_result.set_result(object()) + + def _reap_worker_task(self: AutoscaledPool, task: asyncio.Task) -> None: + """A callback for finished worker tasks. + + - It interrupts the run in case of an exception, + - keeps track of tasks in progress, + - notifies the orchestrator + """ + self._worker_tasks_updated.set() + self._worker_tasks.remove(task) + + if not task.cancelled() and (exception := task.exception()) and not self._run_result.done(): + self._run_result.set_exception(exception) + + async def _worker_task(self: AutoscaledPool) -> None: + try: + await asyncio.wait_for( + self._run_task_function(), + timeout=self._task_timeout.total_seconds() if self._task_timeout is not None else None, + ) + except asyncio.TimeoutError: + timeout_str = self._task_timeout.total_seconds() if self._task_timeout is not None else '*not set*' + logger.warning(f'Task timed out after {timeout_str} seconds') + finally: + logger.debug('Worker task finished') diff --git a/src/crawlee/autoscaling/types.py b/src/crawlee/autoscaling/types.py index 54563e5303..99815d57d6 100644 --- a/src/crawlee/autoscaling/types.py +++ b/src/crawlee/autoscaling/types.py @@ -45,6 +45,16 @@ def is_system_idle(self: SystemInfo) -> bool: and not self.client_info.is_overloaded ) + def __str__(self: SystemInfo) -> str: + """Get a string representation of the system info.""" + stats = { + 'cpu': self.cpu_info.actual_ratio if self.cpu_info else '-', + 'mem': self.mem_info.actual_ratio if self.mem_info else '-', + 'event_lopp': self.event_loop_info.actual_ratio if self.event_loop_info else '-', + 'client_info': self.client_info.actual_ratio if self.client_info else '-', + } + return ';'.join(f'{name} = {ratio}' for name, ratio in stats.items()) + @dataclass class FinalStatistics: diff --git a/tests/unit/_utils/test_measure_time.py b/tests/unit/_utils/test_measure_time.py new file mode 100644 index 0000000000..400e563e05 --- /dev/null +++ b/tests/unit/_utils/test_measure_time.py @@ -0,0 +1,39 @@ +import asyncio +import time + +import pytest + +from crawlee._utils.measure_time import measure_time + + +def test_measure_time_wall_sync() -> None: + with measure_time() as elapsed: + time.sleep(0.1) + + assert elapsed.cpu is not None + assert elapsed.wall is not None + assert elapsed.wall >= 0.09 + + +def test_measure_time_cpu_sync() -> None: + with measure_time() as elapsed: + start = time.time() + acc = 0 + + while time.time() - start < 0.1: + acc += 1 + acc *= acc + + assert elapsed.cpu is not None + assert elapsed.wall is not None + assert elapsed.cpu >= 0.05 + + +@pytest.mark.asyncio() +async def test_measure_time_wall_async() -> None: + with measure_time() as elapsed: + await asyncio.sleep(0.1) + + assert elapsed.cpu is not None + assert elapsed.wall is not None + assert elapsed.wall >= 0.09 diff --git a/tests/unit/autoscaling/test_autoscaled_pool.py b/tests/unit/autoscaling/test_autoscaled_pool.py new file mode 100644 index 0000000000..c62a98716f --- /dev/null +++ b/tests/unit/autoscaling/test_autoscaled_pool.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import asyncio +from contextlib import suppress +from datetime import datetime, timedelta, timezone +from typing import cast +from unittest.mock import Mock + +import pytest + +from crawlee._utils.measure_time import measure_time +from crawlee.autoscaling.autoscaled_pool import AutoscaledPool +from crawlee.autoscaling.system_status import SystemStatus +from crawlee.autoscaling.types import LoadRatioInfo, SystemInfo + +pytestmark = pytest.mark.asyncio() + + +@pytest.fixture() +def system_status() -> SystemStatus | Mock: + return Mock(spec=SystemStatus) + + +async def test_runs_concurrently(system_status: SystemStatus | Mock) -> None: + done_count = 0 + + async def run() -> None: + await asyncio.sleep(0.1) + nonlocal done_count + done_count += 1 + + pool = AutoscaledPool( + system_status=system_status, + run_task_function=run, + is_task_ready_function=lambda: True, + is_finished_function=lambda: done_count >= 10, + min_concurrency=10, + max_concurrency=10, + ) + + with measure_time() as elapsed: + await pool.run() + + assert elapsed.wall is not None + assert elapsed.wall < 0.3 + + assert done_count >= 10 + + +async def test_abort_works(system_status: SystemStatus | Mock) -> None: + async def run() -> None: + await asyncio.sleep(60) + + pool = AutoscaledPool( + system_status=system_status, + run_task_function=run, + is_task_ready_function=lambda: True, + is_finished_function=lambda: False, + min_concurrency=10, + max_concurrency=10, + ) + + with measure_time() as elapsed: + run_task = asyncio.create_task(pool.run()) + await asyncio.sleep(0.1) + assert pool.current_concurrency == 10 + await pool.abort() + assert pool.current_concurrency == 0 + await run_task + + assert elapsed.wall is not None + assert elapsed.wall < 0.3 + + +async def test_propagates_exceptions(system_status: SystemStatus | Mock) -> None: + done_count = 0 + + async def run() -> None: + await asyncio.sleep(0.1) + nonlocal done_count + done_count += 1 + + if done_count > 5: + raise RuntimeError('Scheduled crash') + + pool = AutoscaledPool( + system_status=system_status, + run_task_function=run, + is_task_ready_function=lambda: True, + is_finished_function=lambda: done_count >= 20, + min_concurrency=10, + max_concurrency=10, + ) + + with pytest.raises(RuntimeError, match='Scheduled crash'): + await pool.run() + + assert done_count < 20 + + +async def test_autoscales(system_status: SystemStatus | Mock) -> None: + done_count = 0 + + async def run() -> None: + await asyncio.sleep(0.1) + nonlocal done_count + done_count += 1 + + start = datetime.now(timezone.utc) + + def get_historical_status() -> SystemInfo: + result = SystemInfo( + cpu_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), + mem_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), + event_loop_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), + client_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), + mem_current_bytes=1024, + ) + + # 0.5 seconds after the start of the test, pretend the CPU became overloaded + if result.created_at - start >= timedelta(seconds=0.5): + result.cpu_info = LoadRatioInfo(limit_ratio=0.9, actual_ratio=1.0) + + return result + + cast(Mock, system_status.get_historical_status).side_effect = get_historical_status + + pool = AutoscaledPool( + system_status=system_status, + run_task_function=run, + is_task_ready_function=lambda: True, + is_finished_function=lambda: False, + min_concurrency=1, + desired_concurrency=1, + max_concurrency=4, + autoscale_interval=timedelta(seconds=0.1), + ) + + pool_run_task = asyncio.create_task(pool.run(), name='pool run task') + + try: + # After 0.2s, there should be an increase in concurrency + await asyncio.sleep(0.2) + assert pool.desired_concurrency > 1 + + # After 0.5s, the concurrency should reach max concurrency + await asyncio.sleep(0.3) + assert pool.desired_concurrency == 4 + + # The concurrency should guarantee completion of more than 10 tasks (a single worker would complete ~5) + assert done_count > 10 + + # After 0.7s, the pretend overload should have kicked in and there should be a drop in desired concurrency + await asyncio.sleep(0.2) + assert pool.desired_concurrency < 4 + + # After a full second, the pool should scale down all the way to 1 + await asyncio.sleep(0.3) + assert pool.desired_concurrency == 1 + finally: + pool_run_task.cancel() + with suppress(asyncio.CancelledError): + await pool_run_task + + +async def test_max_tasks_per_minute_works(system_status: SystemStatus | Mock) -> None: + done_count = 0 + + async def run() -> None: + await asyncio.sleep(0.1) + nonlocal done_count + done_count += 1 + + pool = AutoscaledPool( + system_status=system_status, + run_task_function=run, + is_task_ready_function=lambda: True, + is_finished_function=lambda: False, + min_concurrency=1, + desired_concurrency=1, + max_concurrency=1, + max_tasks_per_minute=120, + ) + + pool_run_task = asyncio.create_task(pool.run(), name='pool run task') + try: + await asyncio.sleep(0.5) + assert done_count <= 1 + finally: + pool_run_task.cancel() + with suppress(asyncio.CancelledError): + await pool_run_task