From 403816ff2340840f0e09b1c1bfc86ce6fe833e4f Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 20 Feb 2024 15:05:00 +0100 Subject: [PATCH 01/23] Initial AutoscaledPool implementation --- src/crawlee/autoscaling/autoscaled_pool.py | 150 +++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 src/crawlee/autoscaling/autoscaled_pool.py diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py new file mode 100644 index 0000000000..80beb04017 --- /dev/null +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -0,0 +1,150 @@ +# Inspiration: https://github.com/apify/crawlee/blob/v3.7.3/packages/core/src/autoscaling/autoscaled_pool.ts + +from __future__ import annotations + +import asyncio +from datetime import timedelta +from typing import TYPE_CHECKING, Awaitable, Callable + +from crawlee._utils.recurring_task import RecurringTask + +if TYPE_CHECKING: + from crawlee.autoscaling.system_status import SystemStatus + + +class AutoscaledPool: + """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel. + + The pool only starts new tasks if there is enough free CPU and memory available. + """ + + def __init__( + self: AutoscaledPool, + *, + system_status: SystemStatus, + run_task_function: Callable[..., Awaitable], + is_task_ready_function: Callable[..., bool], + is_finished_function: Callable[..., bool], + task_timeout: timedelta | None = None, + autoscale_interval: timedelta = timedelta(seconds=10), + logging_interval: timedelta = timedelta(minutes=1), + desired_concurrency: int | None = None, + desired_concurrency_ratio: float = 0.9, + min_concurrency: int = 1, + max_concurrency: int = 200, + scale_up_step_ratio: float = 0.05, + scale_down_step_ratio: float = 0.05, + max_tasks_per_minute: int | None = None, + ) -> None: + self.system_status = system_status + + self._run_task_function = run_task_function + self._is_task_ready_function = is_task_ready_function + self._is_finished_function = is_finished_function + + self._task_timeout = task_timeout + self._logging_interval = logging_interval # TODO: implement logging + + self._autoscale_task = RecurringTask(self._autoscale, autoscale_interval) + self._worker_tasks = list[asyncio.Task]() + self._worker_tasks_updated = asyncio.Event() + + if desired_concurrency is not None and desired_concurrency < 1: + raise ValueError('desired_concurrency must be 1 or larger') + + if min_concurrency < 1: + raise ValueError('min_concurrency must be 1 or larger') + + if max_concurrency < min_concurrency: + raise ValueError('max_concurrency cannot be less than min_concurrency') + + if desired_concurrency_ratio < 0 or desired_concurrency_ratio > 1: + raise ValueError('desired_concurrency_ratio must be between 0 and 1 (non-inclusive)') + + self._desired_concurrency_ratio = desired_concurrency_ratio + self._desired_concurrency = desired_concurrency if desired_concurrency is not None else min_concurrency + self._max_concurrency = max_concurrency + self._min_concurrency = min_concurrency + + self._scale_up_step_ratio = scale_up_step_ratio + self._scale_down_step_ratio = scale_down_step_ratio + + self._max_tasks_per_minute = max_tasks_per_minute # TODO: implement concurrency limiting + self._is_paused = False + + async def run(self: AutoscaledPool) -> None: + """Start the autoscaled pool and return when all tasks are completed and `is_finished_function` returns True. + + If there is an exception in one of the tasks, it will be re-raised. + """ + self._autoscale() + self._autoscale_task.start() + + while not self._is_finished_function(): + wait_for_workers_update = asyncio.create_task( + self._worker_tasks_updated.wait(), name='wait for worker tasks update' + ) + wait_for_worker_tasks = asyncio.create_task( + asyncio.wait(self._worker_tasks, return_when=asyncio.FIRST_EXCEPTION), + name='wait for worker tasks to complete', + ) + + self._worker_tasks_updated.clear() + + try: + await asyncio.wait( + [wait_for_workers_update, wait_for_worker_tasks], + return_when=asyncio.FIRST_COMPLETED, + ) + finally: + if not wait_for_worker_tasks.done(): + wait_for_worker_tasks.cancel() + + if not wait_for_workers_update.done(): + wait_for_workers_update.cancel() + + for task in self._worker_tasks: + if task.done(): + exception = task.exception() + if exception is not None: + raise exception + + async def abort(self: AutoscaledPool) -> None: + """Interrupt the autoscaled pool and all the tasks in progress.""" + self._is_paused = True + await self._autoscale_task.stop() + + for task in self._worker_tasks: + task.cancel() + + async def pause(self: AutoscaledPool) -> None: + """Pause the autoscaled pool so that it does not start new tasks.""" + self._is_paused = True + + async def resume(self: AutoscaledPool) -> None: + """Resume a paused autoscaled pool so that it continues starting new tasks.""" + self._is_paused = False + + def _autoscale(self: AutoscaledPool) -> None: + # TODO: adjust desired_concurrency based on system_status + + if len(self._worker_tasks) > self._desired_concurrency: + for _ in range(len(self._worker_tasks) - self._desired_concurrency): + self._worker_tasks.pop().cancel() + + elif len(self._worker_tasks) < self._desired_concurrency: + for i in range(len(self._worker_tasks), self._desired_concurrency): + task = asyncio.create_task(self._worker_task(), name=f'worker task #{i + 1}') + self._worker_tasks.append(task) + + async def _worker_task(self: AutoscaledPool) -> None: + while not self._is_finished_function(): + await asyncio.sleep(0) + + if self._is_paused: + continue + + await asyncio.wait_for( + self._run_task_function(), + timeout=self._task_timeout.total_seconds() if self._task_timeout is not None else None, + ) From 355aaacb26818c303c537b41b2504bd5597c0090 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 21 Feb 2024 11:40:45 +0100 Subject: [PATCH 02/23] Initial tests --- src/crawlee/_utils/measure_time.py | 25 +++++++ src/crawlee/autoscaling/autoscaled_pool.py | 59 ++++++++-------- .../unit/autoscaling/test_autoscaled_pool.py | 69 +++++++++++++++++++ 3 files changed, 126 insertions(+), 27 deletions(-) create mode 100644 src/crawlee/_utils/measure_time.py create mode 100644 tests/unit/autoscaling/test_autoscaled_pool.py diff --git a/src/crawlee/_utils/measure_time.py b/src/crawlee/_utils/measure_time.py new file mode 100644 index 0000000000..f6829dadac --- /dev/null +++ b/src/crawlee/_utils/measure_time.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import time +from contextlib import contextmanager +from typing import Iterator + + +class TimerResult: + wall: float | None + cpu: float | None + + +@contextmanager +def measure_time() -> Iterator[TimerResult]: + result = TimerResult() + before_wall = time.monotonic() + before_cpu = time.thread_time() + + try: + yield result + finally: + after_wall = time.monotonic() + after_cpu = time.thread_time() + result.wall = after_wall - before_wall + result.cpu = after_cpu - before_cpu diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 80beb04017..4cc7ae6bdf 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -80,34 +80,39 @@ async def run(self: AutoscaledPool) -> None: self._autoscale() self._autoscale_task.start() - while not self._is_finished_function(): - wait_for_workers_update = asyncio.create_task( - self._worker_tasks_updated.wait(), name='wait for worker tasks update' - ) - wait_for_worker_tasks = asyncio.create_task( - asyncio.wait(self._worker_tasks, return_when=asyncio.FIRST_EXCEPTION), - name='wait for worker tasks to complete', - ) - - self._worker_tasks_updated.clear() - - try: - await asyncio.wait( - [wait_for_workers_update, wait_for_worker_tasks], - return_when=asyncio.FIRST_COMPLETED, + try: + while not self._is_finished_function(): + wait_for_workers_update = asyncio.create_task( + self._worker_tasks_updated.wait(), name='wait for worker tasks update' + ) + wait_for_worker_tasks = asyncio.create_task( + asyncio.wait(self._worker_tasks, return_when=asyncio.FIRST_EXCEPTION), + name='wait for worker tasks to complete', ) - finally: - if not wait_for_worker_tasks.done(): - wait_for_worker_tasks.cancel() - - if not wait_for_workers_update.done(): - wait_for_workers_update.cancel() - - for task in self._worker_tasks: - if task.done(): - exception = task.exception() - if exception is not None: - raise exception + + self._worker_tasks_updated.clear() + + try: + await asyncio.wait( + [wait_for_workers_update, wait_for_worker_tasks], + return_when=asyncio.FIRST_COMPLETED, + ) + finally: + if not wait_for_worker_tasks.done(): + wait_for_worker_tasks.cancel() + + if not wait_for_workers_update.done(): + wait_for_workers_update.cancel() + + for task in self._worker_tasks: + if task.done(): + exception = task.exception() + if exception is not None: + raise exception + finally: + await self._autoscale_task.stop() + self._desired_concurrency = 0 + self._autoscale() async def abort(self: AutoscaledPool) -> None: """Interrupt the autoscaled pool and all the tasks in progress.""" diff --git a/tests/unit/autoscaling/test_autoscaled_pool.py b/tests/unit/autoscaling/test_autoscaled_pool.py new file mode 100644 index 0000000000..7b83fc48d5 --- /dev/null +++ b/tests/unit/autoscaling/test_autoscaled_pool.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import asyncio +from unittest.mock import MagicMock, Mock + +import pytest + +from crawlee._utils.measure_time import measure_time +from crawlee.autoscaling.autoscaled_pool import AutoscaledPool +from crawlee.autoscaling.system_status import SystemStatus + + +@pytest.fixture() +def system_status() -> SystemStatus | Mock: + return MagicMock(spec=SystemStatus) + + +@pytest.mark.asyncio() +async def test_runs_concurrently(system_status: SystemStatus | Mock) -> None: + done_count = 0 + + async def run() -> None: + await asyncio.sleep(0.1) + nonlocal done_count + done_count += 1 + + pool = AutoscaledPool( + system_status=system_status, + run_task_function=run, + is_task_ready_function=lambda: True, + is_finished_function=lambda: done_count >= 10, + min_concurrency=10, + max_concurrency=10, + ) + + with measure_time() as elapsed: + await pool.run() + + assert elapsed.wall is not None + assert elapsed.wall < 0.3 + + assert done_count >= 10 + + +@pytest.mark.asyncio() +async def test_propagates_exceptions(system_status: SystemStatus | Mock) -> None: + done_count = 0 + + async def run() -> None: + await asyncio.sleep(0.1) + nonlocal done_count + done_count += 1 + + if done_count > 5: + raise RuntimeError('Scheduled crash') + + pool = AutoscaledPool( + system_status=system_status, + run_task_function=run, + is_task_ready_function=lambda: True, + is_finished_function=lambda: done_count >= 20, + min_concurrency=10, + max_concurrency=10, + ) + + with pytest.raises(RuntimeError): + await pool.run() + + assert done_count < 20 From e1cc456c1d1328d47ea887bf6b26ece37fc327c7 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 21 Feb 2024 15:56:42 +0100 Subject: [PATCH 03/23] Implement SystemInfo-driven autoscaling --- src/crawlee/autoscaling/autoscaled_pool.py | 33 +++++--- .../unit/autoscaling/test_autoscaled_pool.py | 75 ++++++++++++++++++- 2 files changed, 97 insertions(+), 11 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 4cc7ae6bdf..deae6228fe 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio +import math +from contextlib import suppress from datetime import timedelta from typing import TYPE_CHECKING, Awaitable, Callable @@ -29,7 +31,6 @@ def __init__( autoscale_interval: timedelta = timedelta(seconds=10), logging_interval: timedelta = timedelta(minutes=1), desired_concurrency: int | None = None, - desired_concurrency_ratio: float = 0.9, min_concurrency: int = 1, max_concurrency: int = 200, scale_up_step_ratio: float = 0.05, @@ -58,10 +59,6 @@ def __init__( if max_concurrency < min_concurrency: raise ValueError('max_concurrency cannot be less than min_concurrency') - if desired_concurrency_ratio < 0 or desired_concurrency_ratio > 1: - raise ValueError('desired_concurrency_ratio must be between 0 and 1 (non-inclusive)') - - self._desired_concurrency_ratio = desired_concurrency_ratio self._desired_concurrency = desired_concurrency if desired_concurrency is not None else min_concurrency self._max_concurrency = max_concurrency self._min_concurrency = min_concurrency @@ -77,7 +74,7 @@ async def run(self: AutoscaledPool) -> None: If there is an exception in one of the tasks, it will be re-raised. """ - self._autoscale() + self._ensure_desired_concurrency() self._autoscale_task.start() try: @@ -110,9 +107,11 @@ async def run(self: AutoscaledPool) -> None: if exception is not None: raise exception finally: - await self._autoscale_task.stop() + with suppress(asyncio.CancelledError): + await self._autoscale_task.stop() + self._desired_concurrency = 0 - self._autoscale() + self._ensure_desired_concurrency() async def abort(self: AutoscaledPool) -> None: """Interrupt the autoscaled pool and all the tasks in progress.""" @@ -130,17 +129,33 @@ async def resume(self: AutoscaledPool) -> None: """Resume a paused autoscaled pool so that it continues starting new tasks.""" self._is_paused = False + @property + def desired_concurrency(self: AutoscaledPool) -> int: + """The current desired concurrency, possibly updated by the pool according to system load.""" + return self._desired_concurrency + def _autoscale(self: AutoscaledPool) -> None: - # TODO: adjust desired_concurrency based on system_status + status = self.system_status.get_historical_status() + if status.is_system_idle and self._desired_concurrency < self._max_concurrency: + step = math.ceil(self._scale_up_step_ratio * self._desired_concurrency) + self._desired_concurrency = min(self._max_concurrency, self._desired_concurrency + step) + elif not status.is_system_idle and self._desired_concurrency > self._min_concurrency: + step = math.ceil(self._scale_down_step_ratio * self._desired_concurrency) + self._desired_concurrency = max(self._min_concurrency, self._desired_concurrency - step) + + self._ensure_desired_concurrency() + def _ensure_desired_concurrency(self: AutoscaledPool) -> None: if len(self._worker_tasks) > self._desired_concurrency: for _ in range(len(self._worker_tasks) - self._desired_concurrency): self._worker_tasks.pop().cancel() + self._worker_tasks_updated.set() elif len(self._worker_tasks) < self._desired_concurrency: for i in range(len(self._worker_tasks), self._desired_concurrency): task = asyncio.create_task(self._worker_task(), name=f'worker task #{i + 1}') self._worker_tasks.append(task) + self._worker_tasks_updated.set() async def _worker_task(self: AutoscaledPool) -> None: while not self._is_finished_function(): diff --git a/tests/unit/autoscaling/test_autoscaled_pool.py b/tests/unit/autoscaling/test_autoscaled_pool.py index 7b83fc48d5..734c7edde6 100644 --- a/tests/unit/autoscaling/test_autoscaled_pool.py +++ b/tests/unit/autoscaling/test_autoscaled_pool.py @@ -1,6 +1,9 @@ from __future__ import annotations import asyncio +from contextlib import suppress +from datetime import datetime, timedelta, timezone +from typing import cast from unittest.mock import MagicMock, Mock import pytest @@ -8,6 +11,9 @@ from crawlee._utils.measure_time import measure_time from crawlee.autoscaling.autoscaled_pool import AutoscaledPool from crawlee.autoscaling.system_status import SystemStatus +from crawlee.autoscaling.types import LoadRatioInfo, SystemInfo + +pytestmark = pytest.mark.asyncio() @pytest.fixture() @@ -15,7 +21,6 @@ def system_status() -> SystemStatus | Mock: return MagicMock(spec=SystemStatus) -@pytest.mark.asyncio() async def test_runs_concurrently(system_status: SystemStatus | Mock) -> None: done_count = 0 @@ -42,7 +47,6 @@ async def run() -> None: assert done_count >= 10 -@pytest.mark.asyncio() async def test_propagates_exceptions(system_status: SystemStatus | Mock) -> None: done_count = 0 @@ -67,3 +71,70 @@ async def run() -> None: await pool.run() assert done_count < 20 + + +async def test_autoscales(system_status: SystemStatus | Mock) -> None: + done_count = 0 + + async def run() -> None: + await asyncio.sleep(0.1) + nonlocal done_count + done_count += 1 + + start = datetime.now(timezone.utc) + + def get_historical_status() -> SystemInfo: + now = datetime.now(timezone.utc) + result = SystemInfo( + now, + cpu_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), + mem_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), + event_loop_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), + client_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), + mem_current_bytes=1024, + ) + + # 0.5 seconds after the start of the test, pretend the CPU became overloaded + if now - start >= timedelta(seconds=0.5): + result.cpu_info = LoadRatioInfo(limit_ratio=0.9, actual_ratio=1.0) + + return result + + cast(Mock, system_status.get_historical_status).side_effect = get_historical_status + + pool = AutoscaledPool( + system_status=system_status, + run_task_function=run, + is_task_ready_function=lambda: True, + is_finished_function=lambda: False, + min_concurrency=1, + desired_concurrency=1, + max_concurrency=4, + autoscale_interval=timedelta(seconds=0.1), + ) + + pool_run_task = asyncio.create_task(pool.run(), name='pool run task') + + try: + # After 0.2s, there should be an increase in concurrency + await asyncio.sleep(0.2) + assert pool.desired_concurrency > 1 + + # After 0.5s, the concurrency should reach max concurrency + await asyncio.sleep(0.3) + assert pool.desired_concurrency == 4 + + # The concurrency should guarantee completion of more than 10 tasks (a single worker would complete ~5) + assert done_count > 10 + + # After 0.7s, the pretend overload should have kicked in and there should be a drop in desired concurrency + await asyncio.sleep(0.2) + assert pool.desired_concurrency < 4 + + # After a full second, the pool should scale down all the way to 1 + await asyncio.sleep(0.3) + assert pool.desired_concurrency == 1 + finally: + pool_run_task.cancel() + with suppress(asyncio.CancelledError): + await pool_run_task From 3ff330dd7c199ed0b308bd008e3bb7f765419d0c Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Wed, 21 Feb 2024 16:34:11 +0100 Subject: [PATCH 04/23] Implement max_tasks_per_minute --- src/crawlee/autoscaling/autoscaled_pool.py | 12 ++++++-- .../unit/autoscaling/test_autoscaled_pool.py | 29 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index deae6228fe..a7b1bb0ef2 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -66,7 +66,7 @@ def __init__( self._scale_up_step_ratio = scale_up_step_ratio self._scale_down_step_ratio = scale_down_step_ratio - self._max_tasks_per_minute = max_tasks_per_minute # TODO: implement concurrency limiting + self._max_tasks_per_minute = max_tasks_per_minute self._is_paused = False async def run(self: AutoscaledPool) -> None: @@ -159,11 +159,19 @@ def _ensure_desired_concurrency(self: AutoscaledPool) -> None: async def _worker_task(self: AutoscaledPool) -> None: while not self._is_finished_function(): - await asyncio.sleep(0) + if self._max_tasks_per_minute is not None: + delay = 60 / self._max_tasks_per_minute / self._desired_concurrency + else: + delay = 0 + + await asyncio.sleep(delay) if self._is_paused: continue + if not self._is_task_ready_function(): + continue + await asyncio.wait_for( self._run_task_function(), timeout=self._task_timeout.total_seconds() if self._task_timeout is not None else None, diff --git a/tests/unit/autoscaling/test_autoscaled_pool.py b/tests/unit/autoscaling/test_autoscaled_pool.py index 734c7edde6..a91c58c42f 100644 --- a/tests/unit/autoscaling/test_autoscaled_pool.py +++ b/tests/unit/autoscaling/test_autoscaled_pool.py @@ -138,3 +138,32 @@ def get_historical_status() -> SystemInfo: pool_run_task.cancel() with suppress(asyncio.CancelledError): await pool_run_task + + +async def test_max_tasks_per_minute_works(system_status: SystemStatus | Mock) -> None: + done_count = 0 + + async def run() -> None: + await asyncio.sleep(0.1) + nonlocal done_count + done_count += 1 + + pool = AutoscaledPool( + system_status=system_status, + run_task_function=run, + is_task_ready_function=lambda: True, + is_finished_function=lambda: False, + min_concurrency=1, + desired_concurrency=1, + max_concurrency=1, + max_tasks_per_minute=120, + ) + + pool_run_task = asyncio.create_task(pool.run(), name='pool run task') + try: + await asyncio.sleep(0.5) + assert done_count <= 1 + finally: + pool_run_task.cancel() + with suppress(asyncio.CancelledError): + await pool_run_task From e0efcb4f593cd4187e271cde802af822e0442ffb Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Thu, 22 Feb 2024 13:19:28 +0100 Subject: [PATCH 05/23] Add logging --- src/crawlee/autoscaling/autoscaled_pool.py | 44 +++++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index a7b1bb0ef2..bfb3f48bc7 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -6,6 +6,7 @@ import math from contextlib import suppress from datetime import timedelta +from logging import getLogger from typing import TYPE_CHECKING, Awaitable, Callable from crawlee._utils.recurring_task import RecurringTask @@ -13,6 +14,8 @@ if TYPE_CHECKING: from crawlee.autoscaling.system_status import SystemStatus +logger = getLogger(__name__) + class AutoscaledPool: """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel. @@ -44,9 +47,12 @@ def __init__( self._is_finished_function = is_finished_function self._task_timeout = task_timeout - self._logging_interval = logging_interval # TODO: implement logging + + self._logging_interval = logging_interval + self._log_system_status_task = RecurringTask(self._log_system_status, logging_interval) self._autoscale_task = RecurringTask(self._autoscale, autoscale_interval) + self._worker_tasks = list[asyncio.Task]() self._worker_tasks_updated = asyncio.Event() @@ -63,6 +69,8 @@ def __init__( self._max_concurrency = max_concurrency self._min_concurrency = min_concurrency + self._current_concurrency = 0 + self._scale_up_step_ratio = scale_up_step_ratio self._scale_down_step_ratio = scale_down_step_ratio @@ -76,6 +84,9 @@ async def run(self: AutoscaledPool) -> None: """ self._ensure_desired_concurrency() self._autoscale_task.start() + self._log_system_status_task.start() + + logger.debug('Starting the pool') try: while not self._is_finished_function(): @@ -109,10 +120,14 @@ async def run(self: AutoscaledPool) -> None: finally: with suppress(asyncio.CancelledError): await self._autoscale_task.stop() + with suppress(asyncio.CancelledError): + await self._log_system_status_task.stop() self._desired_concurrency = 0 self._ensure_desired_concurrency() + logger.debug('Pool cleanup finished') + async def abort(self: AutoscaledPool) -> None: """Interrupt the autoscaled pool and all the tasks in progress.""" self._is_paused = True @@ -157,6 +172,11 @@ def _ensure_desired_concurrency(self: AutoscaledPool) -> None: self._worker_tasks.append(task) self._worker_tasks_updated.set() + def _log_system_status(self: AutoscaledPool) -> None: + logger.info( + f'current_concurrency = {self._current_concurrency}; desired_concurrency = {self._desired_concurrency}' + ) + async def _worker_task(self: AutoscaledPool) -> None: while not self._is_finished_function(): if self._max_tasks_per_minute is not None: @@ -167,12 +187,26 @@ async def _worker_task(self: AutoscaledPool) -> None: await asyncio.sleep(delay) if self._is_paused: + logger.debug('Paused - not executing a task') continue if not self._is_task_ready_function(): + logger.debug('No task is ready yet') continue - await asyncio.wait_for( - self._run_task_function(), - timeout=self._task_timeout.total_seconds() if self._task_timeout is not None else None, - ) + self._current_concurrency += 1 + try: + await asyncio.wait_for( + self._run_task_function(), + timeout=self._task_timeout.total_seconds() if self._task_timeout is not None else None, + ) + except asyncio.TimeoutError: + logger.warning( + f"Task timed out after { + self._task_timeout.total_seconds() if self._task_timeout is not None else '*not set*' + } seconds" + ) + finally: + self._current_concurrency -= 1 + + logger.debug('Worker task finished') From 9077d395662b0abeeb299188e8be2f253ce1ae14 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Thu, 22 Feb 2024 13:28:02 +0100 Subject: [PATCH 06/23] Check current concurrency when scaling up --- src/crawlee/autoscaling/autoscaled_pool.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index bfb3f48bc7..efa1c4f5d5 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -34,6 +34,7 @@ def __init__( autoscale_interval: timedelta = timedelta(seconds=10), logging_interval: timedelta = timedelta(minutes=1), desired_concurrency: int | None = None, + desired_concurrency_ratio: float = 0.9, min_concurrency: int = 1, max_concurrency: int = 200, scale_up_step_ratio: float = 0.05, @@ -65,6 +66,10 @@ def __init__( if max_concurrency < min_concurrency: raise ValueError('max_concurrency cannot be less than min_concurrency') + if desired_concurrency_ratio < 0 or desired_concurrency_ratio > 1: + raise ValueError('desired_concurrency_ratio must be between 0 and 1 (non-inclusive)') + + self._desired_concurrency_ratio = desired_concurrency_ratio self._desired_concurrency = desired_concurrency if desired_concurrency is not None else min_concurrency self._max_concurrency = max_concurrency self._min_concurrency = min_concurrency @@ -151,7 +156,13 @@ def desired_concurrency(self: AutoscaledPool) -> int: def _autoscale(self: AutoscaledPool) -> None: status = self.system_status.get_historical_status() - if status.is_system_idle and self._desired_concurrency < self._max_concurrency: + min_current_concurrency = math.floor(self._desired_concurrency_ratio * self._current_concurrency) + + if ( + status.is_system_idle + and self._desired_concurrency < self._max_concurrency + and self._current_concurrency >= min_current_concurrency + ): step = math.ceil(self._scale_up_step_ratio * self._desired_concurrency) self._desired_concurrency = min(self._max_concurrency, self._desired_concurrency + step) elif not status.is_system_idle and self._desired_concurrency > self._min_concurrency: From c3b619189c83fd0f7b9396ea85fc7f9c820ee646 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Thu, 22 Feb 2024 13:28:48 +0100 Subject: [PATCH 07/23] Bug --- src/crawlee/autoscaling/autoscaled_pool.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index efa1c4f5d5..5d0d440b29 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -212,11 +212,8 @@ async def _worker_task(self: AutoscaledPool) -> None: timeout=self._task_timeout.total_seconds() if self._task_timeout is not None else None, ) except asyncio.TimeoutError: - logger.warning( - f"Task timed out after { - self._task_timeout.total_seconds() if self._task_timeout is not None else '*not set*' - } seconds" - ) + timeout_str = self._task_timeout.total_seconds() if self._task_timeout is not None else '*not set*' + logger.warning(f'Task timed out after {timeout_str} seconds') finally: self._current_concurrency -= 1 From 2575749fadd6190a25aaeef787e857f63cd11ecd Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Thu, 22 Feb 2024 14:17:17 +0100 Subject: [PATCH 08/23] Add system status to periodic log message --- src/crawlee/autoscaling/autoscaled_pool.py | 10 +++++++--- src/crawlee/autoscaling/types.py | 10 ++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 5d0d440b29..d817d79b97 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -41,7 +41,7 @@ def __init__( scale_down_step_ratio: float = 0.05, max_tasks_per_minute: int | None = None, ) -> None: - self.system_status = system_status + self._system_status = system_status self._run_task_function = run_task_function self._is_task_ready_function = is_task_ready_function @@ -155,7 +155,7 @@ def desired_concurrency(self: AutoscaledPool) -> int: return self._desired_concurrency def _autoscale(self: AutoscaledPool) -> None: - status = self.system_status.get_historical_status() + status = self._system_status.get_historical_status() min_current_concurrency = math.floor(self._desired_concurrency_ratio * self._current_concurrency) if ( @@ -184,8 +184,12 @@ def _ensure_desired_concurrency(self: AutoscaledPool) -> None: self._worker_tasks_updated.set() def _log_system_status(self: AutoscaledPool) -> None: + system_status = self._system_status.get_historical_status() + logger.info( - f'current_concurrency = {self._current_concurrency}; desired_concurrency = {self._desired_concurrency}' + f'current_concurrency = {self._current_concurrency}; ' + f'desired_concurrency = {self._desired_concurrency}; ' + f'{system_status!s}' ) async def _worker_task(self: AutoscaledPool) -> None: diff --git a/src/crawlee/autoscaling/types.py b/src/crawlee/autoscaling/types.py index 4a66a71691..8b7d7a1da6 100644 --- a/src/crawlee/autoscaling/types.py +++ b/src/crawlee/autoscaling/types.py @@ -44,6 +44,16 @@ def is_system_idle(self: SystemInfo) -> bool: and not self.client_info.is_overloaded ) + def __str__(self: SystemInfo) -> str: + """Get a string representation of the system info.""" + stats = { + 'cpu': self.cpu_info.actual_ratio if self.cpu_info else '-', + 'mem': self.mem_info.actual_ratio if self.mem_info else '-', + 'event_lopp': self.event_loop_info.actual_ratio if self.event_loop_info else '-', + 'client_info': self.client_info.actual_ratio if self.client_info else '-', + } + return ';'.join(f'{name} = {ratio}' for name, ratio in stats.items()) + @dataclass class FinalStatistics: From 7d0ec6e57ad24c8a6b490463c10c0a281c7735ee Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Thu, 22 Feb 2024 15:54:22 +0100 Subject: [PATCH 09/23] Avoid interrupting tasks when scaling down --- src/crawlee/autoscaling/autoscaled_pool.py | 43 +++++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index d817d79b97..72f56487d3 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -54,8 +54,11 @@ def __init__( self._autoscale_task = RecurringTask(self._autoscale, autoscale_interval) - self._worker_tasks = list[asyncio.Task]() + self._worker_tasks = list[tuple[asyncio.Task, asyncio.Event]]() + """A list of workers tasks and events to signal that the task should terminate""" + self._worker_tasks_updated = asyncio.Event() + self._tasks_for_cleanup = list[asyncio.Task]() if desired_concurrency is not None and desired_concurrency < 1: raise ValueError('desired_concurrency must be 1 or larger') @@ -99,7 +102,7 @@ async def run(self: AutoscaledPool) -> None: self._worker_tasks_updated.wait(), name='wait for worker tasks update' ) wait_for_worker_tasks = asyncio.create_task( - asyncio.wait(self._worker_tasks, return_when=asyncio.FIRST_EXCEPTION), + asyncio.wait([task for task, _ in self._worker_tasks], return_when=asyncio.FIRST_EXCEPTION), name='wait for worker tasks to complete', ) @@ -117,7 +120,7 @@ async def run(self: AutoscaledPool) -> None: if not wait_for_workers_update.done(): wait_for_workers_update.cancel() - for task in self._worker_tasks: + for task, _ in self._worker_tasks: if task.done(): exception = task.exception() if exception is not None: @@ -131,6 +134,11 @@ async def run(self: AutoscaledPool) -> None: self._desired_concurrency = 0 self._ensure_desired_concurrency() + for task in self._tasks_for_cleanup: + if not task.done(): + with suppress(asyncio.CancelledError): + await task + logger.debug('Pool cleanup finished') async def abort(self: AutoscaledPool) -> None: @@ -138,7 +146,7 @@ async def abort(self: AutoscaledPool) -> None: self._is_paused = True await self._autoscale_task.stop() - for task in self._worker_tasks: + for task, _ in self._worker_tasks: task.cancel() async def pause(self: AutoscaledPool) -> None: @@ -174,15 +182,26 @@ def _autoscale(self: AutoscaledPool) -> None: def _ensure_desired_concurrency(self: AutoscaledPool) -> None: if len(self._worker_tasks) > self._desired_concurrency: for _ in range(len(self._worker_tasks) - self._desired_concurrency): - self._worker_tasks.pop().cancel() + task, terminate_event = self._worker_tasks.pop() + self._mark_task_for_cleanup(task) + terminate_event.set() self._worker_tasks_updated.set() elif len(self._worker_tasks) < self._desired_concurrency: for i in range(len(self._worker_tasks), self._desired_concurrency): - task = asyncio.create_task(self._worker_task(), name=f'worker task #{i + 1}') - self._worker_tasks.append(task) + terminate_event = asyncio.Event() + task = asyncio.create_task(self._worker_task(terminate_event), name=f'worker task #{i + 1}') + self._worker_tasks.append((task, terminate_event)) self._worker_tasks_updated.set() + def _mark_task_for_cleanup(self: AutoscaledPool, task: asyncio.Task) -> None: + def cleanup(_: asyncio.Future) -> None: + task.exception() + self._tasks_for_cleanup.remove(task) + + self._tasks_for_cleanup.append(task) + task.add_done_callback(cleanup) + def _log_system_status(self: AutoscaledPool) -> None: system_status = self._system_status.get_historical_status() @@ -192,14 +211,18 @@ def _log_system_status(self: AutoscaledPool) -> None: f'{system_status!s}' ) - async def _worker_task(self: AutoscaledPool) -> None: + async def _worker_task(self: AutoscaledPool, terminate_event: asyncio.Event) -> None: while not self._is_finished_function(): - if self._max_tasks_per_minute is not None: + if self._max_tasks_per_minute is not None and self._desired_concurrency > 0: delay = 60 / self._max_tasks_per_minute / self._desired_concurrency else: delay = 0 - await asyncio.sleep(delay) + with suppress(asyncio.TimeoutError): + await asyncio.wait_for(terminate_event.wait(), delay) + + if terminate_event.is_set() or self._is_finished_function(): + break if self._is_paused: logger.debug('Paused - not executing a task') From 0f34f2c040885c6e5b5727a6771e6a4ae4d60662 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 23 Feb 2024 10:04:38 +0100 Subject: [PATCH 10/23] Add doc and unit test for measure_time --- src/crawlee/_utils/measure_time.py | 7 +++-- tests/unit/_utils/test_measure_time.py | 39 ++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 tests/unit/_utils/test_measure_time.py diff --git a/src/crawlee/_utils/measure_time.py b/src/crawlee/_utils/measure_time.py index f6829dadac..082ef9f869 100644 --- a/src/crawlee/_utils/measure_time.py +++ b/src/crawlee/_utils/measure_time.py @@ -2,16 +2,19 @@ import time from contextlib import contextmanager +from dataclasses import dataclass from typing import Iterator +@dataclass class TimerResult: - wall: float | None - cpu: float | None + wall: float | None = None + cpu: float | None = None @contextmanager def measure_time() -> Iterator[TimerResult]: + """Measure the execution time (wall-clock and CPU) between the start and end of the with-block.""" result = TimerResult() before_wall = time.monotonic() before_cpu = time.thread_time() diff --git a/tests/unit/_utils/test_measure_time.py b/tests/unit/_utils/test_measure_time.py new file mode 100644 index 0000000000..1e521bd2b8 --- /dev/null +++ b/tests/unit/_utils/test_measure_time.py @@ -0,0 +1,39 @@ +import asyncio +import time + +import pytest + +from crawlee._utils.measure_time import measure_time + + +def test_measure_time_wall_sync() -> None: + with measure_time() as elapsed: + time.sleep(0.1) + + assert elapsed.cpu is not None + assert elapsed.wall is not None + assert elapsed.wall >= 0.1 + + +def test_measure_time_cpu_sync() -> None: + with measure_time() as elapsed: + start = time.time() + acc = 0 + + while time.time() - start < 0.1: + acc += 1 + acc *= acc + + assert elapsed.cpu is not None + assert elapsed.wall is not None + assert elapsed.cpu >= 0.05 + + +@pytest.mark.asyncio() +async def test_measure_time_wall_async() -> None: + with measure_time() as elapsed: + await asyncio.sleep(0.1) + + assert elapsed.cpu is not None + assert elapsed.wall is not None + assert elapsed.wall >= 0.1 From 23f7cf47519ebc83eff02131c74b4318273b337c Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 23 Feb 2024 11:32:39 +0100 Subject: [PATCH 11/23] Less strict measure_time tests --- tests/unit/_utils/test_measure_time.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/_utils/test_measure_time.py b/tests/unit/_utils/test_measure_time.py index 1e521bd2b8..400e563e05 100644 --- a/tests/unit/_utils/test_measure_time.py +++ b/tests/unit/_utils/test_measure_time.py @@ -12,7 +12,7 @@ def test_measure_time_wall_sync() -> None: assert elapsed.cpu is not None assert elapsed.wall is not None - assert elapsed.wall >= 0.1 + assert elapsed.wall >= 0.09 def test_measure_time_cpu_sync() -> None: @@ -36,4 +36,4 @@ async def test_measure_time_wall_async() -> None: assert elapsed.cpu is not None assert elapsed.wall is not None - assert elapsed.wall >= 0.1 + assert elapsed.wall >= 0.09 From 49e571af8584cefc9282cf54893f8179740c1b96 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 23 Feb 2024 13:05:30 +0100 Subject: [PATCH 12/23] Apply suggestions from code review Co-authored-by: Vlada Dusek --- src/crawlee/autoscaling/autoscaled_pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 72f56487d3..c0a68d5e10 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -149,11 +149,11 @@ async def abort(self: AutoscaledPool) -> None: for task, _ in self._worker_tasks: task.cancel() - async def pause(self: AutoscaledPool) -> None: + def pause(self: AutoscaledPool) -> None: """Pause the autoscaled pool so that it does not start new tasks.""" self._is_paused = True - async def resume(self: AutoscaledPool) -> None: + def resume(self: AutoscaledPool) -> None: """Resume a paused autoscaled pool so that it continues starting new tasks.""" self._is_paused = False From 56326fc14fdbdc3968d271400a935cd9b605e0aa Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 23 Feb 2024 13:06:37 +0100 Subject: [PATCH 13/23] Update tests/unit/autoscaling/test_autoscaled_pool.py Co-authored-by: Vlada Dusek --- tests/unit/autoscaling/test_autoscaled_pool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/autoscaling/test_autoscaled_pool.py b/tests/unit/autoscaling/test_autoscaled_pool.py index a91c58c42f..1ae8b9a701 100644 --- a/tests/unit/autoscaling/test_autoscaled_pool.py +++ b/tests/unit/autoscaling/test_autoscaled_pool.py @@ -67,7 +67,7 @@ async def run() -> None: max_concurrency=10, ) - with pytest.raises(RuntimeError): + with pytest.raises(RuntimeError, match='Scheduled crash'): await pool.run() assert done_count < 20 From c0ef8640bdbd312ee63fbb439213b46a22e46a2c Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 23 Feb 2024 13:09:06 +0100 Subject: [PATCH 14/23] Exploit default value of SystemInfo.created_at --- tests/unit/autoscaling/test_autoscaled_pool.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/unit/autoscaling/test_autoscaled_pool.py b/tests/unit/autoscaling/test_autoscaled_pool.py index 1ae8b9a701..a03835f876 100644 --- a/tests/unit/autoscaling/test_autoscaled_pool.py +++ b/tests/unit/autoscaling/test_autoscaled_pool.py @@ -84,9 +84,7 @@ async def run() -> None: start = datetime.now(timezone.utc) def get_historical_status() -> SystemInfo: - now = datetime.now(timezone.utc) result = SystemInfo( - now, cpu_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), mem_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), event_loop_info=LoadRatioInfo(limit_ratio=0.9, actual_ratio=0.3), @@ -95,7 +93,7 @@ def get_historical_status() -> SystemInfo: ) # 0.5 seconds after the start of the test, pretend the CPU became overloaded - if now - start >= timedelta(seconds=0.5): + if result.created_at - start >= timedelta(seconds=0.5): result.cpu_info = LoadRatioInfo(limit_ratio=0.9, actual_ratio=1.0) return result From 7a4eccf821e7f68a7c894c1819d58cd756609132 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 23 Feb 2024 21:25:15 +0100 Subject: [PATCH 15/23] Document __init__ args --- src/crawlee/autoscaling/autoscaled_pool.py | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index c0a68d5e10..2f36af7946 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -21,6 +21,32 @@ class AutoscaledPool: """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel. The pool only starts new tasks if there is enough free CPU and memory available. + + Args: + system_status: Provides data about system utilization (load). + run_task_function: A function that performs an asynchronous resource-intensive task. + is_task_ready_function: A function that indicates whether `runTaskFunction` should be called. This function + is called every time there is free capacity for a new task and it should indicate whether it should start + a new task or not by resolving to either `true` or `false`. Besides its obvious use, it is also useful for + task throttling to save resources. is_finished_function: A function that is called only when there are no + tasks to be processed. If it resolves to `true` then the pool's run finishes. Being called only when there + are no tasks being processed means that as long as `isTaskReadyFunction()` keeps resolving to `true`, + `isFinishedFunction()` will never be called. To abort a run, use the abort method. + task_timeout: Timeout in which the `runTaskFunction` needs to finish. + autoscale_interval: Defines how often the pool should attempt to adjust the desired concurrency based on + the latest system status. Setting it lower than 1 might have a severe impact on performance. We suggest + using a value from 5 to 20. + logging_interval: Specifies a period in which the instance logs its state, in seconds. + desired_concurrency: The desired number of tasks that should be running parallel on the start of the pool, if + there is a large enough supply of them. By default, it is `minConcurrency`. + desired_concurrency_ratio: Minimum level of desired concurrency to reach before more scaling up is allowed. + min_concurrency: The minimum number of tasks running in parallel. If you set this value too high with respect + to the available system memory and CPU, your code might run extremely slow or crash. + max_concurrency: The maximum number of tasks running in parallel. + scale_up_step_ratio: Defines the fractional amount of desired concurrency to be added with each scaling up. + scale_down_step_ratio: Defines the amount of desired concurrency to be subtracted with each scaling down. + max_tasks_per_minute: The maximum number of tasks per minute the pool can run. By default, this is set to + `Infinity`, but you can pass any positive, non-zero integer. """ def __init__( From b724e7ac22ec02a775ca322f1c85dd49014ad266 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 23 Feb 2024 21:27:51 +0100 Subject: [PATCH 16/23] Remove unnecessary MagicMock --- tests/unit/autoscaling/test_autoscaled_pool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/autoscaling/test_autoscaled_pool.py b/tests/unit/autoscaling/test_autoscaled_pool.py index a03835f876..110d355aa7 100644 --- a/tests/unit/autoscaling/test_autoscaled_pool.py +++ b/tests/unit/autoscaling/test_autoscaled_pool.py @@ -4,7 +4,7 @@ from contextlib import suppress from datetime import datetime, timedelta, timezone from typing import cast -from unittest.mock import MagicMock, Mock +from unittest.mock import Mock import pytest @@ -18,7 +18,7 @@ @pytest.fixture() def system_status() -> SystemStatus | Mock: - return MagicMock(spec=SystemStatus) + return Mock(spec=SystemStatus) async def test_runs_concurrently(system_status: SystemStatus | Mock) -> None: From 284c4e97c206b2de82627d00650508c61a0542d8 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Fri, 23 Feb 2024 21:43:36 +0100 Subject: [PATCH 17/23] Put argument descriptions where they belong --- src/crawlee/autoscaling/autoscaled_pool.py | 58 ++++++++++++---------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 2f36af7946..805e4844f0 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -21,32 +21,6 @@ class AutoscaledPool: """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel. The pool only starts new tasks if there is enough free CPU and memory available. - - Args: - system_status: Provides data about system utilization (load). - run_task_function: A function that performs an asynchronous resource-intensive task. - is_task_ready_function: A function that indicates whether `runTaskFunction` should be called. This function - is called every time there is free capacity for a new task and it should indicate whether it should start - a new task or not by resolving to either `true` or `false`. Besides its obvious use, it is also useful for - task throttling to save resources. is_finished_function: A function that is called only when there are no - tasks to be processed. If it resolves to `true` then the pool's run finishes. Being called only when there - are no tasks being processed means that as long as `isTaskReadyFunction()` keeps resolving to `true`, - `isFinishedFunction()` will never be called. To abort a run, use the abort method. - task_timeout: Timeout in which the `runTaskFunction` needs to finish. - autoscale_interval: Defines how often the pool should attempt to adjust the desired concurrency based on - the latest system status. Setting it lower than 1 might have a severe impact on performance. We suggest - using a value from 5 to 20. - logging_interval: Specifies a period in which the instance logs its state, in seconds. - desired_concurrency: The desired number of tasks that should be running parallel on the start of the pool, if - there is a large enough supply of them. By default, it is `minConcurrency`. - desired_concurrency_ratio: Minimum level of desired concurrency to reach before more scaling up is allowed. - min_concurrency: The minimum number of tasks running in parallel. If you set this value too high with respect - to the available system memory and CPU, your code might run extremely slow or crash. - max_concurrency: The maximum number of tasks running in parallel. - scale_up_step_ratio: Defines the fractional amount of desired concurrency to be added with each scaling up. - scale_down_step_ratio: Defines the amount of desired concurrency to be subtracted with each scaling down. - max_tasks_per_minute: The maximum number of tasks per minute the pool can run. By default, this is set to - `Infinity`, but you can pass any positive, non-zero integer. """ def __init__( @@ -67,6 +41,38 @@ def __init__( scale_down_step_ratio: float = 0.05, max_tasks_per_minute: int | None = None, ) -> None: + """Initialize the AutoscaledPool. + + Args: + system_status: Provides data about system utilization (load). + run_task_function: A function that performs an asynchronous resource-intensive task. + is_task_ready_function: A function that indicates whether `run_task_function` should be called. This function + is called every time there is free capacity for a new task and it should indicate whether it should start + a new task or not by resolving to either `True` or `False`. Besides its obvious use, it is also useful for + task throttling to save resources. is_finished_function: A function that is called only when there are no + tasks to be processed. If it resolves to `True` then the pool's run finishes. Being called only when there + are no tasks being processed means that as long as `is_task_ready_function()` keeps resolving to `True`, + `is_finished_function()` will never be called. To abort a run, use the `abort` method. + is_finished_function: A function that is called only when there are no tasks to be processed. If it resolves to + `true` then the pool's run finishes. Being called only when there are no tasks being processed means that + as long as `isTaskReadyFunction()` keeps resolving to `true`, `isFinishedFunction()` will never be called. + To abort a run, use the `abort` method. + task_timeout: Timeout in which the `run_task_function` needs to finish. + autoscale_interval: Defines how often the pool should attempt to adjust the desired concurrency based on + the latest system status. Setting it lower than 1 might have a severe impact on performance. We suggest + using a value from 5 to 20. + logging_interval: Specifies a period in which the instance logs its state, in seconds. + desired_concurrency: The desired number of tasks that should be running parallel on the start of the pool, if + there is a large enough supply of them. By default, it is `min_concurrency`. + desired_concurrency_ratio: Minimum level of desired concurrency to reach before more scaling up is allowed. + min_concurrency: The minimum number of tasks running in parallel. If you set this value too high with respect + to the available system memory and CPU, your code might run extremely slow or crash. + max_concurrency: The maximum number of tasks running in parallel. + scale_up_step_ratio: Defines the fractional amount of desired concurrency to be added with each scaling up. + scale_down_step_ratio: Defines the amount of desired concurrency to be subtracted with each scaling down. + max_tasks_per_minute: The maximum number of tasks per minute the pool can run. By default, this is set to + `Infinity`, but you can pass any positive, non-zero integer. + """ self._system_status = system_status self._run_task_function = run_task_function From 287e66e739edb785691e8c70ebe6bca494ed1fc2 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 26 Feb 2024 11:30:50 +0100 Subject: [PATCH 18/23] Better AutoscaledPool docblock formatting --- src/crawlee/autoscaling/autoscaled_pool.py | 69 +++++++++++++--------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 805e4844f0..8459980876 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -44,34 +44,47 @@ def __init__( """Initialize the AutoscaledPool. Args: - system_status: Provides data about system utilization (load). - run_task_function: A function that performs an asynchronous resource-intensive task. - is_task_ready_function: A function that indicates whether `run_task_function` should be called. This function - is called every time there is free capacity for a new task and it should indicate whether it should start - a new task or not by resolving to either `True` or `False`. Besides its obvious use, it is also useful for - task throttling to save resources. is_finished_function: A function that is called only when there are no - tasks to be processed. If it resolves to `True` then the pool's run finishes. Being called only when there - are no tasks being processed means that as long as `is_task_ready_function()` keeps resolving to `True`, - `is_finished_function()` will never be called. To abort a run, use the `abort` method. - is_finished_function: A function that is called only when there are no tasks to be processed. If it resolves to - `true` then the pool's run finishes. Being called only when there are no tasks being processed means that - as long as `isTaskReadyFunction()` keeps resolving to `true`, `isFinishedFunction()` will never be called. - To abort a run, use the `abort` method. - task_timeout: Timeout in which the `run_task_function` needs to finish. - autoscale_interval: Defines how often the pool should attempt to adjust the desired concurrency based on - the latest system status. Setting it lower than 1 might have a severe impact on performance. We suggest - using a value from 5 to 20. - logging_interval: Specifies a period in which the instance logs its state, in seconds. - desired_concurrency: The desired number of tasks that should be running parallel on the start of the pool, if - there is a large enough supply of them. By default, it is `min_concurrency`. - desired_concurrency_ratio: Minimum level of desired concurrency to reach before more scaling up is allowed. - min_concurrency: The minimum number of tasks running in parallel. If you set this value too high with respect - to the available system memory and CPU, your code might run extremely slow or crash. - max_concurrency: The maximum number of tasks running in parallel. - scale_up_step_ratio: Defines the fractional amount of desired concurrency to be added with each scaling up. - scale_down_step_ratio: Defines the amount of desired concurrency to be subtracted with each scaling down. - max_tasks_per_minute: The maximum number of tasks per minute the pool can run. By default, this is set to - `Infinity`, but you can pass any positive, non-zero integer. + system_status: Provides data about system utilization (load). + + run_task_function: A function that performs an asynchronous resource-intensive task. + + is_task_ready_function: A function that indicates whether `run_task_function` should be called. This + function is called every time there is free capacity for a new task and it should indicate whether it should + start a new task or not by resolving to either `True` or `False`. Besides its obvious use, it is also useful + for task throttling to save resources. is_finished_function: A function that is called only when there are + no tasks to be processed. If it resolves to `True` then the pool's run finishes. Being called only when + there are no tasks being processed means that as long as `is_task_ready_function()` keeps resolving to + `True`, `is_finished_function()` will never be called. To abort a run, use the `abort` method. + + is_finished_function: A function that is called only when there are no tasks to be processed. If it resolves + to `true` then the pool's run finishes. Being called only when there are no tasks being processed means + that as long as `isTaskReadyFunction()` keeps resolving to `true`, `isFinishedFunction()` will never + be called. To abort a run, use the `abort` method. + + task_timeout: Timeout in which the `run_task_function` needs to finish. + + autoscale_interval: Defines how often the pool should attempt to adjust the desired concurrency based on + the latest system status. Setting it lower than 1 might have a severe impact on performance. We suggest + using a value from 5 to 20. + + logging_interval: Specifies a period in which the instance logs its state, in seconds. + + desired_concurrency: The desired number of tasks that should be running parallel on the start of the pool, + if there is a large enough supply of them. By default, it is `min_concurrency`. + + desired_concurrency_ratio: Minimum level of desired concurrency to reach before more scaling up is allowed. + + min_concurrency: The minimum number of tasks running in parallel. If you set this value too high with + respect to the available system memory and CPU, your code might run extremely slow or crash. + + max_concurrency: The maximum number of tasks running in parallel. + + scale_up_step_ratio: Defines the fractional amount of desired concurrency to be added with each scaling up. + + scale_down_step_ratio: Defines the amount of desired concurrency to be subtracted with each scaling down. + + max_tasks_per_minute: The maximum number of tasks per minute the pool can run. By default, this is set to + `Infinity`, but you can pass any positive, non-zero integer. """ self._system_status = system_status From 7d846ba5374e1f595ee5d307359c9cdab761cb8d Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Mon, 26 Feb 2024 23:35:31 +0100 Subject: [PATCH 19/23] Simplify the concurrency model --- src/crawlee/autoscaling/autoscaled_pool.py | 201 ++++++++++----------- 1 file changed, 93 insertions(+), 108 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 8459980876..75362580f3 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -1,5 +1,4 @@ # Inspiration: https://github.com/apify/crawlee/blob/v3.7.3/packages/core/src/autoscaling/autoscaled_pool.ts - from __future__ import annotations import asyncio @@ -99,11 +98,11 @@ def __init__( self._autoscale_task = RecurringTask(self._autoscale, autoscale_interval) - self._worker_tasks = list[tuple[asyncio.Task, asyncio.Event]]() - """A list of workers tasks and events to signal that the task should terminate""" + self._worker_tasks = list[asyncio.Task]() + """A list of worker tasks currently in progress""" self._worker_tasks_updated = asyncio.Event() - self._tasks_for_cleanup = list[asyncio.Task]() + self._run_result = asyncio.Future() if desired_concurrency is not None and desired_concurrency < 1: raise ValueError('desired_concurrency must be 1 or larger') @@ -122,76 +121,61 @@ def __init__( self._max_concurrency = max_concurrency self._min_concurrency = min_concurrency - self._current_concurrency = 0 - self._scale_up_step_ratio = scale_up_step_ratio self._scale_down_step_ratio = scale_down_step_ratio self._max_tasks_per_minute = max_tasks_per_minute self._is_paused = False + self._is_running = False async def run(self: AutoscaledPool) -> None: """Start the autoscaled pool and return when all tasks are completed and `is_finished_function` returns True. If there is an exception in one of the tasks, it will be re-raised. """ - self._ensure_desired_concurrency() + logger.debug('Starting the pool') + + self._is_running = True + self._autoscale_task.start() self._log_system_status_task.start() - logger.debug('Starting the pool') + result = asyncio.Future() + orchestrator = asyncio.create_task( + self._worker_task_orchestrator(result), name='autoscaled pool worker task orchestrator' + ) try: - while not self._is_finished_function(): - wait_for_workers_update = asyncio.create_task( - self._worker_tasks_updated.wait(), name='wait for worker tasks update' - ) - wait_for_worker_tasks = asyncio.create_task( - asyncio.wait([task for task, _ in self._worker_tasks], return_when=asyncio.FIRST_EXCEPTION), - name='wait for worker tasks to complete', - ) - - self._worker_tasks_updated.clear() - - try: - await asyncio.wait( - [wait_for_workers_update, wait_for_worker_tasks], - return_when=asyncio.FIRST_COMPLETED, - ) - finally: - if not wait_for_worker_tasks.done(): - wait_for_worker_tasks.cancel() - - if not wait_for_workers_update.done(): - wait_for_workers_update.cancel() - - for task, _ in self._worker_tasks: - if task.done(): - exception = task.exception() - if exception is not None: - raise exception + await result finally: with suppress(asyncio.CancelledError): await self._autoscale_task.stop() with suppress(asyncio.CancelledError): await self._log_system_status_task.stop() - self._desired_concurrency = 0 - self._ensure_desired_concurrency() + if not orchestrator.done(): + orchestrator.cancel() + elif not orchestrator.cancelled() and orchestrator.exception() is not None: + logger.error('Exception in worker task orchestrator', exc_info=orchestrator.exception()) + + logger.info('Waiting for remaining tasks to finish') - for task in self._tasks_for_cleanup: + for task in self._worker_tasks: if not task.done(): - with suppress(asyncio.CancelledError): + with suppress(BaseException): await task + self._is_running = False logger.debug('Pool cleanup finished') async def abort(self: AutoscaledPool) -> None: """Interrupt the autoscaled pool and all the tasks in progress.""" self._is_paused = True - await self._autoscale_task.stop() - for task, _ in self._worker_tasks: + with suppress(asyncio.CancelledError): + await self._autoscale_task.stop() + + for task in self._worker_tasks: task.cancel() def pause(self: AutoscaledPool) -> None: @@ -207,86 +191,87 @@ def desired_concurrency(self: AutoscaledPool) -> int: """The current desired concurrency, possibly updated by the pool according to system load.""" return self._desired_concurrency + @property + def current_concurrency(self: AutoscaledPool) -> int: + """The number of concurrent tasks in progress.""" + return len(self._worker_tasks) + def _autoscale(self: AutoscaledPool) -> None: status = self._system_status.get_historical_status() - min_current_concurrency = math.floor(self._desired_concurrency_ratio * self._current_concurrency) - if ( + min_current_concurrency = math.floor(self._desired_concurrency_ratio * self.current_concurrency) + should_scale_up = ( status.is_system_idle and self._desired_concurrency < self._max_concurrency - and self._current_concurrency >= min_current_concurrency - ): + and self.current_concurrency >= min_current_concurrency + ) + + should_scale_down = not status.is_system_idle and self._desired_concurrency > self._min_concurrency + + if should_scale_up: step = math.ceil(self._scale_up_step_ratio * self._desired_concurrency) self._desired_concurrency = min(self._max_concurrency, self._desired_concurrency + step) - elif not status.is_system_idle and self._desired_concurrency > self._min_concurrency: + elif should_scale_down: step = math.ceil(self._scale_down_step_ratio * self._desired_concurrency) self._desired_concurrency = max(self._min_concurrency, self._desired_concurrency - step) - self._ensure_desired_concurrency() - - def _ensure_desired_concurrency(self: AutoscaledPool) -> None: - if len(self._worker_tasks) > self._desired_concurrency: - for _ in range(len(self._worker_tasks) - self._desired_concurrency): - task, terminate_event = self._worker_tasks.pop() - self._mark_task_for_cleanup(task) - terminate_event.set() - self._worker_tasks_updated.set() - - elif len(self._worker_tasks) < self._desired_concurrency: - for i in range(len(self._worker_tasks), self._desired_concurrency): - terminate_event = asyncio.Event() - task = asyncio.create_task(self._worker_task(terminate_event), name=f'worker task #{i + 1}') - self._worker_tasks.append((task, terminate_event)) - self._worker_tasks_updated.set() - - def _mark_task_for_cleanup(self: AutoscaledPool, task: asyncio.Task) -> None: - def cleanup(_: asyncio.Future) -> None: - task.exception() - self._tasks_for_cleanup.remove(task) - - self._tasks_for_cleanup.append(task) - task.add_done_callback(cleanup) - def _log_system_status(self: AutoscaledPool) -> None: system_status = self._system_status.get_historical_status() logger.info( - f'current_concurrency = {self._current_concurrency}; ' - f'desired_concurrency = {self._desired_concurrency}; ' + f'current_concurrency = {self.current_concurrency}; ' + f'desired_concurrency = {self.desired_concurrency}; ' f'{system_status!s}' ) - async def _worker_task(self: AutoscaledPool, terminate_event: asyncio.Event) -> None: - while not self._is_finished_function(): - if self._max_tasks_per_minute is not None and self._desired_concurrency > 0: - delay = 60 / self._max_tasks_per_minute / self._desired_concurrency - else: - delay = 0 - - with suppress(asyncio.TimeoutError): - await asyncio.wait_for(terminate_event.wait(), delay) - - if terminate_event.is_set() or self._is_finished_function(): - break - - if self._is_paused: - logger.debug('Paused - not executing a task') - continue - - if not self._is_task_ready_function(): - logger.debug('No task is ready yet') - continue - - self._current_concurrency += 1 - try: - await asyncio.wait_for( - self._run_task_function(), - timeout=self._task_timeout.total_seconds() if self._task_timeout is not None else None, - ) - except asyncio.TimeoutError: - timeout_str = self._task_timeout.total_seconds() if self._task_timeout is not None else '*not set*' - logger.warning(f'Task timed out after {timeout_str} seconds') - finally: - self._current_concurrency -= 1 - - logger.debug('Worker task finished') + async def _worker_task_orchestrator(self: AutoscaledPool, result: asyncio.Future) -> None: + try: + while not self._is_finished_function() and not result.done(): + self._worker_tasks_updated.clear() + + current_status = self._system_status.get_current_status() + if not current_status.is_system_idle: + logger.debug('Not scheduling new tasks - system is overloaded') + elif self._is_paused: + logger.debug('Not scheduling new tasks - the autoscaled pool is paused') + elif self.current_concurrency >= self.desired_concurrency: + logger.debug('Not scheduling new tasks - already running at desired concurrency') + elif not self._is_task_ready_function(): + logger.debug('Not scheduling new task - no task is ready') + else: + logger.debug('Scheduling a new task') + worker_task = asyncio.create_task(self._worker_task(), name='autoscaled pool worker task') + worker_task.add_done_callback( + lambda _, worker_task=worker_task: self._reap_worker_task(worker_task, result) + ) + self._worker_tasks.append(worker_task) + + if self._max_tasks_per_minute is not None: + await asyncio.sleep(60 / self._max_tasks_per_minute) + + continue + + with suppress(asyncio.TimeoutError): + await asyncio.wait_for(self._worker_tasks_updated.wait(), timeout=0.5) + finally: + if not (result.done() and result.exception() is not None): + result.set_result(object()) + + def _reap_worker_task(self: AutoscaledPool, task: asyncio.Task, result: asyncio.Future) -> None: + self._worker_tasks_updated.set() + self._worker_tasks.remove(task) + + if not task.cancelled() and (exception := task.exception()) and not result.done(): + result.set_exception(exception) + + async def _worker_task(self: AutoscaledPool) -> None: + try: + await asyncio.wait_for( + self._run_task_function(), + timeout=self._task_timeout.total_seconds() if self._task_timeout is not None else None, + ) + except asyncio.TimeoutError: + timeout_str = self._task_timeout.total_seconds() if self._task_timeout is not None else '*not set*' + logger.warning(f'Task timed out after {timeout_str} seconds') + finally: + logger.debug('Worker task finished') From ea9bd60021b27019d3f5e58235fd138b4724fcd2 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 27 Feb 2024 10:18:04 +0100 Subject: [PATCH 20/23] Fix AutoscaledPool.abort() --- src/crawlee/autoscaling/autoscaled_pool.py | 47 +++++++++++-------- .../unit/autoscaling/test_autoscaled_pool.py | 25 ++++++++++ 2 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 75362580f3..770f963d83 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -16,6 +16,10 @@ logger = getLogger(__name__) +class AbortError(Exception): + """Raised when an AutoscaledPool run is aborted. Not for direct use.""" + + class AutoscaledPool: """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel. @@ -102,6 +106,7 @@ def __init__( """A list of worker tasks currently in progress""" self._worker_tasks_updated = asyncio.Event() + self._cleanup_done = asyncio.Event() self._run_result = asyncio.Future() if desired_concurrency is not None and desired_concurrency < 1: @@ -133,20 +138,26 @@ async def run(self: AutoscaledPool) -> None: If there is an exception in one of the tasks, it will be re-raised. """ - logger.debug('Starting the pool') + if self._is_running: + raise RuntimeError('The pool is already running') self._is_running = True + logger.debug('Starting the pool') self._autoscale_task.start() self._log_system_status_task.start() - result = asyncio.Future() orchestrator = asyncio.create_task( - self._worker_task_orchestrator(result), name='autoscaled pool worker task orchestrator' + self._worker_task_orchestrator(), name='autoscaled pool worker task orchestrator' ) try: - await result + await self._run_result + except AbortError: + orchestrator.cancel() + for task in self._worker_tasks: + if not task.done(): + task.cancel() finally: with suppress(asyncio.CancelledError): await self._autoscale_task.stop() @@ -165,18 +176,16 @@ async def run(self: AutoscaledPool) -> None: with suppress(BaseException): await task + self._run_result = asyncio.Future() + self._cleanup_done.set() self._is_running = False + logger.debug('Pool cleanup finished') async def abort(self: AutoscaledPool) -> None: """Interrupt the autoscaled pool and all the tasks in progress.""" - self._is_paused = True - - with suppress(asyncio.CancelledError): - await self._autoscale_task.stop() - - for task in self._worker_tasks: - task.cancel() + self._run_result.set_exception(AbortError()) + await self._cleanup_done.wait() def pause(self: AutoscaledPool) -> None: """Pause the autoscaled pool so that it does not start new tasks.""" @@ -224,9 +233,9 @@ def _log_system_status(self: AutoscaledPool) -> None: f'{system_status!s}' ) - async def _worker_task_orchestrator(self: AutoscaledPool, result: asyncio.Future) -> None: + async def _worker_task_orchestrator(self: AutoscaledPool) -> None: try: - while not self._is_finished_function() and not result.done(): + while not self._is_finished_function() and not self._run_result.done(): self._worker_tasks_updated.clear() current_status = self._system_status.get_current_status() @@ -242,7 +251,7 @@ async def _worker_task_orchestrator(self: AutoscaledPool, result: asyncio.Future logger.debug('Scheduling a new task') worker_task = asyncio.create_task(self._worker_task(), name='autoscaled pool worker task') worker_task.add_done_callback( - lambda _, worker_task=worker_task: self._reap_worker_task(worker_task, result) + lambda _, worker_task=worker_task: self._reap_worker_task(worker_task) ) self._worker_tasks.append(worker_task) @@ -254,15 +263,15 @@ async def _worker_task_orchestrator(self: AutoscaledPool, result: asyncio.Future with suppress(asyncio.TimeoutError): await asyncio.wait_for(self._worker_tasks_updated.wait(), timeout=0.5) finally: - if not (result.done() and result.exception() is not None): - result.set_result(object()) + if not (self._run_result.done() and self._run_result.exception() is not None): + self._run_result.set_result(object()) - def _reap_worker_task(self: AutoscaledPool, task: asyncio.Task, result: asyncio.Future) -> None: + def _reap_worker_task(self: AutoscaledPool, task: asyncio.Task) -> None: self._worker_tasks_updated.set() self._worker_tasks.remove(task) - if not task.cancelled() and (exception := task.exception()) and not result.done(): - result.set_exception(exception) + if not task.cancelled() and (exception := task.exception()) and not self._run_result.done(): + self._run_result.set_exception(exception) async def _worker_task(self: AutoscaledPool) -> None: try: diff --git a/tests/unit/autoscaling/test_autoscaled_pool.py b/tests/unit/autoscaling/test_autoscaled_pool.py index 110d355aa7..c62a98716f 100644 --- a/tests/unit/autoscaling/test_autoscaled_pool.py +++ b/tests/unit/autoscaling/test_autoscaled_pool.py @@ -47,6 +47,31 @@ async def run() -> None: assert done_count >= 10 +async def test_abort_works(system_status: SystemStatus | Mock) -> None: + async def run() -> None: + await asyncio.sleep(60) + + pool = AutoscaledPool( + system_status=system_status, + run_task_function=run, + is_task_ready_function=lambda: True, + is_finished_function=lambda: False, + min_concurrency=10, + max_concurrency=10, + ) + + with measure_time() as elapsed: + run_task = asyncio.create_task(pool.run()) + await asyncio.sleep(0.1) + assert pool.current_concurrency == 10 + await pool.abort() + assert pool.current_concurrency == 0 + await run_task + + assert elapsed.wall is not None + assert elapsed.wall < 0.3 + + async def test_propagates_exceptions(system_status: SystemStatus | Mock) -> None: done_count = 0 From 27e3d56b4e74e540fdb95a3ee413ec79aa17b423 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 27 Feb 2024 10:25:51 +0100 Subject: [PATCH 21/23] Add documentation --- src/crawlee/autoscaling/autoscaled_pool.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 770f963d83..1ceac26985 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -23,7 +23,8 @@ class AbortError(Exception): class AutoscaledPool: """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel. - The pool only starts new tasks if there is enough free CPU and memory available. + The pool only starts new tasks if there is enough free CPU and memory available. If an exception is thrown in + any of the tasks, it is propagated and the pool is stopped. """ def __init__( @@ -206,6 +207,7 @@ def current_concurrency(self: AutoscaledPool) -> int: return len(self._worker_tasks) def _autoscale(self: AutoscaledPool) -> None: + """Inspect system load status and adjust desired concurrency if necessary. Do not call directly.""" status = self._system_status.get_historical_status() min_current_concurrency = math.floor(self._desired_concurrency_ratio * self.current_concurrency) @@ -234,6 +236,10 @@ def _log_system_status(self: AutoscaledPool) -> None: ) async def _worker_task_orchestrator(self: AutoscaledPool) -> None: + """Launches worker tasks whenever there is free capacity and a task is ready. + + Exits when `is_finished_function` returns True. + """ try: while not self._is_finished_function() and not self._run_result.done(): self._worker_tasks_updated.clear() @@ -267,6 +273,12 @@ async def _worker_task_orchestrator(self: AutoscaledPool) -> None: self._run_result.set_result(object()) def _reap_worker_task(self: AutoscaledPool, task: asyncio.Task) -> None: + """A callback for finished worker tasks. + + - It interrupts the run in case of an exception, + - keeps track of tasks in progress, + - notifies the orchestrator + """ self._worker_tasks_updated.set() self._worker_tasks.remove(task) From f1396b204b56496fabb69f8d70d397de8d4b70fc Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 27 Feb 2024 10:34:06 +0100 Subject: [PATCH 22/23] Fix mypy issues --- src/crawlee/autoscaling/autoscaled_pool.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 1ceac26985..7eb3b315ad 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -108,7 +108,7 @@ def __init__( self._worker_tasks_updated = asyncio.Event() self._cleanup_done = asyncio.Event() - self._run_result = asyncio.Future() + self._run_result: asyncio.Future = asyncio.Future() if desired_concurrency is not None and desired_concurrency < 1: raise ValueError('desired_concurrency must be 1 or larger') @@ -256,9 +256,7 @@ async def _worker_task_orchestrator(self: AutoscaledPool) -> None: else: logger.debug('Scheduling a new task') worker_task = asyncio.create_task(self._worker_task(), name='autoscaled pool worker task') - worker_task.add_done_callback( - lambda _, worker_task=worker_task: self._reap_worker_task(worker_task) - ) + worker_task.add_done_callback(lambda task: self._reap_worker_task(task)) self._worker_tasks.append(worker_task) if self._max_tasks_per_minute is not None: From 5e1ed6264d074033939ba61199aeb59e309b26a4 Mon Sep 17 00:00:00 2001 From: Jan Buchar Date: Tue, 27 Feb 2024 15:35:13 +0100 Subject: [PATCH 23/23] Apply suggestions from code review Co-authored-by: Vlada Dusek --- src/crawlee/autoscaling/autoscaled_pool.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/crawlee/autoscaling/autoscaled_pool.py b/src/crawlee/autoscaling/autoscaled_pool.py index 7eb3b315ad..ce6b094766 100644 --- a/src/crawlee/autoscaling/autoscaled_pool.py +++ b/src/crawlee/autoscaling/autoscaled_pool.py @@ -1,4 +1,5 @@ # Inspiration: https://github.com/apify/crawlee/blob/v3.7.3/packages/core/src/autoscaling/autoscaled_pool.ts + from __future__ import annotations import asyncio @@ -53,17 +54,18 @@ def __init__( run_task_function: A function that performs an asynchronous resource-intensive task. is_task_ready_function: A function that indicates whether `run_task_function` should be called. This - function is called every time there is free capacity for a new task and it should indicate whether it should - start a new task or not by resolving to either `True` or `False`. Besides its obvious use, it is also useful - for task throttling to save resources. is_finished_function: A function that is called only when there are - no tasks to be processed. If it resolves to `True` then the pool's run finishes. Being called only when - there are no tasks being processed means that as long as `is_task_ready_function()` keeps resolving to - `True`, `is_finished_function()` will never be called. To abort a run, use the `abort` method. + function is called every time there is free capacity for a new task and it should indicate whether + it should start a new task or not by resolving to either `True` or `False`. Besides its obvious use, + it is also useful for task throttling to save resources. is_finished_function: A function that is + called only when there are no tasks to be processed. If it resolves to `True` then the pool's run + finishes. Being called only when there are no tasks being processed means that as long as + `is_task_ready_function()` keeps resolving to `True`, `is_finished_function()` will never be called. + To abort a run, use the `abort` method. is_finished_function: A function that is called only when there are no tasks to be processed. If it resolves - to `true` then the pool's run finishes. Being called only when there are no tasks being processed means - that as long as `isTaskReadyFunction()` keeps resolving to `true`, `isFinishedFunction()` will never - be called. To abort a run, use the `abort` method. + to `true` then the pool's run finishes. Being called only when there are no tasks being processed means + that as long as `isTaskReadyFunction()` keeps resolving to `true`, `isFinishedFunction()` will never + be called. To abort a run, use the `abort` method. task_timeout: Timeout in which the `run_task_function` needs to finish.