Repository navigation
feat: AutoscaledPool implementation #55
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
24 commits
Select commit
Hold shift + click to select a range
403816f
Initial AutoscaledPool implementation
janbuchar 355aaac
Initial tests
janbuchar e1cc456
Implement SystemInfo-driven autoscaling
janbuchar 3ff330d
Implement max_tasks_per_minute
janbuchar e0efcb4
Add logging
janbuchar 9077d39
Check current concurrency when scaling up
janbuchar c3b6191
Bug
janbuchar 2575749
Add system status to periodic log message
janbuchar 7d0ec6e
Avoid interrupting tasks when scaling down
janbuchar 0f34f2c
Add doc and unit test for measure_time
janbuchar 23f7cf4
Less strict measure_time tests
janbuchar 49e571a
Apply suggestions from code review
janbuchar 56326fc
Update tests/unit/autoscaling/test_autoscaled_pool.py
janbuchar c4d59a9
Merge remote-tracking branch 'origin/master' into autoscaled-pool
janbuchar c0ef864
Exploit default value of SystemInfo.created_at
janbuchar 7a4eccf
Document __init__ args
janbuchar b724e7a
Remove unnecessary MagicMock
janbuchar 284c4e9
Put argument descriptions where they belong
janbuchar 287e66e
Better AutoscaledPool docblock formatting
janbuchar 7d846ba
Simplify the concurrency model
janbuchar ea9bd60
Fix AutoscaledPool.abort()
janbuchar 27e3d56
Add documentation
janbuchar f1396b2
Fix mypy issues
janbuchar 5e1ed62
Apply suggestions from code review
janbuchar File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| from __future__ import annotations | ||
|
|
||
| import time | ||
| from contextlib import contextmanager | ||
| from dataclasses import dataclass | ||
| from typing import Iterator | ||
|
|
||
|
|
||
| @dataclass | ||
| class TimerResult: | ||
|
janbuchar marked this conversation as resolved.
Resolved
|
||
| wall: float | None = None | ||
| cpu: float | None = None | ||
|
|
||
|
|
||
| @contextmanager | ||
| def measure_time() -> Iterator[TimerResult]: | ||
|
janbuchar marked this conversation as resolved.
Resolved
|
||
| """Measure the execution time (wall-clock and CPU) between the start and end of the with-block.""" | ||
| result = TimerResult() | ||
| before_wall = time.monotonic() | ||
| before_cpu = time.thread_time() | ||
|
|
||
| try: | ||
| yield result | ||
| finally: | ||
| after_wall = time.monotonic() | ||
| after_cpu = time.thread_time() | ||
| result.wall = after_wall - before_wall | ||
| result.cpu = after_cpu - before_cpu | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,298 @@ | ||
| # Inspiration: https://github.com/apify/crawlee/blob/v3.7.3/packages/core/src/autoscaling/autoscaled_pool.ts | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import asyncio | ||
| import math | ||
| from contextlib import suppress | ||
| from datetime import timedelta | ||
| from logging import getLogger | ||
| from typing import TYPE_CHECKING, Awaitable, Callable | ||
|
|
||
| from crawlee._utils.recurring_task import RecurringTask | ||
|
|
||
| if TYPE_CHECKING: | ||
| from crawlee.autoscaling.system_status import SystemStatus | ||
|
|
||
| logger = getLogger(__name__) | ||
|
|
||
|
|
||
| class AbortError(Exception): | ||
| """Raised when an AutoscaledPool run is aborted. Not for direct use.""" | ||
|
|
||
|
|
||
| class AutoscaledPool: | ||
| """Manages a pool of asynchronous resource-intensive tasks that are executed in parallel. | ||
|
|
||
| The pool only starts new tasks if there is enough free CPU and memory available. If an exception is thrown in | ||
| any of the tasks, it is propagated and the pool is stopped. | ||
| """ | ||
|
|
||
| def __init__( | ||
| self: AutoscaledPool, | ||
| *, | ||
| system_status: SystemStatus, | ||
| run_task_function: Callable[..., Awaitable], | ||
| is_task_ready_function: Callable[..., bool], | ||
|
janbuchar marked this conversation as resolved.
Resolved
|
||
| is_finished_function: Callable[..., bool], | ||
| task_timeout: timedelta | None = None, | ||
| autoscale_interval: timedelta = timedelta(seconds=10), | ||
| logging_interval: timedelta = timedelta(minutes=1), | ||
| desired_concurrency: int | None = None, | ||
| desired_concurrency_ratio: float = 0.9, | ||
| min_concurrency: int = 1, | ||
| max_concurrency: int = 200, | ||
| scale_up_step_ratio: float = 0.05, | ||
| scale_down_step_ratio: float = 0.05, | ||
| max_tasks_per_minute: int | None = None, | ||
| ) -> None: | ||
| """Initialize the AutoscaledPool. | ||
|
|
||
| Args: | ||
| system_status: Provides data about system utilization (load). | ||
|
|
||
| run_task_function: A function that performs an asynchronous resource-intensive task. | ||
|
|
||
| is_task_ready_function: A function that indicates whether `run_task_function` should be called. This | ||
| function is called every time there is free capacity for a new task and it should indicate whether | ||
| it should start a new task or not by resolving to either `True` or `False`. Besides its obvious use, | ||
| it is also useful for task throttling to save resources. is_finished_function: A function that is | ||
| called only when there are no tasks to be processed. If it resolves to `True` then the pool's run | ||
| finishes. Being called only when there are no tasks being processed means that as long as | ||
| `is_task_ready_function()` keeps resolving to `True`, `is_finished_function()` will never be called. | ||
| To abort a run, use the `abort` method. | ||
|
|
||
| is_finished_function: A function that is called only when there are no tasks to be processed. If it resolves | ||
| to `true` then the pool's run finishes. Being called only when there are no tasks being processed means | ||
| that as long as `isTaskReadyFunction()` keeps resolving to `true`, `isFinishedFunction()` will never | ||
| be called. To abort a run, use the `abort` method. | ||
|
|
||
| task_timeout: Timeout in which the `run_task_function` needs to finish. | ||
|
|
||
| autoscale_interval: Defines how often the pool should attempt to adjust the desired concurrency based on | ||
| the latest system status. Setting it lower than 1 might have a severe impact on performance. We suggest | ||
| using a value from 5 to 20. | ||
|
|
||
| logging_interval: Specifies a period in which the instance logs its state, in seconds. | ||
|
|
||
| desired_concurrency: The desired number of tasks that should be running parallel on the start of the pool, | ||
| if there is a large enough supply of them. By default, it is `min_concurrency`. | ||
|
|
||
| desired_concurrency_ratio: Minimum level of desired concurrency to reach before more scaling up is allowed. | ||
|
|
||
| min_concurrency: The minimum number of tasks running in parallel. If you set this value too high with | ||
| respect to the available system memory and CPU, your code might run extremely slow or crash. | ||
|
|
||
| max_concurrency: The maximum number of tasks running in parallel. | ||
|
|
||
| scale_up_step_ratio: Defines the fractional amount of desired concurrency to be added with each scaling up. | ||
|
|
||
| scale_down_step_ratio: Defines the amount of desired concurrency to be subtracted with each scaling down. | ||
|
|
||
| max_tasks_per_minute: The maximum number of tasks per minute the pool can run. By default, this is set to | ||
| `Infinity`, but you can pass any positive, non-zero integer. | ||
| """ | ||
| self._system_status = system_status | ||
|
|
||
| self._run_task_function = run_task_function | ||
| self._is_task_ready_function = is_task_ready_function | ||
| self._is_finished_function = is_finished_function | ||
|
|
||
| self._task_timeout = task_timeout | ||
|
|
||
| self._logging_interval = logging_interval | ||
| self._log_system_status_task = RecurringTask(self._log_system_status, logging_interval) | ||
|
|
||
| self._autoscale_task = RecurringTask(self._autoscale, autoscale_interval) | ||
|
|
||
| self._worker_tasks = list[asyncio.Task]() | ||
| """A list of worker tasks currently in progress""" | ||
|
vdusek marked this conversation as resolved.
Resolved
|
||
|
|
||
| self._worker_tasks_updated = asyncio.Event() | ||
| self._cleanup_done = asyncio.Event() | ||
| self._run_result: asyncio.Future = asyncio.Future() | ||
|
|
||
| if desired_concurrency is not None and desired_concurrency < 1: | ||
| raise ValueError('desired_concurrency must be 1 or larger') | ||
|
|
||
| if min_concurrency < 1: | ||
| raise ValueError('min_concurrency must be 1 or larger') | ||
|
|
||
| if max_concurrency < min_concurrency: | ||
| raise ValueError('max_concurrency cannot be less than min_concurrency') | ||
|
|
||
| if desired_concurrency_ratio < 0 or desired_concurrency_ratio > 1: | ||
| raise ValueError('desired_concurrency_ratio must be between 0 and 1 (non-inclusive)') | ||
|
|
||
| self._desired_concurrency_ratio = desired_concurrency_ratio | ||
| self._desired_concurrency = desired_concurrency if desired_concurrency is not None else min_concurrency | ||
| self._max_concurrency = max_concurrency | ||
| self._min_concurrency = min_concurrency | ||
|
|
||
| self._scale_up_step_ratio = scale_up_step_ratio | ||
| self._scale_down_step_ratio = scale_down_step_ratio | ||
|
|
||
| self._max_tasks_per_minute = max_tasks_per_minute | ||
| self._is_paused = False | ||
| self._is_running = False | ||
|
|
||
| async def run(self: AutoscaledPool) -> None: | ||
| """Start the autoscaled pool and return when all tasks are completed and `is_finished_function` returns True. | ||
|
|
||
| If there is an exception in one of the tasks, it will be re-raised. | ||
| """ | ||
| if self._is_running: | ||
| raise RuntimeError('The pool is already running') | ||
|
|
||
| self._is_running = True | ||
| logger.debug('Starting the pool') | ||
|
|
||
| self._autoscale_task.start() | ||
| self._log_system_status_task.start() | ||
|
|
||
| orchestrator = asyncio.create_task( | ||
| self._worker_task_orchestrator(), name='autoscaled pool worker task orchestrator' | ||
| ) | ||
|
|
||
| try: | ||
| await self._run_result | ||
| except AbortError: | ||
| orchestrator.cancel() | ||
| for task in self._worker_tasks: | ||
| if not task.done(): | ||
| task.cancel() | ||
| finally: | ||
| with suppress(asyncio.CancelledError): | ||
| await self._autoscale_task.stop() | ||
| with suppress(asyncio.CancelledError): | ||
| await self._log_system_status_task.stop() | ||
|
janbuchar marked this conversation as resolved.
Resolved
|
||
|
|
||
| if not orchestrator.done(): | ||
| orchestrator.cancel() | ||
| elif not orchestrator.cancelled() and orchestrator.exception() is not None: | ||
| logger.error('Exception in worker task orchestrator', exc_info=orchestrator.exception()) | ||
|
|
||
| logger.info('Waiting for remaining tasks to finish') | ||
|
|
||
| for task in self._worker_tasks: | ||
| if not task.done(): | ||
| with suppress(BaseException): | ||
| await task | ||
|
|
||
| self._run_result = asyncio.Future() | ||
| self._cleanup_done.set() | ||
| self._is_running = False | ||
|
|
||
| logger.debug('Pool cleanup finished') | ||
|
|
||
| async def abort(self: AutoscaledPool) -> None: | ||
| """Interrupt the autoscaled pool and all the tasks in progress.""" | ||
| self._run_result.set_exception(AbortError()) | ||
| await self._cleanup_done.wait() | ||
|
|
||
| def pause(self: AutoscaledPool) -> None: | ||
| """Pause the autoscaled pool so that it does not start new tasks.""" | ||
| self._is_paused = True | ||
|
|
||
| def resume(self: AutoscaledPool) -> None: | ||
| """Resume a paused autoscaled pool so that it continues starting new tasks.""" | ||
| self._is_paused = False | ||
|
|
||
| @property | ||
| def desired_concurrency(self: AutoscaledPool) -> int: | ||
| """The current desired concurrency, possibly updated by the pool according to system load.""" | ||
| return self._desired_concurrency | ||
|
vdusek marked this conversation as resolved.
Resolved
|
||
|
|
||
| @property | ||
| def current_concurrency(self: AutoscaledPool) -> int: | ||
| """The number of concurrent tasks in progress.""" | ||
| return len(self._worker_tasks) | ||
|
|
||
| def _autoscale(self: AutoscaledPool) -> None: | ||
| """Inspect system load status and adjust desired concurrency if necessary. Do not call directly.""" | ||
| status = self._system_status.get_historical_status() | ||
|
|
||
| min_current_concurrency = math.floor(self._desired_concurrency_ratio * self.current_concurrency) | ||
| should_scale_up = ( | ||
| status.is_system_idle | ||
| and self._desired_concurrency < self._max_concurrency | ||
| and self.current_concurrency >= min_current_concurrency | ||
| ) | ||
|
|
||
| should_scale_down = not status.is_system_idle and self._desired_concurrency > self._min_concurrency | ||
|
|
||
| if should_scale_up: | ||
| step = math.ceil(self._scale_up_step_ratio * self._desired_concurrency) | ||
| self._desired_concurrency = min(self._max_concurrency, self._desired_concurrency + step) | ||
| elif should_scale_down: | ||
| step = math.ceil(self._scale_down_step_ratio * self._desired_concurrency) | ||
| self._desired_concurrency = max(self._min_concurrency, self._desired_concurrency - step) | ||
|
|
||
| def _log_system_status(self: AutoscaledPool) -> None: | ||
| system_status = self._system_status.get_historical_status() | ||
|
|
||
| logger.info( | ||
| f'current_concurrency = {self.current_concurrency}; ' | ||
| f'desired_concurrency = {self.desired_concurrency}; ' | ||
| f'{system_status!s}' | ||
| ) | ||
|
|
||
| async def _worker_task_orchestrator(self: AutoscaledPool) -> None: | ||
| """Launches worker tasks whenever there is free capacity and a task is ready. | ||
|
|
||
| Exits when `is_finished_function` returns True. | ||
| """ | ||
| try: | ||
| while not self._is_finished_function() and not self._run_result.done(): | ||
| self._worker_tasks_updated.clear() | ||
|
|
||
| current_status = self._system_status.get_current_status() | ||
| if not current_status.is_system_idle: | ||
| logger.debug('Not scheduling new tasks - system is overloaded') | ||
| elif self._is_paused: | ||
| logger.debug('Not scheduling new tasks - the autoscaled pool is paused') | ||
| elif self.current_concurrency >= self.desired_concurrency: | ||
| logger.debug('Not scheduling new tasks - already running at desired concurrency') | ||
| elif not self._is_task_ready_function(): | ||
| logger.debug('Not scheduling new task - no task is ready') | ||
| else: | ||
| logger.debug('Scheduling a new task') | ||
| worker_task = asyncio.create_task(self._worker_task(), name='autoscaled pool worker task') | ||
| worker_task.add_done_callback(lambda task: self._reap_worker_task(task)) | ||
| self._worker_tasks.append(worker_task) | ||
|
|
||
| if self._max_tasks_per_minute is not None: | ||
| await asyncio.sleep(60 / self._max_tasks_per_minute) | ||
|
|
||
| continue | ||
|
|
||
| with suppress(asyncio.TimeoutError): | ||
| await asyncio.wait_for(self._worker_tasks_updated.wait(), timeout=0.5) | ||
| finally: | ||
| if not (self._run_result.done() and self._run_result.exception() is not None): | ||
| self._run_result.set_result(object()) | ||
|
|
||
| def _reap_worker_task(self: AutoscaledPool, task: asyncio.Task) -> None: | ||
| """A callback for finished worker tasks. | ||
|
|
||
| - It interrupts the run in case of an exception, | ||
| - keeps track of tasks in progress, | ||
| - notifies the orchestrator | ||
| """ | ||
| self._worker_tasks_updated.set() | ||
| self._worker_tasks.remove(task) | ||
|
|
||
| if not task.cancelled() and (exception := task.exception()) and not self._run_result.done(): | ||
| self._run_result.set_exception(exception) | ||
|
|
||
| async def _worker_task(self: AutoscaledPool) -> None: | ||
| try: | ||
| await asyncio.wait_for( | ||
| self._run_task_function(), | ||
| timeout=self._task_timeout.total_seconds() if self._task_timeout is not None else None, | ||
| ) | ||
| except asyncio.TimeoutError: | ||
| timeout_str = self._task_timeout.total_seconds() if self._task_timeout is not None else '*not set*' | ||
| logger.warning(f'Task timed out after {timeout_str} seconds') | ||
| finally: | ||
| logger.debug('Worker task finished') | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,39 @@ | ||
| import asyncio | ||
| import time | ||
|
|
||
| import pytest | ||
|
|
||
| from crawlee._utils.measure_time import measure_time | ||
|
|
||
|
|
||
| def test_measure_time_wall_sync() -> None: | ||
| with measure_time() as elapsed: | ||
| time.sleep(0.1) | ||
|
|
||
| assert elapsed.cpu is not None | ||
| assert elapsed.wall is not None | ||
| assert elapsed.wall >= 0.09 | ||
|
|
||
|
|
||
| def test_measure_time_cpu_sync() -> None: | ||
| with measure_time() as elapsed: | ||
| start = time.time() | ||
| acc = 0 | ||
|
|
||
| while time.time() - start < 0.1: | ||
| acc += 1 | ||
| acc *= acc | ||
|
|
||
| assert elapsed.cpu is not None | ||
| assert elapsed.wall is not None | ||
| assert elapsed.cpu >= 0.05 | ||
|
|
||
|
|
||
| @pytest.mark.asyncio() | ||
| async def test_measure_time_wall_async() -> None: | ||
| with measure_time() as elapsed: | ||
| await asyncio.sleep(0.1) | ||
|
|
||
| assert elapsed.cpu is not None | ||
| assert elapsed.wall is not None | ||
| assert elapsed.wall >= 0.09 |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.