Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
403816f
Initial AutoscaledPool implementation
janbuchar Feb 20, 2024
355aaac
Initial tests
janbuchar Feb 21, 2024
e1cc456
Implement SystemInfo-driven autoscaling
janbuchar Feb 21, 2024
3ff330d
Implement max_tasks_per_minute
janbuchar Feb 21, 2024
e0efcb4
Add logging
janbuchar Feb 22, 2024
9077d39
Check current concurrency when scaling up
janbuchar Feb 22, 2024
c3b6191
Bug
janbuchar Feb 22, 2024
2575749
Add system status to periodic log message
janbuchar Feb 22, 2024
7d0ec6e
Avoid interrupting tasks when scaling down
janbuchar Feb 22, 2024
0f34f2c
Add doc and unit test for measure_time
janbuchar Feb 23, 2024
23f7cf4
Less strict measure_time tests
janbuchar Feb 23, 2024
49e571a
Apply suggestions from code review
janbuchar Feb 23, 2024
56326fc
Update tests/unit/autoscaling/test_autoscaled_pool.py
janbuchar Feb 23, 2024
c4d59a9
Merge remote-tracking branch 'origin/master' into autoscaled-pool
janbuchar Feb 23, 2024
c0ef864
Exploit default value of SystemInfo.created_at
janbuchar Feb 23, 2024
7a4eccf
Document __init__ args
janbuchar Feb 23, 2024
b724e7a
Remove unnecessary MagicMock
janbuchar Feb 23, 2024
284c4e9
Put argument descriptions where they belong
janbuchar Feb 23, 2024
287e66e
Better AutoscaledPool docblock formatting
janbuchar Feb 26, 2024
7d846ba
Simplify the concurrency model
janbuchar Feb 26, 2024
ea9bd60
Fix AutoscaledPool.abort()
janbuchar Feb 27, 2024
27e3d56
Add documentation
janbuchar Feb 27, 2024
f1396b2
Fix mypy issues
janbuchar Feb 27, 2024
5e1ed62
Apply suggestions from code review
janbuchar Feb 27, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions src/crawlee/_utils/measure_time.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
from __future__ import annotations
Comment thread
janbuchar marked this conversation as resolved.
Resolved

import time
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Iterator


@dataclass
class TimerResult:
Comment thread
janbuchar marked this conversation as resolved.
Resolved
wall: float | None = None
cpu: float | None = None


@contextmanager
def measure_time() -> Iterator[TimerResult]:
Comment thread
janbuchar marked this conversation as resolved.
Resolved
"""Measure the execution time (wall-clock and CPU) between the start and end of the with-block."""
result = TimerResult()
before_wall = time.monotonic()
before_cpu = time.thread_time()

try:
yield result
finally:
after_wall = time.monotonic()
after_cpu = time.thread_time()
result.wall = after_wall - before_wall
result.cpu = after_cpu - before_cpu
298 changes: 298 additions & 0 deletions src/crawlee/autoscaling/autoscaled_pool.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,298 @@
# Inspiration: https://github.com/apify/crawlee/blob/v3.7.3/packages/core/src/autoscaling/autoscaled_pool.ts

from __future__ import annotations

import asyncio
import math
from contextlib import suppress
from datetime import timedelta
from logging import getLogger
from typing import TYPE_CHECKING, Awaitable, Callable

from crawlee._utils.recurring_task import RecurringTask

if TYPE_CHECKING:
from crawlee.autoscaling.system_status import SystemStatus

logger = getLogger(__name__)


class AbortError(Exception):
"""Raised when an AutoscaledPool run is aborted. Not for direct use."""


class AutoscaledPool:
"""Manages a pool of asynchronous resource-intensive tasks that are executed in parallel.

The pool only starts new tasks if there is enough free CPU and memory available. If an exception is thrown in
any of the tasks, it is propagated and the pool is stopped.
"""

def __init__(
self: AutoscaledPool,
*,
system_status: SystemStatus,
run_task_function: Callable[..., Awaitable],
is_task_ready_function: Callable[..., bool],
Comment thread
janbuchar marked this conversation as resolved.
Resolved
is_finished_function: Callable[..., bool],
task_timeout: timedelta | None = None,
autoscale_interval: timedelta = timedelta(seconds=10),
logging_interval: timedelta = timedelta(minutes=1),
desired_concurrency: int | None = None,
desired_concurrency_ratio: float = 0.9,
min_concurrency: int = 1,
max_concurrency: int = 200,
scale_up_step_ratio: float = 0.05,
scale_down_step_ratio: float = 0.05,
max_tasks_per_minute: int | None = None,
) -> None:
"""Initialize the AutoscaledPool.

Args:
system_status: Provides data about system utilization (load).

run_task_function: A function that performs an asynchronous resource-intensive task.

is_task_ready_function: A function that indicates whether `run_task_function` should be called. This
function is called every time there is free capacity for a new task and it should indicate whether
it should start a new task or not by resolving to either `True` or `False`. Besides its obvious use,
it is also useful for task throttling to save resources. is_finished_function: A function that is
called only when there are no tasks to be processed. If it resolves to `True` then the pool's run
finishes. Being called only when there are no tasks being processed means that as long as
`is_task_ready_function()` keeps resolving to `True`, `is_finished_function()` will never be called.
To abort a run, use the `abort` method.

is_finished_function: A function that is called only when there are no tasks to be processed. If it resolves
to `true` then the pool's run finishes. Being called only when there are no tasks being processed means
that as long as `isTaskReadyFunction()` keeps resolving to `true`, `isFinishedFunction()` will never
be called. To abort a run, use the `abort` method.

task_timeout: Timeout in which the `run_task_function` needs to finish.

autoscale_interval: Defines how often the pool should attempt to adjust the desired concurrency based on
the latest system status. Setting it lower than 1 might have a severe impact on performance. We suggest
using a value from 5 to 20.

logging_interval: Specifies a period in which the instance logs its state, in seconds.

desired_concurrency: The desired number of tasks that should be running parallel on the start of the pool,
if there is a large enough supply of them. By default, it is `min_concurrency`.

desired_concurrency_ratio: Minimum level of desired concurrency to reach before more scaling up is allowed.

min_concurrency: The minimum number of tasks running in parallel. If you set this value too high with
respect to the available system memory and CPU, your code might run extremely slow or crash.

max_concurrency: The maximum number of tasks running in parallel.

scale_up_step_ratio: Defines the fractional amount of desired concurrency to be added with each scaling up.

scale_down_step_ratio: Defines the amount of desired concurrency to be subtracted with each scaling down.

max_tasks_per_minute: The maximum number of tasks per minute the pool can run. By default, this is set to
`Infinity`, but you can pass any positive, non-zero integer.
"""
self._system_status = system_status

self._run_task_function = run_task_function
self._is_task_ready_function = is_task_ready_function
self._is_finished_function = is_finished_function

self._task_timeout = task_timeout

self._logging_interval = logging_interval
self._log_system_status_task = RecurringTask(self._log_system_status, logging_interval)

self._autoscale_task = RecurringTask(self._autoscale, autoscale_interval)

self._worker_tasks = list[asyncio.Task]()
"""A list of worker tasks currently in progress"""
Comment thread
vdusek marked this conversation as resolved.
Resolved

self._worker_tasks_updated = asyncio.Event()
self._cleanup_done = asyncio.Event()
self._run_result: asyncio.Future = asyncio.Future()

if desired_concurrency is not None and desired_concurrency < 1:
raise ValueError('desired_concurrency must be 1 or larger')

if min_concurrency < 1:
raise ValueError('min_concurrency must be 1 or larger')

if max_concurrency < min_concurrency:
raise ValueError('max_concurrency cannot be less than min_concurrency')

if desired_concurrency_ratio < 0 or desired_concurrency_ratio > 1:
raise ValueError('desired_concurrency_ratio must be between 0 and 1 (non-inclusive)')

self._desired_concurrency_ratio = desired_concurrency_ratio
self._desired_concurrency = desired_concurrency if desired_concurrency is not None else min_concurrency
self._max_concurrency = max_concurrency
self._min_concurrency = min_concurrency

self._scale_up_step_ratio = scale_up_step_ratio
self._scale_down_step_ratio = scale_down_step_ratio

self._max_tasks_per_minute = max_tasks_per_minute
self._is_paused = False
self._is_running = False

async def run(self: AutoscaledPool) -> None:
"""Start the autoscaled pool and return when all tasks are completed and `is_finished_function` returns True.

If there is an exception in one of the tasks, it will be re-raised.
"""
if self._is_running:
raise RuntimeError('The pool is already running')

self._is_running = True
logger.debug('Starting the pool')

self._autoscale_task.start()
self._log_system_status_task.start()

orchestrator = asyncio.create_task(
self._worker_task_orchestrator(), name='autoscaled pool worker task orchestrator'
)

try:
await self._run_result
except AbortError:
orchestrator.cancel()
for task in self._worker_tasks:
if not task.done():
task.cancel()
finally:
with suppress(asyncio.CancelledError):
await self._autoscale_task.stop()
with suppress(asyncio.CancelledError):
await self._log_system_status_task.stop()
Comment thread
janbuchar marked this conversation as resolved.
Resolved

if not orchestrator.done():
orchestrator.cancel()
elif not orchestrator.cancelled() and orchestrator.exception() is not None:
logger.error('Exception in worker task orchestrator', exc_info=orchestrator.exception())

logger.info('Waiting for remaining tasks to finish')

for task in self._worker_tasks:
if not task.done():
with suppress(BaseException):
await task

self._run_result = asyncio.Future()
self._cleanup_done.set()
self._is_running = False

logger.debug('Pool cleanup finished')

async def abort(self: AutoscaledPool) -> None:
"""Interrupt the autoscaled pool and all the tasks in progress."""
self._run_result.set_exception(AbortError())
await self._cleanup_done.wait()

def pause(self: AutoscaledPool) -> None:
"""Pause the autoscaled pool so that it does not start new tasks."""
self._is_paused = True

def resume(self: AutoscaledPool) -> None:
"""Resume a paused autoscaled pool so that it continues starting new tasks."""
self._is_paused = False

@property
def desired_concurrency(self: AutoscaledPool) -> int:
"""The current desired concurrency, possibly updated by the pool according to system load."""
return self._desired_concurrency
Comment thread
vdusek marked this conversation as resolved.
Resolved

@property
def current_concurrency(self: AutoscaledPool) -> int:
"""The number of concurrent tasks in progress."""
return len(self._worker_tasks)

def _autoscale(self: AutoscaledPool) -> None:
"""Inspect system load status and adjust desired concurrency if necessary. Do not call directly."""
status = self._system_status.get_historical_status()

min_current_concurrency = math.floor(self._desired_concurrency_ratio * self.current_concurrency)
should_scale_up = (
status.is_system_idle
and self._desired_concurrency < self._max_concurrency
and self.current_concurrency >= min_current_concurrency
)

should_scale_down = not status.is_system_idle and self._desired_concurrency > self._min_concurrency

if should_scale_up:
step = math.ceil(self._scale_up_step_ratio * self._desired_concurrency)
self._desired_concurrency = min(self._max_concurrency, self._desired_concurrency + step)
elif should_scale_down:
step = math.ceil(self._scale_down_step_ratio * self._desired_concurrency)
self._desired_concurrency = max(self._min_concurrency, self._desired_concurrency - step)

def _log_system_status(self: AutoscaledPool) -> None:
system_status = self._system_status.get_historical_status()

logger.info(
f'current_concurrency = {self.current_concurrency}; '
f'desired_concurrency = {self.desired_concurrency}; '
f'{system_status!s}'
)

async def _worker_task_orchestrator(self: AutoscaledPool) -> None:
"""Launches worker tasks whenever there is free capacity and a task is ready.

Exits when `is_finished_function` returns True.
"""
try:
while not self._is_finished_function() and not self._run_result.done():
self._worker_tasks_updated.clear()

current_status = self._system_status.get_current_status()
if not current_status.is_system_idle:
logger.debug('Not scheduling new tasks - system is overloaded')
elif self._is_paused:
logger.debug('Not scheduling new tasks - the autoscaled pool is paused')
elif self.current_concurrency >= self.desired_concurrency:
logger.debug('Not scheduling new tasks - already running at desired concurrency')
elif not self._is_task_ready_function():
logger.debug('Not scheduling new task - no task is ready')
else:
logger.debug('Scheduling a new task')
worker_task = asyncio.create_task(self._worker_task(), name='autoscaled pool worker task')
worker_task.add_done_callback(lambda task: self._reap_worker_task(task))
self._worker_tasks.append(worker_task)

if self._max_tasks_per_minute is not None:
await asyncio.sleep(60 / self._max_tasks_per_minute)

continue

with suppress(asyncio.TimeoutError):
await asyncio.wait_for(self._worker_tasks_updated.wait(), timeout=0.5)
finally:
if not (self._run_result.done() and self._run_result.exception() is not None):
self._run_result.set_result(object())

def _reap_worker_task(self: AutoscaledPool, task: asyncio.Task) -> None:
"""A callback for finished worker tasks.

- It interrupts the run in case of an exception,
- keeps track of tasks in progress,
- notifies the orchestrator
"""
self._worker_tasks_updated.set()
self._worker_tasks.remove(task)

if not task.cancelled() and (exception := task.exception()) and not self._run_result.done():
self._run_result.set_exception(exception)

async def _worker_task(self: AutoscaledPool) -> None:
try:
await asyncio.wait_for(
self._run_task_function(),
timeout=self._task_timeout.total_seconds() if self._task_timeout is not None else None,
)
except asyncio.TimeoutError:
timeout_str = self._task_timeout.total_seconds() if self._task_timeout is not None else '*not set*'
logger.warning(f'Task timed out after {timeout_str} seconds')
finally:
logger.debug('Worker task finished')
10 changes: 10 additions & 0 deletions src/crawlee/autoscaling/types.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,16 @@ def is_system_idle(self: SystemInfo) -> bool:
and not self.client_info.is_overloaded
)

def __str__(self: SystemInfo) -> str:
"""Get a string representation of the system info."""
stats = {
'cpu': self.cpu_info.actual_ratio if self.cpu_info else '-',
'mem': self.mem_info.actual_ratio if self.mem_info else '-',
'event_lopp': self.event_loop_info.actual_ratio if self.event_loop_info else '-',
'client_info': self.client_info.actual_ratio if self.client_info else '-',
}
return ';'.join(f'{name} = {ratio}' for name, ratio in stats.items())


@dataclass
class FinalStatistics:
Expand Down
39 changes: 39 additions & 0 deletions tests/unit/_utils/test_measure_time.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import asyncio
import time

import pytest

from crawlee._utils.measure_time import measure_time


def test_measure_time_wall_sync() -> None:
with measure_time() as elapsed:
time.sleep(0.1)

assert elapsed.cpu is not None
assert elapsed.wall is not None
assert elapsed.wall >= 0.09


def test_measure_time_cpu_sync() -> None:
with measure_time() as elapsed:
start = time.time()
acc = 0

while time.time() - start < 0.1:
acc += 1
acc *= acc

assert elapsed.cpu is not None
assert elapsed.wall is not None
assert elapsed.cpu >= 0.05


@pytest.mark.asyncio()
async def test_measure_time_wall_async() -> None:
with measure_time() as elapsed:
await asyncio.sleep(0.1)

assert elapsed.cpu is not None
assert elapsed.wall is not None
assert elapsed.wall >= 0.09
Loading