From 6d0cb719daeddd6081b7569c95811702a4afaf37 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 12 Nov 2024 16:42:59 +0100 Subject: [PATCH 1/9] Fix memory usage calcualtion Aditional reverse was changing order of snapshots and the rest of code expected latest snapshots last. Simplified ratio calculation. Remove fake 0.001s for zero timed measurements. Snapshot of zero duration should not contribute to ratio calculation. --- src/crawlee/_autoscaling/snapshotter.py | 2 +- src/crawlee/_autoscaling/system_status.py | 18 +++++++++--------- tests/unit/_autoscaling/test_system_status.py | 8 ++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/crawlee/_autoscaling/snapshotter.py b/src/crawlee/_autoscaling/snapshotter.py index 161ce2a63a..19cd074310 100644 --- a/src/crawlee/_autoscaling/snapshotter.py +++ b/src/crawlee/_autoscaling/snapshotter.py @@ -194,7 +194,7 @@ def _get_sample(snapshots: list[Snapshot], duration: timedelta | None = None) -> return [] latest_time = snapshots[-1].created_at - return [snapshot for snapshot in reversed(snapshots) if latest_time - snapshot.created_at <= duration] + return [snapshot for snapshot in snapshots if latest_time - snapshot.created_at <= duration] def _snapshot_cpu(self, event_data: EventSystemInfoData) -> None: """Captures a snapshot of the current CPU usage. diff --git a/src/crawlee/_autoscaling/system_status.py b/src/crawlee/_autoscaling/system_status.py index 737b0eec07..bbcc2aa613 100644 --- a/src/crawlee/_autoscaling/system_status.py +++ b/src/crawlee/_autoscaling/system_status.py @@ -10,7 +10,6 @@ from crawlee._autoscaling.types import LoadRatioInfo, Snapshot, SystemInfo from crawlee._utils.docs import docs_group -from crawlee._utils.math import compute_weighted_avg if TYPE_CHECKING: from crawlee._autoscaling import Snapshotter @@ -182,17 +181,18 @@ def _is_sample_overloaded(self, sample: list[Snapshot], threshold: float) -> Loa if len(sample) == 1: return LoadRatioInfo(limit_ratio=threshold, actual_ratio=float(sample[0].is_overloaded)) - weights, values = [], [] - + overloaded_time = 0.0 + non_overloaded_time = 0.0 for previous, current in pairwise(sample): - weight = (current.created_at - previous.created_at).total_seconds() or 0.001 # Avoid zero - value = float(current.is_overloaded) - weights.append(weight) - values.append(value) + time = (current.created_at - previous.created_at).total_seconds() + if current.is_overloaded: + overloaded_time += time + else: + non_overloaded_time += time try: - weighted_avg = compute_weighted_avg(values, weights) + overloaded_ratio = overloaded_time / (overloaded_time + non_overloaded_time) except ValueError as exc: raise ValueError('Failed to compute weighted average for the sample.') from exc - return LoadRatioInfo(limit_ratio=threshold, actual_ratio=round(weighted_avg, 3)) + return LoadRatioInfo(limit_ratio=threshold, actual_ratio=round(overloaded_ratio, 3)) diff --git a/tests/unit/_autoscaling/test_system_status.py b/tests/unit/_autoscaling/test_system_status.py index 7da6cb6004..1b61c31478 100644 --- a/tests/unit/_autoscaling/test_system_status.py +++ b/tests/unit/_autoscaling/test_system_status.py @@ -92,19 +92,19 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None: current_size=ByteSize.from_gb(4), max_memory_size=ByteSize.from_gb(12), max_used_memory_ratio=0.8, - created_at=now - timedelta(minutes=3), + created_at=now - timedelta(seconds=90), ), MemorySnapshot( current_size=ByteSize.from_gb(7), max_memory_size=ByteSize.from_gb(8), max_used_memory_ratio=0.8, - created_at=now - timedelta(minutes=2), + created_at=now - timedelta(seconds=60), ), MemorySnapshot( current_size=ByteSize.from_gb(28), max_memory_size=ByteSize.from_gb(30), max_used_memory_ratio=0.8, - created_at=now - timedelta(minutes=1), + created_at=now - timedelta(seconds=30), ), MemorySnapshot( current_size=ByteSize.from_gb(48), @@ -150,7 +150,7 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None: current_system_info = system_status.get_current_system_info() assert current_system_info == SystemInfo( cpu_info=LoadRatioInfo(limit_ratio=system_status._cpu_overload_threshold, actual_ratio=1.0), - memory_info=LoadRatioInfo(limit_ratio=system_status._memory_overload_threshold, actual_ratio=1.0), + memory_info=LoadRatioInfo(limit_ratio=system_status._memory_overload_threshold, actual_ratio=0.5), event_loop_info=LoadRatioInfo(limit_ratio=system_status._event_loop_overload_threshold, actual_ratio=0), client_info=LoadRatioInfo(limit_ratio=system_status._client_overload_threshold, actual_ratio=0), created_at=current_system_info.created_at, From 89d5cf3a518a0d7057e738a71bd1ab10662eb8ac Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 12 Nov 2024 18:19:21 +0100 Subject: [PATCH 2/9] WIP exploring order of samples and the lack of guarantee of their ordering --- src/crawlee/_autoscaling/system_status.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/crawlee/_autoscaling/system_status.py b/src/crawlee/_autoscaling/system_status.py index bbcc2aa613..066c1cc144 100644 --- a/src/crawlee/_autoscaling/system_status.py +++ b/src/crawlee/_autoscaling/system_status.py @@ -162,6 +162,7 @@ def _is_client_overloaded(self, sample_duration: timedelta | None = None) -> Loa Client load ratio information. """ sample = self._snapshotter.get_client_sample(sample_duration) + # TODO: DATA IS NOT GUARATEED TO BE SORTED! Either guarantee order or have to sort on demand return self._is_sample_overloaded(sample, self._client_overload_threshold) def _is_sample_overloaded(self, sample: list[Snapshot], threshold: float) -> LoadRatioInfo: @@ -183,6 +184,7 @@ def _is_sample_overloaded(self, sample: list[Snapshot], threshold: float) -> Loa overloaded_time = 0.0 non_overloaded_time = 0.0 + for previous, current in pairwise(sample): time = (current.created_at - previous.created_at).total_seconds() if current.is_overloaded: @@ -190,9 +192,15 @@ def _is_sample_overloaded(self, sample: list[Snapshot], threshold: float) -> Loa else: non_overloaded_time += time - try: + if (overloaded_time + non_overloaded_time) == 0: + overloaded_ratio = 0 + else: overloaded_ratio = overloaded_time / (overloaded_time + non_overloaded_time) - except ValueError as exc: - raise ValueError('Failed to compute weighted average for the sample.') from exc + + if overloaded_ratio > 0: + try: + logger.info(f"Current mem load: {sample[-1].current_size / sample[-1].max_memory_size}, overloaded_ratio: {overloaded_ratio}") + except: + pass return LoadRatioInfo(limit_ratio=threshold, actual_ratio=round(overloaded_ratio, 3)) From 895e2767126b80549a6b2cc0181b74035205dda0 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 13 Nov 2024 08:39:13 +0100 Subject: [PATCH 3/9] WIP for testing on platform Original with extra debug --- src/crawlee/_autoscaling/snapshotter.py | 2 +- src/crawlee/_autoscaling/system_status.py | 35 ++++++++++--------- tests/unit/_autoscaling/test_system_status.py | 8 ++--- 3 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/crawlee/_autoscaling/snapshotter.py b/src/crawlee/_autoscaling/snapshotter.py index 19cd074310..161ce2a63a 100644 --- a/src/crawlee/_autoscaling/snapshotter.py +++ b/src/crawlee/_autoscaling/snapshotter.py @@ -194,7 +194,7 @@ def _get_sample(snapshots: list[Snapshot], duration: timedelta | None = None) -> return [] latest_time = snapshots[-1].created_at - return [snapshot for snapshot in snapshots if latest_time - snapshot.created_at <= duration] + return [snapshot for snapshot in reversed(snapshots) if latest_time - snapshot.created_at <= duration] def _snapshot_cpu(self, event_data: EventSystemInfoData) -> None: """Captures a snapshot of the current CPU usage. diff --git a/src/crawlee/_autoscaling/system_status.py b/src/crawlee/_autoscaling/system_status.py index 066c1cc144..f9db06ccbf 100644 --- a/src/crawlee/_autoscaling/system_status.py +++ b/src/crawlee/_autoscaling/system_status.py @@ -8,8 +8,10 @@ from more_itertools import pairwise +import crawlee from crawlee._autoscaling.types import LoadRatioInfo, Snapshot, SystemInfo from crawlee._utils.docs import docs_group +from crawlee._utils.math import compute_weighted_avg if TYPE_CHECKING: from crawlee._autoscaling import Snapshotter @@ -162,7 +164,6 @@ def _is_client_overloaded(self, sample_duration: timedelta | None = None) -> Loa Client load ratio information. """ sample = self._snapshotter.get_client_sample(sample_duration) - # TODO: DATA IS NOT GUARATEED TO BE SORTED! Either guarantee order or have to sort on demand return self._is_sample_overloaded(sample, self._client_overload_threshold) def _is_sample_overloaded(self, sample: list[Snapshot], threshold: float) -> LoadRatioInfo: @@ -182,25 +183,25 @@ def _is_sample_overloaded(self, sample: list[Snapshot], threshold: float) -> Loa if len(sample) == 1: return LoadRatioInfo(limit_ratio=threshold, actual_ratio=float(sample[0].is_overloaded)) - overloaded_time = 0.0 - non_overloaded_time = 0.0 + weights, values = [], [] for previous, current in pairwise(sample): - time = (current.created_at - previous.created_at).total_seconds() - if current.is_overloaded: - overloaded_time += time - else: - non_overloaded_time += time - - if (overloaded_time + non_overloaded_time) == 0: - overloaded_ratio = 0 - else: - overloaded_ratio = overloaded_time / (overloaded_time + non_overloaded_time) - - if overloaded_ratio > 0: + weight = (current.created_at - previous.created_at).total_seconds() or 0.001 # Avoid zero + value = float(current.is_overloaded) + weights.append(weight) + values.append(value) + + try: + weighted_avg = compute_weighted_avg(values, weights) + except ValueError as exc: + raise ValueError('Failed to compute weighted average for the sample.') from exc + + if weighted_avg > 0: try: - logger.info(f"Current mem load: {sample[-1].current_size / sample[-1].max_memory_size}, overloaded_ratio: {overloaded_ratio}") + if type(sample[-1])== crawlee._autoscaling.types.MemorySnapshot: + logger.info(f"Current load: {sample[-1].current_size / sample[-1].max_memory_size}, overloaded_ratio: {weighted_avg}, sample size={len(sample)}") except: pass - return LoadRatioInfo(limit_ratio=threshold, actual_ratio=round(overloaded_ratio, 3)) + + return LoadRatioInfo(limit_ratio=threshold, actual_ratio=round(weighted_avg, 3)) diff --git a/tests/unit/_autoscaling/test_system_status.py b/tests/unit/_autoscaling/test_system_status.py index 1b61c31478..7da6cb6004 100644 --- a/tests/unit/_autoscaling/test_system_status.py +++ b/tests/unit/_autoscaling/test_system_status.py @@ -92,19 +92,19 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None: current_size=ByteSize.from_gb(4), max_memory_size=ByteSize.from_gb(12), max_used_memory_ratio=0.8, - created_at=now - timedelta(seconds=90), + created_at=now - timedelta(minutes=3), ), MemorySnapshot( current_size=ByteSize.from_gb(7), max_memory_size=ByteSize.from_gb(8), max_used_memory_ratio=0.8, - created_at=now - timedelta(seconds=60), + created_at=now - timedelta(minutes=2), ), MemorySnapshot( current_size=ByteSize.from_gb(28), max_memory_size=ByteSize.from_gb(30), max_used_memory_ratio=0.8, - created_at=now - timedelta(seconds=30), + created_at=now - timedelta(minutes=1), ), MemorySnapshot( current_size=ByteSize.from_gb(48), @@ -150,7 +150,7 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None: current_system_info = system_status.get_current_system_info() assert current_system_info == SystemInfo( cpu_info=LoadRatioInfo(limit_ratio=system_status._cpu_overload_threshold, actual_ratio=1.0), - memory_info=LoadRatioInfo(limit_ratio=system_status._memory_overload_threshold, actual_ratio=0.5), + memory_info=LoadRatioInfo(limit_ratio=system_status._memory_overload_threshold, actual_ratio=1.0), event_loop_info=LoadRatioInfo(limit_ratio=system_status._event_loop_overload_threshold, actual_ratio=0), client_info=LoadRatioInfo(limit_ratio=system_status._client_overload_threshold, actual_ratio=0), created_at=current_system_info.created_at, From fbb8c1ab601104db63c51168ddda8b8eb44d291f Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 13 Nov 2024 14:38:07 +0100 Subject: [PATCH 4/9] Small fixes to snapshot handling. -Ensure time ordering. -Remove fake time weight for samples from same time. -Remove unnecesary reversing of snapshot. Add test for memory and cpu snapshot ordering. --- src/crawlee/_autoscaling/snapshotter.py | 30 ++++++++++++---- src/crawlee/_autoscaling/system_status.py | 36 ++++++++----------- src/crawlee/_autoscaling/types.py | 1 + tests/unit/_autoscaling/test_snapshotter.py | 34 +++++++++++++++++- tests/unit/_autoscaling/test_system_status.py | 8 ++--- 5 files changed, 75 insertions(+), 34 deletions(-) diff --git a/src/crawlee/_autoscaling/snapshotter.py b/src/crawlee/_autoscaling/snapshotter.py index 161ce2a63a..b0a569ba32 100644 --- a/src/crawlee/_autoscaling/snapshotter.py +++ b/src/crawlee/_autoscaling/snapshotter.py @@ -2,13 +2,22 @@ from __future__ import annotations +from bisect import insort from datetime import datetime, timedelta, timezone from logging import getLogger +from operator import attrgetter from typing import TYPE_CHECKING, cast import psutil -from crawlee._autoscaling.types import ClientSnapshot, CpuSnapshot, EventLoopSnapshot, MemorySnapshot, Snapshot +from crawlee._autoscaling.types import ( + ClientSnapshot, + CpuSnapshot, + EventLoopSnapshot, + ListOfSnapshots, + MemorySnapshot, + Snapshot, +) from crawlee._utils.byte_size import ByteSize from crawlee._utils.docs import docs_group from crawlee._utils.recurring_task import RecurringTask @@ -194,7 +203,7 @@ def _get_sample(snapshots: list[Snapshot], duration: timedelta | None = None) -> return [] latest_time = snapshots[-1].created_at - return [snapshot for snapshot in reversed(snapshots) if latest_time - snapshot.created_at <= duration] + return [snapshot for snapshot in snapshots if latest_time - snapshot.created_at <= duration] def _snapshot_cpu(self, event_data: EventSystemInfoData) -> None: """Captures a snapshot of the current CPU usage. @@ -213,7 +222,7 @@ def _snapshot_cpu(self, event_data: EventSystemInfoData) -> None: snapshots = cast(list[Snapshot], self._cpu_snapshots) self._prune_snapshots(snapshots, event_data.cpu_info.created_at) - self._cpu_snapshots.append(snapshot) + self._time_sorted_insert_snapshot(self._cpu_snapshots, snapshot) def _snapshot_memory(self, event_data: EventSystemInfoData) -> None: """Captures a snapshot of the current memory usage. @@ -233,8 +242,7 @@ def _snapshot_memory(self, event_data: EventSystemInfoData) -> None: snapshots = cast(list[Snapshot], self._memory_snapshots) self._prune_snapshots(snapshots, snapshot.created_at) - self._memory_snapshots.append(snapshot) - + self._time_sorted_insert_snapshot(self._memory_snapshots, snapshot) self._evaluate_memory_load(event_data.memory_info.current_size, event_data.memory_info.created_at) def _snapshot_event_loop(self) -> None: @@ -254,7 +262,7 @@ def _snapshot_event_loop(self) -> None: snapshots = cast(list[Snapshot], self._event_loop_snapshots) self._prune_snapshots(snapshots, snapshot.created_at) - self._event_loop_snapshots.append(snapshot) + self._time_sorted_insert_snapshot(self._event_loop_snapshots, snapshot) def _snapshot_client(self) -> None: """Captures a snapshot of the current API state by checking for rate limit errors (HTTP 429). @@ -271,7 +279,7 @@ def _snapshot_client(self) -> None: snapshots = cast(list[Snapshot], self._client_snapshots) self._prune_snapshots(snapshots, snapshot.created_at) - self._client_snapshots.append(snapshot) + self._time_sorted_insert_snapshot(self._client_snapshots, snapshot) def _prune_snapshots(self, snapshots: list[Snapshot], now: datetime) -> None: """Removes snapshots that are older than the `self._snapshot_history`. @@ -322,3 +330,11 @@ def _evaluate_memory_load(self, current_memory_usage_size: ByteSize, snapshot_ti 'Consider increasing available memory.' ) self._timestamp_of_last_memory_warning = snapshot_timestamp + + @staticmethod + def _time_sorted_insert_snapshot(sorted_list: ListOfSnapshots, snapshot: Snapshot) -> None: + """Sorted insert of snapshot to ordered_list of snapshots based on created_at. + + Incoming snapshot order can't be relied on and explicit sorting is necessary. + """ + insort(sorted_list, snapshot, key=attrgetter('created_at')) # type: ignore # Mypy struggles with this function. diff --git a/src/crawlee/_autoscaling/system_status.py b/src/crawlee/_autoscaling/system_status.py index f9db06ccbf..c14ab74e4b 100644 --- a/src/crawlee/_autoscaling/system_status.py +++ b/src/crawlee/_autoscaling/system_status.py @@ -8,10 +8,8 @@ from more_itertools import pairwise -import crawlee from crawlee._autoscaling.types import LoadRatioInfo, Snapshot, SystemInfo from crawlee._utils.docs import docs_group -from crawlee._utils.math import compute_weighted_avg if TYPE_CHECKING: from crawlee._autoscaling import Snapshotter @@ -183,25 +181,19 @@ def _is_sample_overloaded(self, sample: list[Snapshot], threshold: float) -> Loa if len(sample) == 1: return LoadRatioInfo(limit_ratio=threshold, actual_ratio=float(sample[0].is_overloaded)) - weights, values = [], [] + overloaded_time = 0.0 + non_overloaded_time = 0.0 for previous, current in pairwise(sample): - weight = (current.created_at - previous.created_at).total_seconds() or 0.001 # Avoid zero - value = float(current.is_overloaded) - weights.append(weight) - values.append(value) - - try: - weighted_avg = compute_weighted_avg(values, weights) - except ValueError as exc: - raise ValueError('Failed to compute weighted average for the sample.') from exc - - if weighted_avg > 0: - try: - if type(sample[-1])== crawlee._autoscaling.types.MemorySnapshot: - logger.info(f"Current load: {sample[-1].current_size / sample[-1].max_memory_size}, overloaded_ratio: {weighted_avg}, sample size={len(sample)}") - except: - pass - - - return LoadRatioInfo(limit_ratio=threshold, actual_ratio=round(weighted_avg, 3)) + time = (current.created_at - previous.created_at).total_seconds() + if current.is_overloaded: + overloaded_time += time + else: + non_overloaded_time += time + + if (overloaded_time + non_overloaded_time) == 0: + overloaded_ratio = 0 + else: + overloaded_ratio = overloaded_time / (overloaded_time + non_overloaded_time) + + return LoadRatioInfo(limit_ratio=threshold, actual_ratio=round(overloaded_ratio, 3)) diff --git a/src/crawlee/_autoscaling/types.py b/src/crawlee/_autoscaling/types.py index dbf1e828ad..5dc3c5cdeb 100644 --- a/src/crawlee/_autoscaling/types.py +++ b/src/crawlee/_autoscaling/types.py @@ -150,3 +150,4 @@ def is_overloaded(self) -> bool: Snapshot = Union[MemorySnapshot, CpuSnapshot, EventLoopSnapshot, ClientSnapshot] +ListOfSnapshots = Union[list[MemorySnapshot], list[CpuSnapshot], list[EventLoopSnapshot], list[ClientSnapshot]] diff --git a/tests/unit/_autoscaling/test_snapshotter.py b/tests/unit/_autoscaling/test_snapshotter.py index c1addba7b2..a5136be651 100644 --- a/tests/unit/_autoscaling/test_snapshotter.py +++ b/tests/unit/_autoscaling/test_snapshotter.py @@ -12,7 +12,7 @@ from crawlee._utils.byte_size import ByteSize from crawlee._utils.system import CpuInfo, MemoryInfo from crawlee.events import EventManager, LocalEventManager -from crawlee.events._types import EventSystemInfoData +from crawlee.events._types import Event, EventSystemInfoData @pytest.fixture @@ -207,3 +207,35 @@ def test_memory_load_evaluation_silent_on_acceptable_usage( ) assert mock_logger_warn.call_count == 0 + + +async def test_snapshots_time_ordered() -> None: + # All internal snapshot list should be ordered by creation time in ascending order. + # Scenario where older emitted event arrives after newer event. + # Snapshotter should not trust the event order and check events' times. + time_new = datetime.now(tz=timezone.utc) + time_old = datetime.now(tz=timezone.utc) - timedelta(milliseconds=50) + + def create_event_data(creation_time: datetime) -> EventSystemInfoData: + return EventSystemInfoData( + cpu_info=CpuInfo(used_ratio=0.5, created_at=creation_time), + memory_info=MemoryInfo( + current_size=ByteSize(bytes=1), created_at=creation_time, total_size=ByteSize(bytes=2) + ), + ) + + async with ( + LocalEventManager() as event_manager, + Snapshotter(event_manager, available_memory_ratio=0.25) as snapshotter, + ): + event_manager.emit(event=Event.SYSTEM_INFO, event_data=create_event_data(time_new)) + await event_manager.wait_for_all_listeners_to_complete() + event_manager.emit(event=Event.SYSTEM_INFO, event_data=create_event_data(time_old)) + await event_manager.wait_for_all_listeners_to_complete() + + memory_samples = snapshotter.get_memory_sample() + cpu_samples = snapshotter.get_cpu_sample() + assert memory_samples[0].created_at == time_old + assert cpu_samples[0].created_at == time_old + assert memory_samples[1].created_at == time_new + assert cpu_samples[1].created_at == time_new diff --git a/tests/unit/_autoscaling/test_system_status.py b/tests/unit/_autoscaling/test_system_status.py index 7da6cb6004..1b61c31478 100644 --- a/tests/unit/_autoscaling/test_system_status.py +++ b/tests/unit/_autoscaling/test_system_status.py @@ -92,19 +92,19 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None: current_size=ByteSize.from_gb(4), max_memory_size=ByteSize.from_gb(12), max_used_memory_ratio=0.8, - created_at=now - timedelta(minutes=3), + created_at=now - timedelta(seconds=90), ), MemorySnapshot( current_size=ByteSize.from_gb(7), max_memory_size=ByteSize.from_gb(8), max_used_memory_ratio=0.8, - created_at=now - timedelta(minutes=2), + created_at=now - timedelta(seconds=60), ), MemorySnapshot( current_size=ByteSize.from_gb(28), max_memory_size=ByteSize.from_gb(30), max_used_memory_ratio=0.8, - created_at=now - timedelta(minutes=1), + created_at=now - timedelta(seconds=30), ), MemorySnapshot( current_size=ByteSize.from_gb(48), @@ -150,7 +150,7 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None: current_system_info = system_status.get_current_system_info() assert current_system_info == SystemInfo( cpu_info=LoadRatioInfo(limit_ratio=system_status._cpu_overload_threshold, actual_ratio=1.0), - memory_info=LoadRatioInfo(limit_ratio=system_status._memory_overload_threshold, actual_ratio=1.0), + memory_info=LoadRatioInfo(limit_ratio=system_status._memory_overload_threshold, actual_ratio=0.5), event_loop_info=LoadRatioInfo(limit_ratio=system_status._event_loop_overload_threshold, actual_ratio=0), client_info=LoadRatioInfo(limit_ratio=system_status._client_overload_threshold, actual_ratio=0), created_at=current_system_info.created_at, From fd9b7c24e9b6832c0469e85277550d3a49545433 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 13 Nov 2024 16:43:50 +0100 Subject: [PATCH 5/9] Use sortedlist instead of bisect.insort bisect.insort does not support 'key' keyword untill python 3.10 sortedcollections are already in part of project, so no extra dependency --- poetry.lock | 59 ++++++++++++++++++++++++- pyproject.toml | 1 + src/crawlee/_autoscaling/snapshotter.py | 27 ++++------- 3 files changed, 68 insertions(+), 19 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4294e84f30..ee15f5c6ed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -249,6 +249,10 @@ files = [ {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d"}, {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0"}, {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec"}, {file = "Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2"}, {file = "Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128"}, {file = "Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc"}, @@ -261,8 +265,14 @@ files = [ {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9"}, {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265"}, {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b"}, {file = "Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50"}, {file = "Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1"}, + {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28"}, + {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f"}, {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409"}, {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2"}, {file = "Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451"}, @@ -273,8 +283,24 @@ files = [ {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180"}, {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248"}, {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839"}, {file = "Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0"}, {file = "Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951"}, + {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5"}, + {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7"}, + {file = "Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0"}, + {file = "Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b"}, {file = "Brotli-1.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1"}, {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d"}, {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b"}, @@ -284,6 +310,10 @@ files = [ {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2"}, {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354"}, {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:aea440a510e14e818e67bfc4027880e2fb500c2ccb20ab21c7a7c8b5b4703d75"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:6974f52a02321b36847cd19d1b8e381bf39939c21efd6ee2fc13a28b0d99348c"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:a7e53012d2853a07a4a79c00643832161a910674a893d296c9f1259859a289d2"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:d7702622a8b40c49bffb46e1e3ba2e81268d5c04a34f460978c6b5517a34dd52"}, {file = "Brotli-1.1.0-cp36-cp36m-win32.whl", hash = "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460"}, {file = "Brotli-1.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579"}, {file = "Brotli-1.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c"}, @@ -295,6 +325,10 @@ files = [ {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74"}, {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b"}, {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:cb1dac1770878ade83f2ccdf7d25e494f05c9165f5246b46a621cc849341dc01"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:3ee8a80d67a4334482d9712b8e83ca6b1d9bc7e351931252ebef5d8f7335a547"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5e55da2c8724191e5b557f8e18943b1b4839b8efc3ef60d65985bcf6f587dd38"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:d342778ef319e1026af243ed0a07c97acf3bad33b9f29e7ae6a1f68fd083e90c"}, {file = "Brotli-1.1.0-cp37-cp37m-win32.whl", hash = "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95"}, {file = "Brotli-1.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68"}, {file = "Brotli-1.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3"}, @@ -307,6 +341,10 @@ files = [ {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a"}, {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:141bd4d93984070e097521ed07e2575b46f817d08f9fa42b16b9b5f27b5ac088"}, {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d2b35ca2c7f81d173d2fadc2f4f31e88cc5f7a39ae5b6db5513cf3383b0e0ec7"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:af6fa6817889314555aede9a919612b23739395ce767fe7fcbea9a80bf140fe5"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:2feb1d960f760a575dbc5ab3b1c00504b24caaf6986e2dc2b01c09c87866a943"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4410f84b33374409552ac9b6903507cdb31cd30d2501fc5ca13d18f73548444a"}, {file = "Brotli-1.1.0-cp38-cp38-win32.whl", hash = "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b"}, {file = "Brotli-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0"}, {file = "Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a"}, @@ -319,6 +357,10 @@ files = [ {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c"}, {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d"}, {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb"}, {file = "Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64"}, {file = "Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467"}, {file = "Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724"}, @@ -2752,6 +2794,21 @@ files = [ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] +[[package]] +name = "sortedcontainers-stubs" +version = "2.4.2" +description = "Type stubs for sortedcontainers" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "sortedcontainers_stubs-2.4.2-py3-none-any.whl", hash = "sha256:6cdbcf9c71198729dd6c0a8a7dc9742768f25b49384b91965d716a9d16ca7b2a"}, + {file = "sortedcontainers_stubs-2.4.2.tar.gz", hash = "sha256:c24e33877effa5a3eb85357112496ba33a0b681c05e0d5d3745664fd3be75d82"}, +] + +[package.dependencies] +sortedcontainers = ">=2,<3" +typing-extensions = ">=4.1.0,<5.0.0" + [[package]] name = "soupsieve" version = "2.6" @@ -3294,4 +3351,4 @@ playwright = ["playwright"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "d49bb55e1dc149d480273ebb907db7f7061d0200ab947a28e540ad94d1b4615e" +content-hash = "bea33bb28a5a8ec0f945c0f5936d44e1fafefefff293a05b2d0840fe2b0f18c2" diff --git a/pyproject.toml b/pyproject.toml index 8dec5a615f..9af0fcf1ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,6 +90,7 @@ types-beautifulsoup4 = "~4.12.0.20240229" types-colorama = "~0.4.15.20240106" types-psutil = "~5.9.5.20240205" types-python-dateutil = "~2.9.0.20240316" +sortedcontainers-stubs = "^2.4.2" [tool.poetry.extras] all = ["apify", "beautifulsoup4", "lxml", "html5lib", "curl-cffi", "playwright"] diff --git a/src/crawlee/_autoscaling/snapshotter.py b/src/crawlee/_autoscaling/snapshotter.py index b0a569ba32..1ccb493255 100644 --- a/src/crawlee/_autoscaling/snapshotter.py +++ b/src/crawlee/_autoscaling/snapshotter.py @@ -2,19 +2,18 @@ from __future__ import annotations -from bisect import insort from datetime import datetime, timedelta, timezone from logging import getLogger from operator import attrgetter from typing import TYPE_CHECKING, cast import psutil +from sortedcontainers import SortedList from crawlee._autoscaling.types import ( ClientSnapshot, CpuSnapshot, EventLoopSnapshot, - ListOfSnapshots, MemorySnapshot, Snapshot, ) @@ -103,10 +102,10 @@ def __init__( cast(float, available_memory_ratio) ) - self._cpu_snapshots: list[CpuSnapshot] = [] - self._event_loop_snapshots: list[EventLoopSnapshot] = [] - self._memory_snapshots: list[MemorySnapshot] = [] - self._client_snapshots: list[ClientSnapshot] = [] + self._cpu_snapshots: SortedList[CpuSnapshot] = SortedList([], key=attrgetter('created_at')) + self._event_loop_snapshots: SortedList[EventLoopSnapshot] = SortedList([], key=attrgetter('created_at')) + self._memory_snapshots: SortedList[MemorySnapshot] = SortedList([], key=attrgetter('created_at')) + self._client_snapshots: SortedList[ClientSnapshot] = SortedList([], key=attrgetter('created_at')) self._snapshot_event_loop_task = RecurringTask(self._snapshot_event_loop, self._event_loop_snapshot_interval) self._snapshot_client_task = RecurringTask(self._snapshot_client, self._client_snapshot_interval) @@ -222,7 +221,7 @@ def _snapshot_cpu(self, event_data: EventSystemInfoData) -> None: snapshots = cast(list[Snapshot], self._cpu_snapshots) self._prune_snapshots(snapshots, event_data.cpu_info.created_at) - self._time_sorted_insert_snapshot(self._cpu_snapshots, snapshot) + self._cpu_snapshots.add(snapshot) def _snapshot_memory(self, event_data: EventSystemInfoData) -> None: """Captures a snapshot of the current memory usage. @@ -242,7 +241,7 @@ def _snapshot_memory(self, event_data: EventSystemInfoData) -> None: snapshots = cast(list[Snapshot], self._memory_snapshots) self._prune_snapshots(snapshots, snapshot.created_at) - self._time_sorted_insert_snapshot(self._memory_snapshots, snapshot) + self._memory_snapshots.add(snapshot) self._evaluate_memory_load(event_data.memory_info.current_size, event_data.memory_info.created_at) def _snapshot_event_loop(self) -> None: @@ -262,7 +261,7 @@ def _snapshot_event_loop(self) -> None: snapshots = cast(list[Snapshot], self._event_loop_snapshots) self._prune_snapshots(snapshots, snapshot.created_at) - self._time_sorted_insert_snapshot(self._event_loop_snapshots, snapshot) + self._event_loop_snapshots.add(snapshot) def _snapshot_client(self) -> None: """Captures a snapshot of the current API state by checking for rate limit errors (HTTP 429). @@ -279,7 +278,7 @@ def _snapshot_client(self) -> None: snapshots = cast(list[Snapshot], self._client_snapshots) self._prune_snapshots(snapshots, snapshot.created_at) - self._time_sorted_insert_snapshot(self._client_snapshots, snapshot) + self._client_snapshots.add(snapshot) def _prune_snapshots(self, snapshots: list[Snapshot], now: datetime) -> None: """Removes snapshots that are older than the `self._snapshot_history`. @@ -330,11 +329,3 @@ def _evaluate_memory_load(self, current_memory_usage_size: ByteSize, snapshot_ti 'Consider increasing available memory.' ) self._timestamp_of_last_memory_warning = snapshot_timestamp - - @staticmethod - def _time_sorted_insert_snapshot(sorted_list: ListOfSnapshots, snapshot: Snapshot) -> None: - """Sorted insert of snapshot to ordered_list of snapshots based on created_at. - - Incoming snapshot order can't be relied on and explicit sorting is necessary. - """ - insort(sorted_list, snapshot, key=attrgetter('created_at')) # type: ignore # Mypy struggles with this function. From c6db51743338ce596601ada8aa508deb6893020a Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 14 Nov 2024 09:48:58 +0100 Subject: [PATCH 6/9] Update tests with SortedDict --- src/crawlee/_autoscaling/snapshotter.py | 16 +- src/crawlee/_autoscaling/system_status.py | 2 +- src/crawlee/_autoscaling/types.py | 1 - tests/unit/_autoscaling/test_snapshotter.py | 52 +++--- tests/unit/_autoscaling/test_system_status.py | 156 ++++++++++-------- 5 files changed, 126 insertions(+), 101 deletions(-) diff --git a/src/crawlee/_autoscaling/snapshotter.py b/src/crawlee/_autoscaling/snapshotter.py index 1ccb493255..102a243cff 100644 --- a/src/crawlee/_autoscaling/snapshotter.py +++ b/src/crawlee/_autoscaling/snapshotter.py @@ -5,7 +5,7 @@ from datetime import datetime, timedelta, timezone from logging import getLogger from operator import attrgetter -from typing import TYPE_CHECKING, cast +from typing import TYPE_CHECKING, TypeVar, cast import psutil from sortedcontainers import SortedList @@ -29,6 +29,8 @@ logger = getLogger(__name__) +T = TypeVar('T') + @docs_group('Classes') class Snapshotter: @@ -102,16 +104,20 @@ def __init__( cast(float, available_memory_ratio) ) - self._cpu_snapshots: SortedList[CpuSnapshot] = SortedList([], key=attrgetter('created_at')) - self._event_loop_snapshots: SortedList[EventLoopSnapshot] = SortedList([], key=attrgetter('created_at')) - self._memory_snapshots: SortedList[MemorySnapshot] = SortedList([], key=attrgetter('created_at')) - self._client_snapshots: SortedList[ClientSnapshot] = SortedList([], key=attrgetter('created_at')) + self._cpu_snapshots: SortedList[CpuSnapshot] = self._get_sorted_list_by_created_at([]) + self._event_loop_snapshots: SortedList[EventLoopSnapshot] = self._get_sorted_list_by_created_at([]) + self._memory_snapshots: SortedList[MemorySnapshot] = self._get_sorted_list_by_created_at([]) + self._client_snapshots: SortedList[ClientSnapshot] = self._get_sorted_list_by_created_at([]) self._snapshot_event_loop_task = RecurringTask(self._snapshot_event_loop, self._event_loop_snapshot_interval) self._snapshot_client_task = RecurringTask(self._snapshot_client, self._client_snapshot_interval) self._timestamp_of_last_memory_warning: datetime = datetime.now(timezone.utc) - timedelta(hours=1) + @staticmethod + def _get_sorted_list_by_created_at(input_list: list[T]) -> SortedList[T]: + return SortedList(input_list, key=attrgetter('created_at')) + @staticmethod def _get_default_max_memory_size(available_memory_ratio: float) -> ByteSize: """Default `memory_max_size` is 1/4 of the total system memory.""" diff --git a/src/crawlee/_autoscaling/system_status.py b/src/crawlee/_autoscaling/system_status.py index c14ab74e4b..50af0a054a 100644 --- a/src/crawlee/_autoscaling/system_status.py +++ b/src/crawlee/_autoscaling/system_status.py @@ -192,7 +192,7 @@ def _is_sample_overloaded(self, sample: list[Snapshot], threshold: float) -> Loa non_overloaded_time += time if (overloaded_time + non_overloaded_time) == 0: - overloaded_ratio = 0 + overloaded_ratio = 0.0 else: overloaded_ratio = overloaded_time / (overloaded_time + non_overloaded_time) diff --git a/src/crawlee/_autoscaling/types.py b/src/crawlee/_autoscaling/types.py index 5dc3c5cdeb..dbf1e828ad 100644 --- a/src/crawlee/_autoscaling/types.py +++ b/src/crawlee/_autoscaling/types.py @@ -150,4 +150,3 @@ def is_overloaded(self) -> bool: Snapshot = Union[MemorySnapshot, CpuSnapshot, EventLoopSnapshot, ClientSnapshot] -ListOfSnapshots = Union[list[MemorySnapshot], list[CpuSnapshot], list[EventLoopSnapshot], list[ClientSnapshot]] diff --git a/tests/unit/_autoscaling/test_snapshotter.py b/tests/unit/_autoscaling/test_snapshotter.py index a5136be651..b94f0ca4e8 100644 --- a/tests/unit/_autoscaling/test_snapshotter.py +++ b/tests/unit/_autoscaling/test_snapshotter.py @@ -50,9 +50,11 @@ def test_snapshot_memory(snapshotter: Snapshotter, event_system_data_info: Event def test_snapshot_event_loop(snapshotter: Snapshotter) -> None: - snapshotter._event_loop_snapshots = [ - EventLoopSnapshot(delay=timedelta(milliseconds=100), max_delay=timedelta(milliseconds=500)), - ] + snapshotter._event_loop_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + EventLoopSnapshot(delay=timedelta(milliseconds=100), max_delay=timedelta(milliseconds=500)), + ] + ) snapshotter._snapshot_event_loop() assert len(snapshotter._event_loop_snapshots) == 2 @@ -65,14 +67,16 @@ def test_snapshot_client(snapshotter: Snapshotter) -> None: def test_get_cpu_sample(snapshotter: Snapshotter) -> None: now = datetime.now(timezone.utc) - cpu_snapshots = [ - CpuSnapshot( - used_ratio=0.5, - max_used_ratio=0.95, - created_at=now - timedelta(hours=delta), - ) - for delta in range(5, 0, -1) - ] + cpu_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot( + used_ratio=0.5, + max_used_ratio=0.95, + created_at=now - timedelta(hours=delta), + ) + for delta in range(5, 0, -1) + ] + ) snapshotter._cpu_snapshots = cpu_snapshots @@ -114,12 +118,14 @@ def test_snapshot_pruning_removes_outdated_records(snapshotter: Snapshotter) -> five_hours_ago = now - timedelta(hours=5) # Create mock snapshots with varying creation times - snapshots = [ - CpuSnapshot(used_ratio=0.5, max_used_ratio=0.95, created_at=five_hours_ago), - CpuSnapshot(used_ratio=0.6, max_used_ratio=0.95, created_at=three_hours_ago), - CpuSnapshot(used_ratio=0.7, max_used_ratio=0.95, created_at=two_hours_ago), - CpuSnapshot(used_ratio=0.8, max_used_ratio=0.95, created_at=now), - ] + snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot(used_ratio=0.5, max_used_ratio=0.95, created_at=five_hours_ago), + CpuSnapshot(used_ratio=0.6, max_used_ratio=0.95, created_at=three_hours_ago), + CpuSnapshot(used_ratio=0.7, max_used_ratio=0.95, created_at=two_hours_ago), + CpuSnapshot(used_ratio=0.8, max_used_ratio=0.95, created_at=now), + ] + ) # Assign these snapshots to one of the lists (e.g., CPU snapshots) snapshotter._cpu_snapshots = snapshots @@ -136,7 +142,7 @@ def test_snapshot_pruning_removes_outdated_records(snapshotter: Snapshotter) -> def test_pruning_empty_snapshot_list_remains_empty(snapshotter: Snapshotter) -> None: now = datetime.now(timezone.utc) - snapshotter._cpu_snapshots = [] + snapshotter._cpu_snapshots = Snapshotter._get_sorted_list_by_created_at([]) snapshots_casted = cast(list[Snapshot], snapshotter._cpu_snapshots) snapshotter._prune_snapshots(snapshots_casted, now) assert snapshotter._cpu_snapshots == [] @@ -150,10 +156,12 @@ def test_snapshot_pruning_keeps_recent_records_unaffected(snapshotter: Snapshott one_hour_ago = now - timedelta(hours=1) # Create mock snapshots with varying creation times - snapshots = [ - CpuSnapshot(used_ratio=0.7, max_used_ratio=0.95, created_at=one_hour_ago), - CpuSnapshot(used_ratio=0.8, max_used_ratio=0.95, created_at=now), - ] + snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot(used_ratio=0.7, max_used_ratio=0.95, created_at=one_hour_ago), + CpuSnapshot(used_ratio=0.8, max_used_ratio=0.95, created_at=now), + ] + ) # Assign these snapshots to one of the lists (e.g., CPU snapshots) snapshotter._cpu_snapshots = snapshots diff --git a/tests/unit/_autoscaling/test_system_status.py b/tests/unit/_autoscaling/test_system_status.py index 1b61c31478..7409ed0a6a 100644 --- a/tests/unit/_autoscaling/test_system_status.py +++ b/tests/unit/_autoscaling/test_system_status.py @@ -42,12 +42,14 @@ async def test_start_stop_lifecycle() -> None: def test_cpu_is_overloaded(snapshotter: Snapshotter, now: datetime) -> None: system_status = SystemStatus(snapshotter, cpu_overload_threshold=0.5) - system_status._snapshotter._cpu_snapshots = [ - CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), - CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), - CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), - CpuSnapshot(used_ratio=0.9, max_used_ratio=0.75, created_at=now), - ] + system_status._snapshotter._cpu_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), + CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), + CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), + CpuSnapshot(used_ratio=0.9, max_used_ratio=0.75, created_at=now), + ] + ) cpu_info = system_status._is_cpu_overloaded() assert cpu_info == LoadRatioInfo(limit_ratio=0.5, actual_ratio=0.667) @@ -56,12 +58,14 @@ def test_cpu_is_overloaded(snapshotter: Snapshotter, now: datetime) -> None: def test_cpu_is_not_overloaded(snapshotter: Snapshotter, now: datetime) -> None: system_status = SystemStatus(snapshotter, cpu_overload_threshold=0.5) - system_status._snapshotter._cpu_snapshots = [ - CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), - CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), - CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), - CpuSnapshot(used_ratio=0.5, max_used_ratio=0.75, created_at=now), - ] + system_status._snapshotter._cpu_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), + CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), + CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), + CpuSnapshot(used_ratio=0.5, max_used_ratio=0.75, created_at=now), + ] + ) cpu_info = system_status._is_cpu_overloaded() assert cpu_info == LoadRatioInfo(limit_ratio=0.5, actual_ratio=0.333) @@ -79,72 +83,80 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None: ) # Add CPU snapshots - system_status._snapshotter._cpu_snapshots = [ - CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), - CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), - CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), - CpuSnapshot(used_ratio=0.9, max_used_ratio=0.75, created_at=now), - ] + system_status._snapshotter._cpu_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), + CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), + CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), + CpuSnapshot(used_ratio=0.9, max_used_ratio=0.75, created_at=now), + ] + ) # Add memory snapshots - system_status._snapshotter._memory_snapshots = [ - MemorySnapshot( - current_size=ByteSize.from_gb(4), - max_memory_size=ByteSize.from_gb(12), - max_used_memory_ratio=0.8, - created_at=now - timedelta(seconds=90), - ), - MemorySnapshot( - current_size=ByteSize.from_gb(7), - max_memory_size=ByteSize.from_gb(8), - max_used_memory_ratio=0.8, - created_at=now - timedelta(seconds=60), - ), - MemorySnapshot( - current_size=ByteSize.from_gb(28), - max_memory_size=ByteSize.from_gb(30), - max_used_memory_ratio=0.8, - created_at=now - timedelta(seconds=30), - ), - MemorySnapshot( - current_size=ByteSize.from_gb(48), - max_memory_size=ByteSize.from_gb(60), - max_used_memory_ratio=0.8, - created_at=now, - ), - ] + system_status._snapshotter._memory_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + MemorySnapshot( + current_size=ByteSize.from_gb(4), + max_memory_size=ByteSize.from_gb(12), + max_used_memory_ratio=0.8, + created_at=now - timedelta(seconds=90), + ), + MemorySnapshot( + current_size=ByteSize.from_gb(7), + max_memory_size=ByteSize.from_gb(8), + max_used_memory_ratio=0.8, + created_at=now - timedelta(seconds=60), + ), + MemorySnapshot( + current_size=ByteSize.from_gb(28), + max_memory_size=ByteSize.from_gb(30), + max_used_memory_ratio=0.8, + created_at=now - timedelta(seconds=30), + ), + MemorySnapshot( + current_size=ByteSize.from_gb(48), + max_memory_size=ByteSize.from_gb(60), + max_used_memory_ratio=0.8, + created_at=now, + ), + ] + ) # Add event loop snapshots - system_status._snapshotter._event_loop_snapshots = [ - EventLoopSnapshot( - delay=timedelta(milliseconds=700), - max_delay=timedelta(milliseconds=500), - created_at=now - timedelta(minutes=3), - ), - EventLoopSnapshot( - delay=timedelta(milliseconds=600), - max_delay=timedelta(milliseconds=500), - created_at=now - timedelta(minutes=2), - ), - EventLoopSnapshot( - delay=timedelta(milliseconds=200), - max_delay=timedelta(milliseconds=500), - created_at=now - timedelta(minutes=1), - ), - EventLoopSnapshot( - delay=timedelta(milliseconds=100), - max_delay=timedelta(milliseconds=500), - created_at=now, - ), - ] + system_status._snapshotter._event_loop_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + EventLoopSnapshot( + delay=timedelta(milliseconds=700), + max_delay=timedelta(milliseconds=500), + created_at=now - timedelta(minutes=3), + ), + EventLoopSnapshot( + delay=timedelta(milliseconds=600), + max_delay=timedelta(milliseconds=500), + created_at=now - timedelta(minutes=2), + ), + EventLoopSnapshot( + delay=timedelta(milliseconds=200), + max_delay=timedelta(milliseconds=500), + created_at=now - timedelta(minutes=1), + ), + EventLoopSnapshot( + delay=timedelta(milliseconds=100), + max_delay=timedelta(milliseconds=500), + created_at=now, + ), + ] + ) # Add client snapshots - system_status._snapshotter._client_snapshots = [ - ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=3)), - ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=2)), - ClientSnapshot(error_count=2, max_error_count=2, created_at=now - timedelta(minutes=1)), - ClientSnapshot(error_count=0, max_error_count=2, created_at=now), - ] + system_status._snapshotter._client_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=3)), + ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=2)), + ClientSnapshot(error_count=2, max_error_count=2, created_at=now - timedelta(minutes=1)), + ClientSnapshot(error_count=0, max_error_count=2, created_at=now), + ] + ) # Test current system info current_system_info = system_status.get_current_system_info() From 8a6acc35e0345731614fc33c3f4be93f0b5ca294 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 14 Nov 2024 11:47:46 +0100 Subject: [PATCH 7/9] Address review comments Remove unused compute_weighted_avg and tests. --- pyproject.toml | 2 +- src/crawlee/_autoscaling/snapshotter.py | 8 ++--- src/crawlee/_utils/math.py | 30 ---------------- tests/unit/_utils/test_math.py | 46 ------------------------- 4 files changed, 5 insertions(+), 81 deletions(-) delete mode 100644 src/crawlee/_utils/math.py delete mode 100644 tests/unit/_utils/test_math.py diff --git a/pyproject.toml b/pyproject.toml index 9af0fcf1ca..e56810abe3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,11 +86,11 @@ pytest-xdist = "~3.6.0" respx = "~0.21.0" ruff = "~0.7.0" setuptools = "~75.3.0" # setuptools are used by pytest, but not explicitly required +sortedcontainers-stubs = "^2.4.2" types-beautifulsoup4 = "~4.12.0.20240229" types-colorama = "~0.4.15.20240106" types-psutil = "~5.9.5.20240205" types-python-dateutil = "~2.9.0.20240316" -sortedcontainers-stubs = "^2.4.2" [tool.poetry.extras] all = ["apify", "beautifulsoup4", "lxml", "html5lib", "curl-cffi", "playwright"] diff --git a/src/crawlee/_autoscaling/snapshotter.py b/src/crawlee/_autoscaling/snapshotter.py index 102a243cff..3bb9c98098 100644 --- a/src/crawlee/_autoscaling/snapshotter.py +++ b/src/crawlee/_autoscaling/snapshotter.py @@ -104,10 +104,10 @@ def __init__( cast(float, available_memory_ratio) ) - self._cpu_snapshots: SortedList[CpuSnapshot] = self._get_sorted_list_by_created_at([]) - self._event_loop_snapshots: SortedList[EventLoopSnapshot] = self._get_sorted_list_by_created_at([]) - self._memory_snapshots: SortedList[MemorySnapshot] = self._get_sorted_list_by_created_at([]) - self._client_snapshots: SortedList[ClientSnapshot] = self._get_sorted_list_by_created_at([]) + self._cpu_snapshots = self._get_sorted_list_by_created_at(list[CpuSnapshot]()) + self._event_loop_snapshots = self._get_sorted_list_by_created_at(list[EventLoopSnapshot]()) + self._memory_snapshots = self._get_sorted_list_by_created_at(list[MemorySnapshot]()) + self._client_snapshots = self._get_sorted_list_by_created_at(list[ClientSnapshot]()) self._snapshot_event_loop_task = RecurringTask(self._snapshot_event_loop, self._event_loop_snapshot_interval) self._snapshot_client_task = RecurringTask(self._snapshot_client, self._client_snapshot_interval) diff --git a/src/crawlee/_utils/math.py b/src/crawlee/_utils/math.py deleted file mode 100644 index 00ff73f707..0000000000 --- a/src/crawlee/_utils/math.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - - -def compute_weighted_avg(values: list[float], weights: list[float]) -> float: - """Computes a weighted average of an array of numbers, complemented by an array of weights. - - Args: - values: List of values. - weights: List of weights. - - Raises: - ValueError: If total weight is zero. - - Returns: - float: Weighted average. - """ - if not values or not weights: - raise ValueError('Values and weights lists must not be empty') - - if len(values) != len(weights): - raise ValueError('Values and weights must be of equal length') - - # zip(..., strict=True) can be used once support for Python 3.9 is dropped - result = sum(value * weight for value, weight in zip(values, weights)) - total_weight = sum(weights) - - if total_weight == 0: - raise ValueError('Total weight cannot be zero') - - return result / total_weight diff --git a/tests/unit/_utils/test_math.py b/tests/unit/_utils/test_math.py deleted file mode 100644 index d9376f1281..0000000000 --- a/tests/unit/_utils/test_math.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -import pytest - -from crawlee._utils.math import compute_weighted_avg - - -@pytest.mark.parametrize( - ('values', 'weights', 'expected'), - [ - ([20, 40, 50], [2, 3, 5], 41), - ([1, 2, 3], [0.5, 0.25, 0.25], 1.75), - ([4, 4, 4], [1, 0, 1], 4.0), - ([1, 2, 3], [0.33, 0.33, 0.33], 2), - ([1, 2, 3], [0.2, -0.3, 0.5], 2.75), - ], - ids=['basic', 'fractional_weights', 'zero_weight', 'all_equal_weights', 'negative_weights'], -) -def test_compute_weighted_avg_various_conditions(values: list[float], weights: list[float], expected: float) -> None: - assert compute_weighted_avg(values, weights) == expected - - -def test_compute_weighted_avg_raises_error_for_empty_lists() -> None: - values: list[float] = [] - weights: list[float] = [] - with pytest.raises(ValueError, match='Values and weights lists must not be empty'): - compute_weighted_avg(values, weights) - - -@pytest.mark.parametrize( - ('values', 'weights'), - [ - ([3, 2], [10]), - ([2], [1, 5, 7]), - ], -) -def test_compute_weighted_avg_raises_error_for_unequal_length_lists(values: list[float], weights: list[float]) -> None: - with pytest.raises(ValueError, match='Values and weights must be of equal length'): - compute_weighted_avg(values, weights) - - -def test_compute_weighted_avg_raises_error_for_zero_total_weight() -> None: - values: list[float] = [1, 2, 3] - weights: list[float] = [0, 0, 0] - with pytest.raises(ValueError, match='Total weight cannot be zero'): - compute_weighted_avg(values, weights) From d825e0c6e373553caacdb7dfeacfb4e090ed3259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Proch=C3=A1zka?= Date: Thu, 14 Nov 2024 11:57:07 +0100 Subject: [PATCH 8/9] Update tests/unit/_autoscaling/test_snapshotter.py Co-authored-by: Vlada Dusek --- tests/unit/_autoscaling/test_snapshotter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/_autoscaling/test_snapshotter.py b/tests/unit/_autoscaling/test_snapshotter.py index b94f0ca4e8..21f80e1d64 100644 --- a/tests/unit/_autoscaling/test_snapshotter.py +++ b/tests/unit/_autoscaling/test_snapshotter.py @@ -142,7 +142,7 @@ def test_snapshot_pruning_removes_outdated_records(snapshotter: Snapshotter) -> def test_pruning_empty_snapshot_list_remains_empty(snapshotter: Snapshotter) -> None: now = datetime.now(timezone.utc) - snapshotter._cpu_snapshots = Snapshotter._get_sorted_list_by_created_at([]) + snapshotter._cpu_snapshots = Snapshotter._get_sorted_list_by_created_at(list[CpuSnapshot]()) snapshots_casted = cast(list[Snapshot], snapshotter._cpu_snapshots) snapshotter._prune_snapshots(snapshots_casted, now) assert snapshotter._cpu_snapshots == [] From 691e4f3622b34190ef82ac609912a364527e13a0 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Fri, 15 Nov 2024 08:53:03 +0100 Subject: [PATCH 9/9] Review comments 2 --- src/crawlee/_autoscaling/system_status.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/crawlee/_autoscaling/system_status.py b/src/crawlee/_autoscaling/system_status.py index 50af0a054a..2fb4d7d45d 100644 --- a/src/crawlee/_autoscaling/system_status.py +++ b/src/crawlee/_autoscaling/system_status.py @@ -186,14 +186,16 @@ def _is_sample_overloaded(self, sample: list[Snapshot], threshold: float) -> Loa for previous, current in pairwise(sample): time = (current.created_at - previous.created_at).total_seconds() + if time < 0: + raise ValueError('Negative time. Code assumptions are not valid. Expected time sorted samples.') if current.is_overloaded: overloaded_time += time else: non_overloaded_time += time - if (overloaded_time + non_overloaded_time) == 0: + if (total_time := overloaded_time + non_overloaded_time) == 0: overloaded_ratio = 0.0 else: - overloaded_ratio = overloaded_time / (overloaded_time + non_overloaded_time) + overloaded_ratio = overloaded_time / total_time return LoadRatioInfo(limit_ratio=threshold, actual_ratio=round(overloaded_ratio, 3))