diff --git a/poetry.lock b/poetry.lock index 4294e84f30..ee15f5c6ed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -249,6 +249,10 @@ files = [ {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d"}, {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0"}, {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec"}, {file = "Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2"}, {file = "Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128"}, {file = "Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc"}, @@ -261,8 +265,14 @@ files = [ {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9"}, {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265"}, {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b"}, {file = "Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50"}, {file = "Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1"}, + {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28"}, + {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f"}, {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409"}, {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2"}, {file = "Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451"}, @@ -273,8 +283,24 @@ files = [ {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180"}, {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248"}, {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839"}, {file = "Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0"}, {file = "Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951"}, + {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5"}, + {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7"}, + {file = "Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0"}, + {file = "Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b"}, {file = "Brotli-1.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1"}, {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d"}, {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b"}, @@ -284,6 +310,10 @@ files = [ {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2"}, {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354"}, {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:aea440a510e14e818e67bfc4027880e2fb500c2ccb20ab21c7a7c8b5b4703d75"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:6974f52a02321b36847cd19d1b8e381bf39939c21efd6ee2fc13a28b0d99348c"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:a7e53012d2853a07a4a79c00643832161a910674a893d296c9f1259859a289d2"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:d7702622a8b40c49bffb46e1e3ba2e81268d5c04a34f460978c6b5517a34dd52"}, {file = "Brotli-1.1.0-cp36-cp36m-win32.whl", hash = "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460"}, {file = "Brotli-1.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579"}, {file = "Brotli-1.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c"}, @@ -295,6 +325,10 @@ files = [ {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74"}, {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b"}, {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:cb1dac1770878ade83f2ccdf7d25e494f05c9165f5246b46a621cc849341dc01"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:3ee8a80d67a4334482d9712b8e83ca6b1d9bc7e351931252ebef5d8f7335a547"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5e55da2c8724191e5b557f8e18943b1b4839b8efc3ef60d65985bcf6f587dd38"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:d342778ef319e1026af243ed0a07c97acf3bad33b9f29e7ae6a1f68fd083e90c"}, {file = "Brotli-1.1.0-cp37-cp37m-win32.whl", hash = "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95"}, {file = "Brotli-1.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68"}, {file = "Brotli-1.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3"}, @@ -307,6 +341,10 @@ files = [ {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a"}, {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:141bd4d93984070e097521ed07e2575b46f817d08f9fa42b16b9b5f27b5ac088"}, {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d2b35ca2c7f81d173d2fadc2f4f31e88cc5f7a39ae5b6db5513cf3383b0e0ec7"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:af6fa6817889314555aede9a919612b23739395ce767fe7fcbea9a80bf140fe5"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:2feb1d960f760a575dbc5ab3b1c00504b24caaf6986e2dc2b01c09c87866a943"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4410f84b33374409552ac9b6903507cdb31cd30d2501fc5ca13d18f73548444a"}, {file = "Brotli-1.1.0-cp38-cp38-win32.whl", hash = "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b"}, {file = "Brotli-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0"}, {file = "Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a"}, @@ -319,6 +357,10 @@ files = [ {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c"}, {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d"}, {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb"}, {file = "Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64"}, {file = "Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467"}, {file = "Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724"}, @@ -2752,6 +2794,21 @@ files = [ {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, ] +[[package]] +name = "sortedcontainers-stubs" +version = "2.4.2" +description = "Type stubs for sortedcontainers" +optional = false +python-versions = ">=3.8,<4.0" +files = [ + {file = "sortedcontainers_stubs-2.4.2-py3-none-any.whl", hash = "sha256:6cdbcf9c71198729dd6c0a8a7dc9742768f25b49384b91965d716a9d16ca7b2a"}, + {file = "sortedcontainers_stubs-2.4.2.tar.gz", hash = "sha256:c24e33877effa5a3eb85357112496ba33a0b681c05e0d5d3745664fd3be75d82"}, +] + +[package.dependencies] +sortedcontainers = ">=2,<3" +typing-extensions = ">=4.1.0,<5.0.0" + [[package]] name = "soupsieve" version = "2.6" @@ -3294,4 +3351,4 @@ playwright = ["playwright"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "d49bb55e1dc149d480273ebb907db7f7061d0200ab947a28e540ad94d1b4615e" +content-hash = "bea33bb28a5a8ec0f945c0f5936d44e1fafefefff293a05b2d0840fe2b0f18c2" diff --git a/pyproject.toml b/pyproject.toml index 8dec5a615f..e56810abe3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ pytest-xdist = "~3.6.0" respx = "~0.21.0" ruff = "~0.7.0" setuptools = "~75.3.0" # setuptools are used by pytest, but not explicitly required +sortedcontainers-stubs = "^2.4.2" types-beautifulsoup4 = "~4.12.0.20240229" types-colorama = "~0.4.15.20240106" types-psutil = "~5.9.5.20240205" diff --git a/src/crawlee/_autoscaling/snapshotter.py b/src/crawlee/_autoscaling/snapshotter.py index 161ce2a63a..3bb9c98098 100644 --- a/src/crawlee/_autoscaling/snapshotter.py +++ b/src/crawlee/_autoscaling/snapshotter.py @@ -4,11 +4,19 @@ from datetime import datetime, timedelta, timezone from logging import getLogger -from typing import TYPE_CHECKING, cast +from operator import attrgetter +from typing import TYPE_CHECKING, TypeVar, cast import psutil - -from crawlee._autoscaling.types import ClientSnapshot, CpuSnapshot, EventLoopSnapshot, MemorySnapshot, Snapshot +from sortedcontainers import SortedList + +from crawlee._autoscaling.types import ( + ClientSnapshot, + CpuSnapshot, + EventLoopSnapshot, + MemorySnapshot, + Snapshot, +) from crawlee._utils.byte_size import ByteSize from crawlee._utils.docs import docs_group from crawlee._utils.recurring_task import RecurringTask @@ -21,6 +29,8 @@ logger = getLogger(__name__) +T = TypeVar('T') + @docs_group('Classes') class Snapshotter: @@ -94,16 +104,20 @@ def __init__( cast(float, available_memory_ratio) ) - self._cpu_snapshots: list[CpuSnapshot] = [] - self._event_loop_snapshots: list[EventLoopSnapshot] = [] - self._memory_snapshots: list[MemorySnapshot] = [] - self._client_snapshots: list[ClientSnapshot] = [] + self._cpu_snapshots = self._get_sorted_list_by_created_at(list[CpuSnapshot]()) + self._event_loop_snapshots = self._get_sorted_list_by_created_at(list[EventLoopSnapshot]()) + self._memory_snapshots = self._get_sorted_list_by_created_at(list[MemorySnapshot]()) + self._client_snapshots = self._get_sorted_list_by_created_at(list[ClientSnapshot]()) self._snapshot_event_loop_task = RecurringTask(self._snapshot_event_loop, self._event_loop_snapshot_interval) self._snapshot_client_task = RecurringTask(self._snapshot_client, self._client_snapshot_interval) self._timestamp_of_last_memory_warning: datetime = datetime.now(timezone.utc) - timedelta(hours=1) + @staticmethod + def _get_sorted_list_by_created_at(input_list: list[T]) -> SortedList[T]: + return SortedList(input_list, key=attrgetter('created_at')) + @staticmethod def _get_default_max_memory_size(available_memory_ratio: float) -> ByteSize: """Default `memory_max_size` is 1/4 of the total system memory.""" @@ -194,7 +208,7 @@ def _get_sample(snapshots: list[Snapshot], duration: timedelta | None = None) -> return [] latest_time = snapshots[-1].created_at - return [snapshot for snapshot in reversed(snapshots) if latest_time - snapshot.created_at <= duration] + return [snapshot for snapshot in snapshots if latest_time - snapshot.created_at <= duration] def _snapshot_cpu(self, event_data: EventSystemInfoData) -> None: """Captures a snapshot of the current CPU usage. @@ -213,7 +227,7 @@ def _snapshot_cpu(self, event_data: EventSystemInfoData) -> None: snapshots = cast(list[Snapshot], self._cpu_snapshots) self._prune_snapshots(snapshots, event_data.cpu_info.created_at) - self._cpu_snapshots.append(snapshot) + self._cpu_snapshots.add(snapshot) def _snapshot_memory(self, event_data: EventSystemInfoData) -> None: """Captures a snapshot of the current memory usage. @@ -233,8 +247,7 @@ def _snapshot_memory(self, event_data: EventSystemInfoData) -> None: snapshots = cast(list[Snapshot], self._memory_snapshots) self._prune_snapshots(snapshots, snapshot.created_at) - self._memory_snapshots.append(snapshot) - + self._memory_snapshots.add(snapshot) self._evaluate_memory_load(event_data.memory_info.current_size, event_data.memory_info.created_at) def _snapshot_event_loop(self) -> None: @@ -254,7 +267,7 @@ def _snapshot_event_loop(self) -> None: snapshots = cast(list[Snapshot], self._event_loop_snapshots) self._prune_snapshots(snapshots, snapshot.created_at) - self._event_loop_snapshots.append(snapshot) + self._event_loop_snapshots.add(snapshot) def _snapshot_client(self) -> None: """Captures a snapshot of the current API state by checking for rate limit errors (HTTP 429). @@ -271,7 +284,7 @@ def _snapshot_client(self) -> None: snapshots = cast(list[Snapshot], self._client_snapshots) self._prune_snapshots(snapshots, snapshot.created_at) - self._client_snapshots.append(snapshot) + self._client_snapshots.add(snapshot) def _prune_snapshots(self, snapshots: list[Snapshot], now: datetime) -> None: """Removes snapshots that are older than the `self._snapshot_history`. diff --git a/src/crawlee/_autoscaling/system_status.py b/src/crawlee/_autoscaling/system_status.py index 737b0eec07..2fb4d7d45d 100644 --- a/src/crawlee/_autoscaling/system_status.py +++ b/src/crawlee/_autoscaling/system_status.py @@ -10,7 +10,6 @@ from crawlee._autoscaling.types import LoadRatioInfo, Snapshot, SystemInfo from crawlee._utils.docs import docs_group -from crawlee._utils.math import compute_weighted_avg if TYPE_CHECKING: from crawlee._autoscaling import Snapshotter @@ -182,17 +181,21 @@ def _is_sample_overloaded(self, sample: list[Snapshot], threshold: float) -> Loa if len(sample) == 1: return LoadRatioInfo(limit_ratio=threshold, actual_ratio=float(sample[0].is_overloaded)) - weights, values = [], [] + overloaded_time = 0.0 + non_overloaded_time = 0.0 for previous, current in pairwise(sample): - weight = (current.created_at - previous.created_at).total_seconds() or 0.001 # Avoid zero - value = float(current.is_overloaded) - weights.append(weight) - values.append(value) - - try: - weighted_avg = compute_weighted_avg(values, weights) - except ValueError as exc: - raise ValueError('Failed to compute weighted average for the sample.') from exc - - return LoadRatioInfo(limit_ratio=threshold, actual_ratio=round(weighted_avg, 3)) + time = (current.created_at - previous.created_at).total_seconds() + if time < 0: + raise ValueError('Negative time. Code assumptions are not valid. Expected time sorted samples.') + if current.is_overloaded: + overloaded_time += time + else: + non_overloaded_time += time + + if (total_time := overloaded_time + non_overloaded_time) == 0: + overloaded_ratio = 0.0 + else: + overloaded_ratio = overloaded_time / total_time + + return LoadRatioInfo(limit_ratio=threshold, actual_ratio=round(overloaded_ratio, 3)) diff --git a/src/crawlee/_utils/math.py b/src/crawlee/_utils/math.py deleted file mode 100644 index 00ff73f707..0000000000 --- a/src/crawlee/_utils/math.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - - -def compute_weighted_avg(values: list[float], weights: list[float]) -> float: - """Computes a weighted average of an array of numbers, complemented by an array of weights. - - Args: - values: List of values. - weights: List of weights. - - Raises: - ValueError: If total weight is zero. - - Returns: - float: Weighted average. - """ - if not values or not weights: - raise ValueError('Values and weights lists must not be empty') - - if len(values) != len(weights): - raise ValueError('Values and weights must be of equal length') - - # zip(..., strict=True) can be used once support for Python 3.9 is dropped - result = sum(value * weight for value, weight in zip(values, weights)) - total_weight = sum(weights) - - if total_weight == 0: - raise ValueError('Total weight cannot be zero') - - return result / total_weight diff --git a/tests/unit/_autoscaling/test_snapshotter.py b/tests/unit/_autoscaling/test_snapshotter.py index c1addba7b2..21f80e1d64 100644 --- a/tests/unit/_autoscaling/test_snapshotter.py +++ b/tests/unit/_autoscaling/test_snapshotter.py @@ -12,7 +12,7 @@ from crawlee._utils.byte_size import ByteSize from crawlee._utils.system import CpuInfo, MemoryInfo from crawlee.events import EventManager, LocalEventManager -from crawlee.events._types import EventSystemInfoData +from crawlee.events._types import Event, EventSystemInfoData @pytest.fixture @@ -50,9 +50,11 @@ def test_snapshot_memory(snapshotter: Snapshotter, event_system_data_info: Event def test_snapshot_event_loop(snapshotter: Snapshotter) -> None: - snapshotter._event_loop_snapshots = [ - EventLoopSnapshot(delay=timedelta(milliseconds=100), max_delay=timedelta(milliseconds=500)), - ] + snapshotter._event_loop_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + EventLoopSnapshot(delay=timedelta(milliseconds=100), max_delay=timedelta(milliseconds=500)), + ] + ) snapshotter._snapshot_event_loop() assert len(snapshotter._event_loop_snapshots) == 2 @@ -65,14 +67,16 @@ def test_snapshot_client(snapshotter: Snapshotter) -> None: def test_get_cpu_sample(snapshotter: Snapshotter) -> None: now = datetime.now(timezone.utc) - cpu_snapshots = [ - CpuSnapshot( - used_ratio=0.5, - max_used_ratio=0.95, - created_at=now - timedelta(hours=delta), - ) - for delta in range(5, 0, -1) - ] + cpu_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot( + used_ratio=0.5, + max_used_ratio=0.95, + created_at=now - timedelta(hours=delta), + ) + for delta in range(5, 0, -1) + ] + ) snapshotter._cpu_snapshots = cpu_snapshots @@ -114,12 +118,14 @@ def test_snapshot_pruning_removes_outdated_records(snapshotter: Snapshotter) -> five_hours_ago = now - timedelta(hours=5) # Create mock snapshots with varying creation times - snapshots = [ - CpuSnapshot(used_ratio=0.5, max_used_ratio=0.95, created_at=five_hours_ago), - CpuSnapshot(used_ratio=0.6, max_used_ratio=0.95, created_at=three_hours_ago), - CpuSnapshot(used_ratio=0.7, max_used_ratio=0.95, created_at=two_hours_ago), - CpuSnapshot(used_ratio=0.8, max_used_ratio=0.95, created_at=now), - ] + snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot(used_ratio=0.5, max_used_ratio=0.95, created_at=five_hours_ago), + CpuSnapshot(used_ratio=0.6, max_used_ratio=0.95, created_at=three_hours_ago), + CpuSnapshot(used_ratio=0.7, max_used_ratio=0.95, created_at=two_hours_ago), + CpuSnapshot(used_ratio=0.8, max_used_ratio=0.95, created_at=now), + ] + ) # Assign these snapshots to one of the lists (e.g., CPU snapshots) snapshotter._cpu_snapshots = snapshots @@ -136,7 +142,7 @@ def test_snapshot_pruning_removes_outdated_records(snapshotter: Snapshotter) -> def test_pruning_empty_snapshot_list_remains_empty(snapshotter: Snapshotter) -> None: now = datetime.now(timezone.utc) - snapshotter._cpu_snapshots = [] + snapshotter._cpu_snapshots = Snapshotter._get_sorted_list_by_created_at(list[CpuSnapshot]()) snapshots_casted = cast(list[Snapshot], snapshotter._cpu_snapshots) snapshotter._prune_snapshots(snapshots_casted, now) assert snapshotter._cpu_snapshots == [] @@ -150,10 +156,12 @@ def test_snapshot_pruning_keeps_recent_records_unaffected(snapshotter: Snapshott one_hour_ago = now - timedelta(hours=1) # Create mock snapshots with varying creation times - snapshots = [ - CpuSnapshot(used_ratio=0.7, max_used_ratio=0.95, created_at=one_hour_ago), - CpuSnapshot(used_ratio=0.8, max_used_ratio=0.95, created_at=now), - ] + snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot(used_ratio=0.7, max_used_ratio=0.95, created_at=one_hour_ago), + CpuSnapshot(used_ratio=0.8, max_used_ratio=0.95, created_at=now), + ] + ) # Assign these snapshots to one of the lists (e.g., CPU snapshots) snapshotter._cpu_snapshots = snapshots @@ -207,3 +215,35 @@ def test_memory_load_evaluation_silent_on_acceptable_usage( ) assert mock_logger_warn.call_count == 0 + + +async def test_snapshots_time_ordered() -> None: + # All internal snapshot list should be ordered by creation time in ascending order. + # Scenario where older emitted event arrives after newer event. + # Snapshotter should not trust the event order and check events' times. + time_new = datetime.now(tz=timezone.utc) + time_old = datetime.now(tz=timezone.utc) - timedelta(milliseconds=50) + + def create_event_data(creation_time: datetime) -> EventSystemInfoData: + return EventSystemInfoData( + cpu_info=CpuInfo(used_ratio=0.5, created_at=creation_time), + memory_info=MemoryInfo( + current_size=ByteSize(bytes=1), created_at=creation_time, total_size=ByteSize(bytes=2) + ), + ) + + async with ( + LocalEventManager() as event_manager, + Snapshotter(event_manager, available_memory_ratio=0.25) as snapshotter, + ): + event_manager.emit(event=Event.SYSTEM_INFO, event_data=create_event_data(time_new)) + await event_manager.wait_for_all_listeners_to_complete() + event_manager.emit(event=Event.SYSTEM_INFO, event_data=create_event_data(time_old)) + await event_manager.wait_for_all_listeners_to_complete() + + memory_samples = snapshotter.get_memory_sample() + cpu_samples = snapshotter.get_cpu_sample() + assert memory_samples[0].created_at == time_old + assert cpu_samples[0].created_at == time_old + assert memory_samples[1].created_at == time_new + assert cpu_samples[1].created_at == time_new diff --git a/tests/unit/_autoscaling/test_system_status.py b/tests/unit/_autoscaling/test_system_status.py index 7da6cb6004..7409ed0a6a 100644 --- a/tests/unit/_autoscaling/test_system_status.py +++ b/tests/unit/_autoscaling/test_system_status.py @@ -42,12 +42,14 @@ async def test_start_stop_lifecycle() -> None: def test_cpu_is_overloaded(snapshotter: Snapshotter, now: datetime) -> None: system_status = SystemStatus(snapshotter, cpu_overload_threshold=0.5) - system_status._snapshotter._cpu_snapshots = [ - CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), - CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), - CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), - CpuSnapshot(used_ratio=0.9, max_used_ratio=0.75, created_at=now), - ] + system_status._snapshotter._cpu_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), + CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), + CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), + CpuSnapshot(used_ratio=0.9, max_used_ratio=0.75, created_at=now), + ] + ) cpu_info = system_status._is_cpu_overloaded() assert cpu_info == LoadRatioInfo(limit_ratio=0.5, actual_ratio=0.667) @@ -56,12 +58,14 @@ def test_cpu_is_overloaded(snapshotter: Snapshotter, now: datetime) -> None: def test_cpu_is_not_overloaded(snapshotter: Snapshotter, now: datetime) -> None: system_status = SystemStatus(snapshotter, cpu_overload_threshold=0.5) - system_status._snapshotter._cpu_snapshots = [ - CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), - CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), - CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), - CpuSnapshot(used_ratio=0.5, max_used_ratio=0.75, created_at=now), - ] + system_status._snapshotter._cpu_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), + CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), + CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), + CpuSnapshot(used_ratio=0.5, max_used_ratio=0.75, created_at=now), + ] + ) cpu_info = system_status._is_cpu_overloaded() assert cpu_info == LoadRatioInfo(limit_ratio=0.5, actual_ratio=0.333) @@ -79,78 +83,86 @@ def test_get_system_info(snapshotter: Snapshotter, now: datetime) -> None: ) # Add CPU snapshots - system_status._snapshotter._cpu_snapshots = [ - CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), - CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), - CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), - CpuSnapshot(used_ratio=0.9, max_used_ratio=0.75, created_at=now), - ] + system_status._snapshotter._cpu_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + CpuSnapshot(used_ratio=0.6, max_used_ratio=0.75, created_at=now - timedelta(minutes=3)), + CpuSnapshot(used_ratio=0.7, max_used_ratio=0.75, created_at=now - timedelta(minutes=2)), + CpuSnapshot(used_ratio=0.8, max_used_ratio=0.75, created_at=now - timedelta(minutes=1)), + CpuSnapshot(used_ratio=0.9, max_used_ratio=0.75, created_at=now), + ] + ) # Add memory snapshots - system_status._snapshotter._memory_snapshots = [ - MemorySnapshot( - current_size=ByteSize.from_gb(4), - max_memory_size=ByteSize.from_gb(12), - max_used_memory_ratio=0.8, - created_at=now - timedelta(minutes=3), - ), - MemorySnapshot( - current_size=ByteSize.from_gb(7), - max_memory_size=ByteSize.from_gb(8), - max_used_memory_ratio=0.8, - created_at=now - timedelta(minutes=2), - ), - MemorySnapshot( - current_size=ByteSize.from_gb(28), - max_memory_size=ByteSize.from_gb(30), - max_used_memory_ratio=0.8, - created_at=now - timedelta(minutes=1), - ), - MemorySnapshot( - current_size=ByteSize.from_gb(48), - max_memory_size=ByteSize.from_gb(60), - max_used_memory_ratio=0.8, - created_at=now, - ), - ] + system_status._snapshotter._memory_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + MemorySnapshot( + current_size=ByteSize.from_gb(4), + max_memory_size=ByteSize.from_gb(12), + max_used_memory_ratio=0.8, + created_at=now - timedelta(seconds=90), + ), + MemorySnapshot( + current_size=ByteSize.from_gb(7), + max_memory_size=ByteSize.from_gb(8), + max_used_memory_ratio=0.8, + created_at=now - timedelta(seconds=60), + ), + MemorySnapshot( + current_size=ByteSize.from_gb(28), + max_memory_size=ByteSize.from_gb(30), + max_used_memory_ratio=0.8, + created_at=now - timedelta(seconds=30), + ), + MemorySnapshot( + current_size=ByteSize.from_gb(48), + max_memory_size=ByteSize.from_gb(60), + max_used_memory_ratio=0.8, + created_at=now, + ), + ] + ) # Add event loop snapshots - system_status._snapshotter._event_loop_snapshots = [ - EventLoopSnapshot( - delay=timedelta(milliseconds=700), - max_delay=timedelta(milliseconds=500), - created_at=now - timedelta(minutes=3), - ), - EventLoopSnapshot( - delay=timedelta(milliseconds=600), - max_delay=timedelta(milliseconds=500), - created_at=now - timedelta(minutes=2), - ), - EventLoopSnapshot( - delay=timedelta(milliseconds=200), - max_delay=timedelta(milliseconds=500), - created_at=now - timedelta(minutes=1), - ), - EventLoopSnapshot( - delay=timedelta(milliseconds=100), - max_delay=timedelta(milliseconds=500), - created_at=now, - ), - ] + system_status._snapshotter._event_loop_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + EventLoopSnapshot( + delay=timedelta(milliseconds=700), + max_delay=timedelta(milliseconds=500), + created_at=now - timedelta(minutes=3), + ), + EventLoopSnapshot( + delay=timedelta(milliseconds=600), + max_delay=timedelta(milliseconds=500), + created_at=now - timedelta(minutes=2), + ), + EventLoopSnapshot( + delay=timedelta(milliseconds=200), + max_delay=timedelta(milliseconds=500), + created_at=now - timedelta(minutes=1), + ), + EventLoopSnapshot( + delay=timedelta(milliseconds=100), + max_delay=timedelta(milliseconds=500), + created_at=now, + ), + ] + ) # Add client snapshots - system_status._snapshotter._client_snapshots = [ - ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=3)), - ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=2)), - ClientSnapshot(error_count=2, max_error_count=2, created_at=now - timedelta(minutes=1)), - ClientSnapshot(error_count=0, max_error_count=2, created_at=now), - ] + system_status._snapshotter._client_snapshots = Snapshotter._get_sorted_list_by_created_at( + [ + ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=3)), + ClientSnapshot(error_count=1, max_error_count=2, created_at=now - timedelta(minutes=2)), + ClientSnapshot(error_count=2, max_error_count=2, created_at=now - timedelta(minutes=1)), + ClientSnapshot(error_count=0, max_error_count=2, created_at=now), + ] + ) # Test current system info current_system_info = system_status.get_current_system_info() assert current_system_info == SystemInfo( cpu_info=LoadRatioInfo(limit_ratio=system_status._cpu_overload_threshold, actual_ratio=1.0), - memory_info=LoadRatioInfo(limit_ratio=system_status._memory_overload_threshold, actual_ratio=1.0), + memory_info=LoadRatioInfo(limit_ratio=system_status._memory_overload_threshold, actual_ratio=0.5), event_loop_info=LoadRatioInfo(limit_ratio=system_status._event_loop_overload_threshold, actual_ratio=0), client_info=LoadRatioInfo(limit_ratio=system_status._client_overload_threshold, actual_ratio=0), created_at=current_system_info.created_at, diff --git a/tests/unit/_utils/test_math.py b/tests/unit/_utils/test_math.py deleted file mode 100644 index d9376f1281..0000000000 --- a/tests/unit/_utils/test_math.py +++ /dev/null @@ -1,46 +0,0 @@ -from __future__ import annotations - -import pytest - -from crawlee._utils.math import compute_weighted_avg - - -@pytest.mark.parametrize( - ('values', 'weights', 'expected'), - [ - ([20, 40, 50], [2, 3, 5], 41), - ([1, 2, 3], [0.5, 0.25, 0.25], 1.75), - ([4, 4, 4], [1, 0, 1], 4.0), - ([1, 2, 3], [0.33, 0.33, 0.33], 2), - ([1, 2, 3], [0.2, -0.3, 0.5], 2.75), - ], - ids=['basic', 'fractional_weights', 'zero_weight', 'all_equal_weights', 'negative_weights'], -) -def test_compute_weighted_avg_various_conditions(values: list[float], weights: list[float], expected: float) -> None: - assert compute_weighted_avg(values, weights) == expected - - -def test_compute_weighted_avg_raises_error_for_empty_lists() -> None: - values: list[float] = [] - weights: list[float] = [] - with pytest.raises(ValueError, match='Values and weights lists must not be empty'): - compute_weighted_avg(values, weights) - - -@pytest.mark.parametrize( - ('values', 'weights'), - [ - ([3, 2], [10]), - ([2], [1, 5, 7]), - ], -) -def test_compute_weighted_avg_raises_error_for_unequal_length_lists(values: list[float], weights: list[float]) -> None: - with pytest.raises(ValueError, match='Values and weights must be of equal length'): - compute_weighted_avg(values, weights) - - -def test_compute_weighted_avg_raises_error_for_zero_total_weight() -> None: - values: list[float] = [1, 2, 3] - weights: list[float] = [0, 0, 0] - with pytest.raises(ValueError, match='Total weight cannot be zero'): - compute_weighted_avg(values, weights)