From 39a3bf71d29c0242c3a7cf972351107d0cd607a0 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Thu, 13 Feb 2025 09:42:50 +0100 Subject: [PATCH 1/9] WIP --- src/crawlee/statistics/_error_tracker.py | 12 ++++++++++-- tests/unit/_statistics/test_error_tracker.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) create mode 100644 tests/unit/_statistics/test_error_tracker.py diff --git a/src/crawlee/statistics/_error_tracker.py b/src/crawlee/statistics/_error_tracker.py index 64f9b74d4e..b44f7a7cf2 100644 --- a/src/crawlee/statistics/_error_tracker.py +++ b/src/crawlee/statistics/_error_tracker.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections import Counter +from collections import Counter, defaultdict from dataclasses import dataclass @@ -16,7 +16,15 @@ class ErrorGroup: class ErrorTracker: """Track errors and aggregates their counts by similarity.""" - def __init__(self) -> None: + def __init__(self, + show_error_code: bool = True, + show_error_name: bool = True, + show_stack_trace: bool = True, + show_full_stack: bool = False, + show_error_message: bool = True, + show_full_message: bool = False, + save_error_snapshots: bool = False, + ) -> None: self._errors = Counter[ErrorGroup]() def add(self, error: Exception) -> None: diff --git a/tests/unit/_statistics/test_error_tracker.py b/tests/unit/_statistics/test_error_tracker.py new file mode 100644 index 0000000000..2ba0f619ce --- /dev/null +++ b/tests/unit/_statistics/test_error_tracker.py @@ -0,0 +1,15 @@ +from crawlee.statistics._error_tracker import ErrorTracker + + +def test_error_tracker(): + error_tracker = ErrorTracker() + error_1 = ValueError("Some value error") + error_2 = ValueError("Some value error") + error_3 = ValueError("Another value error") + + error_tracker.add(error_1) + error_tracker.add(error_2) + error_tracker.add(error_3) + + assert error_tracker.total == 3 + assert error_tracker.unique_error_count == 1 From 682eacc5cd543223a7851acc625780d733be5316 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 18 Feb 2025 17:37:21 +0100 Subject: [PATCH 2/9] Basic structure TODO: Message matching and replacing with wildcards Extra args - full stack and full message Tests --- src/crawlee/statistics/_error_tracker.py | 52 +++++++++++++++----- tests/unit/_statistics/test_error_tracker.py | 14 +++--- 2 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/crawlee/statistics/_error_tracker.py b/src/crawlee/statistics/_error_tracker.py index b44f7a7cf2..2baa79c936 100644 --- a/src/crawlee/statistics/_error_tracker.py +++ b/src/crawlee/statistics/_error_tracker.py @@ -2,35 +2,59 @@ from __future__ import annotations +import traceback from collections import Counter, defaultdict from dataclasses import dataclass -@dataclass(frozen=True, unsafe_hash=True) -class ErrorGroup: - """Identifies a group of similar errors.""" - class_name: str | None +# Errors are internally tracked in deeply nested dict where individual error groups count can be accessed like this: +# count = ErrorStackTraceGroups["some_stack_trace"]["some_error_code"]["some_error_name"]["some_error_message"] +# if any of the filtering keys was not used it will be `None` instead. +# count = ErrorStackTraceGroups["some_stack_trace"][None]["some_error_name"][None] + +ErrorMessageGroups = Counter[str|None] +ErrorTypeGroups = dict[str|None, ErrorMessageGroups] +#ErrorCodeGroups = dict[str|None, ErrorNameGroups] # No error codes in Python +ErrorStackTraceGroups = dict[str|None, ErrorTypeGroups] + class ErrorTracker: """Track errors and aggregates their counts by similarity.""" def __init__(self, - show_error_code: bool = True, show_error_name: bool = True, show_stack_trace: bool = True, - show_full_stack: bool = False, + show_full_stack: bool = False, # Todo, what does it do in JS? show_error_message: bool = True, - show_full_message: bool = False, - save_error_snapshots: bool = False, + show_full_message: bool = False, # Todo, seems like only first line otherwise + save_error_snapshots: bool = False, # Todo, what does it do in JS? ) -> None: - self._errors = Counter[ErrorGroup]() + self.show_error_name = show_error_name + self.show_stack_trace = show_stack_trace + self.show_full_stack = show_full_stack + self.show_error_message = show_error_message + self.show_full_message = show_full_message + + self._errors: ErrorStackTraceGroups = defaultdict(lambda: defaultdict(Counter)) def add(self, error: Exception) -> None: """Include an error in the statistics.""" - error_group = ErrorGroup(class_name=error.__class__.__name__) - self._errors[error_group] += 1 + # Check for similar + error_group_name = error.__class__.__name__ if self.show_error_name else None + error_group_message = str(error) if self.show_error_message else None + if self.show_stack_trace: + error_traceback = traceback.extract_tb(error.__traceback__) + error_group_stack_trace = f"{error_traceback[0].filename}:{error_traceback[0].lineno}" + # More extraction if full stack... + else: + error_group_stack_trace = None + + # TODO perform special message grouping + # First just exact match + + self._errors[error_group_stack_trace][error_group_name].update([error_group_message]) @property def unique_error_count(self) -> int: @@ -41,3 +65,9 @@ def unique_error_count(self) -> int: def total(self) -> int: """Total number of errors.""" return sum(self._errors.values()) + + def _add_to_existing_similar_error_group(self): + pass + + def _are_errors_similar(self) -> bool: + pass diff --git a/tests/unit/_statistics/test_error_tracker.py b/tests/unit/_statistics/test_error_tracker.py index 2ba0f619ce..ef1fcf0488 100644 --- a/tests/unit/_statistics/test_error_tracker.py +++ b/tests/unit/_statistics/test_error_tracker.py @@ -1,15 +1,15 @@ +import traceback + from crawlee.statistics._error_tracker import ErrorTracker def test_error_tracker(): error_tracker = ErrorTracker() - error_1 = ValueError("Some value error") - error_2 = ValueError("Some value error") - error_3 = ValueError("Another value error") - - error_tracker.add(error_1) - error_tracker.add(error_2) - error_tracker.add(error_3) + for error in [ValueError("Some value error"), ValueError("Some value error"), ValueError("Another value error")]: + try: + raise error + except Exception: + error_tracker.add(error) assert error_tracker.total == 3 assert error_tracker.unique_error_count == 1 From 038888694c3ef975b9634e02e68bb830cbc31bc8 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 19 Feb 2025 14:31:37 +0100 Subject: [PATCH 3/9] Basic filtering working. Todo: Add get_most_popular_errors Add extended configuration parameters --- src/crawlee/statistics/_error_tracker.py | 62 +++++++++++++++++--- tests/unit/_statistics/test_error_tracker.py | 55 ++++++++++++++--- 2 files changed, 101 insertions(+), 16 deletions(-) diff --git a/src/crawlee/statistics/_error_tracker.py b/src/crawlee/statistics/_error_tracker.py index 2baa79c936..e5d149546b 100644 --- a/src/crawlee/statistics/_error_tracker.py +++ b/src/crawlee/statistics/_error_tracker.py @@ -5,8 +5,7 @@ import traceback from collections import Counter, defaultdict from dataclasses import dataclass - - +from itertools import zip_longest # Errors are internally tracked in deeply nested dict where individual error groups count can be accessed like this: # count = ErrorStackTraceGroups["some_stack_trace"]["some_error_code"]["some_error_name"]["some_error_message"] @@ -36,7 +35,6 @@ def __init__(self, self.show_full_stack = show_full_stack self.show_error_message = show_error_message self.show_full_message = show_full_message - self._errors: ErrorStackTraceGroups = defaultdict(lambda: defaultdict(Counter)) def add(self, error: Exception) -> None: @@ -54,20 +52,66 @@ def add(self, error: Exception) -> None: # TODO perform special message grouping # First just exact match - self._errors[error_group_stack_trace][error_group_name].update([error_group_message]) + # All groups, except the lowest level is matched. + specific_groups = self._errors[error_group_stack_trace][error_group_name] + + if error_group_message in specific_groups: + # Exact match + specific_groups.update([error_group_message]) + else: + for known_error_group_message in specific_groups: + if new_group_name:=self._create_generic_message(known_error_group_message, error_group_message): + # Replace old name + specific_groups[new_group_name]=specific_groups.pop(known_error_group_message) + # Increment + specific_groups.update([new_group_name]) + break + else: + # No similar message found. Create new group. + self._errors[error_group_stack_trace][error_group_name].update([error_group_message]) + @property def unique_error_count(self) -> int: """Number of distinct kinds of errors.""" - return len(self._errors) + unique_error_count = 0 + for stack_group in self._errors.values(): + for name_group in stack_group.values(): + unique_error_count+=len(name_group) + return unique_error_count @property def total(self) -> int: """Total number of errors.""" - return sum(self._errors.values()) + error_count = 0 + for stack_group in self._errors.values(): + for name_group in stack_group.values(): + error_count+=sum(name_group.values()) + return error_count - def _add_to_existing_similar_error_group(self): - pass + def get_most_popular_errors(self, n=3) -> str: - def _are_errors_similar(self) -> bool: + def _add_to_existing_similar_message_group(self): + # Check to if exactly same error group already exists + # If not, try to assign to similar group. Iterate one by one and try to merge if possible pass + + def _create_generic_message(self, message_1: str, message_2: str) -> str: + replacement_symbol = "_" + replacement_count = 0 + message_1_parts = message_1.split(" ") + message_2_parts = message_2.split(" ") + generic_message_parts = [] + parts_count = max(len(message_1_parts), len(message_2_parts)) + + for message_1_part, message_2_part in zip_longest(message_1_parts, message_2_parts, fillvalue=""): + if message_1_part != message_2_part: + generic_message_parts.append(replacement_symbol) + replacement_count+=1 + if replacement_count >= parts_count/2: + # Messages are too different. + return "" + else: + generic_message_parts.append(message_1_part) + return " ".join(generic_message_parts) + diff --git a/tests/unit/_statistics/test_error_tracker.py b/tests/unit/_statistics/test_error_tracker.py index ef1fcf0488..6d7d43e444 100644 --- a/tests/unit/_statistics/test_error_tracker.py +++ b/tests/unit/_statistics/test_error_tracker.py @@ -1,15 +1,56 @@ -import traceback +import pytest + from crawlee.statistics._error_tracker import ErrorTracker -def test_error_tracker(): +@pytest.mark.parametrize("error_tracker, expected_unique_errors", [ + (ErrorTracker(), 4), + (ErrorTracker(show_stack_trace=False), 3), + (ErrorTracker(show_error_name=False), 3), + (ErrorTracker(show_error_message=False), 3), + (ErrorTracker(show_stack_trace=False, show_error_name=False), 2), + (ErrorTracker(show_stack_trace=False, show_error_message=False), 2), + (ErrorTracker(show_stack_trace=False, show_error_name=False, show_error_message=False), 1), + +]) +def test_error_tracker_counts(error_tracker: ErrorTracker, expected_unique_errors: int): + """Use different settings of `error_tracker` and test unique errors count.""" + + for error in [ + Exception("Some value error abc"), + ValueError("Some value error abc"), # Different type, different error + ValueError("Some value error cde"), # Same type and similar message to previous, considered the same. + ValueError("Another value error efg") # Same type, but too different message to previous, considered different. + ]: + try: + raise error # Errors raised on same line + except Exception as e: + error_tracker.add(e) + + try: + raise ValueError("Some value error abc") # Same as one previous error, but different line. + except Exception as e: + error_tracker.add(e) + + assert error_tracker.total == 5 + assert error_tracker.unique_error_count == expected_unique_errors + +@pytest.mark.parametrize("message_1, message_2, expected_generic_message", [ + ("Some error number 123", "Some error number 456", "Some error number _"), +]) +def test_error_tracker_similar_messages(message_1, message_2, expected_generic_message): + """Test that similar messages collapse into same group with generic name that contains wildcards.""" error_tracker = ErrorTracker() - for error in [ValueError("Some value error"), ValueError("Some value error"), ValueError("Another value error")]: + for error in [ + ValueError(message_1), + ValueError(message_2), + ]: try: - raise error - except Exception: - error_tracker.add(error) + raise error # Errors raised on same line + except Exception as e: + error_tracker.add(e) - assert error_tracker.total == 3 + assert error_tracker.total == 2 assert error_tracker.unique_error_count == 1 + # TODO test after most popular errors From 433dadb021ec5bbeb3168c62045debf9cca575cd Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 19 Feb 2025 16:11:47 +0100 Subject: [PATCH 4/9] Add most common errors TODO: Add extended configuration parameters --- src/crawlee/statistics/_error_tracker.py | 54 +++++++++++--------- tests/unit/_statistics/test_error_tracker.py | 47 +++++++++++------ 2 files changed, 59 insertions(+), 42 deletions(-) diff --git a/src/crawlee/statistics/_error_tracker.py b/src/crawlee/statistics/_error_tracker.py index e5d149546b..20839b347f 100644 --- a/src/crawlee/statistics/_error_tracker.py +++ b/src/crawlee/statistics/_error_tracker.py @@ -4,31 +4,23 @@ import traceback from collections import Counter, defaultdict -from dataclasses import dataclass from itertools import zip_longest -# Errors are internally tracked in deeply nested dict where individual error groups count can be accessed like this: -# count = ErrorStackTraceGroups["some_stack_trace"]["some_error_code"]["some_error_name"]["some_error_message"] -# if any of the filtering keys was not used it will be `None` instead. -# count = ErrorStackTraceGroups["some_stack_trace"][None]["some_error_name"][None] - ErrorMessageGroups = Counter[str|None] ErrorTypeGroups = dict[str|None, ErrorMessageGroups] -#ErrorCodeGroups = dict[str|None, ErrorNameGroups] # No error codes in Python ErrorStackTraceGroups = dict[str|None, ErrorTypeGroups] - class ErrorTracker: """Track errors and aggregates their counts by similarity.""" - def __init__(self, + def __init__(self,*, show_error_name: bool = True, show_stack_trace: bool = True, - show_full_stack: bool = False, # Todo, what does it do in JS? + show_full_stack: bool = False, # TODO, what does it do in JS? show_error_message: bool = True, - show_full_message: bool = False, # Todo, seems like only first line otherwise - save_error_snapshots: bool = False, # Todo, what does it do in JS? + show_full_message: bool = False, # TODO, seems like only first line otherwise + save_error_snapshots: bool = False, # TODO, what does it do in JS? ) -> None: self.show_error_name = show_error_name self.show_stack_trace = show_stack_trace @@ -41,10 +33,10 @@ def add(self, error: Exception) -> None: """Include an error in the statistics.""" # Check for similar error_group_name = error.__class__.__name__ if self.show_error_name else None - error_group_message = str(error) if self.show_error_message else None + error_group_message = error.args[0] if self.show_error_message else None if self.show_stack_trace: error_traceback = traceback.extract_tb(error.__traceback__) - error_group_stack_trace = f"{error_traceback[0].filename}:{error_traceback[0].lineno}" + error_group_stack_trace = f'{error_traceback[0].filename}:{error_traceback[0].lineno}' # More extraction if full stack... else: error_group_stack_trace = None @@ -89,29 +81,41 @@ def total(self) -> int: error_count+=sum(name_group.values()) return error_count - def get_most_popular_errors(self, n=3) -> str: + def get_most_popular_errors(self, n: int=3) -> str: + all_errors = Counter() + for stack_group_name, stack_group in self._errors.items(): + for name_group_name, name_group in stack_group.items(): + for message_group_name, count in name_group.items(): + all_errors[self._get_error_repr( + stack_group_name, name_group_name, message_group_name)]=count + return all_errors.most_common(n) + + + + def _get_error_repr(self, stack: str, name: str, message: str) -> str: + """Get the most specific error representation.""" + stack_part = f'{stack}:' if stack else '' + name_part = f'{name}:' if name else '' + message_part = f'{message}' if message else '' + return f'{stack_part}{name_part}{message_part}' - def _add_to_existing_similar_message_group(self): - # Check to if exactly same error group already exists - # If not, try to assign to similar group. Iterate one by one and try to merge if possible - pass def _create_generic_message(self, message_1: str, message_2: str) -> str: - replacement_symbol = "_" + replacement_symbol = '_' replacement_count = 0 - message_1_parts = message_1.split(" ") - message_2_parts = message_2.split(" ") + message_1_parts = message_1.split(' ') + message_2_parts = message_2.split(' ') generic_message_parts = [] parts_count = max(len(message_1_parts), len(message_2_parts)) - for message_1_part, message_2_part in zip_longest(message_1_parts, message_2_parts, fillvalue=""): + for message_1_part, message_2_part in zip_longest(message_1_parts, message_2_parts, fillvalue=''): if message_1_part != message_2_part: generic_message_parts.append(replacement_symbol) replacement_count+=1 if replacement_count >= parts_count/2: # Messages are too different. - return "" + return '' else: generic_message_parts.append(message_1_part) - return " ".join(generic_message_parts) + return ' '.join(generic_message_parts) diff --git a/tests/unit/_statistics/test_error_tracker.py b/tests/unit/_statistics/test_error_tracker.py index 6d7d43e444..f71e7d1e48 100644 --- a/tests/unit/_statistics/test_error_tracker.py +++ b/tests/unit/_statistics/test_error_tracker.py @@ -1,10 +1,11 @@ -import pytest +import traceback +import pytest from crawlee.statistics._error_tracker import ErrorTracker -@pytest.mark.parametrize("error_tracker, expected_unique_errors", [ +@pytest.mark.parametrize(('error_tracker', 'expected_unique_errors'), [ (ErrorTracker(), 4), (ErrorTracker(show_stack_trace=False), 3), (ErrorTracker(show_error_name=False), 3), @@ -14,43 +15,55 @@ (ErrorTracker(show_stack_trace=False, show_error_name=False, show_error_message=False), 1), ]) -def test_error_tracker_counts(error_tracker: ErrorTracker, expected_unique_errors: int): +def test_error_tracker_counts(error_tracker: ErrorTracker, expected_unique_errors: int) -> None: """Use different settings of `error_tracker` and test unique errors count.""" for error in [ - Exception("Some value error abc"), - ValueError("Some value error abc"), # Different type, different error - ValueError("Some value error cde"), # Same type and similar message to previous, considered the same. - ValueError("Another value error efg") # Same type, but too different message to previous, considered different. + Exception('Some value error abc'), + ValueError('Some value error abc'), # Different type, different error + ValueError('Some value error cde'), # Same type and similar message to previous, considered the same. + ValueError('Another value error efg') # Same type, but too different message to previous, considered different. ]: try: raise error # Errors raised on same line - except Exception as e: + except Exception as e: # noqa:PERF203 error_tracker.add(e) try: - raise ValueError("Some value error abc") # Same as one previous error, but different line. + raise ValueError('Some value error abc') # Same as one previous error, but different line. except Exception as e: error_tracker.add(e) assert error_tracker.total == 5 assert error_tracker.unique_error_count == expected_unique_errors -@pytest.mark.parametrize("message_1, message_2, expected_generic_message", [ - ("Some error number 123", "Some error number 456", "Some error number _"), +@pytest.mark.parametrize(('message_1', 'message_2', 'expected_generic_message'), [ + ('Some error number 123', 'Some error number 456', 'Some error number _'), + ('Some error number 123 456', 'Some error number 123 456 789', 'Some error number 123 456 _'), + ('Some error number 0 0 0', 'Some error number 1 0 1', 'Some error number _ 0 _'), ]) -def test_error_tracker_similar_messages(message_1, message_2, expected_generic_message): +def test_error_tracker_similar_messages(message_1: str, message_2: str, expected_generic_message: str) -> None: """Test that similar messages collapse into same group with generic name that contains wildcards.""" + line: int error_tracker = ErrorTracker() for error in [ + KeyError(message_1), + KeyError(message_1), + KeyError(message_1), ValueError(message_1), ValueError(message_2), + RuntimeError(message_2), ]: try: - raise error # Errors raised on same line - except Exception as e: + raise error # Errors raised on the same line + except Exception as e: # noqa:PERF203 error_tracker.add(e) + line = traceback.extract_tb(e.__traceback__)[0].lineno - assert error_tracker.total == 2 - assert error_tracker.unique_error_count == 1 - # TODO test after most popular errors + errors = error_tracker.get_most_popular_errors() + assert errors[0][0] == f'{__file__}:{line}:KeyError:{message_1}' + assert errors[0][1] == 3 + assert errors[1][0] == f'{__file__}:{line}:ValueError:{expected_generic_message}' + assert errors[1][1] == 2 + assert errors[2][0] == f'{__file__}:{line}:RuntimeError:{message_2}' + assert errors[2][1] == 1 From dacc81062777b065afe538525a43142fbe675aee Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Mon, 24 Feb 2025 13:09:48 +0100 Subject: [PATCH 5/9] Add detailed message option. Do not port option for full trace - not used in JS version. --- src/crawlee/statistics/_error_tracker.py | 101 +++++++++++-------- tests/unit/_statistics/test_error_tracker.py | 75 +++++++++----- 2 files changed, 110 insertions(+), 66 deletions(-) diff --git a/src/crawlee/statistics/_error_tracker.py b/src/crawlee/statistics/_error_tracker.py index 20839b347f..a21ad56969 100644 --- a/src/crawlee/statistics/_error_tracker.py +++ b/src/crawlee/statistics/_error_tracker.py @@ -5,63 +5,73 @@ import traceback from collections import Counter, defaultdict from itertools import zip_longest +from typing import Union -ErrorMessageGroups = Counter[str|None] -ErrorTypeGroups = dict[str|None, ErrorMessageGroups] -ErrorStackTraceGroups = dict[str|None, ErrorTypeGroups] +GroupName = Union[str, None] +ErrorMessageGroups = Counter[GroupName] +ErrorTypeGroups = dict[GroupName, ErrorMessageGroups] +ErrorStackTraceGroups = dict[GroupName, ErrorTypeGroups] class ErrorTracker: """Track errors and aggregates their counts by similarity.""" - def __init__(self,*, - show_error_name: bool = True, - show_stack_trace: bool = True, - show_full_stack: bool = False, # TODO, what does it do in JS? - show_error_message: bool = True, - show_full_message: bool = False, # TODO, seems like only first line otherwise - save_error_snapshots: bool = False, # TODO, what does it do in JS? + def __init__( + self, + *, + show_error_name: bool = True, + show_stack_trace: bool = True, + show_error_message: bool = True, + show_full_message: bool = False, ) -> None: self.show_error_name = show_error_name self.show_stack_trace = show_stack_trace - self.show_full_stack = show_full_stack self.show_error_message = show_error_message + if show_full_message and not show_error_message: + raise ValueError('`show_error_message` must be `True` if `show_full_message` is set to `True`') self.show_full_message = show_full_message self._errors: ErrorStackTraceGroups = defaultdict(lambda: defaultdict(Counter)) def add(self, error: Exception) -> None: """Include an error in the statistics.""" - # Check for similar error_group_name = error.__class__.__name__ if self.show_error_name else None - error_group_message = error.args[0] if self.show_error_message else None - if self.show_stack_trace: - error_traceback = traceback.extract_tb(error.__traceback__) - error_group_stack_trace = f'{error_traceback[0].filename}:{error_traceback[0].lineno}' - # More extraction if full stack... - else: - error_group_stack_trace = None - - # TODO perform special message grouping - # First just exact match + error_group_message = self._get_error_message(error) + error_group_stack_trace = self._get_traceback_text(error) - # All groups, except the lowest level is matched. + # All groups, except the lowest level is matched exactly. specific_groups = self._errors[error_group_stack_trace][error_group_name] + # Lowest level is matched by similarity if error_group_message in specific_groups: # Exact match specific_groups.update([error_group_message]) else: - for known_error_group_message in specific_groups: - if new_group_name:=self._create_generic_message(known_error_group_message, error_group_message): + for existing_error_group_message in specific_groups: + # Add to first group with similar text. Modify text with wildcard characters if necessary. + if new_error_group_message := self._create_generic_message( + existing_error_group_message, error_group_message + ): # Replace old name - specific_groups[new_group_name]=specific_groups.pop(known_error_group_message) + specific_groups[new_error_group_message] = specific_groups.pop(existing_error_group_message) # Increment - specific_groups.update([new_group_name]) + specific_groups.update([new_error_group_message]) break else: # No similar message found. Create new group. self._errors[error_group_stack_trace][error_group_name].update([error_group_message]) + def _get_traceback_text(self, error: Exception) -> str | None: + if self.show_stack_trace: + error_traceback = traceback.extract_tb(error.__traceback__) + return f'{error_traceback[0].filename.split("/")[-1]}:{error_traceback[0].lineno}' + return None + + def _get_error_message(self, error: Exception) -> str | None: + if self.show_error_message: + if self.show_full_message: + return str(error.args[0]) + return str(error.args[0]).split('\n')[0] + return None @property def unique_error_count(self) -> int: @@ -69,7 +79,7 @@ def unique_error_count(self) -> int: unique_error_count = 0 for stack_group in self._errors.values(): for name_group in stack_group.values(): - unique_error_count+=len(name_group) + unique_error_count += len(name_group) return unique_error_count @property @@ -78,44 +88,49 @@ def total(self) -> int: error_count = 0 for stack_group in self._errors.values(): for name_group in stack_group.values(): - error_count+=sum(name_group.values()) + error_count += sum(name_group.values()) return error_count - def get_most_popular_errors(self, n: int=3) -> str: - all_errors = Counter() - for stack_group_name, stack_group in self._errors.items(): - for name_group_name, name_group in stack_group.items(): + def get_most_common_errors(self, n: int = 3) -> list[tuple[str | None, int]]: + """Return n most common errors.""" + all_errors: ErrorMessageGroups = Counter() + for stack_group_name, stack_group in self._errors.items(): + for name_group_name, name_group in stack_group.items(): for message_group_name, count in name_group.items(): - all_errors[self._get_error_repr( - stack_group_name, name_group_name, message_group_name)]=count + all_errors[self._get_error_repr(stack_group_name, name_group_name, message_group_name)] = count return all_errors.most_common(n) - - - def _get_error_repr(self, stack: str, name: str, message: str) -> str: + def _get_error_repr(self, stack: str | None, name: str | None, message: str | None) -> str: """Get the most specific error representation.""" stack_part = f'{stack}:' if stack else '' name_part = f'{name}:' if name else '' message_part = f'{message}' if message else '' return f'{stack_part}{name_part}{message_part}' + @staticmethod + def _create_generic_message(message_1: str | None, message_2: str | None) -> str | None: + """Create a generic error message from two messages, if they are similar enough. + + Different parts of similar messages are replaced by `_`. + """ + if message_1 is None or message_2 is None: + return None - def _create_generic_message(self, message_1: str, message_2: str) -> str: replacement_symbol = '_' replacement_count = 0 + + generic_message_parts = [] message_1_parts = message_1.split(' ') message_2_parts = message_2.split(' ') - generic_message_parts = [] parts_count = max(len(message_1_parts), len(message_2_parts)) for message_1_part, message_2_part in zip_longest(message_1_parts, message_2_parts, fillvalue=''): if message_1_part != message_2_part: generic_message_parts.append(replacement_symbol) - replacement_count+=1 - if replacement_count >= parts_count/2: + replacement_count += 1 + if replacement_count >= parts_count / 2: # Messages are too different. return '' else: generic_message_parts.append(message_1_part) return ' '.join(generic_message_parts) - diff --git a/tests/unit/_statistics/test_error_tracker.py b/tests/unit/_statistics/test_error_tracker.py index f71e7d1e48..e9b9e1d0f2 100644 --- a/tests/unit/_statistics/test_error_tracker.py +++ b/tests/unit/_statistics/test_error_tracker.py @@ -5,16 +5,18 @@ from crawlee.statistics._error_tracker import ErrorTracker -@pytest.mark.parametrize(('error_tracker', 'expected_unique_errors'), [ - (ErrorTracker(), 4), - (ErrorTracker(show_stack_trace=False), 3), - (ErrorTracker(show_error_name=False), 3), - (ErrorTracker(show_error_message=False), 3), - (ErrorTracker(show_stack_trace=False, show_error_name=False), 2), - (ErrorTracker(show_stack_trace=False, show_error_message=False), 2), - (ErrorTracker(show_stack_trace=False, show_error_name=False, show_error_message=False), 1), - -]) +@pytest.mark.parametrize( + ('error_tracker', 'expected_unique_errors'), + [ + (ErrorTracker(), 4), + (ErrorTracker(show_stack_trace=False), 3), + (ErrorTracker(show_error_name=False), 3), + (ErrorTracker(show_error_message=False), 3), + (ErrorTracker(show_stack_trace=False, show_error_name=False), 2), + (ErrorTracker(show_stack_trace=False, show_error_message=False), 2), + (ErrorTracker(show_stack_trace=False, show_error_name=False, show_error_message=False), 1), + ], +) def test_error_tracker_counts(error_tracker: ErrorTracker, expected_unique_errors: int) -> None: """Use different settings of `error_tracker` and test unique errors count.""" @@ -22,7 +24,9 @@ def test_error_tracker_counts(error_tracker: ErrorTracker, expected_unique_error Exception('Some value error abc'), ValueError('Some value error abc'), # Different type, different error ValueError('Some value error cde'), # Same type and similar message to previous, considered the same. - ValueError('Another value error efg') # Same type, but too different message to previous, considered different. + ValueError( + 'Another value error efg' + ), # Same type, but too different message to previous, considered different. ]: try: raise error # Errors raised on same line @@ -37,14 +41,19 @@ def test_error_tracker_counts(error_tracker: ErrorTracker, expected_unique_error assert error_tracker.total == 5 assert error_tracker.unique_error_count == expected_unique_errors -@pytest.mark.parametrize(('message_1', 'message_2', 'expected_generic_message'), [ - ('Some error number 123', 'Some error number 456', 'Some error number _'), - ('Some error number 123 456', 'Some error number 123 456 789', 'Some error number 123 456 _'), - ('Some error number 0 0 0', 'Some error number 1 0 1', 'Some error number _ 0 _'), -]) -def test_error_tracker_similar_messages(message_1: str, message_2: str, expected_generic_message: str) -> None: - """Test that similar messages collapse into same group with generic name that contains wildcards.""" - line: int + +@pytest.mark.parametrize( + ('message_1', 'message_2', 'expected_generic_message'), + [ + ('Some error number 123', 'Some error number 456', 'Some error number _'), + ('Some error number 123 456', 'Some error number 123 456 789', 'Some error number 123 456 _'), + ('Some error number 0 0 0', 'Some error number 1 0 1', 'Some error number _ 0 _'), + ], +) +def test_error_tracker_similar_messages_full_stack( + message_1: str, message_2: str, expected_generic_message: str +) -> None: + """Test that similar messages collapse into same group with generic name that contains wildcard symbols.""" error_tracker = ErrorTracker() for error in [ KeyError(message_1), @@ -60,10 +69,30 @@ def test_error_tracker_similar_messages(message_1: str, message_2: str, expected error_tracker.add(e) line = traceback.extract_tb(e.__traceback__)[0].lineno - errors = error_tracker.get_most_popular_errors() - assert errors[0][0] == f'{__file__}:{line}:KeyError:{message_1}' + file_name = __file__.split('/')[-1] + errors = error_tracker.get_most_common_errors() + assert errors[0][0] == f'{file_name}:{line}:KeyError:{message_1}' assert errors[0][1] == 3 - assert errors[1][0] == f'{__file__}:{line}:ValueError:{expected_generic_message}' + assert errors[1][0] == f'{file_name}:{line}:ValueError:{expected_generic_message}' assert errors[1][1] == 2 - assert errors[2][0] == f'{__file__}:{line}:RuntimeError:{message_2}' + assert errors[2][0] == f'{file_name}:{line}:RuntimeError:{message_2}' assert errors[2][1] == 1 + + +@pytest.mark.parametrize( + ('show_full_message', 'expected_message'), + [ + (True, 'Error line 1\n Error line 2'), + (False, 'Error line 1'), + ], +) +def test_show_full_message(*, show_full_message: bool, expected_message: str) -> None: + """Test error message settings with both options of `show_full_message`.""" + error_tracker = ErrorTracker(show_stack_trace=False, show_error_name=False, show_full_message=show_full_message) + + try: + raise RuntimeError('Error line 1\n Error line 2') # Errors raised on the same line + except Exception as e: + error_tracker.add(e) + + assert error_tracker.get_most_common_errors()[0][0] == expected_message From 104cf26fd62a229ca61f605de06e167b459d5bf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Proch=C3=A1zka?= Date: Mon, 24 Feb 2025 13:59:40 +0100 Subject: [PATCH 6/9] Update comments --- src/crawlee/statistics/_error_tracker.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/crawlee/statistics/_error_tracker.py b/src/crawlee/statistics/_error_tracker.py index a21ad56969..3033fb0f67 100644 --- a/src/crawlee/statistics/_error_tracker.py +++ b/src/crawlee/statistics/_error_tracker.py @@ -38,12 +38,12 @@ def add(self, error: Exception) -> None: error_group_message = self._get_error_message(error) error_group_stack_trace = self._get_traceback_text(error) - # All groups, except the lowest level is matched exactly. + # First two levels are grouped only in case of exact match. specific_groups = self._errors[error_group_stack_trace][error_group_name] - # Lowest level is matched by similarity + # Lowest level group is matched by similarity. if error_group_message in specific_groups: - # Exact match + # Exact match. specific_groups.update([error_group_message]) else: for existing_error_group_message in specific_groups: @@ -51,9 +51,9 @@ def add(self, error: Exception) -> None: if new_error_group_message := self._create_generic_message( existing_error_group_message, error_group_message ): - # Replace old name + # Replace old name. specific_groups[new_error_group_message] = specific_groups.pop(existing_error_group_message) - # Increment + # Increment. specific_groups.update([new_error_group_message]) break else: From 96b5c51b1d825cee863101001d2de77133712f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josef=20Proch=C3=A1zka?= Date: Mon, 24 Feb 2025 14:02:54 +0100 Subject: [PATCH 7/9] Update _error_tracker.py --- src/crawlee/statistics/_error_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crawlee/statistics/_error_tracker.py b/src/crawlee/statistics/_error_tracker.py index 3033fb0f67..7606b165dd 100644 --- a/src/crawlee/statistics/_error_tracker.py +++ b/src/crawlee/statistics/_error_tracker.py @@ -122,7 +122,7 @@ def _create_generic_message(message_1: str | None, message_2: str | None) -> str generic_message_parts = [] message_1_parts = message_1.split(' ') message_2_parts = message_2.split(' ') - parts_count = max(len(message_1_parts), len(message_2_parts)) + parts_count = min(len(message_1_parts), len(message_2_parts)) for message_1_part, message_2_part in zip_longest(message_1_parts, message_2_parts, fillvalue=''): if message_1_part != message_2_part: From 39c5c28c9e40d6fdeec20f44954e49641553eb7b Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 5 Mar 2025 09:42:41 +0100 Subject: [PATCH 8/9] Review comments --- src/crawlee/statistics/_error_tracker.py | 22 +++++++++----------- tests/unit/_statistics/test_error_tracker.py | 18 +++++++++------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/crawlee/statistics/_error_tracker.py b/src/crawlee/statistics/_error_tracker.py index 7606b165dd..e224aa12d7 100644 --- a/src/crawlee/statistics/_error_tracker.py +++ b/src/crawlee/statistics/_error_tracker.py @@ -8,9 +8,7 @@ from typing import Union GroupName = Union[str, None] -ErrorMessageGroups = Counter[GroupName] -ErrorTypeGroups = dict[GroupName, ErrorMessageGroups] -ErrorStackTraceGroups = dict[GroupName, ErrorTypeGroups] +ErrorFilenameGroups = dict[GroupName, dict[GroupName, Counter[GroupName]]] class ErrorTracker: @@ -20,23 +18,23 @@ def __init__( self, *, show_error_name: bool = True, - show_stack_trace: bool = True, + show_file_and_line_number: bool = True, show_error_message: bool = True, show_full_message: bool = False, ) -> None: self.show_error_name = show_error_name - self.show_stack_trace = show_stack_trace + self.show_file_and_line_number = show_file_and_line_number self.show_error_message = show_error_message if show_full_message and not show_error_message: raise ValueError('`show_error_message` must be `True` if `show_full_message` is set to `True`') self.show_full_message = show_full_message - self._errors: ErrorStackTraceGroups = defaultdict(lambda: defaultdict(Counter)) + self._errors: ErrorFilenameGroups = defaultdict(lambda: defaultdict(Counter)) def add(self, error: Exception) -> None: """Include an error in the statistics.""" error_group_name = error.__class__.__name__ if self.show_error_name else None error_group_message = self._get_error_message(error) - error_group_stack_trace = self._get_traceback_text(error) + error_group_stack_trace = self._get_file_and_line(error) # First two levels are grouped only in case of exact match. specific_groups = self._errors[error_group_stack_trace][error_group_name] @@ -60,8 +58,8 @@ def add(self, error: Exception) -> None: # No similar message found. Create new group. self._errors[error_group_stack_trace][error_group_name].update([error_group_message]) - def _get_traceback_text(self, error: Exception) -> str | None: - if self.show_stack_trace: + def _get_file_and_line(self, error: Exception) -> str | None: + if self.show_file_and_line_number: error_traceback = traceback.extract_tb(error.__traceback__) return f'{error_traceback[0].filename.split("/")[-1]}:{error_traceback[0].lineno}' return None @@ -93,7 +91,7 @@ def total(self) -> int: def get_most_common_errors(self, n: int = 3) -> list[tuple[str | None, int]]: """Return n most common errors.""" - all_errors: ErrorMessageGroups = Counter() + all_errors: Counter[GroupName] = Counter() for stack_group_name, stack_group in self._errors.items(): for name_group_name, name_group in stack_group.items(): for message_group_name, count in name_group.items(): @@ -116,7 +114,7 @@ def _create_generic_message(message_1: str | None, message_2: str | None) -> str if message_1 is None or message_2 is None: return None - replacement_symbol = '_' + replacement_string = '***' replacement_count = 0 generic_message_parts = [] @@ -126,7 +124,7 @@ def _create_generic_message(message_1: str | None, message_2: str | None) -> str for message_1_part, message_2_part in zip_longest(message_1_parts, message_2_parts, fillvalue=''): if message_1_part != message_2_part: - generic_message_parts.append(replacement_symbol) + generic_message_parts.append(replacement_string) replacement_count += 1 if replacement_count >= parts_count / 2: # Messages are too different. diff --git a/tests/unit/_statistics/test_error_tracker.py b/tests/unit/_statistics/test_error_tracker.py index e9b9e1d0f2..1895671267 100644 --- a/tests/unit/_statistics/test_error_tracker.py +++ b/tests/unit/_statistics/test_error_tracker.py @@ -9,12 +9,12 @@ ('error_tracker', 'expected_unique_errors'), [ (ErrorTracker(), 4), - (ErrorTracker(show_stack_trace=False), 3), + (ErrorTracker(show_file_and_line_number=False), 3), (ErrorTracker(show_error_name=False), 3), (ErrorTracker(show_error_message=False), 3), - (ErrorTracker(show_stack_trace=False, show_error_name=False), 2), - (ErrorTracker(show_stack_trace=False, show_error_message=False), 2), - (ErrorTracker(show_stack_trace=False, show_error_name=False, show_error_message=False), 1), + (ErrorTracker(show_error_name=False, show_file_and_line_number=False), 2), + (ErrorTracker(show_file_and_line_number=False, show_error_message=False), 2), + (ErrorTracker(show_error_name=False, show_file_and_line_number=False, show_error_message=False), 1), ], ) def test_error_tracker_counts(error_tracker: ErrorTracker, expected_unique_errors: int) -> None: @@ -45,9 +45,9 @@ def test_error_tracker_counts(error_tracker: ErrorTracker, expected_unique_error @pytest.mark.parametrize( ('message_1', 'message_2', 'expected_generic_message'), [ - ('Some error number 123', 'Some error number 456', 'Some error number _'), - ('Some error number 123 456', 'Some error number 123 456 789', 'Some error number 123 456 _'), - ('Some error number 0 0 0', 'Some error number 1 0 1', 'Some error number _ 0 _'), + ('Some error number 123', 'Some error number 456', 'Some error number ***'), + ('Some error number 123 456', 'Some error number 123 456 789', 'Some error number 123 456 ***'), + ('Some error number 0 0 0', 'Some error number 1 0 1', 'Some error number *** 0 ***'), ], ) def test_error_tracker_similar_messages_full_stack( @@ -88,7 +88,9 @@ def test_error_tracker_similar_messages_full_stack( ) def test_show_full_message(*, show_full_message: bool, expected_message: str) -> None: """Test error message settings with both options of `show_full_message`.""" - error_tracker = ErrorTracker(show_stack_trace=False, show_error_name=False, show_full_message=show_full_message) + error_tracker = ErrorTracker( + show_error_name=False, show_file_and_line_number=False, show_full_message=show_full_message + ) try: raise RuntimeError('Error line 1\n Error line 2') # Errors raised on the same line From 6784a0e781b8e320d17f3a37f7e21336a0a22812 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Wed, 5 Mar 2025 09:48:08 +0100 Subject: [PATCH 9/9] Remove stack from variable names --- src/crawlee/statistics/_error_tracker.py | 28 +++++++++++++----------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/src/crawlee/statistics/_error_tracker.py b/src/crawlee/statistics/_error_tracker.py index e224aa12d7..e607093b50 100644 --- a/src/crawlee/statistics/_error_tracker.py +++ b/src/crawlee/statistics/_error_tracker.py @@ -34,10 +34,10 @@ def add(self, error: Exception) -> None: """Include an error in the statistics.""" error_group_name = error.__class__.__name__ if self.show_error_name else None error_group_message = self._get_error_message(error) - error_group_stack_trace = self._get_file_and_line(error) + error_group_file_and_line = self._get_file_and_line(error) # First two levels are grouped only in case of exact match. - specific_groups = self._errors[error_group_stack_trace][error_group_name] + specific_groups = self._errors[error_group_file_and_line][error_group_name] # Lowest level group is matched by similarity. if error_group_message in specific_groups: @@ -56,7 +56,7 @@ def add(self, error: Exception) -> None: break else: # No similar message found. Create new group. - self._errors[error_group_stack_trace][error_group_name].update([error_group_message]) + self._errors[error_group_file_and_line][error_group_name].update([error_group_message]) def _get_file_and_line(self, error: Exception) -> str | None: if self.show_file_and_line_number: @@ -75,8 +75,8 @@ def _get_error_message(self, error: Exception) -> str | None: def unique_error_count(self) -> int: """Number of distinct kinds of errors.""" unique_error_count = 0 - for stack_group in self._errors.values(): - for name_group in stack_group.values(): + for file_and_line_group in self._errors.values(): + for name_group in file_and_line_group.values(): unique_error_count += len(name_group) return unique_error_count @@ -84,26 +84,28 @@ def unique_error_count(self) -> int: def total(self) -> int: """Total number of errors.""" error_count = 0 - for stack_group in self._errors.values(): - for name_group in stack_group.values(): + for file_and_line_group in self._errors.values(): + for name_group in file_and_line_group.values(): error_count += sum(name_group.values()) return error_count def get_most_common_errors(self, n: int = 3) -> list[tuple[str | None, int]]: """Return n most common errors.""" all_errors: Counter[GroupName] = Counter() - for stack_group_name, stack_group in self._errors.items(): - for name_group_name, name_group in stack_group.items(): + for file_and_line_group_name, file_and_line_group in self._errors.items(): + for name_group_name, name_group in file_and_line_group.items(): for message_group_name, count in name_group.items(): - all_errors[self._get_error_repr(stack_group_name, name_group_name, message_group_name)] = count + all_errors[self._get_error_repr(file_and_line_group_name, name_group_name, message_group_name)] = ( + count + ) return all_errors.most_common(n) - def _get_error_repr(self, stack: str | None, name: str | None, message: str | None) -> str: + def _get_error_repr(self, file_and_line: str | None, name: str | None, message: str | None) -> str: """Get the most specific error representation.""" - stack_part = f'{stack}:' if stack else '' + file_and_line_part = f'{file_and_line}:' if file_and_line else '' name_part = f'{name}:' if name else '' message_part = f'{message}' if message else '' - return f'{stack_part}{name_part}{message_part}' + return f'{file_and_line_part}{name_part}{message_part}' @staticmethod def _create_generic_message(message_1: str | None, message_2: str | None) -> str | None: