Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ pyee = "^11.1.0"
python-dateutil = "^2.9.0"
sortedcollections = "^2.1.0"
typing-extensions = "^4.1.0"
tldextract = "^5.1.2"

[tool.poetry.group.dev.dependencies]
build = "~1.2.0"
Expand Down Expand Up @@ -190,4 +191,8 @@ ignore_missing_imports = true
ignore_missing_imports = true

[tool.coverage.report]
exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:"]
exclude_lines = [
"pragma: no cover",
"if TYPE_CHECKING:",
"assert_never()"
]
1 change: 1 addition & 0 deletions src/crawlee/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
from ._utils.globs import Glob
141 changes: 141 additions & 0 deletions src/crawlee/_utils/globs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
from __future__ import annotations

import os
import re
from typing import Sequence


class Glob:
"""Wraps a glob pattern (supports the `*`, `**`, `?` wildcards)."""

def __init__(self, glob: str) -> None:
self.glob = glob
self.regexp = re.compile(_translate(self.glob, recursive=True))


def _translate(
pat: str, *, recursive: bool = False, include_hidden: bool = False, seps: Sequence[str] | None = None
) -> str:
"""Translate a pathname with shell wildcards to a regular expression.

If `recursive` is true, the pattern segment '**' will match any number of
path segments.

If `include_hidden` is true, wildcards can match path segments beginning
with a dot ('.').

If a sequence of separator characters is given to `seps`, they will be
used to split the pattern into segments and match path separators. If not
given, os.path.sep and os.path.altsep (where available) are used.

HACK: This function is copied from CPython stdlib source. It will be released in Python 3.13 as `glob.translate`
"""
if not seps:
seps = (os.path.sep, os.path.altsep) if os.path.altsep else os.path.sep

escaped_seps = ''.join(map(re.escape, seps))
any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps
not_sep = f'[^{escaped_seps}]'
if include_hidden:
one_last_segment = f'{not_sep}+'
one_segment = f'{one_last_segment}{any_sep}'
any_segments = f'(?:.+{any_sep})?'
any_last_segments = '.*'
else:
one_last_segment = f'[^{escaped_seps}.]{not_sep}*'
one_segment = f'{one_last_segment}{any_sep}'
any_segments = f'(?:{one_segment})*'
any_last_segments = f'{any_segments}(?:{one_last_segment})?'

results = []
parts = re.split(any_sep, pat)
last_part_idx = len(parts) - 1
for idx, part in enumerate(parts):
if part == '*':
results.append(one_segment if idx < last_part_idx else one_last_segment)
elif recursive and part == '**':
if idx < last_part_idx:
if parts[idx + 1] != '**':
results.append(any_segments)
else:
results.append(any_last_segments)
else:
if part:
if not include_hidden and part[0] in '*?':
results.append(r'(?!\.)')
results.extend(_fnmatch_translate(part, f'{not_sep}*', not_sep))
if idx < last_part_idx:
results.append(any_sep)
res = ''.join(results)
return rf'(?s:{res})\Z'


def _fnmatch_translate(pat: str, star: str, question_mark: str) -> list[str]: # noqa: PLR0912
"""Copy of fnmatch._translate from Python 3.13."""
res = list[str]()
add = res.append
i, n = 0, len(pat)
while i < n:
c = pat[i]
i = i + 1
if c == '*':
# compress consecutive `*` into one
if (not res) or res[-1] is not star:
add(star)
elif c == '?':
add(question_mark)
elif c == '[':
j = i
if j < n and pat[j] == '!':
j = j + 1
if j < n and pat[j] == ']':
j = j + 1
while j < n and pat[j] != ']':
j = j + 1
if j >= n:
add('\\[')
else:
stuff = pat[i:j]
if '-' not in stuff:
stuff = stuff.replace('\\', r'\\')
else:
chunks = []
k = i + 2 if pat[i] == '!' else i + 1
while True:
k = pat.find('-', k, j)
if k < 0:
break
chunks.append(pat[i:k])
i = k + 1
k = k + 3
chunk = pat[i:j]
if chunk:
chunks.append(chunk)
else:
chunks[-1] += '-'
# Remove empty ranges -- invalid in RE.
for k in range(len(chunks) - 1, 0, -1):
if chunks[k - 1][-1] > chunks[k][0]:
chunks[k - 1] = chunks[k - 1][:-1] + chunks[k][1:]
del chunks[k]
# Escape backslashes and hyphens for set difference (--).
# Hyphens that create ranges shouldn't be escaped.
stuff = '-'.join(s.replace('\\', r'\\').replace('-', r'\-') for s in chunks)
# Escape set operations (&&, ~~ and ||).
stuff = re.sub(r'([&~|])', r'\\\1', stuff)
i = j + 1
if not stuff:
# Empty range: never match.
add('(?!)')
elif stuff == '!':
# Negated empty range: match any character.
add('.')
else:
if stuff[0] == '!':
stuff = '^' + stuff[1:]
elif stuff[0] in ('^', '['):
stuff = '\\' + stuff
add(f'[{stuff}]')
else:
add(re.escape(c))
return res
Loading