Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion src/crawlee/_utils/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@

def is_url_absolute(url: str) -> bool:
"""Check if a URL is absolute."""
return bool(urlparse(url).netloc)
url_parsed = urlparse(url)
return bool(url_parsed.scheme) and bool(url_parsed.netloc)


def convert_to_absolute_url(base_url: str, relative_url: str) -> str:
Expand Down
13 changes: 12 additions & 1 deletion src/crawlee/beautifulsoup_crawler/beautifulsoup_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import TYPE_CHECKING, Any, AsyncGenerator, Iterable, Literal

from bs4 import BeautifulSoup, Tag
from pydantic import ValidationError
from typing_extensions import Unpack

from crawlee._utils.blocked import RETRY_CSS_SELECTORS
Expand Down Expand Up @@ -136,7 +137,17 @@ async def enqueue_links(
if not is_url_absolute(url):
url = convert_to_absolute_url(context.request.url, url)

requests.append(BaseRequestData.from_url(url, user_data=link_user_data))
try:
request = BaseRequestData.from_url(url, user_data=link_user_data)
except ValidationError as exc:
context.log.debug(
f'Skipping URL "{url}" due to invalid format: {exc}. '
'This may be caused by a malformed URL or unsupported URL scheme. '
'Please ensure the URL is correct and retry.'
)
continue

requests.append(request)

await context.add_requests(requests, **kwargs)

Expand Down
13 changes: 12 additions & 1 deletion src/crawlee/parsel_crawler/parsel_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from typing import TYPE_CHECKING, Any, AsyncGenerator, Iterable

from parsel import Selector
from pydantic import ValidationError
from typing_extensions import Unpack

from crawlee._utils.blocked import RETRY_CSS_SELECTORS
Expand Down Expand Up @@ -133,7 +134,17 @@ async def enqueue_links(
if not is_url_absolute(url):
url = str(convert_to_absolute_url(context.request.url, url))

requests.append(BaseRequestData.from_url(url, user_data=link_user_data))
try:
request = BaseRequestData.from_url(url, user_data=link_user_data)
except ValidationError as exc:
context.log.debug(
f'Skipping URL "{url}" due to invalid format: {exc}. '
'This may be caused by a malformed URL or unsupported URL scheme. '
'Please ensure the URL is correct and retry.'
)
continue

requests.append(request)

await context.add_requests(requests, **kwargs)

Expand Down
12 changes: 11 additions & 1 deletion src/crawlee/playwright_crawler/playwright_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import logging
from typing import TYPE_CHECKING, Literal

from pydantic import ValidationError
from typing_extensions import Unpack

from crawlee._utils.blocked import RETRY_CSS_SELECTORS
Expand Down Expand Up @@ -141,7 +142,16 @@ async def enqueue_links(
if label is not None:
link_user_data.setdefault('label', label)

request = BaseRequestData.from_url(url, user_data=link_user_data)
try:
request = BaseRequestData.from_url(url, user_data=link_user_data)
except ValidationError as exc:
context.log.debug(
f'Skipping URL "{url}" due to invalid format: {exc}. '
'This may be caused by a malformed URL or unsupported URL scheme. '
'Please ensure the URL is correct and retry.'
)
continue

requests.append(request)

await context.add_requests(requests, **kwargs)
Expand Down
6 changes: 6 additions & 0 deletions tests/unit/_utils/test_urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ def test_is_url_absolute() -> None:
assert is_url_absolute('http://example.com/path') is True
assert is_url_absolute('https://example.com/path') is True
assert is_url_absolute('ftp://example.com/path') is True
assert is_url_absolute('//example.com/path') is False
assert is_url_absolute('/path/to/resource') is False
assert is_url_absolute('relative/path/to/resource') is False
assert is_url_absolute('example.com/path') is False
Expand All @@ -21,6 +22,11 @@ def test_convert_to_absolute_url() -> None:
absolute_url = convert_to_absolute_url(base_url, relative_url)
assert absolute_url == 'http://example.com/path/to/resource'

base_url = 'http://example.com'
relative_url = '//example.com/path/to/resource'
absolute_url = convert_to_absolute_url(base_url, relative_url)
assert absolute_url == 'http://example.com/path/to/resource'

base_url = 'http://example.com/base/'
relative_url = '../path/to/resource'
absolute_url = convert_to_absolute_url(base_url, relative_url)
Expand Down