Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/crawlee/crawlers/_basic/_basic_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1129,7 +1129,7 @@ async def _commit_request_handler_result(self, context: BasicCrawlingContext) ->
and self._check_enqueue_strategy(
add_requests_call.get('strategy', 'all'),
target_url=urlparse(dst_request.url),
origin_url=urlparse(origin),
origin_url=urlparse(context.request.url),

Copilot AI May 16, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The tests assign the original URL to context.request.loaded_url but the production code uses context.request.url for the enqueue strategy check. Ensure that the correct property (either 'url' or 'loaded_url') is used consistently across the system to fix the redirect issue.

Suggested change
origin_url=urlparse(context.request.url),
origin_url=urlparse(context.request.loaded_url or context.request.url),

Copilot uses AI. Check for mistakes.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Tests assign context.request.loaded_url to simulate a redirect. The start url should be used when we check the strategy.

)
and self._check_url_patterns(
dst_request.url,
Expand Down
80 changes: 72 additions & 8 deletions tests/unit/crawlers/_basic/test_basic_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,6 +328,7 @@ async def handler(context: BasicCrawlingContext) -> None:
@dataclass
class AddRequestsTestInput:
start_url: str
loaded_url: str
requests: Sequence[str | Request]
expected_urls: Sequence[str]
kwargs: EnqueueLinksKwargs
Expand All @@ -337,6 +338,7 @@ class AddRequestsTestInput:
'https://someplace.com/',
'http://someplace.com/index.html',
'https://blog.someplace.com/index.html',
'https://redirect.someplace.com',
'https://other.place.com/index.html',
)

Expand All @@ -356,6 +358,7 @@ class AddRequestsTestInput:
pytest.param(
AddRequestsTestInput(
start_url='https://a.com/',
loaded_url='https://a.com/',
requests=[
'https://a.com/',
Request.from_url('http://b.com/'),
Expand All @@ -370,52 +373,109 @@ class AddRequestsTestInput:
pytest.param(
AddRequestsTestInput(
start_url=STRATEGY_TEST_URLS[0],
loaded_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS,
kwargs=EnqueueLinksKwargs(),
expected_urls=STRATEGY_TEST_URLS[1:],
),
id='enqueue_strategy_1',
id='enqueue_strategy_default',
),
pytest.param(
AddRequestsTestInput(
start_url=STRATEGY_TEST_URLS[0],
loaded_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS,
kwargs=EnqueueLinksKwargs(strategy='all'),
expected_urls=STRATEGY_TEST_URLS[1:],
),
id='enqueue_strategy_2',
id='enqueue_strategy_all',
),
pytest.param(
AddRequestsTestInput(
start_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS,
loaded_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS[:4],
kwargs=EnqueueLinksKwargs(strategy='same-domain'),
expected_urls=STRATEGY_TEST_URLS[1:3],
expected_urls=STRATEGY_TEST_URLS[1:4],
),
id='enqueue_strategy_3',
id='enqueue_strategy_same_domain',
),
pytest.param(
AddRequestsTestInput(
start_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS,
loaded_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS[:4],
kwargs=EnqueueLinksKwargs(strategy='same-hostname'),
expected_urls=[STRATEGY_TEST_URLS[1]],
),
id='enqueue_strategy_4',
id='enqueue_strategy_same_hostname',
),
pytest.param(
AddRequestsTestInput(
start_url=STRATEGY_TEST_URLS[0],
loaded_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS[:4],
kwargs=EnqueueLinksKwargs(strategy='same-origin'),
expected_urls=[],
),
id='enqueue_strategy_same_origin',
),
# Enqueue strategy with redirect
pytest.param(
AddRequestsTestInput(
start_url=STRATEGY_TEST_URLS[3],
loaded_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS,
kwargs=EnqueueLinksKwargs(),
expected_urls=STRATEGY_TEST_URLS[:3] + STRATEGY_TEST_URLS[4:],
),
id='redirect_enqueue_strategy_default',
),
pytest.param(
AddRequestsTestInput(
start_url=STRATEGY_TEST_URLS[3],
loaded_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS,
kwargs=EnqueueLinksKwargs(strategy='all'),
expected_urls=STRATEGY_TEST_URLS[:3] + STRATEGY_TEST_URLS[4:],
),
id='redirect_enqueue_strategy_all',
),
pytest.param(
AddRequestsTestInput(
start_url=STRATEGY_TEST_URLS[3],
loaded_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS,
kwargs=EnqueueLinksKwargs(strategy='same-domain'),
expected_urls=STRATEGY_TEST_URLS[:3],
),
id='redirect_enqueue_strategy_same_domain',
),
pytest.param(
AddRequestsTestInput(
start_url=STRATEGY_TEST_URLS[3],
loaded_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS,
kwargs=EnqueueLinksKwargs(strategy='same-hostname'),
expected_urls=[],
),
id='redirect_enqueue_strategy_same_hostname',
),
pytest.param(
AddRequestsTestInput(
start_url=STRATEGY_TEST_URLS[3],
loaded_url=STRATEGY_TEST_URLS[0],
requests=STRATEGY_TEST_URLS,
kwargs=EnqueueLinksKwargs(strategy='same-origin'),
expected_urls=[],
),
id='enqueue_strategy_5',
id='redirect_enqueue_strategy_same_origin',
),
# Include/exclude
pytest.param(
AddRequestsTestInput(
start_url=INCLUDE_TEST_URLS[0],
loaded_url=INCLUDE_TEST_URLS[0],
requests=INCLUDE_TEST_URLS,
kwargs=EnqueueLinksKwargs(include=[Glob('https://someplace.com/**/cats')]),
expected_urls=[INCLUDE_TEST_URLS[1], INCLUDE_TEST_URLS[4]],
Expand All @@ -425,6 +485,7 @@ class AddRequestsTestInput:
pytest.param(
AddRequestsTestInput(
start_url=INCLUDE_TEST_URLS[0],
loaded_url=INCLUDE_TEST_URLS[0],
requests=INCLUDE_TEST_URLS,
kwargs=EnqueueLinksKwargs(exclude=[Glob('https://someplace.com/**/cats')]),
expected_urls=[INCLUDE_TEST_URLS[2], INCLUDE_TEST_URLS[3]],
Expand All @@ -434,6 +495,7 @@ class AddRequestsTestInput:
pytest.param(
AddRequestsTestInput(
start_url=INCLUDE_TEST_URLS[0],
loaded_url=INCLUDE_TEST_URLS[0],
requests=INCLUDE_TEST_URLS,
kwargs=EnqueueLinksKwargs(
include=[Glob('https://someplace.com/**/cats')], exclude=[Glob('https://**/archive/**')]
Expand All @@ -451,6 +513,8 @@ async def test_enqueue_strategy(test_input: AddRequestsTestInput) -> None:

@crawler.router.handler('start')
async def start_handler(context: BasicCrawlingContext) -> None:
# Assign test value to loaded_url - BasicCrawler does not do any navigation by itself
context.request.loaded_url = test_input.loaded_url

Copilot AI May 16, 2025

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The test assigns the original URL to context.request.loaded_url but if the commit handler is intended to use the original URL for enqueue strategy testing, consider aligning the property used (for example, updating context.request.url) to match the production logic.

Suggested change
context.request.loaded_url = test_input.loaded_url
context.request.url = test_input.loaded_url

Copilot uses AI. Check for mistakes.

@Mantisus Mantisus May 16, 2025 •

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is correct, we do this to simulate a redirect.

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps we should add a comment here? Copilot has a point, this looks sort of fishy 🙂

Something along the lines of "Assign test value to loaded_url - BasicCrawler does not do any navigation by itself".

await context.add_requests(
test_input.requests,
**test_input.kwargs,
Expand Down