Repository navigation
fix: fix match check for specified enqueue strategy for requests with redirect #1199
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -328,6 +328,7 @@ async def handler(context: BasicCrawlingContext) -> None: | |||||
| @dataclass | ||||||
| class AddRequestsTestInput: | ||||||
| start_url: str | ||||||
| loaded_url: str | ||||||
| requests: Sequence[str | Request] | ||||||
| expected_urls: Sequence[str] | ||||||
| kwargs: EnqueueLinksKwargs | ||||||
|
|
@@ -337,6 +338,7 @@ class AddRequestsTestInput: | |||||
| 'https://someplace.com/', | ||||||
| 'http://someplace.com/index.html', | ||||||
| 'https://blog.someplace.com/index.html', | ||||||
| 'https://redirect.someplace.com', | ||||||
| 'https://other.place.com/index.html', | ||||||
| ) | ||||||
|
|
||||||
|
|
@@ -356,6 +358,7 @@ class AddRequestsTestInput: | |||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url='https://a.com/', | ||||||
| loaded_url='https://a.com/', | ||||||
| requests=[ | ||||||
| 'https://a.com/', | ||||||
| Request.from_url('http://b.com/'), | ||||||
|
|
@@ -370,52 +373,109 @@ class AddRequestsTestInput: | |||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=STRATEGY_TEST_URLS[0], | ||||||
| loaded_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS, | ||||||
| kwargs=EnqueueLinksKwargs(), | ||||||
| expected_urls=STRATEGY_TEST_URLS[1:], | ||||||
| ), | ||||||
| id='enqueue_strategy_1', | ||||||
| id='enqueue_strategy_default', | ||||||
| ), | ||||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=STRATEGY_TEST_URLS[0], | ||||||
| loaded_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS, | ||||||
| kwargs=EnqueueLinksKwargs(strategy='all'), | ||||||
| expected_urls=STRATEGY_TEST_URLS[1:], | ||||||
| ), | ||||||
| id='enqueue_strategy_2', | ||||||
| id='enqueue_strategy_all', | ||||||
| ), | ||||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS, | ||||||
| loaded_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS[:4], | ||||||
| kwargs=EnqueueLinksKwargs(strategy='same-domain'), | ||||||
| expected_urls=STRATEGY_TEST_URLS[1:3], | ||||||
| expected_urls=STRATEGY_TEST_URLS[1:4], | ||||||
| ), | ||||||
| id='enqueue_strategy_3', | ||||||
| id='enqueue_strategy_same_domain', | ||||||
| ), | ||||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS, | ||||||
| loaded_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS[:4], | ||||||
| kwargs=EnqueueLinksKwargs(strategy='same-hostname'), | ||||||
| expected_urls=[STRATEGY_TEST_URLS[1]], | ||||||
| ), | ||||||
| id='enqueue_strategy_4', | ||||||
| id='enqueue_strategy_same_hostname', | ||||||
| ), | ||||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=STRATEGY_TEST_URLS[0], | ||||||
| loaded_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS[:4], | ||||||
| kwargs=EnqueueLinksKwargs(strategy='same-origin'), | ||||||
| expected_urls=[], | ||||||
| ), | ||||||
| id='enqueue_strategy_same_origin', | ||||||
| ), | ||||||
| # Enqueue strategy with redirect | ||||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=STRATEGY_TEST_URLS[3], | ||||||
| loaded_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS, | ||||||
| kwargs=EnqueueLinksKwargs(), | ||||||
| expected_urls=STRATEGY_TEST_URLS[:3] + STRATEGY_TEST_URLS[4:], | ||||||
| ), | ||||||
| id='redirect_enqueue_strategy_default', | ||||||
| ), | ||||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=STRATEGY_TEST_URLS[3], | ||||||
| loaded_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS, | ||||||
| kwargs=EnqueueLinksKwargs(strategy='all'), | ||||||
| expected_urls=STRATEGY_TEST_URLS[:3] + STRATEGY_TEST_URLS[4:], | ||||||
| ), | ||||||
| id='redirect_enqueue_strategy_all', | ||||||
| ), | ||||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=STRATEGY_TEST_URLS[3], | ||||||
| loaded_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS, | ||||||
| kwargs=EnqueueLinksKwargs(strategy='same-domain'), | ||||||
| expected_urls=STRATEGY_TEST_URLS[:3], | ||||||
| ), | ||||||
| id='redirect_enqueue_strategy_same_domain', | ||||||
| ), | ||||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=STRATEGY_TEST_URLS[3], | ||||||
| loaded_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS, | ||||||
| kwargs=EnqueueLinksKwargs(strategy='same-hostname'), | ||||||
| expected_urls=[], | ||||||
| ), | ||||||
| id='redirect_enqueue_strategy_same_hostname', | ||||||
| ), | ||||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=STRATEGY_TEST_URLS[3], | ||||||
| loaded_url=STRATEGY_TEST_URLS[0], | ||||||
| requests=STRATEGY_TEST_URLS, | ||||||
| kwargs=EnqueueLinksKwargs(strategy='same-origin'), | ||||||
| expected_urls=[], | ||||||
| ), | ||||||
| id='enqueue_strategy_5', | ||||||
| id='redirect_enqueue_strategy_same_origin', | ||||||
| ), | ||||||
| # Include/exclude | ||||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=INCLUDE_TEST_URLS[0], | ||||||
| loaded_url=INCLUDE_TEST_URLS[0], | ||||||
| requests=INCLUDE_TEST_URLS, | ||||||
| kwargs=EnqueueLinksKwargs(include=[Glob('https://someplace.com/**/cats')]), | ||||||
| expected_urls=[INCLUDE_TEST_URLS[1], INCLUDE_TEST_URLS[4]], | ||||||
|
|
@@ -425,6 +485,7 @@ class AddRequestsTestInput: | |||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=INCLUDE_TEST_URLS[0], | ||||||
| loaded_url=INCLUDE_TEST_URLS[0], | ||||||
| requests=INCLUDE_TEST_URLS, | ||||||
| kwargs=EnqueueLinksKwargs(exclude=[Glob('https://someplace.com/**/cats')]), | ||||||
| expected_urls=[INCLUDE_TEST_URLS[2], INCLUDE_TEST_URLS[3]], | ||||||
|
|
@@ -434,6 +495,7 @@ class AddRequestsTestInput: | |||||
| pytest.param( | ||||||
| AddRequestsTestInput( | ||||||
| start_url=INCLUDE_TEST_URLS[0], | ||||||
| loaded_url=INCLUDE_TEST_URLS[0], | ||||||
| requests=INCLUDE_TEST_URLS, | ||||||
| kwargs=EnqueueLinksKwargs( | ||||||
| include=[Glob('https://someplace.com/**/cats')], exclude=[Glob('https://**/archive/**')] | ||||||
|
|
@@ -451,6 +513,8 @@ async def test_enqueue_strategy(test_input: AddRequestsTestInput) -> None: | |||||
|
|
||||||
| @crawler.router.handler('start') | ||||||
| async def start_handler(context: BasicCrawlingContext) -> None: | ||||||
| # Assign test value to loaded_url - BasicCrawler does not do any navigation by itself | ||||||
| context.request.loaded_url = test_input.loaded_url | ||||||
|
||||||
| context.request.loaded_url = test_input.loaded_url | |
| context.request.url = test_input.loaded_url |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is correct, we do this to simulate a redirect.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Perhaps we should add a comment here? Copilot has a point, this looks sort of fishy 🙂
Something along the lines of "Assign test value to loaded_url - BasicCrawler does not do any navigation by itself".
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The tests assign the original URL to context.request.loaded_url but the production code uses context.request.url for the enqueue strategy check. Ensure that the correct property (either 'url' or 'loaded_url') is used consistently across the system to fix the redirect issue.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Tests assign
context.request.loaded_urlto simulate a redirect. The start url should be used when we check the strategy.