Skip to content

IndexError: list index out of range #334

Closed

Description

@bingbingyu
import asyncio

from crawlee.playwright_crawler import (
    PlaywrightCrawler,
    PlaywrightCrawlingContext,
)


async def main() -> None:
    crawler = PlaywrightCrawler(
        # Limit the crawl to max requests. Remove or increase it for crawling all links.
        max_requests_per_crawl=10,
    )

    # Define the default request handler, which will be called for every request.
    @crawler.router.default_handler
    async def request_handler(context: PlaywrightCrawlingContext) -> None:
        context.log.info(f"Processing {context.request.url} ...")

        # Extract data from the page.
        data = {
            "url": context.request.url,
            "title": context.page.title.string if context.page.title else None,
        }

        # Enqueue all links found on the page.
        await context.enqueue_links()

        # Push the extracted data to the default dataset.
        await context.push_data(data)

    # Run the crawler with the initial list of URLs.
    await crawler.run(["https://crawlee.dev"])

    # Export the entire dataset to a CSV file.
    await crawler.export_data("results.csv")


if __name__ == "__main__":
    asyncio.run(main())

TERMINAL:

Traceback (most recent call last):
  File "g:\python\python_experiment\my-crawler\my-crawler\routes.py", line 40, in <module>
    asyncio.run(main())
  File "D:\python3.9\lib\asyncio\runners.py", line 44, in run 
    return loop.run_until_complete(main)
  File "D:\python3.9\lib\asyncio\base_events.py", line 642, in run_until_complete
    return future.result()
  File "g:\python\python_experiment\my-crawler\my-crawler\routes.py", line 36, in main
    await crawler.export_data("results.csv")
  File "g:\python\python_experiment\venv\lib\site-packages\crawlee\basic_crawler\basic_crawler.py", line 474, in export_data

    return await dataset.write_to(content_type, path.open('w', newline=''))
  File "g:\python\python_experiment\venv\lib\site-packages\crawlee\storages\dataset.py", line 213, in write_to
    writer.writerows([items[0].keys(), *[item.values() for item in items]])
IndexError: list index out of range

Activity

  1. janbuchar commented on Jul 21, 2024

    @janbuchar
    Collaborator

    I assume your dataset is empty, which causes the error. We will handle this and print a warning instead.

  2. added
    bugSomething isn't working.
    t-toolingIssues with this label are in the ownership of the tooling team.
    on Jul 21, 2024
  3. self-assigned this
    on Jul 22, 2024
  4. added a commit that references this issue on Jul 24, 2024
    22b95d1
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment

Metadata

Metadata

Assignees

Labels

bugSomething isn't working.t-toolingIssues with this label are in the ownership of the tooling team.

Type

No type

Projects

No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions