From b003d21df6354f1a37551ec0cd9893a7d694b0c2 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 5 Mar 2025 00:45:22 +0000 Subject: [PATCH 1/3] update template --- src/crawlee/project_template/hooks/pre_gen_project.py | 2 +- .../project_template/{{cookiecutter.project_name}}/Dockerfile | 4 ++-- uv.lock | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crawlee/project_template/hooks/pre_gen_project.py b/src/crawlee/project_template/hooks/pre_gen_project.py index 5e1d1f8db0..c2a343577e 100644 --- a/src/crawlee/project_template/hooks/pre_gen_project.py +++ b/src/crawlee/project_template/hooks/pre_gen_project.py @@ -8,5 +8,5 @@ raise RuntimeError('You chose to use the Poetry package manager, but it does not seem to be installed') from exc if not re.match(r'Poetry \(version 1\..*\)', version): - raise RuntimeError(f'Poetry 1.x is required, but "{version}" is installed') + raise RuntimeError(f'Poetry below 3.x is required, but "{version}" is installed') # % endif diff --git a/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile b/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile index baf51e2bc6..f42adfb43b 100644 --- a/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile +++ b/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile @@ -14,8 +14,8 @@ RUN apt update && apt install -yq git && rm -rf /var/lib/apt/lists/* # % if cookiecutter.package_manager == 'poetry' RUN pip install -U pip setuptools \ - && pip install 'poetry<2' \ - && poetry self add 'poetry-plugin-export<1.9.0' + && pip install 'poetry<3' \ + && poetry self add 'poetry-plugin-export' # Second, copy just poetry.lock and pyproject.toml into the Actor image, # since those should be the only files that affects the dependency install in the next step, diff --git a/uv.lock b/uv.lock index ae6ca9c0a2..b55a7192fc 100644 --- a/uv.lock +++ b/uv.lock @@ -578,7 +578,7 @@ toml = [ [[package]] name = "crawlee" -version = "0.6.0" +version = "0.6.1" source = { editable = "." } dependencies = [ { name = "browserforge" }, From 6c5ee5479fc736b54acb3452d982981fb40c9134 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Wed, 5 Mar 2025 13:56:00 +0000 Subject: [PATCH 2/3] update template pyproject.toml for PEP-621 --- .../project_template/hooks/pre_gen_project.py | 4 +- .../_pyproject.toml | 40 +++++++++---------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/src/crawlee/project_template/hooks/pre_gen_project.py b/src/crawlee/project_template/hooks/pre_gen_project.py index c2a343577e..9830ee93c6 100644 --- a/src/crawlee/project_template/hooks/pre_gen_project.py +++ b/src/crawlee/project_template/hooks/pre_gen_project.py @@ -7,6 +7,6 @@ except OSError as exc: raise RuntimeError('You chose to use the Poetry package manager, but it does not seem to be installed') from exc -if not re.match(r'Poetry \(version 1\..*\)', version): - raise RuntimeError(f'Poetry below 3.x is required, but "{version}" is installed') +if not re.match(r'Poetry \(version 2\..*\)', version): + raise RuntimeError(f'Poetry 2.x is required, but "{version}" is installed') # % endif diff --git a/src/crawlee/project_template/{{cookiecutter.project_name}}/_pyproject.toml b/src/crawlee/project_template/{{cookiecutter.project_name}}/_pyproject.toml index bfc8295b88..0ae75055b5 100644 --- a/src/crawlee/project_template/{{cookiecutter.project_name}}/_pyproject.toml +++ b/src/crawlee/project_template/{{cookiecutter.project_name}}/_pyproject.toml @@ -7,34 +7,30 @@ # % do extras.append('curl-impersonate') # % endif -# % if cookiecutter.package_manager == 'poetry' -[tool.poetry] +[project] name = "{{cookiecutter.project_name}}" version = "0.0.1" description = "" -authors = ["Your Name "] +authors = [ + {name = "Your Name",email = "you@example.com"} +] readme = "README.md" -package-mode = false +requires-python = ">=3.9,<4.0" +dependencies = [ + "crawlee[{{ extras|join(',') }}]", + # % if cookiecutter.crawler_type == 'playwright-camoufox' + "camoufox[geoip]>=0.4.5", + # % endif + # % if cookiecutter.enable_apify_integration + "apify", + # % endif +] -[tool.poetry.dependencies] -python = "^3.9" -# % if cookiecutter.crawler_type == 'playwright-camoufox' -camoufox = {version= ">=0.4.5", extras = ["geoip"]} -# % endif -# % if cookiecutter.enable_apify_integration -apify = "*" -# % endif -crawlee = {version = "*", extras = {{ extras | tojson }}} +# % if cookiecutter.package_manager == 'poetry' +[tool.poetry] +package-mode = false [build-system] -requires = ["poetry-core"] +requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" -# % else -[project] -name = "{{cookiecutter.project_name}}" -version = "0.0.1" -description = "" -authors = ["Your Name "] -readme = "README.md" -requires-python = ">=3.9" # % endif From a84a876534912752e824e89bb4612c250d58eb1b Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 6 Mar 2025 12:30:53 +0000 Subject: [PATCH 3/3] add uv template --- src/crawlee/_cli.py | 4 +-- .../project_template/cookiecutter.json | 2 +- .../hooks/post_gen_project.py | 12 +++++-- .../project_template/hooks/pre_gen_project.py | 23 ++++++++++---- .../{{cookiecutter.project_name}}/Dockerfile | 22 +++++++++++++ .../{{cookiecutter.project_name}}/README.md | 31 +++++++++++++++++++ 6 files changed, 83 insertions(+), 11 deletions(-) diff --git a/src/crawlee/_cli.py b/src/crawlee/_cli.py index 395a086ff9..6a71478c83 100644 --- a/src/crawlee/_cli.py +++ b/src/crawlee/_cli.py @@ -226,10 +226,10 @@ def create( f'activate the virtual environment in ".venv" ("source .venv/bin/activate") ' f'and run your project using "python -m {package_name}".' ) - elif package_manager == 'poetry': + elif package_manager in {'poetry', 'uv'}: typer.echo( f'To run it, navigate to the directory: "cd {project_name}", ' - f'and run it using "poetry run python -m {package_name}".' + f'and run it using "{package_manager} run python -m {package_name}".' ) typer.echo(f'See the "{project_name}/README.md" for more information.') diff --git a/src/crawlee/project_template/cookiecutter.json b/src/crawlee/project_template/cookiecutter.json index 8afc215207..a32f376736 100644 --- a/src/crawlee/project_template/cookiecutter.json +++ b/src/crawlee/project_template/cookiecutter.json @@ -4,7 +4,7 @@ "crawler_type": ["beautifulsoup", "parsel", "playwright", "playwright-camoufox"], "__crawler_type": "{{ cookiecutter.crawler_type|lower|replace('-', '_') }}", "http_client": ["httpx", "curl-impersonate"], - "package_manager": ["poetry", "pip", "manual"], + "package_manager": ["poetry", "pip", "uv", "manual"], "enable_apify_integration": false, "start_url": "https://crawlee.dev", "_jinja2_env_vars": { diff --git a/src/crawlee/project_template/hooks/post_gen_project.py b/src/crawlee/project_template/hooks/post_gen_project.py index a62f99031b..877f234b6e 100644 --- a/src/crawlee/project_template/hooks/post_gen_project.py +++ b/src/crawlee/project_template/hooks/post_gen_project.py @@ -4,13 +4,21 @@ Path('_pyproject.toml').rename('pyproject.toml') -# % if cookiecutter.package_manager == 'poetry' +# % if cookiecutter.package_manager in ['poetry', 'uv'] Path('requirements.txt').unlink() +# % if cookiecutter.package_manager == 'poetry' subprocess.check_call(['poetry', 'install']) +# % elif cookiecutter.package_manager == 'uv' +subprocess.check_call(['uv', 'sync']) +# % endif + # % if cookiecutter.crawler_type == 'playwright' -subprocess.check_call(['poetry', 'run', 'playwright', 'install']) +manager = "{{ cookiecutter.package_manager }}" +subprocess.check_call([manager, 'run', 'playwright', 'install']) # % endif + + # % elif cookiecutter.package_manager == 'pip' import venv # noqa: E402 diff --git a/src/crawlee/project_template/hooks/pre_gen_project.py b/src/crawlee/project_template/hooks/pre_gen_project.py index 9830ee93c6..e85b59d4d9 100644 --- a/src/crawlee/project_template/hooks/pre_gen_project.py +++ b/src/crawlee/project_template/hooks/pre_gen_project.py @@ -1,12 +1,23 @@ -# % if cookiecutter.package_manager == 'poetry' +# % if cookiecutter.package_manager in ['poetry', 'uv'] import subprocess import re +manager = "{{cookiecutter.package_manager}}" +manager_text = manager.title() +# % if cookiecutter.package_manager == 'poetry' +version_regex = r'Poetry \(version 2\..*\)' +r_version = '2.x' +# % elif cookiecutter.package_manager == 'uv' +version_regex = r'uv (0\..*)' +r_version = '0.x' +# % endif + try: - version = subprocess.check_output(['poetry', '--version']).decode().strip() + version = subprocess.check_output([manager, '--version']).decode().strip() except OSError as exc: - raise RuntimeError('You chose to use the Poetry package manager, but it does not seem to be installed') from exc - -if not re.match(r'Poetry \(version 2\..*\)', version): - raise RuntimeError(f'Poetry 2.x is required, but "{version}" is installed') + raise RuntimeError(f'You chose to use the {manager_text} package manager, but it does not seem to be installed') from exc +print(version) +print(version_regex) +if not re.match(version_regex, version): + raise RuntimeError(f'{manager_text} {r_version} is required, but "{version}" is installed') # % endif diff --git a/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile b/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile index f42adfb43b..65383fa740 100644 --- a/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile +++ b/src/crawlee/project_template/{{cookiecutter.project_name}}/Dockerfile @@ -35,6 +35,28 @@ RUN echo "Python version:" \ pip install -r /dev/stdin --no-dependencies \ && echo "All installed Python packages:" \ && pip freeze +# % elif cookiecutter.package_manager == 'uv' +RUN pip install -U pip setuptools \ + && pip install 'uv<1' + +ENV UV_PROJECT_ENVIRONMENT="/usr/local" + +COPY pyproject.toml uv.lock ./ + +RUN echo "Python version:" \ + && python --version \ + && echo "Installing dependencies:" \ + # Check if playwright is already installed + && PLAYWRIGHT_INSTALLED=$(pip freeze | grep -q playwright && echo "true" || echo "false") \ + && if [ "$PLAYWRIGHT_INSTALLED" = "true" ]; then \ + echo "Playwright already installed, excluding from uv sync" \ + && uv sync --frozen --no-install-project --no-editable -q --no-dev --inexact --no-install-package playwright; \ + else \ + echo "Playwright not found, installing all dependencies" \ + && uv sync --frozen --no-install-project --no-editable -q --no-dev --inexact; \ + fi \ + && echo "All installed Python packages:" \ + && pip freeze # % elif cookiecutter.package_manager == 'pip' RUN pip install -U pip setuptools diff --git a/src/crawlee/project_template/{{cookiecutter.project_name}}/README.md b/src/crawlee/project_template/{{cookiecutter.project_name}}/README.md index 34583553cf..b4e4d159e5 100644 --- a/src/crawlee/project_template/{{cookiecutter.project_name}}/README.md +++ b/src/crawlee/project_template/{{cookiecutter.project_name}}/README.md @@ -31,6 +31,37 @@ python -m pip install . When the dependencies are installed, you may launch the crawler with: +```sh +python -m {{cookiecutter.__package_name}} +``` + +{% elif cookiecutter.package_manager == 'uv' -%} +To get started, ensure you have [UV](https://docs.astral.sh/uv/), a package and dependency management system, installed on your machine. We recommend installing it with the following command: + +```sh +pipx install uv +``` + +Next, install the project dependencies: + +```sh +uv sync +``` + +Finally, launch the crawler with: + +```sh +uv run python -m {{cookiecutter.__package_name}} +``` +{% elif cookiecutter.package_manager == 'pip' -%} +To install dependencies, your can run the following command: + +```sh +python -m pip install . +``` + +When the dependencies are installed, you may launch the crawler with: + ```sh python -m {{cookiecutter.__package_name}} ```