From 367046a6b38b3620959a7bf1d9b359fbc65960ba Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 8 Jul 2025 14:30:57 +0200 Subject: [PATCH 1/2] Add crawlee-cli option to skip project installation Remove `manual` value from package_manager option --- src/crawlee/_cli.py | 37 +++++++++++++++---- .../project_template/cookiecutter.json | 3 +- .../hooks/post_gen_project.py | 4 ++ .../test_static_crawlers_templates.py | 2 +- tests/unit/test_cli.py | 12 ++++++ 5 files changed, 48 insertions(+), 10 deletions(-) diff --git a/src/crawlee/_cli.py b/src/crawlee/_cli.py index e1e76cb8fc..99daa285b2 100644 --- a/src/crawlee/_cli.py +++ b/src/crawlee/_cli.py @@ -29,6 +29,7 @@ http_client_choices = cookiecutter_json['http_client'] package_manager_choices = cookiecutter_json['package_manager'] default_start_url = cookiecutter_json['start_url'] +default_install_project = cookiecutter_json['install_project'] @cli.callback(invoke_without_command=True) @@ -156,6 +157,12 @@ def create( show_default=False, help='Should Apify integration be set up for you? If not given, you will be prompted.', ), + install_project: bool | None = typer.Option( + None, + '--install/--no-install', + show_default=True, + help='Should the project be installed now? If not given, you will be prompted.', + ), ) -> None: """Bootstrap a new Crawlee project.""" try: @@ -182,6 +189,10 @@ def create( if enable_apify_integration is None: enable_apify_integration = _prompt_bool('Should Apify integration be set up for you?', default=False) + # Ask about installing the project + if install_project is None: + install_project = _prompt_bool('Should the project be installed now?', default=True) + if all( [ project_name, @@ -190,6 +201,7 @@ def create( package_manager, start_url, enable_apify_integration is not None, + install_project is not None, ] ): package_name = project_name.replace('-', '_') @@ -213,6 +225,7 @@ def create( 'http_client': http_client, 'enable_apify_integration': enable_apify_integration, 'start_url': start_url, + 'install_project': install_project, }, ) except Exception as exc: @@ -229,21 +242,29 @@ def create( typer.echo(f'Your project "{project_name}" was created.') - if package_manager == 'manual': + if install_project: + if package_manager == 'pip': + typer.echo( + f'To run it, navigate to the directory: "cd {project_name}", ' + f'activate the virtual environment in ".venv" ("source .venv/bin/activate") ' + f'and run your project using "python -m {package_name}".' + ) + else: + typer.echo( + f'To run it, navigate to the directory: "cd {project_name}", ' + f'and run it using "{package_manager} run python -m {package_name}".' + ) + elif package_manager == 'pip': typer.echo( f'To run it, navigate to the directory: "cd {project_name}", ' f'install the dependencies listed in "requirements.txt" ' f'and run it using "python -m {package_name}".' ) - elif package_manager == 'pip': - typer.echo( - f'To run it, navigate to the directory: "cd {project_name}", ' - f'activate the virtual environment in ".venv" ("source .venv/bin/activate") ' - f'and run your project using "python -m {package_name}".' - ) - elif package_manager in {'poetry', 'uv'}: + else: + install_command = 'sync' if package_manager == 'uv' else 'install' typer.echo( f'To run it, navigate to the directory: "cd {project_name}", ' + f'install the project using "{package_manager} {install_command}", ' f'and run it using "{package_manager} run python -m {package_name}".' ) diff --git a/src/crawlee/project_template/cookiecutter.json b/src/crawlee/project_template/cookiecutter.json index a32f376736..3b0b3700e7 100644 --- a/src/crawlee/project_template/cookiecutter.json +++ b/src/crawlee/project_template/cookiecutter.json @@ -4,8 +4,9 @@ "crawler_type": ["beautifulsoup", "parsel", "playwright", "playwright-camoufox"], "__crawler_type": "{{ cookiecutter.crawler_type|lower|replace('-', '_') }}", "http_client": ["httpx", "curl-impersonate"], - "package_manager": ["poetry", "pip", "uv", "manual"], + "package_manager": ["poetry", "pip", "uv"], "enable_apify_integration": false, + "install_project": true, "start_url": "https://crawlee.dev", "_jinja2_env_vars": { "line_statement_prefix": "# %" diff --git a/src/crawlee/project_template/hooks/post_gen_project.py b/src/crawlee/project_template/hooks/post_gen_project.py index c0495a724d..dc90ec516c 100644 --- a/src/crawlee/project_template/hooks/post_gen_project.py +++ b/src/crawlee/project_template/hooks/post_gen_project.py @@ -5,6 +5,7 @@ # % if cookiecutter.package_manager in ['poetry', 'uv'] Path('requirements.txt').unlink() +# % if cookiecutter.install_project == True # % if cookiecutter.package_manager == 'poetry' subprocess.check_call(['poetry', 'install']) # % elif cookiecutter.package_manager == 'uv' @@ -15,6 +16,7 @@ manager = "{{ cookiecutter.package_manager }}" subprocess.check_call([manager, 'run', 'playwright', 'install']) # % endif +# % endif # % elif cookiecutter.package_manager == 'pip' @@ -24,6 +26,7 @@ venv_root = Path('.venv') venv.main([str(venv_root)]) +# % if cookiecutter.install_project == True if platform.system() == 'Windows': # noqa: SIM108 path = venv_root / 'Scripts' else: @@ -39,3 +42,4 @@ subprocess.check_call([str(path / 'playwright'), 'install']) # % endif # % endif +# % endif diff --git a/tests/e2e/project_template/test_static_crawlers_templates.py b/tests/e2e/project_template/test_static_crawlers_templates.py index c10a776bd3..ad5c46a8c1 100644 --- a/tests/e2e/project_template/test_static_crawlers_templates.py +++ b/tests/e2e/project_template/test_static_crawlers_templates.py @@ -61,8 +61,8 @@ async def test_static_crawler_actor_at_apify( 'http_client': http_client, 'enable_apify_integration': True, 'start_url': default_start_url, + 'install_project': False, }, - accept_hooks=False, # Do not install the newly created environment. output_dir=tmp_path, ) diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index a452950adc..08ee8a0b23 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -30,6 +30,7 @@ def test_create_interactive(mock_cookiecutter: Mock, monkeypatch: pytest.MonkeyP readchar.key.ENTER, readchar.key.ENTER, readchar.key.ENTER, + readchar.key.ENTER, ] ) monkeypatch.setattr(target=readchar, name='readkey', value=lambda: next(mock_input)) @@ -47,6 +48,7 @@ def test_create_interactive(mock_cookiecutter: Mock, monkeypatch: pytest.MonkeyP 'http_client': 'httpx', 'enable_apify_integration': False, 'start_url': 'https://crawlee.dev', + 'install_project': True, }, ) @@ -62,6 +64,7 @@ def test_create_interactive_non_default_template(mock_cookiecutter: Mock, monkey readchar.key.ENTER, readchar.key.ENTER, readchar.key.ENTER, + readchar.key.ENTER, ] ) monkeypatch.setattr(target=readchar, name='readkey', value=lambda: next(mock_input)) @@ -79,6 +82,7 @@ def test_create_interactive_non_default_template(mock_cookiecutter: Mock, monkey 'http_client': 'httpx', 'enable_apify_integration': False, 'start_url': 'https://crawlee.dev', + 'install_project': True, }, ) @@ -98,6 +102,7 @@ def test_create_non_interactive(mock_cookiecutter: Mock) -> None: '--start-url', 'https://yr.no', '--no-apify', + '--no-install', ], ) @@ -111,6 +116,7 @@ def test_create_non_interactive(mock_cookiecutter: Mock) -> None: 'http_client': 'curl-impersonate', 'start_url': 'https://yr.no', 'enable_apify_integration': False, + 'install_project': False, }, ) @@ -144,6 +150,7 @@ def test_create_existing_folder( '--start-url', 'https://yr.no', '--no-apify', + '--install', ], ) assert 'existing_project already exists' in result.output @@ -158,6 +165,7 @@ def test_create_existing_folder( 'http_client': 'curl-impersonate', 'start_url': 'https://yr.no', 'enable_apify_integration': False, + 'install_project': True, }, ) @@ -175,6 +183,7 @@ def test_create_existing_folder_interactive( readchar.key.ENTER, readchar.key.ENTER, readchar.key.ENTER, + readchar.key.ENTER, ] ) monkeypatch.setattr(target=readchar, name='readkey', value=lambda: next(mock_input)) @@ -196,6 +205,7 @@ def test_create_existing_folder_interactive( 'http_client': 'httpx', 'start_url': 'https://crawlee.dev', 'enable_apify_integration': False, + 'install_project': True, }, ) @@ -215,6 +225,7 @@ def test_create_existing_folder_interactive_multiple_attempts( readchar.key.ENTER, readchar.key.ENTER, readchar.key.ENTER, + readchar.key.ENTER, ] ) monkeypatch.setattr(target=readchar, name='readkey', value=lambda: next(mock_input)) @@ -237,5 +248,6 @@ def test_create_existing_folder_interactive_multiple_attempts( 'http_client': 'httpx', 'start_url': 'https://crawlee.dev', 'enable_apify_integration': False, + 'install_project': True, }, ) From d3e7a50f2960761b9dd3c05a029063e6d89c2c59 Mon Sep 17 00:00:00 2001 From: Josef Prochazka Date: Tue, 8 Jul 2025 16:53:34 +0200 Subject: [PATCH 2/2] AI review comments --- src/crawlee/_cli.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/crawlee/_cli.py b/src/crawlee/_cli.py index 99daa285b2..4aa27f33ee 100644 --- a/src/crawlee/_cli.py +++ b/src/crawlee/_cli.py @@ -29,6 +29,7 @@ http_client_choices = cookiecutter_json['http_client'] package_manager_choices = cookiecutter_json['package_manager'] default_start_url = cookiecutter_json['start_url'] +default_enable_apify_integration = cookiecutter_json['enable_apify_integration'] default_install_project = cookiecutter_json['install_project'] @@ -160,7 +161,7 @@ def create( install_project: bool | None = typer.Option( None, '--install/--no-install', - show_default=True, + show_default=False, help='Should the project be installed now? If not given, you will be prompted.', ), ) -> None: @@ -187,11 +188,13 @@ def create( # Ask about Apify integration if not explicitly configured if enable_apify_integration is None: - enable_apify_integration = _prompt_bool('Should Apify integration be set up for you?', default=False) + enable_apify_integration = _prompt_bool( + 'Should Apify integration be set up for you?', default=default_enable_apify_integration + ) # Ask about installing the project if install_project is None: - install_project = _prompt_bool('Should the project be installed now?', default=True) + install_project = _prompt_bool('Should the project be installed now?', default=default_install_project) if all( [