Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 33 additions & 9 deletions src/crawlee/_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@
http_client_choices = cookiecutter_json['http_client']
package_manager_choices = cookiecutter_json['package_manager']
default_start_url = cookiecutter_json['start_url']
default_enable_apify_integration = cookiecutter_json['enable_apify_integration']
default_install_project = cookiecutter_json['install_project']
Comment thread
Pijukatel marked this conversation as resolved.
Resolved


@cli.callback(invoke_without_command=True)
Expand Down Expand Up @@ -156,6 +158,12 @@ def create(
show_default=False,
help='Should Apify integration be set up for you? If not given, you will be prompted.',
),
install_project: bool | None = typer.Option(
None,
'--install/--no-install',
show_default=False,
help='Should the project be installed now? If not given, you will be prompted.',
),
) -> None:
"""Bootstrap a new Crawlee project."""
try:
Expand All @@ -180,7 +188,13 @@ def create(

# Ask about Apify integration if not explicitly configured
if enable_apify_integration is None:
enable_apify_integration = _prompt_bool('Should Apify integration be set up for you?', default=False)
enable_apify_integration = _prompt_bool(
'Should Apify integration be set up for you?', default=default_enable_apify_integration
)

# Ask about installing the project
if install_project is None:
install_project = _prompt_bool('Should the project be installed now?', default=default_install_project)

if all(
[
Expand All @@ -190,6 +204,7 @@ def create(
package_manager,
start_url,
enable_apify_integration is not None,
install_project is not None,
]
):
package_name = project_name.replace('-', '_')
Expand All @@ -213,6 +228,7 @@ def create(
'http_client': http_client,
'enable_apify_integration': enable_apify_integration,
'start_url': start_url,
'install_project': install_project,
},
)
except Exception as exc:
Expand All @@ -229,21 +245,29 @@ def create(

typer.echo(f'Your project "{project_name}" was created.')

if package_manager == 'manual':
if install_project:
if package_manager == 'pip':
typer.echo(
f'To run it, navigate to the directory: "cd {project_name}", '
f'activate the virtual environment in ".venv" ("source .venv/bin/activate") '
f'and run your project using "python -m {package_name}".'
)
else:
typer.echo(
f'To run it, navigate to the directory: "cd {project_name}", '
f'and run it using "{package_manager} run python -m {package_name}".'
)
elif package_manager == 'pip':
typer.echo(
f'To run it, navigate to the directory: "cd {project_name}", '
f'install the dependencies listed in "requirements.txt" '
f'and run it using "python -m {package_name}".'
)
elif package_manager == 'pip':
typer.echo(
f'To run it, navigate to the directory: "cd {project_name}", '
f'activate the virtual environment in ".venv" ("source .venv/bin/activate") '
f'and run your project using "python -m {package_name}".'
)
elif package_manager in {'poetry', 'uv'}:
else:
install_command = 'sync' if package_manager == 'uv' else 'install'
typer.echo(
f'To run it, navigate to the directory: "cd {project_name}", '
f'install the project using "{package_manager} {install_command}", '
f'and run it using "{package_manager} run python -m {package_name}".'
)

Expand Down
3 changes: 2 additions & 1 deletion src/crawlee/project_template/cookiecutter.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@
"crawler_type": ["beautifulsoup", "parsel", "playwright", "playwright-camoufox"],
"__crawler_type": "{{ cookiecutter.crawler_type|lower|replace('-', '_') }}",
"http_client": ["httpx", "curl-impersonate"],
"package_manager": ["poetry", "pip", "uv", "manual"],
"package_manager": ["poetry", "pip", "uv"],
"enable_apify_integration": false,
"install_project": true,
"start_url": "https://crawlee.dev",
"_jinja2_env_vars": {
"line_statement_prefix": "# %"
Expand Down
4 changes: 4 additions & 0 deletions src/crawlee/project_template/hooks/post_gen_project.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
# % if cookiecutter.package_manager in ['poetry', 'uv']
Path('requirements.txt').unlink()

# % if cookiecutter.install_project == True
# % if cookiecutter.package_manager == 'poetry'
subprocess.check_call(['poetry', 'install'])
# % elif cookiecutter.package_manager == 'uv'
Expand All @@ -15,6 +16,7 @@
manager = "{{ cookiecutter.package_manager }}"
subprocess.check_call([manager, 'run', 'playwright', 'install'])
# % endif
# % endif


# % elif cookiecutter.package_manager == 'pip'
Expand All @@ -24,6 +26,7 @@
venv_root = Path('.venv')
venv.main([str(venv_root)])

# % if cookiecutter.install_project == True
if platform.system() == 'Windows': # noqa: SIM108
path = venv_root / 'Scripts'
else:
Expand All @@ -39,3 +42,4 @@
subprocess.check_call([str(path / 'playwright'), 'install'])
# % endif
# % endif
# % endif
Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,8 @@ async def test_static_crawler_actor_at_apify(
'http_client': http_client,
'enable_apify_integration': True,
'start_url': default_start_url,
'install_project': False,
},
accept_hooks=False, # Do not install the newly created environment.
output_dir=tmp_path,
)

Expand Down
12 changes: 12 additions & 0 deletions tests/unit/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ def test_create_interactive(mock_cookiecutter: Mock, monkeypatch: pytest.MonkeyP
readchar.key.ENTER,
readchar.key.ENTER,
readchar.key.ENTER,
readchar.key.ENTER,
]
)
monkeypatch.setattr(target=readchar, name='readkey', value=lambda: next(mock_input))
Expand All @@ -47,6 +48,7 @@ def test_create_interactive(mock_cookiecutter: Mock, monkeypatch: pytest.MonkeyP
'http_client': 'httpx',
'enable_apify_integration': False,
'start_url': 'https://crawlee.dev',
'install_project': True,
},
)

Expand All @@ -62,6 +64,7 @@ def test_create_interactive_non_default_template(mock_cookiecutter: Mock, monkey
readchar.key.ENTER,
readchar.key.ENTER,
readchar.key.ENTER,
readchar.key.ENTER,
]
)
monkeypatch.setattr(target=readchar, name='readkey', value=lambda: next(mock_input))
Expand All @@ -79,6 +82,7 @@ def test_create_interactive_non_default_template(mock_cookiecutter: Mock, monkey
'http_client': 'httpx',
'enable_apify_integration': False,
'start_url': 'https://crawlee.dev',
'install_project': True,
},
)

Expand All @@ -98,6 +102,7 @@ def test_create_non_interactive(mock_cookiecutter: Mock) -> None:
'--start-url',
'https://yr.no',
'--no-apify',
'--no-install',
],
)

Expand All @@ -111,6 +116,7 @@ def test_create_non_interactive(mock_cookiecutter: Mock) -> None:
'http_client': 'curl-impersonate',
'start_url': 'https://yr.no',
'enable_apify_integration': False,
'install_project': False,
},
)

Expand Down Expand Up @@ -144,6 +150,7 @@ def test_create_existing_folder(
'--start-url',
'https://yr.no',
'--no-apify',
'--install',
],
)
assert 'existing_project already exists' in result.output
Expand All @@ -158,6 +165,7 @@ def test_create_existing_folder(
'http_client': 'curl-impersonate',
'start_url': 'https://yr.no',
'enable_apify_integration': False,
'install_project': True,
},
)

Expand All @@ -175,6 +183,7 @@ def test_create_existing_folder_interactive(
readchar.key.ENTER,
readchar.key.ENTER,
readchar.key.ENTER,
readchar.key.ENTER,
]
)
monkeypatch.setattr(target=readchar, name='readkey', value=lambda: next(mock_input))
Expand All @@ -196,6 +205,7 @@ def test_create_existing_folder_interactive(
'http_client': 'httpx',
'start_url': 'https://crawlee.dev',
'enable_apify_integration': False,
'install_project': True,
},
)

Expand All @@ -215,6 +225,7 @@ def test_create_existing_folder_interactive_multiple_attempts(
readchar.key.ENTER,
readchar.key.ENTER,
readchar.key.ENTER,
readchar.key.ENTER,
]
)
monkeypatch.setattr(target=readchar, name='readkey', value=lambda: next(mock_input))
Expand All @@ -237,5 +248,6 @@ def test_create_existing_folder_interactive_multiple_attempts(
'http_client': 'httpx',
'start_url': 'https://crawlee.dev',
'enable_apify_integration': False,
'install_project': True,
},
)