Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

## [0.0.6](../../releases/tag/v0.0.6) - Unreleased

### Adds

- `Configuration.get_global_configuration` method

### Fixes

- Handling of relative URLs in `add_requests`
Expand Down
2 changes: 1 addition & 1 deletion src/crawlee/basic_crawler/basic_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def __init__(
self._max_session_rotations = max_session_rotations

self._request_provider = request_provider
self._configuration = configuration or Configuration()
self._configuration = configuration or Configuration.get_global_configuration()

self._request_handler_timeout = request_handler_timeout
self._internal_timeout = (
Expand Down
13 changes: 12 additions & 1 deletion src/crawlee/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@
from __future__ import annotations

from datetime import timedelta
from typing import Annotated
from typing import Annotated, ClassVar, cast

from pydantic import AliasChoices, Field
from pydantic_settings import BaseSettings, SettingsConfigDict
from typing_extensions import Self

from crawlee._utils.models import timedelta_ms

Expand All @@ -21,6 +22,8 @@ class Configuration(BaseSettings):
purge_on_start: Whether to purge the storage on start.
"""

_default_instance: ClassVar[Self | None] = None

model_config = SettingsConfigDict(populate_by_name=True)

internal_timeout: Annotated[timedelta | None, Field(alias='crawlee_internal_timeout')] = None
Expand Down Expand Up @@ -204,3 +207,11 @@ class Configuration(BaseSettings):
] = False

in_cloud: Annotated[bool, Field(alias='crawlee_in_cloud')] = False

@classmethod
def get_global_configuration(cls) -> Self:
"""Retrieve the global instance of the configuration."""
if Configuration._default_instance is None:
Configuration._default_instance = cls()

return cast(Self, Configuration._default_instance)
6 changes: 5 additions & 1 deletion src/crawlee/memory_storage_client/memory_storage_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def __init__(self, configuration: Configuration | None = None) -> None:
Args:
configuration: Configuration object to use. If None, a default Configuration object will be created.
"""
self._configuration = configuration or Configuration()
self._explicit_configuration = configuration

self.datasets_handled: list[DatasetClient] = []
self.key_value_stores_handled: list[KeyValueStoreClient] = []
Expand All @@ -60,6 +60,10 @@ def __init__(self, configuration: Configuration | None = None) -> None:
self._purged_on_start = False # Indicates whether a purge was already performed on this instance.
self._purge_lock = asyncio.Lock()

@property
def _configuration(self) -> Configuration:
return self._explicit_configuration or Configuration.get_global_configuration()

@property
def write_metadata(self) -> bool:
"""Whether to write metadata to the storage."""
Expand Down
2 changes: 1 addition & 1 deletion src/crawlee/proxy_configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ def __init__(
tiered_proxy_urls: list[list[str]] | None = None,
configuration: Configuration | None = None,
) -> None:
self._configuration = configuration or Configuration()
self._configuration = configuration or Configuration.get_global_configuration()
self._next_custom_url_index = 0
self._used_proxy_urls = dict[str, URL]()
self._url_validator = TypeAdapter(AnyHttpUrl)
Expand Down
2 changes: 1 addition & 1 deletion src/crawlee/storages/_creation_management.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ async def open_storage(
name: str | None = None,
) -> TResource:
"""Open either a new storage or restore an existing one and return it."""
configuration = configuration or Configuration()
configuration = configuration or Configuration.get_global_configuration()
storage_client = StorageClientManager.get_storage_client(in_cloud=configuration.in_cloud)

# Try to restore the storage from cache by name
Expand Down
3 changes: 3 additions & 0 deletions tests/unit/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,9 @@ def _isolate_test_environment(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -
StorageClientManager._local_client = MemoryStorageClient()
StorageClientManager._cloud_client = None

# Remove global configuration instance - it may contain settings adjusted by a previous test
Configuration._default_instance = None

# Clear creation-related caches to ensure no state is carried over between tests
monkeypatch.setattr(_creation_management, '_cache_dataset_by_id', {})
monkeypatch.setattr(_creation_management, '_cache_dataset_by_name', {})
Expand Down
5 changes: 5 additions & 0 deletions tests/unit/test_configuration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
from crawlee.configuration import Configuration


def test_global_configuration_works() -> None:
assert Configuration.get_global_configuration() is Configuration.get_global_configuration()