Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions src/crawlee/storages/_key_value_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

from typing_extensions import override

from crawlee import service_container
from crawlee._utils.docs import docs_group
from crawlee.base_storage_client._models import KeyValueStoreKeyInfo, KeyValueStoreMetadata
from crawlee.events._types import Event, EventPersistStateData
Expand All @@ -16,6 +15,7 @@
from crawlee._types import JsonSerializable
from crawlee.base_storage_client import BaseStorageClient
from crawlee.configuration import Configuration
from crawlee.events._event_manager import EventManager

T = TypeVar('T')

Expand Down Expand Up @@ -226,12 +226,18 @@ async def _persist_save(self, _event_data: EventPersistStateData | None = None)
for key, value in self._cache.items():
await self.set_value(key, value)

def _get_event_manager(self) -> EventManager:
"""Get event manager from crawlee services."""
from crawlee.service_container import get_event_manager

return get_event_manager() # type: ignore[no-any-return] # Mypy is broken

def _ensure_persist_event(self) -> None:
"""Setup persist state event handling if not already done."""
if self._persist_state_event_started:
return

event_manager = service_container.get_event_manager()
event_manager = self._get_event_manager()
event_manager.on(event=Event.PERSIST_STATE, listener=self._persist_save)
self._persist_state_event_started = True

Expand All @@ -242,7 +248,7 @@ def _clear_cache(self) -> None:
def _drop_persist_state_event(self) -> None:
"""Off event_manager listener and drop event status."""
if self._persist_state_event_started:
event_manager = service_container.get_event_manager()
event_manager = self._get_event_manager()
event_manager.off(event=Event.PERSIST_STATE, listener=self._persist_save)
self._persist_state_event_started = False

Expand Down
2 changes: 1 addition & 1 deletion tests/unit/basic_crawler/test_basic_crawler.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@
import asyncio
import json
import logging
import os
from collections import Counter
from dataclasses import dataclass
from datetime import timedelta
import os
from pathlib import Path
from typing import TYPE_CHECKING, Any
from unittest.mock import AsyncMock, Mock
Expand Down