diff --git a/.rules.md b/.rules.md index 2e6e2708e8..88be674573 100644 --- a/.rules.md +++ b/.rules.md @@ -40,12 +40,12 @@ Each middleware is an async generator that wraps the next handler, enabling setu Three-tier design: - **High-level**: `Dataset`, `KeyValueStore`, `RequestQueue` in `src/crawlee/storages/` -- **Storage clients** (`src/crawlee/storage_clients/`): `FileSystemStorageClient` (default), `MemoryStorageClient`, `SqlStorageClient`, `RedisStorageClient` +- **Storage backends** (`src/crawlee/storage_backends/`): `FileSystemStorageBackend` (default), `MemoryStorageBackend`, `SqlStorageBackend`, `RedisStorageBackend` - **Instance caching**: `StorageInstanceManager` is a global singleton that caches storage instances by ID/name ### Service locator -`src/crawlee/_service_locator.py` is a global singleton managing `Configuration`, `EventManager`, `StorageClient`, and `StorageInstanceManager`. Prevents double-initialization with `ServiceConflictError`. +`src/crawlee/_service_locator.py` is a global singleton managing `Configuration`, `EventManager`, `StorageBackend`, and `StorageInstanceManager`. Prevents double-initialization with `ServiceConflictError`. ### HTTP clients @@ -65,6 +65,6 @@ Pluggable via `HttpClient` interface in `src/crawlee/http_clients/`: - `crawlee._log_config`: the SDK calls `configure_logger(..., remove_old_handlers=True)` and uses `CrawleeLogFormatter` and `get_configured_log_level`. Keep their signatures and behavior stable, even though crawlee itself never passes `remove_old_handlers=True`. - `crawlee._types`, `crawlee._request`, `crawlee._consts`, and helpers under `crawlee._utils` (`crypto`, `urls`, `models`, `file`, `requests`, `byte_size`, `log`, `try_import`). - `crawlee.events._types` and `crawlee.events._event_manager`. -- `crawlee.storage_clients._base` and `crawlee.storage_clients._file_system`, whose file-system clients the SDK subclasses, so their internals count as an interface too. +- `crawlee.storage_backends._base` and `crawlee.storage_backends._file_system`, whose file-system backends the SDK subclasses, so their internals count as an interface too. Before renaming, removing, or changing the behavior of anything in these modules, search apify-sdk-python for its use. diff --git a/docs/deployment/aws_lambda.mdx b/docs/deployment/aws_lambda.mdx index b07d1a2928..43b01adf75 100644 --- a/docs/deployment/aws_lambda.mdx +++ b/docs/deployment/aws_lambda.mdx @@ -22,7 +22,7 @@ For simple crawlers that don't require browser rendering, you can deploy using a ### Updating the code -When instantiating a crawler, use `MemoryStorageClient`. By default, Crawlee uses file-based storage, but the Lambda filesystem is read-only (except for `/tmp`). Using `MemoryStorageClient` tells Crawlee to use in-memory storage instead. +When instantiating a crawler, use `MemoryStorageBackend`. By default, Crawlee uses file-based storage, but the Lambda filesystem is read-only (except for `/tmp`). Using `MemoryStorageBackend` tells Crawlee to use in-memory storage instead. Wrap the crawler logic in a `lambda_handler` function. This is the entry point that AWS will execute. @@ -113,7 +113,7 @@ For crawlers that require browser rendering, you need to deploy using Docker con ### Updating the code -As with `BeautifulSoupCrawler`, use `MemoryStorageClient` and wrap the logic in a `lambda_handler` function. Additionally, configure `browser_launch_options` with flags optimized for serverless environments. These flags disable sandboxing and GPU features that aren't available in Lambda's containerized runtime. +As with `BeautifulSoupCrawler`, use `MemoryStorageBackend` and wrap the logic in a `lambda_handler` function. Additionally, configure `browser_launch_options` with flags optimized for serverless environments. These flags disable sandboxing and GPU features that aren't available in Lambda's containerized runtime. {PlaywrightCrawlerLambda} diff --git a/docs/deployment/code_examples/aws/beautifulsoup_crawler_lambda.py b/docs/deployment/code_examples/aws/beautifulsoup_crawler_lambda.py index 3fb8bfe3b1..14f4d7860a 100644 --- a/docs/deployment/code_examples/aws/beautifulsoup_crawler_lambda.py +++ b/docs/deployment/code_examples/aws/beautifulsoup_crawler_lambda.py @@ -6,22 +6,22 @@ from aws_lambda_powertools.utilities.typing import LambdaContext from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext -from crawlee.storage_clients import MemoryStorageClient +from crawlee.storage_backends import MemoryStorageBackend from crawlee.storages import Dataset, RequestQueue async def main() -> str: # highlight-start # Disable writing storage data to the file system - storage_client = MemoryStorageClient() + storage_backend = MemoryStorageBackend() # highlight-end # Initialize storages - dataset = await Dataset.open(storage_client=storage_client) - request_queue = await RequestQueue.open(storage_client=storage_client) + dataset = await Dataset.open(storage_backend=storage_backend) + request_queue = await RequestQueue.open(storage_backend=storage_backend) crawler = BeautifulSoupCrawler( - storage_client=storage_client, + storage_backend=storage_backend, max_request_retries=1, request_handler_timeout=timedelta(seconds=30), max_requests_per_crawl=10, diff --git a/docs/deployment/code_examples/aws/playwright_crawler_lambda.py b/docs/deployment/code_examples/aws/playwright_crawler_lambda.py index d1c831ef51..93ae8a8bfc 100644 --- a/docs/deployment/code_examples/aws/playwright_crawler_lambda.py +++ b/docs/deployment/code_examples/aws/playwright_crawler_lambda.py @@ -6,22 +6,22 @@ from aws_lambda_powertools.utilities.typing import LambdaContext from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext -from crawlee.storage_clients import MemoryStorageClient +from crawlee.storage_backends import MemoryStorageBackend from crawlee.storages import Dataset, RequestQueue async def main() -> str: # highlight-start # Disable writing storage data to the file system - storage_client = MemoryStorageClient() + storage_backend = MemoryStorageBackend() # highlight-end # Initialize storages - dataset = await Dataset.open(storage_client=storage_client) - request_queue = await RequestQueue.open(storage_client=storage_client) + dataset = await Dataset.open(storage_backend=storage_backend) + request_queue = await RequestQueue.open(storage_backend=storage_backend) crawler = PlaywrightCrawler( - storage_client=storage_client, + storage_backend=storage_backend, max_request_retries=1, request_handler_timeout=timedelta(seconds=30), max_requests_per_crawl=10, diff --git a/docs/deployment/code_examples/google/cloud_run_example.py b/docs/deployment/code_examples/google/cloud_run_example.py index 27d23b99eb..2f03f03275 100644 --- a/docs/deployment/code_examples/google/cloud_run_example.py +++ b/docs/deployment/code_examples/google/cloud_run_example.py @@ -5,7 +5,7 @@ from litestar import Litestar, get from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext -from crawlee.storage_clients import MemoryStorageClient +from crawlee.storage_backends import MemoryStorageBackend @get('/') @@ -13,14 +13,14 @@ async def main() -> str: """The crawler entry point that will be called when the HTTP endpoint is accessed.""" # highlight-start # Disable writing storage data to the file system - storage_client = MemoryStorageClient() + storage_backend = MemoryStorageBackend() # highlight-end crawler = PlaywrightCrawler( headless=True, max_requests_per_crawl=10, browser_type='firefox', - storage_client=storage_client, + storage_backend=storage_backend, ) @crawler.router.default_handler diff --git a/docs/deployment/code_examples/google/google_example.py b/docs/deployment/code_examples/google/google_example.py index 68deac804c..a2041d07ce 100644 --- a/docs/deployment/code_examples/google/google_example.py +++ b/docs/deployment/code_examples/google/google_example.py @@ -6,17 +6,17 @@ from flask import Request, Response from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext -from crawlee.storage_clients import MemoryStorageClient +from crawlee.storage_backends import MemoryStorageBackend async def main() -> str: # highlight-start # Disable writing storage data to the file system - storage_client = MemoryStorageClient() + storage_backend = MemoryStorageBackend() # highlight-end crawler = BeautifulSoupCrawler( - storage_client=storage_client, + storage_backend=storage_backend, max_request_retries=1, request_handler_timeout=timedelta(seconds=30), max_requests_per_crawl=10, diff --git a/docs/examples/run_parallel_crawlers.mdx b/docs/examples/run_parallel_crawlers.mdx index fba5c437b7..a663e12de5 100644 --- a/docs/examples/run_parallel_crawlers.mdx +++ b/docs/examples/run_parallel_crawlers.mdx @@ -12,7 +12,7 @@ This example demonstrates how to run two parallel crawlers where one crawler pro In some situations, you may need different approaches for scraping data from a website. For example, you might use `PlaywrightCrawler` for navigating JavaScript-heavy pages and a faster, more lightweight `ParselCrawler` for processing static pages. One way to solve this is to use `AdaptivePlaywrightCrawler`, see the [Adaptive Playwright crawler example](./adaptive-playwright-crawler) to learn more. -The code below demonstrates an alternative approach using two separate crawlers. Links are passed between crawlers via `RequestQueue` aliases. The `keep_alive` option allows the Playwright crawler to run in the background and wait for incoming links without stopping when its queue is empty. You can also use different storage clients for each crawler without losing the ability to pass links between queues. Learn more about available storage clients in this [guide](/python/docs/guides/storage-clients). +The code below demonstrates an alternative approach using two separate crawlers. Links are passed between crawlers via `RequestQueue` aliases. The `keep_alive` option allows the Playwright crawler to run in the background and wait for incoming links without stopping when its queue is empty. You can also use different storage backends for each crawler without losing the ability to pass links between queues. Learn more about available storage backends in this [guide](../guides/storage-backends). {RunParallelCrawlersExample} diff --git a/docs/guides/architecture_overview.mdx b/docs/guides/architecture_overview.mdx index 9e61987eb0..f48fc16ef7 100644 --- a/docs/guides/architecture_overview.mdx +++ b/docs/guides/architecture_overview.mdx @@ -200,7 +200,7 @@ They have a similar inheritance structure as the crawlers, with the base class b Storages are the components that manage data in Crawlee. They provide a way to store and retrieve data during the crawling process. Crawlee's storage system consists of two main layers: - **Storages**: High-level interfaces for interacting with different storage types -- **Storage clients**: Backend implementations that handle the actual data persistence and management (you will learn more about them in the next section) +- **Storage backends**: Implementations that handle the actual data persistence and management (you will learn more about them in the next section) Crawlee provides three built-in storage types for managing data: @@ -246,14 +246,14 @@ Storage --|> KeyValueStore Storage --|> RequestQueue ``` -## Storage clients +## Storage backends -Storage clients are the backend implementations for storages that handle interactions with different storage systems. They provide a unified interface for `Dataset`, `KeyValueStore`, and `RequestQueue`, regardless of the underlying storage implementation. +Storage backends are the implementations behind storages that handle interactions with different storage systems. They provide a unified interface for `Dataset`, `KeyValueStore`, and `RequestQueue`, regardless of the underlying storage implementation. -Crawlee provides several built-in storage client implementations: +Crawlee provides several built-in storage backend implementations: -- `MemoryStorageClient` - Stores data in memory with no persistence (ideal for testing and fast operations). -- `FileSystemStorageClient` - Provides persistent file system storage with caching (default client). +- `MemoryStorageBackend` - Stores data in memory with no persistence (ideal for testing and fast operations). +- `FileSystemStorageBackend` - Provides persistent file system storage with caching (default backend). - [`ApifyStorageClient`](https://docs.apify.com/sdk/python/reference/class/ApifyStorageClient) - Manages storage on the [Apify platform](https://apify.com/) (cloud-based). It is implemented in the [Apify SDK](https://github.com/apify/apify-sdk-python). You can find more information about it in the [Apify SDK documentation](https://docs.apify.com/sdk/python/). ```mermaid @@ -269,7 +269,7 @@ classDiagram %% Abstract classes %% ======================== -class StorageClient { +class StorageBackend { <> } @@ -277,9 +277,9 @@ class StorageClient { %% Specific classes %% ======================== -class MemoryStorageClient +class MemoryStorageBackend -class FileSystemStorageClient +class FileSystemStorageBackend class ApifyStorageClient @@ -287,14 +287,14 @@ class ApifyStorageClient %% Inheritance arrows %% ======================== -StorageClient --|> MemoryStorageClient -StorageClient --|> FileSystemStorageClient -StorageClient --|> ApifyStorageClient +StorageBackend --|> MemoryStorageBackend +StorageBackend --|> FileSystemStorageBackend +StorageBackend --|> ApifyStorageClient ``` -Storage clients can be registered globally with the `ServiceLocator` (you will learn more about the `ServiceLocator` in the next section), passed directly to crawlers, or specified when opening individual storage instances. You can also create custom storage clients by implementing the `StorageClient` interface. +Storage backends can be registered globally with the `ServiceLocator` (you will learn more about the `ServiceLocator` in the next section), passed directly to crawlers, or specified when opening individual storage instances. You can also create custom storage backends by implementing the `StorageBackend` interface. -See the [Storage clients guide](./storage-clients) for more details. +See the [Storage backends guide](./storage-backends) for more details. ## Request router @@ -319,7 +319,7 @@ See the [Request router guide](./request-router) for detailed information and ex The `ServiceLocator` is a central registry for global services in Crawlee. It manages and provides access to core services throughout the framework, ensuring consistent configuration across all components. The service locator coordinates these three services: - `Configuration` - Application-wide settings and parameters that control various aspects of Crawlee behavior. -- `StorageClient` - Backend implementation for data storage across datasets, key-value stores, and request queues. +- `StorageBackend` - Backend implementation for data storage across datasets, key-value stores, and request queues. - `EventManager` - Event coordination system for internal framework events and custom user hooks. Services can be registered globally through the `service_locator` singleton instance, passed to crawler constructors, or provided when opening individual storage instances. The service locator includes conflict prevention mechanisms to ensure configuration consistency and prevent accidental service conflicts during runtime. diff --git a/docs/guides/code_examples/service_locator/service_conflicts.py b/docs/guides/code_examples/service_locator/service_conflicts.py index 52bcbbe8e9..1b36018e56 100644 --- a/docs/guides/code_examples/service_locator/service_conflicts.py +++ b/docs/guides/code_examples/service_locator/service_conflicts.py @@ -1,21 +1,21 @@ import asyncio from crawlee import service_locator -from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient +from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend async def main() -> None: - # Register the storage client via service locator. - memory_storage_client = MemoryStorageClient() - service_locator.set_storage_client(memory_storage_client) + # Register the storage backend via service locator. + memory_storage_backend = MemoryStorageBackend() + service_locator.set_storage_backend(memory_storage_backend) - # Retrieve the storage client. - current_storage_client = service_locator.get_storage_client() + # Retrieve the storage backend. + current_storage_backend = service_locator.get_storage_backend() - # Try to set a different storage client, which will raise ServiceConflictError - # if storage client was already retrieved. - file_system_storage_client = FileSystemStorageClient() - service_locator.set_storage_client(file_system_storage_client) + # Try to set a different storage backend, which will raise ServiceConflictError + # if storage backend was already retrieved. + file_system_storage_backend = FileSystemStorageBackend() + service_locator.set_storage_backend(file_system_storage_backend) if __name__ == '__main__': diff --git a/docs/guides/code_examples/service_locator/service_crawler_storage_backend.py b/docs/guides/code_examples/service_locator/service_crawler_storage_backend.py new file mode 100644 index 0000000000..75ac06859c --- /dev/null +++ b/docs/guides/code_examples/service_locator/service_crawler_storage_backend.py @@ -0,0 +1,17 @@ +import asyncio + +from crawlee.crawlers import ParselCrawler +from crawlee.storage_backends import MemoryStorageBackend + + +async def main() -> None: + storage_backend = MemoryStorageBackend() + + # Register storage backend via crawler. + crawler = ParselCrawler( + storage_backend=storage_backend, + ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/service_locator/service_crawler_storage_client.py b/docs/guides/code_examples/service_locator/service_crawler_storage_client.py deleted file mode 100644 index 76fe923877..0000000000 --- a/docs/guides/code_examples/service_locator/service_crawler_storage_client.py +++ /dev/null @@ -1,17 +0,0 @@ -import asyncio - -from crawlee.crawlers import ParselCrawler -from crawlee.storage_clients import MemoryStorageClient - - -async def main() -> None: - storage_client = MemoryStorageClient() - - # Register storage client via crawler. - crawler = ParselCrawler( - storage_client=storage_client, - ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/service_locator/service_locator_storage_backend.py b/docs/guides/code_examples/service_locator/service_locator_storage_backend.py new file mode 100644 index 0000000000..01250c7af5 --- /dev/null +++ b/docs/guides/code_examples/service_locator/service_locator_storage_backend.py @@ -0,0 +1,15 @@ +import asyncio + +from crawlee import service_locator +from crawlee.storage_backends import MemoryStorageBackend + + +async def main() -> None: + storage_backend = MemoryStorageBackend() + + # Register storage backend via service locator. + service_locator.set_storage_backend(storage_backend) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/service_locator/service_locator_storage_client.py b/docs/guides/code_examples/service_locator/service_locator_storage_client.py deleted file mode 100644 index 4dcad08420..0000000000 --- a/docs/guides/code_examples/service_locator/service_locator_storage_client.py +++ /dev/null @@ -1,15 +0,0 @@ -import asyncio - -from crawlee import service_locator -from crawlee.storage_clients import MemoryStorageClient - - -async def main() -> None: - storage_client = MemoryStorageClient() - - # Register storage client via service locator. - service_locator.set_storage_client(storage_client) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/service_locator/service_storage_configuration.py b/docs/guides/code_examples/service_locator/service_storage_configuration.py index 580e6d348f..394f79bceb 100644 --- a/docs/guides/code_examples/service_locator/service_storage_configuration.py +++ b/docs/guides/code_examples/service_locator/service_storage_configuration.py @@ -3,7 +3,7 @@ from crawlee import service_locator from crawlee.configuration import Configuration -from crawlee.storage_clients import MemoryStorageClient +from crawlee.storage_backends import MemoryStorageBackend from crawlee.storages import Dataset @@ -22,7 +22,7 @@ async def main() -> None: # Or set explicitly specific configuration if # you do not want to rely on global defaults. dataset_2 = await Dataset.open( - storage_client=MemoryStorageClient(), configuration=configuration + storage_backend=MemoryStorageBackend(), configuration=configuration ) diff --git a/docs/guides/code_examples/service_locator/service_storage_storage_backend.py b/docs/guides/code_examples/service_locator/service_storage_storage_backend.py new file mode 100644 index 0000000000..d1bd164e8c --- /dev/null +++ b/docs/guides/code_examples/service_locator/service_storage_storage_backend.py @@ -0,0 +1,17 @@ +import asyncio + +from crawlee.storage_backends import MemoryStorageBackend +from crawlee.storages import Dataset + + +async def main() -> None: + storage_backend = MemoryStorageBackend() + + # Pass the storage backend to the dataset (or other storage) when opening it. + dataset = await Dataset.open( + storage_backend=storage_backend, + ) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/guides/code_examples/service_locator/service_storage_storage_client.py b/docs/guides/code_examples/service_locator/service_storage_storage_client.py deleted file mode 100644 index 02a0853d44..0000000000 --- a/docs/guides/code_examples/service_locator/service_storage_storage_client.py +++ /dev/null @@ -1,17 +0,0 @@ -import asyncio - -from crawlee.storage_clients import MemoryStorageClient -from crawlee.storages import Dataset - - -async def main() -> None: - storage_client = MemoryStorageClient() - - # Pass the storage client to the dataset (or other storage) when opening it. - dataset = await Dataset.open( - storage_client=storage_client, - ) - - -if __name__ == '__main__': - asyncio.run(main()) diff --git a/docs/guides/code_examples/storage_backends/custom_storage_backend_example.py b/docs/guides/code_examples/storage_backends/custom_storage_backend_example.py new file mode 100644 index 0000000000..19534986c7 --- /dev/null +++ b/docs/guides/code_examples/storage_backends/custom_storage_backend_example.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from crawlee.storage_backends import StorageBackend +from crawlee.storage_backends._base import ( + DatasetBackend, + KeyValueStoreBackend, + RequestQueueBackend, +) + +if TYPE_CHECKING: + from crawlee.configuration import Configuration + +# Implement the storage type backends with your backend logic. + + +class CustomDatasetBackend(DatasetBackend): + # Implement methods like push_data, get_data, iterate_items, etc. + pass + + +class CustomKeyValueStoreBackend(KeyValueStoreBackend): + # Implement methods like get_value, set_value, delete, etc. + pass + + +class CustomRequestQueueBackend(RequestQueueBackend): + # Implement methods like add_request, fetch_next_request, etc. + pass + + +# Implement the storage backend factory. + + +class CustomStorageBackend(StorageBackend): + async def create_dataset_backend( + self, + *, + id: str | None = None, + name: str | None = None, + configuration: Configuration | None = None, + ) -> CustomDatasetBackend: + # Create and return your custom dataset backend. + pass + + async def create_kvs_backend( + self, + *, + id: str | None = None, + name: str | None = None, + configuration: Configuration | None = None, + ) -> CustomKeyValueStoreBackend: + # Create and return your custom key-value store backend. + pass + + async def create_rq_backend( + self, + *, + id: str | None = None, + name: str | None = None, + configuration: Configuration | None = None, + ) -> CustomRequestQueueBackend: + # Create and return your custom request queue backend. + pass diff --git a/docs/guides/code_examples/storage_backends/file_system_storage_backend_basic_example.py b/docs/guides/code_examples/storage_backends/file_system_storage_backend_basic_example.py new file mode 100644 index 0000000000..a0830fe101 --- /dev/null +++ b/docs/guides/code_examples/storage_backends/file_system_storage_backend_basic_example.py @@ -0,0 +1,8 @@ +from crawlee.crawlers import ParselCrawler +from crawlee.storage_backends import FileSystemStorageBackend + +# Create a new instance of storage backend. +storage_backend = FileSystemStorageBackend() + +# And pass it to the crawler. +crawler = ParselCrawler(storage_backend=storage_backend) diff --git a/docs/guides/code_examples/storage_clients/file_system_storage_client_configuration_example.py b/docs/guides/code_examples/storage_backends/file_system_storage_backend_configuration_example.py similarity index 63% rename from docs/guides/code_examples/storage_clients/file_system_storage_client_configuration_example.py rename to docs/guides/code_examples/storage_backends/file_system_storage_backend_configuration_example.py index 1d3507660f..46fd617d76 100644 --- a/docs/guides/code_examples/storage_clients/file_system_storage_client_configuration_example.py +++ b/docs/guides/code_examples/storage_backends/file_system_storage_backend_configuration_example.py @@ -1,9 +1,9 @@ from crawlee.configuration import Configuration from crawlee.crawlers import ParselCrawler -from crawlee.storage_clients import FileSystemStorageClient +from crawlee.storage_backends import FileSystemStorageBackend -# Create a new instance of storage client. -storage_client = FileSystemStorageClient() +# Create a new instance of storage backend. +storage_backend = FileSystemStorageBackend() # Create a configuration with custom settings. configuration = Configuration( @@ -13,6 +13,6 @@ # And pass them to the crawler. crawler = ParselCrawler( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) diff --git a/docs/guides/code_examples/storage_backends/memory_storage_backend_basic_example.py b/docs/guides/code_examples/storage_backends/memory_storage_backend_basic_example.py new file mode 100644 index 0000000000..2bdafd28b7 --- /dev/null +++ b/docs/guides/code_examples/storage_backends/memory_storage_backend_basic_example.py @@ -0,0 +1,8 @@ +from crawlee.crawlers import ParselCrawler +from crawlee.storage_backends import MemoryStorageBackend + +# Create a new instance of storage backend. +storage_backend = MemoryStorageBackend() + +# And pass it to the crawler. +crawler = ParselCrawler(storage_backend=storage_backend) diff --git a/docs/guides/code_examples/storage_backends/redis_storage_backend_basic_example.py b/docs/guides/code_examples/storage_backends/redis_storage_backend_basic_example.py new file mode 100644 index 0000000000..3798cb8c70 --- /dev/null +++ b/docs/guides/code_examples/storage_backends/redis_storage_backend_basic_example.py @@ -0,0 +1,10 @@ +from crawlee.crawlers import ParselCrawler +from crawlee.storage_backends import RedisStorageBackend + +# Create a new instance of storage backend using connection string. +# 'redis://localhost:6379' is the just placeholder, replace it with your actual +# connection string. +storage_backend = RedisStorageBackend(connection_string='redis://localhost:6379') + +# And pass it to the crawler. +crawler = ParselCrawler(storage_backend=storage_backend) diff --git a/docs/guides/code_examples/storage_clients/redis_storage_client_configuration_example.py b/docs/guides/code_examples/storage_backends/redis_storage_backend_configuration_example.py similarity index 73% rename from docs/guides/code_examples/storage_clients/redis_storage_client_configuration_example.py rename to docs/guides/code_examples/storage_backends/redis_storage_backend_configuration_example.py index ad1863aa23..0c35151763 100644 --- a/docs/guides/code_examples/storage_clients/redis_storage_client_configuration_example.py +++ b/docs/guides/code_examples/storage_backends/redis_storage_backend_configuration_example.py @@ -2,12 +2,12 @@ from crawlee.configuration import Configuration from crawlee.crawlers import ParselCrawler -from crawlee.storage_clients import RedisStorageClient +from crawlee.storage_backends import RedisStorageBackend -# Create a new instance of storage client using a Redis client with custom settings. +# Create a new instance of storage backend using a Redis client with custom settings. # Replace host and port with your actual Redis server configuration. # Other Redis client settings can be adjusted as needed. -storage_client = RedisStorageClient( +storage_backend = RedisStorageBackend( redis=Redis( host='localhost', port=6379, @@ -22,6 +22,6 @@ # And pass them to the crawler. crawler = ParselCrawler( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) diff --git a/docs/guides/code_examples/storage_clients/registering_storage_clients_example.py b/docs/guides/code_examples/storage_backends/registering_storage_backends_example.py similarity index 63% rename from docs/guides/code_examples/storage_clients/registering_storage_clients_example.py rename to docs/guides/code_examples/storage_backends/registering_storage_backends_example.py index b1107ea4ad..8630cb79fb 100644 --- a/docs/guides/code_examples/storage_clients/registering_storage_clients_example.py +++ b/docs/guides/code_examples/storage_backends/registering_storage_backends_example.py @@ -2,26 +2,26 @@ from crawlee import service_locator from crawlee.crawlers import ParselCrawler -from crawlee.storage_clients import MemoryStorageClient +from crawlee.storage_backends import MemoryStorageBackend from crawlee.storages import Dataset async def main() -> None: - # Create custom storage client, MemoryStorageClient for example. - storage_client = MemoryStorageClient() + # Create custom storage backend, MemoryStorageBackend for example. + storage_backend = MemoryStorageBackend() # Register it globally via the service locator. - service_locator.set_storage_client(storage_client) + service_locator.set_storage_backend(storage_backend) # Or pass it directly to the crawler, it will be registered globally # to the service locator under the hood. - crawler = ParselCrawler(storage_client=storage_client) + crawler = ParselCrawler(storage_backend=storage_backend) # Or just provide it when opening a storage (e.g. dataset), it will be used # for this storage only, not globally. dataset = await Dataset.open( name='my-dataset', - storage_client=storage_client, + storage_backend=storage_backend, ) diff --git a/docs/guides/code_examples/storage_clients/sql_storage_client_basic_example.py b/docs/guides/code_examples/storage_backends/sql_storage_backend_basic_example.py similarity index 59% rename from docs/guides/code_examples/storage_clients/sql_storage_client_basic_example.py rename to docs/guides/code_examples/storage_backends/sql_storage_backend_basic_example.py index 90c27f4039..92a533f91a 100644 --- a/docs/guides/code_examples/storage_clients/sql_storage_client_basic_example.py +++ b/docs/guides/code_examples/storage_backends/sql_storage_backend_basic_example.py @@ -1,12 +1,12 @@ from crawlee.crawlers import ParselCrawler -from crawlee.storage_clients import SqlStorageClient +from crawlee.storage_backends import SqlStorageBackend async def main() -> None: - # Create a new instance of storage client. + # Create a new instance of storage backend. # This will create an SQLite database file crawlee.db or created tables in your # database if you pass `connection_string` or `engine` # Use the context manager to ensure that connections are properly cleaned up. - async with SqlStorageClient() as storage_client: + async with SqlStorageBackend() as storage_backend: # And pass it to the crawler. - crawler = ParselCrawler(storage_client=storage_client) + crawler = ParselCrawler(storage_backend=storage_backend) diff --git a/docs/guides/code_examples/storage_clients/sql_storage_client_configuration_example.py b/docs/guides/code_examples/storage_backends/sql_storage_backend_configuration_example.py similarity index 81% rename from docs/guides/code_examples/storage_clients/sql_storage_client_configuration_example.py rename to docs/guides/code_examples/storage_backends/sql_storage_backend_configuration_example.py index 257f392683..879054fc96 100644 --- a/docs/guides/code_examples/storage_clients/sql_storage_client_configuration_example.py +++ b/docs/guides/code_examples/storage_backends/sql_storage_backend_configuration_example.py @@ -2,14 +2,14 @@ from crawlee.configuration import Configuration from crawlee.crawlers import ParselCrawler -from crawlee.storage_clients import SqlStorageClient +from crawlee.storage_backends import SqlStorageBackend async def main() -> None: - # Create a new instance of storage client. + # Create a new instance of storage backend. # On first run, also creates tables in your PostgreSQL database. # Use the context manager to ensure that connections are properly cleaned up. - async with SqlStorageClient( + async with SqlStorageBackend( # Create an `engine` with the desired configuration engine=create_async_engine( 'postgresql+asyncpg://myuser:mypassword@localhost:5432/postgres', @@ -20,7 +20,7 @@ async def main() -> None: pool_pre_ping=True, echo=False, ) - ) as storage_client: + ) as storage_backend: # Create a configuration with custom settings. configuration = Configuration( purge_on_start=False, @@ -28,6 +28,6 @@ async def main() -> None: # And pass them to the crawler. crawler = ParselCrawler( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) diff --git a/docs/guides/code_examples/storage_clients/custom_storage_client_example.py b/docs/guides/code_examples/storage_clients/custom_storage_client_example.py deleted file mode 100644 index 271b83d811..0000000000 --- a/docs/guides/code_examples/storage_clients/custom_storage_client_example.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from crawlee.storage_clients import StorageClient -from crawlee.storage_clients._base import ( - DatasetClient, - KeyValueStoreClient, - RequestQueueClient, -) - -if TYPE_CHECKING: - from crawlee.configuration import Configuration - -# Implement the storage type clients with your backend logic. - - -class CustomDatasetClient(DatasetClient): - # Implement methods like push_data, get_data, iterate_items, etc. - pass - - -class CustomKeyValueStoreClient(KeyValueStoreClient): - # Implement methods like get_value, set_value, delete, etc. - pass - - -class CustomRequestQueueClient(RequestQueueClient): - # Implement methods like add_request, fetch_next_request, etc. - pass - - -# Implement the storage client factory. - - -class CustomStorageClient(StorageClient): - async def create_dataset_client( - self, - *, - id: str | None = None, - name: str | None = None, - configuration: Configuration | None = None, - ) -> CustomDatasetClient: - # Create and return your custom dataset client. - pass - - async def create_kvs_client( - self, - *, - id: str | None = None, - name: str | None = None, - configuration: Configuration | None = None, - ) -> CustomKeyValueStoreClient: - # Create and return your custom key-value store client. - pass - - async def create_rq_client( - self, - *, - id: str | None = None, - name: str | None = None, - configuration: Configuration | None = None, - ) -> CustomRequestQueueClient: - # Create and return your custom request queue client. - pass diff --git a/docs/guides/code_examples/storage_clients/file_system_storage_client_basic_example.py b/docs/guides/code_examples/storage_clients/file_system_storage_client_basic_example.py deleted file mode 100644 index 62969f8024..0000000000 --- a/docs/guides/code_examples/storage_clients/file_system_storage_client_basic_example.py +++ /dev/null @@ -1,8 +0,0 @@ -from crawlee.crawlers import ParselCrawler -from crawlee.storage_clients import FileSystemStorageClient - -# Create a new instance of storage client. -storage_client = FileSystemStorageClient() - -# And pass it to the crawler. -crawler = ParselCrawler(storage_client=storage_client) diff --git a/docs/guides/code_examples/storage_clients/memory_storage_client_basic_example.py b/docs/guides/code_examples/storage_clients/memory_storage_client_basic_example.py deleted file mode 100644 index fe79edc3f4..0000000000 --- a/docs/guides/code_examples/storage_clients/memory_storage_client_basic_example.py +++ /dev/null @@ -1,8 +0,0 @@ -from crawlee.crawlers import ParselCrawler -from crawlee.storage_clients import MemoryStorageClient - -# Create a new instance of storage client. -storage_client = MemoryStorageClient() - -# And pass it to the crawler. -crawler = ParselCrawler(storage_client=storage_client) diff --git a/docs/guides/code_examples/storage_clients/redis_storage_client_basic_example.py b/docs/guides/code_examples/storage_clients/redis_storage_client_basic_example.py deleted file mode 100644 index e787069d94..0000000000 --- a/docs/guides/code_examples/storage_clients/redis_storage_client_basic_example.py +++ /dev/null @@ -1,10 +0,0 @@ -from crawlee.crawlers import ParselCrawler -from crawlee.storage_clients import RedisStorageClient - -# Create a new instance of storage client using connection string. -# 'redis://localhost:6379' is the just placeholder, replace it with your actual -# connection string. -storage_client = RedisStorageClient(connection_string='redis://localhost:6379') - -# And pass it to the crawler. -crawler = ParselCrawler(storage_client=storage_client) diff --git a/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py b/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py index 17911b79d7..2e97827277 100644 --- a/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py +++ b/docs/guides/code_examples/storages/cleaning_purge_explicitly_example.py @@ -4,7 +4,7 @@ async def main() -> None: - # Create storage client with configuration + # Create storage backend with configuration dataset = await Dataset.open(name='my-dataset') # Purge the dataset explicitly - purging will remove all items from the dataset. diff --git a/docs/guides/extending_crawlee.mdx b/docs/guides/extending_crawlee.mdx index 6e4a3d3977..3266469b25 100644 --- a/docs/guides/extending_crawlee.mdx +++ b/docs/guides/extending_crawlee.mdx @@ -54,22 +54,22 @@ class HttpClient { } %% ======================== -%% Storage clients +%% Storage backends %% ======================== -class StorageClient { +class StorageBackend { <> } -class DatasetClient { +class DatasetBackend { <> } -class KeyValueStoreClient { +class KeyValueStoreBackend { <> } -class RequestQueueClient { +class RequestQueueBackend { <> } @@ -100,10 +100,10 @@ BasicCrawler --|> PlaywrightCrawler AbstractHttpCrawler --> AbstractHttpParser : parses with PlaywrightCrawler --|> StagehandCrawler BasicCrawler --> HttpClient : uses -BasicCrawler --> StorageClient : uses -StorageClient --> DatasetClient : creates -StorageClient --> KeyValueStoreClient : creates -StorageClient --> RequestQueueClient : creates +BasicCrawler --> StorageBackend : uses +StorageBackend --> DatasetBackend : creates +StorageBackend --> KeyValueStoreBackend : creates +StorageBackend --> RequestQueueBackend : creates PlaywrightCrawler --> BrowserPool : uses BrowserPool --> BrowserPlugin : manages BrowserPlugin --|> PlaywrightBrowserPlugin @@ -141,19 +141,19 @@ Crawlee ships `ImpitHttpClient`, < See the [HTTP clients guide](./http-clients) for the libraries these clients wrap and their installation requirements. -### Storage clients +### Storage backends -A storage client is the backend behind Crawlee's three storages. `Dataset`, `KeyValueStore`, and `RequestQueue` are the API you write against, and the storage client decides where that data actually lives. That separation is what lets you move a crawler from the local file system to a database or a cloud service without changing crawl code. +A storage backend is the layer behind Crawlee's three storages. `Dataset`, `KeyValueStore`, and `RequestQueue` are the API you write against, and the storage backend decides where that data actually lives. That separation is what lets you move a crawler from the local file system to a database or a cloud service without changing crawl code. -The `StorageClient` contract defines three factory methods: `create_dataset_client`, `create_kvs_client`, and `create_rq_client`. The returned clients define the rest of the contract: +The `StorageBackend` contract defines three factory methods: `create_dataset_backend`, `create_kvs_backend`, and `create_rq_backend`. The returned backends define the rest of the contract: -- `DatasetClient` handles appending and reading items. -- `KeyValueStoreClient` handles record access and iteration. -- `RequestQueueClient` handles adding, fetching, and marking requests as handled. +- `DatasetBackend` handles appending and reading items. +- `KeyValueStoreBackend` handles record access and iteration. +- `RequestQueueBackend` handles adding, fetching, and marking requests as handled. A custom backend implements all four classes. -See the [Storage clients guide](./storage-clients) for the built-in implementations and a custom client example. +See the [Storage backends guide](./storage-backends) for the built-in implementations and a custom backend example. ### Browser plugins @@ -179,7 +179,7 @@ Start with configuration before writing a subclass. You can parse a response wit - If reusable HTTP parsing and the handler context both need to change, extend `AbstractHttpCrawler` and implement `AbstractHttpParser`. - If browser-level orchestration or the handler context needs to change, extend `PlaywrightCrawler`. - If the network transport needs to change while crawler behavior stays the same, implement `HttpClient` and pass it to the crawler. -- If the storage backend needs to change while the storage API stays the same, implement `StorageClient` and its three per-storage clients. +- If the storage backend needs to change while the storage API stays the same, implement `StorageBackend` and its three per-storage backends. - If browser launch needs to change while the Playwright lifecycle stays the same, extend `PlaywrightBrowserPlugin`. Implement `BrowserPlugin` directly only when the Playwright plugin's launch and lifecycle contract needs a different implementation. When more than one fits, pick the narrowest. A custom HTTP client works with every HTTP crawler, so it's easier to maintain than a crawler subclass that hard-codes the same transport. diff --git a/docs/guides/service_locator.mdx b/docs/guides/service_locator.mdx index fe10ce50c2..76e145918e 100644 --- a/docs/guides/service_locator.mdx +++ b/docs/guides/service_locator.mdx @@ -11,21 +11,21 @@ import TabItem from '@theme/TabItem'; import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; import ServiceLocatorConfiguration from '!!raw-loader!roa-loader!./code_examples/service_locator/service_locator_configuration.py'; -import ServiceLocatorStorageClient from '!!raw-loader!roa-loader!./code_examples/service_locator/service_locator_storage_client.py'; +import ServiceLocatorStorageBackend from '!!raw-loader!roa-loader!./code_examples/service_locator/service_locator_storage_backend.py'; import ServiceLocatorEventManager from '!!raw-loader!roa-loader!./code_examples/service_locator/service_locator_event_manager.py'; import ServiceCrawlerConfiguration from '!!raw-loader!roa-loader!./code_examples/service_locator/service_crawler_configuration.py'; -import ServiceCrawlerStorageClient from '!!raw-loader!roa-loader!./code_examples/service_locator/service_crawler_storage_client.py'; +import ServiceCrawlerStorageBackend from '!!raw-loader!roa-loader!./code_examples/service_locator/service_crawler_storage_backend.py'; import ServiceCrawlerEventManager from '!!raw-loader!roa-loader!./code_examples/service_locator/service_crawler_event_manager.py'; import ServiceStorageConfiguration from '!!raw-loader!roa-loader!./code_examples/service_locator/service_storage_configuration.py'; -import ServiceStorageStorageClient from '!!raw-loader!roa-loader!./code_examples/service_locator/service_storage_storage_client.py'; +import ServiceStorageStorageBackend from '!!raw-loader!roa-loader!./code_examples/service_locator/service_storage_storage_backend.py'; import ServiceConflicts from '!!raw-loader!roa-loader!./code_examples/service_locator/service_conflicts.py'; The `ServiceLocator` is a central registry for global services. It manages and provides access to these services throughout the framework, ensuring their consistent configuration and across all components. -The service locator manages three core services: `Configuration`, `EventManager`, and `StorageClient`. All services are initialized lazily with defaults when first accessed. +The service locator manages three core services: `Configuration`, `EventManager`, and `StorageBackend`. All services are initialized lazily with defaults when first accessed. ## Services @@ -35,11 +35,11 @@ There are three core services that are managed by the service locator: `Configuration` is a class that provides access to application-wide settings and parameters. It allows you to configure various aspects of Crawlee, such as timeouts, logging level, persistence intervals, and various other settings. The configuration can be set directly in the code or via environment variables. -### StorageClient +### StorageBackend -`StorageClient` is the backend implementation for storages in Crawlee. It provides a unified interface for `Dataset`, `KeyValueStore`, and `RequestQueue`, regardless of the underlying storage implementation. Storage clients were already explained in the storage clients section. +`StorageBackend` is the backend implementation for storages in Crawlee. It provides a unified interface for `Dataset`, `KeyValueStore`, and `RequestQueue`, regardless of the underlying storage implementation. Storage backends were already explained in the storage backends section. -Refer to the [Storage clients guide](./storage-clients) for more information about storage clients and how to use them. +Refer to the [Storage backends guide](./storage-backends) for more information about storage backends and how to use them. ### EventManager @@ -55,9 +55,9 @@ Services can be registered globally through the + - {ServiceLocatorStorageClient} + {ServiceLocatorStorageBackend} @@ -81,9 +81,9 @@ Alternatively services can be passed to the crawler constructors. They will be r - + - {ServiceCrawlerStorageClient} + {ServiceCrawlerStorageBackend} @@ -107,9 +107,9 @@ Alternatively, services can be provided when opening specific storage instances, - + - {ServiceStorageStorageClient} + {ServiceStorageStorageBackend} diff --git a/docs/guides/storage_backends.mdx b/docs/guides/storage_backends.mdx new file mode 100644 index 0000000000..1514be8782 --- /dev/null +++ b/docs/guides/storage_backends.mdx @@ -0,0 +1,558 @@ +--- +id: storage-backends +title: Storage backends +description: How to work with storage backends in Crawlee, including the built-in backends and how to create your own. +--- + +import ApiLink from '@site/src/components/ApiLink'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; +import CodeBlock from '@theme/CodeBlock'; + +import MemoryStorageBackendBasicExample from '!!raw-loader!roa-loader!./code_examples/storage_backends/memory_storage_backend_basic_example.py'; +import FileSystemStorageBackendBasicExample from '!!raw-loader!roa-loader!./code_examples/storage_backends/file_system_storage_backend_basic_example.py'; +import FileSystemStorageBackendConfigurationExample from '!!raw-loader!roa-loader!./code_examples/storage_backends/file_system_storage_backend_configuration_example.py'; +import CustomStorageBackendExample from '!!raw-loader!roa-loader!./code_examples/storage_backends/custom_storage_backend_example.py'; +import RegisteringStorageBackendsExample from '!!raw-loader!roa-loader!./code_examples/storage_backends/registering_storage_backends_example.py'; +import SQLStorageBackendBasicExample from '!!raw-loader!roa-loader!./code_examples/storage_backends/sql_storage_backend_basic_example.py'; +import SQLStorageBackendConfigurationExample from '!!raw-loader!./code_examples/storage_backends/sql_storage_backend_configuration_example.py'; +import RedisStorageBackendBasicExample from '!!raw-loader!./code_examples/storage_backends/redis_storage_backend_basic_example.py'; +import RedisStorageBackendConfigurationExample from '!!raw-loader!./code_examples/storage_backends/redis_storage_backend_configuration_example.py'; + +Storage backends provide a unified interface for interacting with `Dataset`, `KeyValueStore`, and `RequestQueue`, regardless of the underlying implementation. They handle operations like creating, reading, updating, and deleting storage instances, as well as managing data persistence and cleanup. This abstraction makes it easy to switch between different environments, such as local development and cloud production setups. + +## Built-in storage backends + +Crawlee provides three main storage backend implementations: + +- `FileSystemStorageBackend` - Provides persistent file system storage with in-memory caching. +- `MemoryStorageBackend` - Stores data in memory with no persistence. +- `SqlStorageBackend` - Provides persistent storage using a SQL database ([SQLite](https://sqlite.org/), [PostgreSQL](https://www.postgresql.org/), [MySQL](https://www.mysql.com/) or [MariaDB](https://mariadb.org/)). Requires installing the extra dependency: `crawlee[sql_sqlite]` for SQLite, `crawlee[sql_postgres]` for PostgreSQL or `crawlee[sql_mysql]` for MySQL and MariaDB. +- `RedisStorageBackend` - Provides persistent storage using a [Redis](https://redis.io/) database v8.0+. Requires installing the extra dependency `crawlee[redis]`. +- [`ApifyStorageClient`](https://docs.apify.com/sdk/python/reference/class/ApifyStorageClient) - Manages storage on the [Apify platform](https://apify.com), implemented in the [Apify SDK](https://github.com/apify/apify-sdk-python). + +```mermaid +--- +config: + class: + hideEmptyMembersBox: true +--- + +classDiagram + +%% ======================== +%% Abstract classes +%% ======================== + +class StorageBackend { + <> +} + +%% ======================== +%% Specific classes +%% ======================== + +class FileSystemStorageBackend + +class MemoryStorageBackend + +class SqlStorageBackend + +class RedisStorageBackend + +class ApifyStorageClient + +%% ======================== +%% Inheritance arrows +%% ======================== + +StorageBackend --|> FileSystemStorageBackend +StorageBackend --|> MemoryStorageBackend +StorageBackend --|> SqlStorageBackend +StorageBackend --|> RedisStorageBackend +StorageBackend --|> ApifyStorageClient +``` + +### File system storage backend + +The `FileSystemStorageBackend` provides persistent storage by writing data directly to the file system. It uses intelligent caching and batch processing for better performance while storing data in human-readable JSON format. This is the default storage backend used by Crawlee when no other storage backend is specified, making it ideal for large datasets and long-running operations where data persistence is required. + +:::warning Concurrency limitation +The `FileSystemStorageBackend` is not safe for concurrent access from multiple crawler processes. Use it only when running a single crawler process at a time. +::: + +This storage backend is ideal for large datasets, and long-running operations where data persistence is required. Data can be easily inspected and shared with other tools. + + + {FileSystemStorageBackendBasicExample} + + +Configuration options for the `FileSystemStorageBackend` can be set through environment variables or the `Configuration` class: + +- **`storage_dir`** (env: `CRAWLEE_STORAGE_DIR`, default: `'./storage'`) - The root directory for all storage data. +- **`purge_on_start`** (env: `CRAWLEE_PURGE_ON_START`, default: `True`) - Whether to purge default storages on start. + +Data is stored using the following directory structure: + +```text +{CRAWLEE_STORAGE_DIR}/ +├── datasets/ +│ └── {DATASET_NAME}/ +│ ├── __metadata__.json +│ ├── 000000001.json +│ └── 000000002.json +├── key_value_stores/ +│ └── {KVS_NAME}/ +│ ├── __metadata__.json +│ ├── key1.json +│ ├── key2.txt +│ └── key3.json +└── request_queues/ + └── {RQ_NAME}/ + ├── __metadata__.json + ├── {REQUEST_ID_1}.json + └── {REQUEST_ID_2}.json +``` + +Where: + +- `{CRAWLEE_STORAGE_DIR}` - The root directory for local storage. +- `{DATASET_NAME}`, `{KVS_NAME}`, `{RQ_NAME}` - The unique names for each storage instance (defaults to `"default"`). +- Files are stored directly without additional metadata files for simpler structure. + +Here is an example of how to configure the `FileSystemStorageBackend`: + + + {FileSystemStorageBackendConfigurationExample} + + +### Memory storage backend + +The `MemoryStorageBackend` stores all data in memory using Python data structures. It provides fast access but does not persist data between runs, meaning all data is lost when the program terminates. This storage backend is primarily suitable for testing and development, and is usually not a good fit for production use. However, in some cases where speed is prioritized over persistence, it can make sense. + +:::warning Persistence limitation +The `MemoryStorageBackend` does not persist data between runs. All data is lost when the program terminates. +::: + + + {MemoryStorageBackendBasicExample} + + +### SQL storage backend + +:::warning Experimental feature +The `SqlStorageBackend` is experimental. Its API and behavior may change in future releases. +::: + +The `SqlStorageBackend` provides persistent storage using a SQL database (SQLite by default, or PostgreSQL, MySQL, MariaDB). It supports all Crawlee storage types and enables concurrent access from multiple independent clients or processes. + +:::note dependencies +The `SqlStorageBackend` is not included in the core Crawlee package. +To use it, you need to install Crawlee with the appropriate extra dependency: + +- For SQLite support, run: + pip install 'crawlee[sql_sqlite]' +- For PostgreSQL support, run: + pip install 'crawlee[sql_postgres]' +- For MySQL or MariaDB support, run: + pip install 'crawlee[sql_mysql]' +::: + +By default, SqlStorageBackend uses SQLite. +To use a different database, just provide the appropriate connection string via the `connection_string` parameter. No other code changes are needed—the same backend works for all supported databases. + + + {SQLStorageBackendBasicExample} + + +Data is organized in relational tables. Below are the main tables and columns used for each storage type: + +```mermaid +--- +config: + class: + hideEmptyMembersBox: true +--- + +classDiagram + +%% ======================== +%% Storage Backends +%% ======================== + +class SqlDatasetBackend { + <> +} + +class SqlKeyValueStoreBackend { + <> +} + +%% ======================== +%% Dataset Tables +%% ======================== + +class datasets { + <> + + dataset_id (PK) + + internal_name + + name + + accessed_at + + created_at + + modified_at + + item_count + + buffer_locked_until +} + +class dataset_records { + <
> + + item_id (PK) + + dataset_id (FK) + + data +} + +class dataset_metadata_buffer { + <
> + + id (PK) + + accessed_at + + modified_at + + delta_item_count +} + +%% ======================== +%% Key-Value Store Tables +%% ======================== + +class key_value_stores { + <
> + + key_value_store_id (PK) + + internal_name + + name + + accessed_at + + created_at + + modified_at + + buffer_locked_until +} + +class key_value_store_records { + <
> + + key_value_store_id (FK, PK) + + key (PK) + + value + + content_type + + size +} + +class key_value_store_metadata_buffer { + <
> + + id (PK) + + accessed_at + + modified_at +} + +%% ======================== +%% Backend to Table arrows +%% ======================== + +SqlDatasetBackend --> datasets +SqlDatasetBackend --> dataset_records +SqlDatasetBackend --> dataset_metadata_buffer + +SqlKeyValueStoreBackend --> key_value_stores +SqlKeyValueStoreBackend --> key_value_store_records +SqlKeyValueStoreBackend --> key_value_store_metadata_buffer +``` + +```mermaid +--- +config: + class: + hideEmptyMembersBox: true +--- + +classDiagram + +%% ======================== +%% Storage Backends +%% ======================== + +class SqlRequestQueueBackend { + <> +} + +%% ======================== +%% Request Queue Tables +%% ======================== + +class request_queues { + <
> + + request_queue_id (PK) + + internal_name + + name + + accessed_at + + created_at + + modified_at + + had_multiple_clients + + handled_request_count + + pending_request_count + + total_request_count + + buffer_locked_until +} + +class request_queue_records { + <
> + + request_id (PK) + + request_queue_id (FK, PK) + + data + + sequence_number + + is_handled + + time_blocked_until + + client_key +} + +class request_queue_state { + <
> + + request_queue_id (FK, PK) + + sequence_counter + + forefront_sequence_counter +} + +class request_queue_metadata_buffer { + <
> + + id (PK) + + accessed_at + + modified_at + + client_id + + delta_handled_count + + delta_pending_count + + delta_total_count + + need_recalc +} + +%% ======================== +%% Backend to Table arrows +%% ======================== + +SqlRequestQueueBackend --> request_queues +SqlRequestQueueBackend --> request_queue_records +SqlRequestQueueBackend --> request_queue_state +SqlRequestQueueBackend --> request_queue_metadata_buffer +``` + +Configuration options for the `SqlStorageBackend` can be set through environment variables or the `Configuration` class: + +- **`storage_dir`** (env: `CRAWLEE_STORAGE_DIR`, default: `'./storage'`) - The root directory where the default SQLite database will be created if no connection string is provided. +- **`purge_on_start`** (env: `CRAWLEE_PURGE_ON_START`, default: `True`) - Whether to purge default storages on start. + +Configuration options for the `SqlStorageBackend` can be set via constructor arguments: + +- **`connection_string`** (default: SQLite in `Configuration` storage dir) - SQLAlchemy connection string, e.g. `sqlite+aiosqlite:///my.db`, `postgresql+asyncpg://user:pass@host/db`, `mysql+aiomysql://user:pass@host/db` or `mariadb+aiomysql://user:pass@host/db`. +- **`engine`** - Pre-configured SQLAlchemy AsyncEngine (optional). + +For advanced scenarios, you can configure `SqlStorageBackend` with a custom SQLAlchemy engine and additional options via the `Configuration` class. This is useful, for example, when connecting to an external PostgreSQL database or customizing connection pooling. + +:::warning +If you use MySQL or MariaDB, pass the `isolation_level='READ COMMITTED'` argument to `create_async_engine`. MySQL/MariaDB default to the `REPEATABLE READ` isolation level, which can cause unnecessary locking, deadlocks, or stale reads when multiple Crawlee workers access the same tables concurrently. Using `READ COMMITTED` ensures more predictable row-level locking and visibility semantics for `SqlStorageBackend`. +::: + + + {SQLStorageBackendConfigurationExample} + + +### Redis storage backend + +:::warning Experimental feature +The `RedisStorageBackend` is experimental. Its API and behavior may change in future releases. +::: + +The `RedisStorageBackend` provides persistent storage using [Redis](https://redis.io/) database. It supports concurrent access from multiple independent clients or processes and uses Redis native data structures for efficient operations. + +:::note dependencies +The `RedisStorageBackend` is not included in the core Crawlee package. +To use it, you need to install Crawlee with the Redis extra dependency: + +pip install 'crawlee[redis]' + +Additionally, Redis version 8.0 or higher is required. +::: + +:::note Redis persistence +Data persistence in Redis depends on your [database configuration](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/). +::: + +The backend requires either a Redis connection string or a pre-configured Redis client instance. Use a pre-configured client when you need custom Redis settings such as connection pooling, timeouts, or SSL/TLS encryption. + + + {RedisStorageBackendBasicExample} + + +Data is organized using Redis key patterns. Below are the main data structures used for each storage type: + +```mermaid +--- +config: + class: + hideEmptyMembersBox: true +--- + +classDiagram + +%% ======================== +%% Storage Backends +%% ======================== + +class RedisDatasetBackend { + <> +} + +%% ======================== +%% Dataset Keys +%% ======================== + +class DatasetKeys { + datasets:[name]:items - JSON Array + datasets:[name]:metadata - JSON Object +} + +class DatasetsIndexes { + datasets:id_to_name - Hash + datasets:name_to_id - Hash +} + +%% ======================== +%% Backend to Keys arrows +%% ======================== + +RedisDatasetBackend --> DatasetKeys +RedisDatasetBackend --> DatasetsIndexes +``` + +```mermaid +--- +config: + class: + hideEmptyMembersBox: true +--- + +classDiagram + +%% ======================== +%% Storage Backends +%% ======================== + +class RedisKeyValueStoreBackend { + <> +} + +%% ======================== +%% Key-Value Store Keys +%% ======================== + +class KeyValueStoreKeys { + key_value_stores:[name]:items - Hash + key_value_stores:[name]:metadata_items - Hash + key_value_stores:[name]:metadata - JSON Object +} + +class KeyValueStoresIndexes { + key_value_stores:id_to_name - Hash + key_value_stores:name_to_id - Hash +} + +%% ======================== +%% Backend to Keys arrows +%% ======================== + +RedisKeyValueStoreBackend --> KeyValueStoreKeys +RedisKeyValueStoreBackend --> KeyValueStoresIndexes +``` + +```mermaid +--- +config: + class: + hideEmptyMembersBox: true +--- + +classDiagram + +%% ======================== +%% Storage Backends +%% ======================== + +class RedisRequestQueueBackend { + <> +} + +%% ======================== +%% Request Queue Keys +%% ======================== + +class RequestQueueKeys{ + request_queues:[name]:queue - List + request_queues:[name]:data - Hash + request_queues:[name]:in_progress - Hash + request_queues:[name]:added_bloom_filter - Bloom Filter | bloom queue_dedup_strategy + request_queues:[name]:handled_bloom_filter - Bloom Filter | bloom queue_dedup_strategy + request_queues:[name]:pending_set - Set | default queue_dedup_strategy + request_queues:[name]:handled_set - Set | default queue_dedup_strategy + request_queues:[name]:metadata - JSON Object +} + +class RequestQueuesIndexes { + request_queues:id_to_name - Hash + request_queues:name_to_id - Hash +} + +%% ======================== +%% Backend to Keys arrows +%% ======================== + +RedisRequestQueueBackend --> RequestQueueKeys +RedisRequestQueueBackend --> RequestQueuesIndexes +``` + +Configuration options for the `RedisStorageBackend` can be set through environment variables or the `Configuration` class: + +- **`purge_on_start`** (env: `CRAWLEE_PURGE_ON_START`, default: `True`) - Whether to purge default storages on start. + +Configuration options for the `RedisStorageBackend` can be set via constructor arguments: + +- **`connection_string`** - Redis connection string, e.g. `redis://localhost:6379/0`. +- **`redis`** - Pre-configured Redis client instance (optional). + + + {RedisStorageBackendConfigurationExample} + + +## Creating a custom storage backend + +A storage backend consists of two parts: the storage backend factory and individual storage type backends. The `StorageBackend` acts as a factory that creates the storage type backends (`DatasetBackend`, `KeyValueStoreBackend`, `RequestQueueBackend`) where the actual storage logic is implemented. + +Here is an example of a custom storage backend that implements the `StorageBackend` interface: + + + {CustomStorageBackendExample} + + +Custom storage backends can implement any storage logic, such as connecting to a database, using a cloud storage service, or integrating with other systems. They must implement the required methods for creating, reading, updating, and deleting data in the respective storages. + +## Registering storage backends + +Storage backends can be registered in multiple ways: + +- **Globally** - Using the `ServiceLocator` or passing directly to the crawler. +- **Per storage** - When opening a specific storage instance like `Dataset`, `KeyValueStore`, or `RequestQueue`. + + + {RegisteringStorageBackendsExample} + + +You can also register different storage backends for each storage instance, allowing you to use different backends for different storages. This is useful when you want to use a fast in-memory storage for `RequestQueue` while persisting scraping results in `Dataset` or `KeyValueStore`. + +## Conclusion + +Storage backends in Crawlee decide where your data is stored. Use `MemoryStorageBackend` for testing and fast operations without persistence, or `FileSystemStorageBackend` for environments where data needs to persist. You can also create custom storage backends for specialized storage systems by implementing the `StorageBackend` interface. + +If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/storage_clients.mdx b/docs/guides/storage_clients.mdx deleted file mode 100644 index 3186f906c9..0000000000 --- a/docs/guides/storage_clients.mdx +++ /dev/null @@ -1,558 +0,0 @@ ---- -id: storage-clients -title: Storage clients -description: How to work with storage clients in Crawlee, including the built-in clients and how to create your own. ---- - -import ApiLink from '@site/src/components/ApiLink'; -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; -import RunnableCodeBlock from '@site/src/components/RunnableCodeBlock'; -import CodeBlock from '@theme/CodeBlock'; - -import MemoryStorageClientBasicExample from '!!raw-loader!roa-loader!./code_examples/storage_clients/memory_storage_client_basic_example.py'; -import FileSystemStorageClientBasicExample from '!!raw-loader!roa-loader!./code_examples/storage_clients/file_system_storage_client_basic_example.py'; -import FileSystemStorageClientConfigurationExample from '!!raw-loader!roa-loader!./code_examples/storage_clients/file_system_storage_client_configuration_example.py'; -import CustomStorageClientExample from '!!raw-loader!roa-loader!./code_examples/storage_clients/custom_storage_client_example.py'; -import RegisteringStorageClientsExample from '!!raw-loader!roa-loader!./code_examples/storage_clients/registering_storage_clients_example.py'; -import SQLStorageClientBasicExample from '!!raw-loader!roa-loader!./code_examples/storage_clients/sql_storage_client_basic_example.py'; -import SQLStorageClientConfigurationExample from '!!raw-loader!./code_examples/storage_clients/sql_storage_client_configuration_example.py'; -import RedisStorageClientBasicExample from '!!raw-loader!./code_examples/storage_clients/redis_storage_client_basic_example.py'; -import RedisStorageClientConfigurationExample from '!!raw-loader!./code_examples/storage_clients/redis_storage_client_configuration_example.py'; - -Storage clients provide a unified interface for interacting with `Dataset`, `KeyValueStore`, and `RequestQueue`, regardless of the underlying implementation. They handle operations like creating, reading, updating, and deleting storage instances, as well as managing data persistence and cleanup. This abstraction makes it easy to switch between different environments, such as local development and cloud production setups. - -## Built-in storage clients - -Crawlee provides three main storage client implementations: - -- `FileSystemStorageClient` - Provides persistent file system storage with in-memory caching. -- `MemoryStorageClient` - Stores data in memory with no persistence. -- `SqlStorageClient` - Provides persistent storage using a SQL database ([SQLite](https://sqlite.org/), [PostgreSQL](https://www.postgresql.org/), [MySQL](https://www.mysql.com/) or [MariaDB](https://mariadb.org/)). Requires installing the extra dependency: `crawlee[sql_sqlite]` for SQLite, `crawlee[sql_postgres]` for PostgreSQL or `crawlee[sql_mysql]` for MySQL and MariaDB. -- `RedisStorageClient` - Provides persistent storage using a [Redis](https://redis.io/) database v8.0+. Requires installing the extra dependency `crawlee[redis]`. -- [`ApifyStorageClient`](https://docs.apify.com/sdk/python/reference/class/ApifyStorageClient) - Manages storage on the [Apify platform](https://apify.com), implemented in the [Apify SDK](https://github.com/apify/apify-sdk-python). - -```mermaid ---- -config: - class: - hideEmptyMembersBox: true ---- - -classDiagram - -%% ======================== -%% Abstract classes -%% ======================== - -class StorageClient { - <> -} - -%% ======================== -%% Specific classes -%% ======================== - -class FileSystemStorageClient - -class MemoryStorageClient - -class SqlStorageClient - -class RedisStorageClient - -class ApifyStorageClient - -%% ======================== -%% Inheritance arrows -%% ======================== - -StorageClient --|> FileSystemStorageClient -StorageClient --|> MemoryStorageClient -StorageClient --|> SqlStorageClient -StorageClient --|> RedisStorageClient -StorageClient --|> ApifyStorageClient -``` - -### File system storage client - -The `FileSystemStorageClient` provides persistent storage by writing data directly to the file system. It uses intelligent caching and batch processing for better performance while storing data in human-readable JSON format. This is the default storage client used by Crawlee when no other storage client is specified, making it ideal for large datasets and long-running operations where data persistence is required. - -:::warning Concurrency limitation -The `FileSystemStorageClient` is not safe for concurrent access from multiple crawler processes. Use it only when running a single crawler process at a time. -::: - -This storage client is ideal for large datasets, and long-running operations where data persistence is required. Data can be easily inspected and shared with other tools. - - - {FileSystemStorageClientBasicExample} - - -Configuration options for the `FileSystemStorageClient` can be set through environment variables or the `Configuration` class: - -- **`storage_dir`** (env: `CRAWLEE_STORAGE_DIR`, default: `'./storage'`) - The root directory for all storage data. -- **`purge_on_start`** (env: `CRAWLEE_PURGE_ON_START`, default: `True`) - Whether to purge default storages on start. - -Data is stored using the following directory structure: - -```text -{CRAWLEE_STORAGE_DIR}/ -├── datasets/ -│ └── {DATASET_NAME}/ -│ ├── __metadata__.json -│ ├── 000000001.json -│ └── 000000002.json -├── key_value_stores/ -│ └── {KVS_NAME}/ -│ ├── __metadata__.json -│ ├── key1.json -│ ├── key2.txt -│ └── key3.json -└── request_queues/ - └── {RQ_NAME}/ - ├── __metadata__.json - ├── {REQUEST_ID_1}.json - └── {REQUEST_ID_2}.json -``` - -Where: - -- `{CRAWLEE_STORAGE_DIR}` - The root directory for local storage. -- `{DATASET_NAME}`, `{KVS_NAME}`, `{RQ_NAME}` - The unique names for each storage instance (defaults to `"default"`). -- Files are stored directly without additional metadata files for simpler structure. - -Here is an example of how to configure the `FileSystemStorageClient`: - - - {FileSystemStorageClientConfigurationExample} - - -### Memory storage client - -The `MemoryStorageClient` stores all data in memory using Python data structures. It provides fast access but does not persist data between runs, meaning all data is lost when the program terminates. This storage client is primarily suitable for testing and development, and is usually not a good fit for production use. However, in some cases where speed is prioritized over persistence, it can make sense. - -:::warning Persistence limitation -The `MemoryStorageClient` does not persist data between runs. All data is lost when the program terminates. -::: - - - {MemoryStorageClientBasicExample} - - -### SQL storage client - -:::warning Experimental feature -The `SqlStorageClient` is experimental. Its API and behavior may change in future releases. -::: - -The `SqlStorageClient` provides persistent storage using a SQL database (SQLite by default, or PostgreSQL, MySQL, MariaDB). It supports all Crawlee storage types and enables concurrent access from multiple independent clients or processes. - -:::note dependencies -The `SqlStorageClient` is not included in the core Crawlee package. -To use it, you need to install Crawlee with the appropriate extra dependency: - -- For SQLite support, run: - pip install 'crawlee[sql_sqlite]' -- For PostgreSQL support, run: - pip install 'crawlee[sql_postgres]' -- For MySQL or MariaDB support, run: - pip install 'crawlee[sql_mysql]' -::: - -By default, SqlStorageClient uses SQLite. -To use a different database, just provide the appropriate connection string via the `connection_string` parameter. No other code changes are needed—the same client works for all supported databases. - - - {SQLStorageClientBasicExample} - - -Data is organized in relational tables. Below are the main tables and columns used for each storage type: - -```mermaid ---- -config: - class: - hideEmptyMembersBox: true ---- - -classDiagram - -%% ======================== -%% Storage Clients -%% ======================== - -class SqlDatasetClient { - <> -} - -class SqlKeyValueStoreClient { - <> -} - -%% ======================== -%% Dataset Tables -%% ======================== - -class datasets { - <
> - + dataset_id (PK) - + internal_name - + name - + accessed_at - + created_at - + modified_at - + item_count - + buffer_locked_until -} - -class dataset_records { - <
> - + item_id (PK) - + dataset_id (FK) - + data -} - -class dataset_metadata_buffer { - <
> - + id (PK) - + accessed_at - + modified_at - + delta_item_count -} - -%% ======================== -%% Key-Value Store Tables -%% ======================== - -class key_value_stores { - <
> - + key_value_store_id (PK) - + internal_name - + name - + accessed_at - + created_at - + modified_at - + buffer_locked_until -} - -class key_value_store_records { - <
> - + key_value_store_id (FK, PK) - + key (PK) - + value - + content_type - + size -} - -class key_value_store_metadata_buffer { - <
> - + id (PK) - + accessed_at - + modified_at -} - -%% ======================== -%% Client to Table arrows -%% ======================== - -SqlDatasetClient --> datasets -SqlDatasetClient --> dataset_records -SqlDatasetClient --> dataset_metadata_buffer - -SqlKeyValueStoreClient --> key_value_stores -SqlKeyValueStoreClient --> key_value_store_records -SqlKeyValueStoreClient --> key_value_store_metadata_buffer -``` - -```mermaid ---- -config: - class: - hideEmptyMembersBox: true ---- - -classDiagram - -%% ======================== -%% Storage Clients -%% ======================== - -class SqlRequestQueueClient { - <> -} - -%% ======================== -%% Request Queue Tables -%% ======================== - -class request_queues { - <
> - + request_queue_id (PK) - + internal_name - + name - + accessed_at - + created_at - + modified_at - + had_multiple_clients - + handled_request_count - + pending_request_count - + total_request_count - + buffer_locked_until -} - -class request_queue_records { - <
> - + request_id (PK) - + request_queue_id (FK, PK) - + data - + sequence_number - + is_handled - + time_blocked_until - + client_key -} - -class request_queue_state { - <
> - + request_queue_id (FK, PK) - + sequence_counter - + forefront_sequence_counter -} - -class request_queue_metadata_buffer { - <
> - + id (PK) - + accessed_at - + modified_at - + client_id - + delta_handled_count - + delta_pending_count - + delta_total_count - + need_recalc -} - -%% ======================== -%% Client to Table arrows -%% ======================== - -SqlRequestQueueClient --> request_queues -SqlRequestQueueClient --> request_queue_records -SqlRequestQueueClient --> request_queue_state -SqlRequestQueueClient --> request_queue_metadata_buffer -``` - -Configuration options for the `SqlStorageClient` can be set through environment variables or the `Configuration` class: - -- **`storage_dir`** (env: `CRAWLEE_STORAGE_DIR`, default: `'./storage'`) - The root directory where the default SQLite database will be created if no connection string is provided. -- **`purge_on_start`** (env: `CRAWLEE_PURGE_ON_START`, default: `True`) - Whether to purge default storages on start. - -Configuration options for the `SqlStorageClient` can be set via constructor arguments: - -- **`connection_string`** (default: SQLite in `Configuration` storage dir) - SQLAlchemy connection string, e.g. `sqlite+aiosqlite:///my.db`, `postgresql+asyncpg://user:pass@host/db`, `mysql+aiomysql://user:pass@host/db` or `mariadb+aiomysql://user:pass@host/db`. -- **`engine`** - Pre-configured SQLAlchemy AsyncEngine (optional). - -For advanced scenarios, you can configure `SqlStorageClient` with a custom SQLAlchemy engine and additional options via the `Configuration` class. This is useful, for example, when connecting to an external PostgreSQL database or customizing connection pooling. - -:::warning -If you use MySQL or MariaDB, pass the `isolation_level='READ COMMITTED'` argument to `create_async_engine`. MySQL/MariaDB default to the `REPEATABLE READ` isolation level, which can cause unnecessary locking, deadlocks, or stale reads when multiple Crawlee workers access the same tables concurrently. Using `READ COMMITTED` ensures more predictable row-level locking and visibility semantics for `SqlStorageClient`. -::: - - - {SQLStorageClientConfigurationExample} - - -### Redis storage client - -:::warning Experimental feature -The `RedisStorageClient` is experimental. Its API and behavior may change in future releases. -::: - -The `RedisStorageClient` provides persistent storage using [Redis](https://redis.io/) database. It supports concurrent access from multiple independent clients or processes and uses Redis native data structures for efficient operations. - -:::note dependencies -The `RedisStorageClient` is not included in the core Crawlee package. -To use it, you need to install Crawlee with the Redis extra dependency: - -pip install 'crawlee[redis]' - -Additionally, Redis version 8.0 or higher is required. -::: - -:::note Redis persistence -Data persistence in Redis depends on your [database configuration](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/). -::: - -The client requires either a Redis connection string or a pre-configured Redis client instance. Use a pre-configured client when you need custom Redis settings such as connection pooling, timeouts, or SSL/TLS encryption. - - - {RedisStorageClientBasicExample} - - -Data is organized using Redis key patterns. Below are the main data structures used for each storage type: - -```mermaid ---- -config: - class: - hideEmptyMembersBox: true ---- - -classDiagram - -%% ======================== -%% Storage Client -%% ======================== - -class RedisDatasetClient { - <> -} - -%% ======================== -%% Dataset Keys -%% ======================== - -class DatasetKeys { - datasets:[name]:items - JSON Array - datasets:[name]:metadata - JSON Object -} - -class DatasetsIndexes { - datasets:id_to_name - Hash - datasets:name_to_id - Hash -} - -%% ======================== -%% Client to Keys arrows -%% ======================== - -RedisDatasetClient --> DatasetKeys -RedisDatasetClient --> DatasetsIndexes -``` - -```mermaid ---- -config: - class: - hideEmptyMembersBox: true ---- - -classDiagram - -%% ======================== -%% Storage Clients -%% ======================== - -class RedisKeyValueStoreClient { - <> -} - -%% ======================== -%% Key-Value Store Keys -%% ======================== - -class KeyValueStoreKeys { - key_value_stores:[name]:items - Hash - key_value_stores:[name]:metadata_items - Hash - key_value_stores:[name]:metadata - JSON Object -} - -class KeyValueStoresIndexes { - key_value_stores:id_to_name - Hash - key_value_stores:name_to_id - Hash -} - -%% ======================== -%% Client to Keys arrows -%% ======================== - -RedisKeyValueStoreClient --> KeyValueStoreKeys -RedisKeyValueStoreClient --> KeyValueStoresIndexes -``` - -```mermaid ---- -config: - class: - hideEmptyMembersBox: true ---- - -classDiagram - -%% ======================== -%% Storage Clients -%% ======================== - -class RedisRequestQueueClient { - <> -} - -%% ======================== -%% Request Queue Keys -%% ======================== - -class RequestQueueKeys{ - request_queues:[name]:queue - List - request_queues:[name]:data - Hash - request_queues:[name]:in_progress - Hash - request_queues:[name]:added_bloom_filter - Bloom Filter | bloom queue_dedup_strategy - request_queues:[name]:handled_bloom_filter - Bloom Filter | bloom queue_dedup_strategy - request_queues:[name]:pending_set - Set | default queue_dedup_strategy - request_queues:[name]:handled_set - Set | default queue_dedup_strategy - request_queues:[name]:metadata - JSON Object -} - -class RequestQueuesIndexes { - request_queues:id_to_name - Hash - request_queues:name_to_id - Hash -} - -%% ======================== -%% Client to Keys arrows -%% ======================== - -RedisRequestQueueClient --> RequestQueueKeys -RedisRequestQueueClient --> RequestQueuesIndexes -``` - -Configuration options for the `RedisStorageClient` can be set through environment variables or the `Configuration` class: - -- **`purge_on_start`** (env: `CRAWLEE_PURGE_ON_START`, default: `True`) - Whether to purge default storages on start. - -Configuration options for the `RedisStorageClient` can be set via constructor arguments: - -- **`connection_string`** - Redis connection string, e.g. `redis://localhost:6379/0`. -- **`redis`** - Pre-configured Redis client instance (optional). - - - {RedisStorageClientConfigurationExample} - - -## Creating a custom storage client - -A storage client consists of two parts: the storage client factory and individual storage type clients. The `StorageClient` acts as a factory that creates specific clients (`DatasetClient`, `KeyValueStoreClient`, `RequestQueueClient`) where the actual storage logic is implemented. - -Here is an example of a custom storage client that implements the `StorageClient` interface: - - - {CustomStorageClientExample} - - -Custom storage clients can implement any storage logic, such as connecting to a database, using a cloud storage service, or integrating with other systems. They must implement the required methods for creating, reading, updating, and deleting data in the respective storages. - -## Registering storage clients - -Storage clients can be registered in multiple ways: - -- **Globally** - Using the `ServiceLocator` or passing directly to the crawler. -- **Per storage** - When opening a specific storage instance like `Dataset`, `KeyValueStore`, or `RequestQueue`. - - - {RegisteringStorageClientsExample} - - -You can also register different storage clients for each storage instance, allowing you to use different backends for different storages. This is useful when you want to use a fast in-memory storage for `RequestQueue` while persisting scraping results in `Dataset` or `KeyValueStore`. - -## Conclusion - -Storage clients in Crawlee provide different backends for data storage. Use `MemoryStorageClient` for testing and fast operations without persistence, or `FileSystemStorageClient` for environments where data needs to persist. You can also create custom storage clients for specialized backends by implementing the `StorageClient` interface. - -If you have questions or need assistance, feel free to reach out on our [GitHub](https://github.com/apify/crawlee-python) or join our [Discord community](https://discord.com/invite/jyEM2PRvMU). Happy scraping! diff --git a/docs/guides/storages.mdx b/docs/guides/storages.mdx index 8506396f1b..7a997c6227 100644 --- a/docs/guides/storages.mdx +++ b/docs/guides/storages.mdx @@ -35,9 +35,9 @@ Crawlee offers several storage types for managing and persisting your crawling d Crawlee's storage system consists of two main layers: - **Storages** (`Dataset`, `KeyValueStore`, `RequestQueue`): High-level interfaces for interacting with different storage types. -- **Storage clients** (`MemoryStorageClient`, `FileSystemStorageClient`, etc.): Backend implementations that handle the actual data persistence and management. +- **Storage backends** (`MemoryStorageBackend`, `FileSystemStorageBackend`, etc.): Backend implementations that handle the actual data persistence and management. -For more information about storage clients and their configuration, see the [Storage clients guide](./storage-clients). +For more information about storage backends and their configuration, see the [Storage backends guide](./storage-backends). ```mermaid --- @@ -206,7 +206,7 @@ Crawlee provides the following helper function to simplify interactions with the ## Cleaning up the storages -By default, Crawlee cleans up all unnamed storages (including the default one) once per process, when each one is first opened, so every process begins with a clean state. This behavior is controlled by `Configuration.purge_on_start` (default: True). In contrast, named storages are never purged automatically and persist across runs. The exact behavior may vary depending on the storage client implementation. +By default, Crawlee cleans up all unnamed storages (including the default one) once per process, when each one is first opened, so every process begins with a clean state. This behavior is controlled by `Configuration.purge_on_start` (default: True). In contrast, named storages are never purged automatically and persist across runs. The exact behavior may vary depending on the storage backend implementation. ### When purging happens @@ -232,7 +232,7 @@ Purge on start behavior just triggers the storage's `purge` method, which remove {CleaningPurgeExplicitlyExample} -Note that purging behavior may vary between storage client implementations. For more details on storage configuration and client implementations, see the [Storage clients guide](./storage-clients). +Note that purging behavior may vary between storage backend implementations. For more details on storage configuration and backend implementations, see the [Storage backends guide](./storage-backends). ## Conclusion diff --git a/docs/upgrading/upgrading_to_v2.md b/docs/upgrading/upgrading_to_v2.md index 007ec49986..bbb08d13a8 100644 --- a/docs/upgrading/upgrading_to_v2.md +++ b/docs/upgrading/upgrading_to_v2.md @@ -9,6 +9,35 @@ This page summarizes the breaking changes between Crawlee for Python v1 and v2. We drop support for Python 3.10, which reached its end of life in October 2026. The minimum supported version is now Python 3.11. +## Storage clients are renamed to storage backends + +The pluggable storage implementations are called storage backends, and the `crawlee.storage_clients` package is now `crawlee.storage_backends`. The v1 names don't exist in v2, so update your imports and arguments: + +- `StorageClient`, `MemoryStorageClient`, `FileSystemStorageClient`, `SqlStorageClient` and `RedisStorageClient` are now `StorageBackend`, `MemoryStorageBackend`, `FileSystemStorageBackend`, `SqlStorageBackend` and `RedisStorageBackend`. +- `DatasetClient`, `KeyValueStoreClient` and `RequestQueueClient` are now `DatasetBackend`, `KeyValueStoreBackend` and `RequestQueueBackend`. Their built-in implementations follow the same pattern, for example `FileSystemDatasetClient` is now `FileSystemDatasetBackend`. +- The `storage_client` argument is now `storage_backend`. That covers `BasicCrawler`, `ServiceLocator`, `Dataset.open`, `KeyValueStore.open` and `RequestQueue.open`. The `to_kvs_storage_client` argument of `Dataset.export_to` is now `to_kvs_storage_backend`. A custom request manager opener passed to `ThrottlingRequestManager` must accept `storage_backend`. +- `ServiceLocator.get_storage_client` and `ServiceLocator.set_storage_client` are now `get_storage_backend` and `set_storage_backend`. + +For example: + +```python +# v1 +from crawlee import service_locator +from crawlee.storage_clients import MemoryStorageClient + +service_locator.set_storage_client(MemoryStorageClient()) +crawler = ParselCrawler(storage_client=MemoryStorageClient()) + +# v2 +from crawlee import service_locator +from crawlee.storage_backends import MemoryStorageBackend + +service_locator.set_storage_backend(MemoryStorageBackend()) +crawler = ParselCrawler(storage_backend=MemoryStorageBackend()) +``` + +A custom storage backend has more names to update. The `StorageBackend` methods `create_dataset_client`, `create_kvs_client`, `create_rq_client` and `get_storage_client_cache_key` are now `create_dataset_backend`, `create_kvs_backend`, `create_rq_backend` and `get_storage_backend_cache_key`. The `client` argument of the `Dataset`, `KeyValueStore` and `RequestQueue` constructors is now `backend`. + ## `Event` is a `StrEnum` `Event` now subclasses `enum.StrEnum`. Calling `str()` on a member or formatting it in an f-string returns its value, for example `'systemInfo'`, where v1 returned `'Event.SYSTEM_INFO'`. Comparisons with plain strings work the same way as in v1. @@ -21,9 +50,9 @@ We drop support for Python 3.10, which reached its end of life in October 2026. `StatisticsState.crawler_runtime_for_serialization` is removed, and the persisted `crawlerRuntimeMillis` value is a number of milliseconds instead of an ISO 8601 duration string. Read the runtime from the `crawler_runtime` property instead. Statistics persisted by v1 are still restored. -## `RequestQueueClient.is_finished` is abstract +## `RequestQueueBackend.is_finished` is abstract -`RequestQueueClient.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. A subclass that doesn't implement it raises `TypeError` on instantiation. If you have a custom request queue client, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in clients already implement it. +`RequestQueueBackend.is_finished` is now an abstract method. In v1, it fell back to `is_empty`, which ignores requests still being processed and could report the queue as finished too early. A subclass that doesn't implement it raises `TypeError` on instantiation. If you have a custom request queue backend, implement `is_finished` so that it returns `True` only when the queue is empty and no requests are in progress. The built-in backends already implement it. ## Keyword-only arguments @@ -96,7 +125,7 @@ The `Request.forefront` property is removed. Proxy tiers were the only code that ## Storage data uses camelCase keys -The storage clients now write their JSON data with camelCase keys, the same as Crawlee JS. This covers the metadata, request and record metadata files of `FileSystemStorageClient`, the request and record metadata JSON stored by `RedisStorageClient`, and the request JSON stored by `SqlStorageClient`. For example, `item_count` in a dataset's `__metadata__.json` becomes `itemCount`. The storage metadata documents of `RedisStorageClient` keep their snake_case keys. +The storage backends now write their JSON data with camelCase keys, the same as Crawlee JS. This covers the metadata, request and record metadata files of `FileSystemStorageBackend`, the request and record metadata JSON stored by `RedisStorageBackend`, and the request JSON stored by `SqlStorageBackend`. For example, `item_count` in a dataset's `__metadata__.json` becomes `itemCount`. The storage metadata documents of `RedisStorageBackend` keep their snake_case keys. Data written by v1 still loads, so an existing storage directory, Redis instance or database keeps working. Any tooling that reads these files directly needs to handle the camelCase keys. @@ -201,9 +230,9 @@ for urls in batches: `Configuration.purge_on_start` isn't affected. Unnamed storages are still purged once per process, when they're first opened. -## `RequestQueue.purge` raises when the storage client can't purge +## `RequestQueue.purge` raises when the storage backend can't purge -In v1, `RequestQueue.purge()` worked around a storage client that didn't support purging. It dropped an unnamed queue and opened the default queue in its place, and it skipped a named queue with a warning. In v2, it raises the client's `NotImplementedError`, the same as Crawlee JS. The Apify platform's storage client doesn't support purging, so on the platform, call `drop()` or open a new queue instead. +In v1, `RequestQueue.purge()` worked around a storage backend that didn't support purging. It dropped an unnamed queue and opened the default queue in its place, and it skipped a named queue with a warning. In v2, it raises the backend's `NotImplementedError`, the same as Crawlee JS. The Apify platform's storage backend doesn't support purging, so on the platform, call `drop()` or open a new queue instead. ## `check_readiness` replaces `is_empty` and `is_finished` @@ -216,7 +245,7 @@ In v1, `RequestQueue.purge()` worked around a storage client that didn't support Each of these classes also has a `status` attribute set to `'ready'`, `'waiting'`, `'stalled'` or `'finished'`, so a check doesn't need an import. -The change covers `RequestQueue`, `RequestList`, `SitemapRequestLoader`, `RequestManagerTandem` and `ThrottlingRequestManager`. `ThrottlingRequestManager.get_stall_reason` is removed too, since a stall is now reported as `StalledStatus`. Unlike `get_stall_reason`, the manager reports a stall only while no request outside the stalled domains is in progress or still being added, since such a request may still add work. Storage clients keep `is_empty` and `is_finished`. +The change covers `RequestQueue`, `RequestList`, `SitemapRequestLoader`, `RequestManagerTandem` and `ThrottlingRequestManager`. `ThrottlingRequestManager.get_stall_reason` is removed too, since a stall is now reported as `StalledStatus`. Unlike `get_stall_reason`, the manager reports a stall only while no request outside the stalled domains is in progress or still being added, since such a request may still add work. Storage backends keep `is_empty` and `is_finished`. ```python # v1 @@ -271,7 +300,9 @@ The crawler logs the first message once per run, the next time it looks for a ne When a crawl processes no requests because its request manager holds only handled ones, the crawler logs a new warning. It starts with `This crawl processed no requests` and explains how to crawl the requests again. -`RequestQueue.purge()` no longer logs the `Storage client "..." does not support purging the request queue.` warnings. It raises `NotImplementedError` instead. For details, see [`RequestQueue.purge` raises when the storage client can't purge](#requestqueuepurge-raises-when-the-storage-client-cant-purge). +Messages that mentioned a storage client name the storage backend instead. The DEBUG message about the implicit default backend now reads `No storage backend set, implicitly creating and using default FileSystemStorageBackend.` The warning about unsupported arguments of a dataset backend ends with the backend class name, for example `... of get_data are not supported by MemoryDatasetBackend.` + +`RequestQueue.purge()` no longer logs the `Storage client "..." does not support purging the request queue.` warnings. It raises `NotImplementedError` instead. For details, see [`RequestQueue.purge` raises when the storage backend can't purge](#requestqueuepurge-raises-when-the-storage-backend-cant-purge). The ERROR message the crawler logs before raising `PersistentRateLimitError` now reads "Giving up the crawl because the request manager can't make progress." instead of "Giving up the crawl because a domain keeps rate-limiting every request." diff --git a/pyproject.toml b/pyproject.toml index ec31ad7b97..a104619c73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -288,7 +288,7 @@ python-version = "3.11" include = ["src", "tests", "scripts", "docs", "website"] exclude = [ "src/crawlee/project_template", - "docs/guides/code_examples/storage_clients/custom_storage_client_example.py", + "docs/guides/code_examples/storage_backends/custom_storage_backend_example.py", "website/versioned_docs", ] diff --git a/src/crawlee/_consts.py b/src/crawlee/_consts.py index 9345e53e98..2e51bace5f 100644 --- a/src/crawlee/_consts.py +++ b/src/crawlee/_consts.py @@ -1,4 +1,4 @@ from __future__ import annotations METADATA_FILENAME = '__metadata__.json' -"""The name of the metadata file for storage clients.""" +"""The name of the metadata file for storage backends.""" diff --git a/src/crawlee/_service_locator.py b/src/crawlee/_service_locator.py index c17a3a95c7..babfd73f31 100644 --- a/src/crawlee/_service_locator.py +++ b/src/crawlee/_service_locator.py @@ -6,7 +6,7 @@ from crawlee.configuration import Configuration from crawlee.errors import ServiceConflictError from crawlee.events import EventManager, LocalEventManager -from crawlee.storage_clients import FileSystemStorageClient, StorageClient +from crawlee.storage_backends import FileSystemStorageBackend, StorageBackend if TYPE_CHECKING: from crawlee.storages._storage_instance_manager import StorageInstanceManager @@ -29,11 +29,11 @@ def __init__( self, configuration: Configuration | None = None, event_manager: EventManager | None = None, - storage_client: StorageClient | None = None, + storage_backend: StorageBackend | None = None, ) -> None: self._configuration = configuration self._event_manager = event_manager - self._storage_client = storage_client + self._storage_backend = storage_backend def get_configuration(self) -> Configuration: """Get the configuration.""" @@ -90,35 +90,35 @@ def set_event_manager(self, event_manager: EventManager) -> None: self._event_manager = event_manager - def get_storage_client(self) -> StorageClient: - """Get the storage client.""" - if self._storage_client is None: - logger.debug('No storage client set, implicitly creating and using default FileSystemStorageClient.') + def get_storage_backend(self) -> StorageBackend: + """Get the storage backend.""" + if self._storage_backend is None: + logger.debug('No storage backend set, implicitly creating and using default FileSystemStorageBackend.') if self._configuration is None: logger.warning( - 'Implicit creation of storage client will implicitly set configuration as side effect. ' + 'Implicit creation of storage backend will implicitly set configuration as side effect. ' 'It is advised to explicitly first set the configuration instead.' ) - self._storage_client = FileSystemStorageClient() + self._storage_backend = FileSystemStorageBackend() - return self._storage_client + return self._storage_backend - def set_storage_client(self, storage_client: StorageClient) -> None: - """Set the storage client. + def set_storage_backend(self, storage_backend: StorageBackend) -> None: + """Set the storage backend. Args: - storage_client: The storage client to set. + storage_backend: The storage backend to set. Raises: - ServiceConflictError: If the storage client has already been retrieved before. + ServiceConflictError: If the storage backend has already been retrieved before. """ - if self._storage_client is storage_client: + if self._storage_backend is storage_backend: # Same instance, no need to anything return - if self._storage_client: - raise ServiceConflictError(StorageClient, storage_client, self._storage_client) + if self._storage_backend: + raise ServiceConflictError(StorageBackend, storage_backend, self._storage_backend) - self._storage_client = storage_client + self._storage_backend = storage_backend @property def storage_instance_manager(self) -> StorageInstanceManager: diff --git a/src/crawlee/_types.py b/src/crawlee/_types.py index b973e6b2e2..93b7380e00 100644 --- a/src/crawlee/_types.py +++ b/src/crawlee/_types.py @@ -23,7 +23,7 @@ from crawlee.http_clients import HttpResponse from crawlee.proxy_configuration import ProxyInfo from crawlee.sessions import Session - from crawlee.storage_clients import StorageClient + from crawlee.storage_backends import StorageBackend from crawlee.storages import KeyValueStore JsonSerializable = Mapping[str, 'JsonSerializable'] | Sequence['JsonSerializable'] | str | int | float | bool | None @@ -750,8 +750,8 @@ class ExportToKwargs(TypedDict): to_kvs_name: NotRequired[str] """Name of the key-value store to save the exported file.""" - to_kvs_storage_client: NotRequired[StorageClient] - """The storage client to use for saving the exported file.""" + to_kvs_storage_backend: NotRequired[StorageBackend] + """The storage backend to use for saving the exported file.""" to_kvs_configuration: NotRequired[Configuration] """The configuration to use for saving the exported file.""" diff --git a/src/crawlee/_utils/docs.py b/src/crawlee/_utils/docs.py index abfbbf140b..9882d5d39e 100644 --- a/src/crawlee/_utils/docs.py +++ b/src/crawlee/_utils/docs.py @@ -19,7 +19,7 @@ 'Request loaders', 'Session management', 'Statistics', - 'Storage clients', + 'Storage backends', 'Storage data', 'Storages', 'Other', diff --git a/src/crawlee/_utils/file.py b/src/crawlee/_utils/file.py index 6c1843fc56..45d98927ce 100644 --- a/src/crawlee/_utils/file.py +++ b/src/crawlee/_utils/file.py @@ -246,7 +246,7 @@ async def export_csv_to_stream( 'CSV export dropped %d key(s) not present in the first item: %s. ' 'Pass collect_all_keys=True to include keys from all items as columns.', len(dropped_keys), - # The keys are annotated as `str`, but nothing enforces that at runtime, and a storage client that + # The keys are annotated as `str`, but nothing enforces that at runtime, and a storage backend that # does not round-trip items through JSON hands them over as pushed. Stringify them so building the # message cannot fail on a key that is not a string, or on a set mixing several key types. ', '.join(sorted(str(key) for key in dropped_keys)), diff --git a/src/crawlee/_utils/recoverable_state.py b/src/crawlee/_utils/recoverable_state.py index 5d33ccc1b5..69dab3f088 100644 --- a/src/crawlee/_utils/recoverable_state.py +++ b/src/crawlee/_utils/recoverable_state.py @@ -66,8 +66,8 @@ def __init__( if not persist_state_kvs_factory: logger.debug( 'No explicit key_value_store set for recoverable state. Recovery will use a system-wide KeyValueStore ' - 'based on service_locator configuration, potentially calling service_locator.set_storage_client in the ' - 'process. It is recommended to initialize RecoverableState with explicit key_value_store to avoid ' + 'based on service_locator configuration, potentially calling service_locator.set_storage_backend in ' + 'the process. It is recommended to initialize RecoverableState with explicit key_value_store to avoid ' 'global side effects.' ) diff --git a/src/crawlee/autoscaling/_snapshotter.py b/src/crawlee/autoscaling/_snapshotter.py index be756d1e11..b5176a9887 100644 --- a/src/crawlee/autoscaling/_snapshotter.py +++ b/src/crawlee/autoscaling/_snapshotter.py @@ -387,9 +387,9 @@ async def _snapshot_client(self) -> None: Must be `async` to ensure it is not scheduled to be run in own thread by the event manager, which could cause race conditions in snapshots manipulation(sorting and pruning). """ - client = service_locator.get_storage_client() + storage_backend = service_locator.get_storage_backend() - rate_limit_errors: dict[int, int] = client.get_rate_limit_errors() + rate_limit_errors: dict[int, int] = storage_backend.get_rate_limit_errors() error_count = rate_limit_errors.get(self._CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT, 0) previous_error_count = self._client_snapshots[-1].error_count if self._client_snapshots else 0 diff --git a/src/crawlee/configuration.py b/src/crawlee/configuration.py index 51c5e4f5ba..0b3180eadb 100644 --- a/src/crawlee/configuration.py +++ b/src/crawlee/configuration.py @@ -85,7 +85,7 @@ class Configuration(BaseSettings): ) ), ] = True - """Whether to purge the storage on the start. This option is utilized by the storage clients.""" + """Whether to purge the storage on the start. This option is utilized by the storage backends.""" persist_state_interval: Annotated[ timedelta_ms, @@ -199,7 +199,7 @@ class Configuration(BaseSettings): ), ), ] = './storage' - """The path to the storage directory. This option is utilized by the storage clients.""" + """The path to the storage directory. This option is utilized by the storage backends.""" headless: Annotated[ bool, diff --git a/src/crawlee/crawlers/_basic/_basic_crawler.py b/src/crawlee/crawlers/_basic/_basic_crawler.py index aeb8b9e158..e71c0e8036 100644 --- a/src/crawlee/crawlers/_basic/_basic_crawler.py +++ b/src/crawlee/crawlers/_basic/_basic_crawler.py @@ -101,8 +101,8 @@ from crawlee.request_loaders import RequestManager from crawlee.sessions import Session from crawlee.statistics import FinalStatistics - from crawlee.storage_clients import StorageClient - from crawlee.storage_clients.models import DatasetItemsListPage + from crawlee.storage_backends import StorageBackend + from crawlee.storage_backends.models import DatasetItemsListPage TCrawlingContext = TypeVar('TCrawlingContext', bound=BasicCrawlingContext, default=BasicCrawlingContext) TStatisticsState = TypeVar('TStatisticsState', bound=StatisticsState, default=StatisticsState) @@ -124,8 +124,8 @@ class _BasicCrawlerOptions(TypedDict): event_manager: NotRequired[EventManager] """The event manager for managing events for the crawler and all its components.""" - storage_client: NotRequired[StorageClient] - """The storage client for managing storages for the crawler and all its components.""" + storage_backend: NotRequired[StorageBackend] + """The storage backend for managing storages for the crawler and all its components.""" request_manager: NotRequired[RequestManager] """Manager of requests that should be processed by the crawler.""" @@ -288,7 +288,7 @@ def __init__( *, configuration: Configuration | None = None, event_manager: EventManager | None = None, - storage_client: StorageClient | None = None, + storage_backend: StorageBackend | None = None, request_manager: RequestManager | None = None, session_pool: SessionPool | None = None, proxy_configuration: ProxyConfiguration | None = None, @@ -324,7 +324,7 @@ def __init__( Args: configuration: The `Configuration` instance. Some of its properties are used as defaults for the crawler. event_manager: The event manager for managing events for the crawler and all its components. - storage_client: The storage client for managing storages for the crawler and all its components. + storage_backend: The storage backend for managing storages for the crawler and all its components. request_manager: Manager of requests that should be processed by the crawler. session_pool: A custom `SessionPool` instance, allowing the use of non-default configuration. proxy_configuration: HTTP proxy configuration used when making requests. @@ -393,14 +393,14 @@ def __init__( elif not event_manager: implicit_event_manager_with_explicit_config = True - if not storage_client: - storage_client = service_locator.get_storage_client() + if not storage_backend: + storage_backend = service_locator.get_storage_backend() if not event_manager: event_manager = service_locator.get_event_manager() self._service_locator = ServiceLocator( - configuration=configuration, storage_client=storage_client, event_manager=event_manager + configuration=configuration, storage_backend=storage_backend, event_manager=event_manager ) config = self._service_locator.get_configuration() @@ -651,7 +651,7 @@ async def get_request_manager(self) -> RequestManager: if not self._request_manager: self._request_manager = await RequestQueue.open( alias=self._default_rq_alias, - storage_client=self._service_locator.get_storage_client(), + storage_backend=self._service_locator.get_storage_backend(), configuration=self._service_locator.get_configuration(), ) @@ -669,7 +669,7 @@ async def get_dataset( id=id, name=name, alias=alias, - storage_client=self._service_locator.get_storage_client(), + storage_backend=self._service_locator.get_storage_backend(), configuration=self._service_locator.get_configuration(), ) @@ -685,7 +685,7 @@ async def get_key_value_store( id=id, name=name, alias=alias, - storage_client=self._service_locator.get_storage_client(), + storage_backend=self._service_locator.get_storage_backend(), configuration=self._service_locator.get_configuration(), ) @@ -989,7 +989,7 @@ async def get_data( id=dataset_id, name=dataset_name, alias=dataset_alias, - storage_client=self._service_locator.get_storage_client(), + storage_backend=self._service_locator.get_storage_backend(), configuration=self._service_locator.get_configuration(), ) return await dataset.get_data(**kwargs) @@ -1024,7 +1024,7 @@ async def export_data( id=dataset_id, name=dataset_name, alias=dataset_alias, - storage_client=self._service_locator.get_storage_client(), + storage_backend=self._service_locator.get_storage_backend(), configuration=self._service_locator.get_configuration(), ) @@ -1452,7 +1452,7 @@ async def _add_requests( id=rq_id, name=rq_name, alias=rq_alias, - storage_client=self._service_locator.get_storage_client(), + storage_backend=self._service_locator.get_storage_backend(), configuration=self._service_locator.get_configuration(), ) else: diff --git a/src/crawlee/request_loaders/_request_loader.py b/src/crawlee/request_loaders/_request_loader.py index a0cac2fbc2..747610bdf5 100644 --- a/src/crawlee/request_loaders/_request_loader.py +++ b/src/crawlee/request_loaders/_request_loader.py @@ -10,7 +10,7 @@ from collections.abc import Sequence from crawlee.request_loaders import RequestManager, RequestManagerTandem - from crawlee.storage_clients.models import ProcessedRequest + from crawlee.storage_backends.models import ProcessedRequest from ._types import RequestSourceStatus diff --git a/src/crawlee/request_loaders/_request_manager.py b/src/crawlee/request_loaders/_request_manager.py index 8ecb181a37..0574ee8263 100644 --- a/src/crawlee/request_loaders/_request_manager.py +++ b/src/crawlee/request_loaders/_request_manager.py @@ -6,7 +6,7 @@ from crawlee._utils.docs import docs_group from crawlee.request_loaders._request_loader import RequestLoader -from crawlee.storage_clients.models import ProcessedRequest +from crawlee.storage_backends.models import ProcessedRequest if TYPE_CHECKING: from collections.abc import Sequence diff --git a/src/crawlee/request_loaders/_request_manager_tandem.py b/src/crawlee/request_loaders/_request_manager_tandem.py index 90d55f94c4..fb79f55374 100644 --- a/src/crawlee/request_loaders/_request_manager_tandem.py +++ b/src/crawlee/request_loaders/_request_manager_tandem.py @@ -16,7 +16,7 @@ from crawlee import Request from crawlee.request_loaders import RequestLoader - from crawlee.storage_clients.models import ProcessedRequest + from crawlee.storage_backends.models import ProcessedRequest from ._types import RequestSourceStatus diff --git a/src/crawlee/request_loaders/_sitemap_request_loader.py b/src/crawlee/request_loaders/_sitemap_request_loader.py index 4b4a47c3b3..9f51c458ca 100644 --- a/src/crawlee/request_loaders/_sitemap_request_loader.py +++ b/src/crawlee/request_loaders/_sitemap_request_loader.py @@ -37,7 +37,7 @@ from crawlee._types import EnqueueStrategy from crawlee.http_clients import HttpClient from crawlee.proxy_configuration import ProxyInfo - from crawlee.storage_clients.models import ProcessedRequest + from crawlee.storage_backends.models import ProcessedRequest from ._types import RequestLoaderStatus diff --git a/src/crawlee/request_loaders/_throttling_request_manager.py b/src/crawlee/request_loaders/_throttling_request_manager.py index 9d202a7999..750d11e56b 100644 --- a/src/crawlee/request_loaders/_throttling_request_manager.py +++ b/src/crawlee/request_loaders/_throttling_request_manager.py @@ -23,8 +23,8 @@ from crawlee._request import Request from crawlee.configuration import Configuration - from crawlee.storage_clients import StorageClient - from crawlee.storage_clients.models import ProcessedRequest + from crawlee.storage_backends import StorageBackend + from crawlee.storage_backends.models import ProcessedRequest from ._types import RequestSourceStatus @@ -67,7 +67,7 @@ class ThrottlingRequestManager(RequestManager, Generic[TRequestManager]): The class is generic over the wrapped manager type. The first asynchronous operation opens one sub-manager per configured domain through `request_manager_opener`, so all of them share the subclass and backing store of `inner`; - the synchronous delay methods never open anything. The opener must accept `alias`, `storage_client`, and + the synchronous delay methods never open anything. The opener must accept `alias`, `storage_backend`, and `configuration` keyword arguments (as `RequestQueue.open` does) and return the same concrete subclass as `inner`. Requests a previous run left in a persistent store become visible again at open. The default `purge_on_start=True` @@ -114,7 +114,7 @@ def __init__( and a trailing root dot are all normalized away. Subdomain wildcards such as `*.example.com` are not supported — list each subdomain explicitly if needed. request_manager_opener: Async callable used to open one sub-manager per configured domain on first use. - Must accept `alias`, `storage_client`, and `configuration` keyword arguments and return the same + Must accept `alias`, `storage_backend`, and `configuration` keyword arguments and return the same concrete subclass as `inner` (e.g. `RequestQueue.open` when `inner` is a `RequestQueue`). service_locator: Service locator for creating sub-managers. If not provided, defaults to the global service locator, ensuring consistency with the crawler's storage backend. @@ -543,7 +543,7 @@ async def _open_sub_manager(self, domain: str) -> None: """Open the sub-manager for a single domain using the configured `request_manager_opener`.""" self._sub_managers[domain] = await self._request_manager_opener( alias=f'throttled-{domain}', - storage_client=self._service_locator.get_storage_client(), + storage_backend=self._service_locator.get_storage_backend(), configuration=self._service_locator.get_configuration(), ) @@ -587,7 +587,7 @@ async def _migrate_to_sub_manager(self, request: Request) -> None: """Move a request fetched from `inner` into its domain's sub-manager.""" sub_manager = self._sub_managers[self._extract_domain(request.url)] try: - # A copy, because some storage clients keep the added object and `inner` marks this one handled below. + # A copy, because some storage backends keep the added object and `inner` marks this one handled below. processed = await sub_manager.add_request(request.model_copy(deep=True)) # Includes cancellation, which would otherwise leave the request in progress in `inner`. except BaseException: @@ -651,7 +651,7 @@ async def __call__( self, *, alias: str | None = ..., - storage_client: StorageClient | None = ..., + storage_backend: StorageBackend | None = ..., configuration: Configuration | None = ..., ) -> TRequestManager_co: ... diff --git a/src/crawlee/storage_backends/__init__.py b/src/crawlee/storage_backends/__init__.py new file mode 100644 index 0000000000..cd20e769a0 --- /dev/null +++ b/src/crawlee/storage_backends/__init__.py @@ -0,0 +1,25 @@ +from crawlee._utils.try_import import install_import_hook as _install_import_hook +from crawlee._utils.try_import import try_import as _try_import + +# These imports have only mandatory dependencies, so they are imported directly. +from ._base import StorageBackend +from ._file_system import FileSystemStorageBackend +from ._memory import MemoryStorageBackend + +_install_import_hook(__name__) + +# The following imports are wrapped in try_import to handle optional dependencies, +# ensuring the module can still function even if these dependencies are missing. +with _try_import(__name__, 'SqlStorageBackend', extra_name=['sql_sqlite', 'sql_postgres', 'sql_mysql']): + from ._sql import SqlStorageBackend + +with _try_import(__name__, 'RedisStorageBackend', extra_name='redis'): + from ._redis import RedisStorageBackend + +__all__ = [ + 'FileSystemStorageBackend', + 'MemoryStorageBackend', + 'RedisStorageBackend', + 'SqlStorageBackend', + 'StorageBackend', +] diff --git a/src/crawlee/storage_backends/_base/__init__.py b/src/crawlee/storage_backends/_base/__init__.py new file mode 100644 index 0000000000..b314917304 --- /dev/null +++ b/src/crawlee/storage_backends/_base/__init__.py @@ -0,0 +1,11 @@ +from ._dataset_backend import DatasetBackend +from ._key_value_store_backend import KeyValueStoreBackend +from ._request_queue_backend import RequestQueueBackend +from ._storage_backend import StorageBackend + +__all__ = [ + 'DatasetBackend', + 'KeyValueStoreBackend', + 'RequestQueueBackend', + 'StorageBackend', +] diff --git a/src/crawlee/storage_clients/_base/_dataset_client.py b/src/crawlee/storage_backends/_base/_dataset_backend.py similarity index 89% rename from src/crawlee/storage_clients/_base/_dataset_client.py rename to src/crawlee/storage_backends/_base/_dataset_backend.py index bd13144e52..f086644b16 100644 --- a/src/crawlee/storage_clients/_base/_dataset_client.py +++ b/src/crawlee/storage_backends/_base/_dataset_backend.py @@ -8,20 +8,20 @@ from collections.abc import AsyncIterator, Mapping from crawlee._types import JsonSerializable - from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata + from crawlee.storage_backends.models import DatasetItemsListPage, DatasetMetadata -class DatasetClient(ABC): - """An abstract class for dataset storage clients. +class DatasetBackend(ABC): + """An abstract class for dataset storage backends. - Dataset clients provide an interface for accessing and manipulating dataset storage. They handle + Dataset backends provide an interface for accessing and manipulating dataset storage. They handle operations like adding and getting dataset items across different storage backends. - Storage clients are specific to the type of storage they manage (`Dataset`, `KeyValueStore`, + These backends are specific to the type of storage they manage (`Dataset`, `KeyValueStore`, `RequestQueue`), and can operate with various storage systems including memory, file system, databases, and cloud storage solutions. - This abstract class defines the interface that all specific dataset clients must implement. + This abstract class defines the interface that all specific dataset backends must implement. """ @abstractmethod diff --git a/src/crawlee/storage_clients/_base/_key_value_store_client.py b/src/crawlee/storage_backends/_base/_key_value_store_backend.py similarity index 90% rename from src/crawlee/storage_clients/_base/_key_value_store_client.py rename to src/crawlee/storage_backends/_base/_key_value_store_backend.py index 53d49059eb..acd38dca6b 100644 --- a/src/crawlee/storage_clients/_base/_key_value_store_client.py +++ b/src/crawlee/storage_backends/_base/_key_value_store_backend.py @@ -3,25 +3,25 @@ from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any -from crawlee.storage_clients.models import KeyValueStoreRecordMetadata +from crawlee.storage_backends.models import KeyValueStoreRecordMetadata if TYPE_CHECKING: from collections.abc import AsyncIterator - from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord + from crawlee.storage_backends.models import KeyValueStoreMetadata, KeyValueStoreRecord -class KeyValueStoreClient(ABC): - """An abstract class for key-value store (KVS) storage clients. +class KeyValueStoreBackend(ABC): + """An abstract class for key-value store (KVS) storage backends. - Key-value stores clients provide an interface for accessing and manipulating KVS storage. They handle + Key-value store backends provide an interface for accessing and manipulating KVS storage. They handle operations like getting, setting, deleting KVS values across different storage backends. - Storage clients are specific to the type of storage they manage (`Dataset`, `KeyValueStore`, + These backends are specific to the type of storage they manage (`Dataset`, `KeyValueStore`, `RequestQueue`), and can operate with various storage systems including memory, file system, databases, and cloud storage solutions. - This abstract class defines the interface that all specific KVS clients must implement. + This abstract class defines the interface that all specific KVS backends must implement. """ @abstractmethod diff --git a/src/crawlee/storage_clients/_base/_request_queue_client.py b/src/crawlee/storage_backends/_base/_request_queue_backend.py similarity index 92% rename from src/crawlee/storage_clients/_base/_request_queue_client.py rename to src/crawlee/storage_backends/_base/_request_queue_backend.py index e9efda0d9c..bf50f88234 100644 --- a/src/crawlee/storage_clients/_base/_request_queue_client.py +++ b/src/crawlee/storage_backends/_base/_request_queue_backend.py @@ -7,14 +7,14 @@ from collections.abc import Sequence from crawlee import Request - from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata + from crawlee.storage_backends.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata -class RequestQueueClient(ABC): - """An abstract class for request queue resource clients. +class RequestQueueBackend(ABC): + """An abstract class for request queue backends. - These clients are specific to the type of resource they manage and operate under a designated storage - client, like a memory storage client. + These backends are specific to the type of resource they manage and operate under a designated storage + backend, like a memory storage backend. """ @abstractmethod diff --git a/src/crawlee/storage_clients/_base/_storage_client.py b/src/crawlee/storage_backends/_base/_storage_backend.py similarity index 55% rename from src/crawlee/storage_clients/_base/_storage_client.py rename to src/crawlee/storage_backends/_base/_storage_backend.py index 049e1342c3..6df8d09ac1 100644 --- a/src/crawlee/storage_clients/_base/_storage_client.py +++ b/src/crawlee/storage_backends/_base/_storage_backend.py @@ -10,86 +10,86 @@ from crawlee.configuration import Configuration - from ._dataset_client import DatasetClient - from ._key_value_store_client import KeyValueStoreClient - from ._request_queue_client import RequestQueueClient + from ._dataset_backend import DatasetBackend + from ._key_value_store_backend import KeyValueStoreBackend + from ._request_queue_backend import RequestQueueBackend -@docs_group('Storage clients') -class StorageClient(ABC): - """Base class for storage clients. +@docs_group('Storage backends') +class StorageBackend(ABC): + """Base class for storage backends. - The `StorageClient` serves as an abstract base class that defines the interface for accessing Crawlee's - storage types: datasets, key-value stores, and request queues. It provides methods to open clients for + The `StorageBackend` serves as an abstract base class that defines the interface for accessing Crawlee's + storage types: datasets, key-value stores, and request queues. It provides methods to open backends for each of these storage types and handles common functionality. - Storage clients implementations can be provided for various backends (file system, memory, databases, + Storage backend implementations can be provided for various systems (file system, memory, databases, various cloud providers, etc.) to support different use cases from development to production environments. - Each storage client implementation is responsible for ensuring proper initialization, data persistence + Each storage backend implementation is responsible for ensuring proper initialization, data persistence (where applicable), and consistent access patterns across all storage types it supports. """ - def get_storage_client_cache_key(self, configuration: Configuration) -> Hashable: # noqa: ARG002 - """Return a cache key that can differentiate between different storages of this and other clients. + def get_storage_backend_cache_key(self, configuration: Configuration) -> Hashable: # noqa: ARG002 + """Return a cache key that can differentiate between different storages of this and other backends. - Can be based on configuration or on the client itself. By default, returns a module and name of the client + Can be based on configuration or on the backend itself. By default, returns a module and name of the backend class. """ return f'{self.__class__.__module__}.{self.__class__.__name__}' @abstractmethod - async def create_dataset_client( + async def create_dataset_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> DatasetClient: - """Create a dataset client.""" + ) -> DatasetBackend: + """Create a dataset backend.""" @abstractmethod - async def create_kvs_client( + async def create_kvs_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> KeyValueStoreClient: - """Create a key-value store client.""" + ) -> KeyValueStoreBackend: + """Create a key-value store backend.""" @abstractmethod - async def create_rq_client( + async def create_rq_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> RequestQueueClient: - """Create a request queue client.""" + ) -> RequestQueueBackend: + """Create a request queue backend.""" def get_rate_limit_errors(self) -> dict[int, int]: - """Return statistics about rate limit errors encountered by the HTTP client in storage client.""" + """Return statistics about rate limit errors encountered by the HTTP client in storage backend.""" return {} async def _purge_if_needed( self, - client: DatasetClient | KeyValueStoreClient | RequestQueueClient, + backend: DatasetBackend | KeyValueStoreBackend | RequestQueueBackend, configuration: Configuration, ) -> None: - """Purge the client if needed. + """Purge the storage behind the backend if needed. - The purge is only performed if the configuration indicates that it should be done and the client + The purge is only performed if the configuration indicates that it should be done and the storage is not a named storage. Named storages are considered global and will typically outlive the run, so they are not purged. Args: - client: The storage client to potentially purge. + backend: The dataset, key-value store, or request queue backend to potentially purge. configuration: Configuration that determines whether purging should occur. """ - metadata = await client.get_metadata() + metadata = await backend.get_metadata() if configuration.purge_on_start and metadata.name is None: - await client.purge() + await backend.purge() diff --git a/src/crawlee/storage_clients/_base/py.typed b/src/crawlee/storage_backends/_base/py.typed similarity index 100% rename from src/crawlee/storage_clients/_base/py.typed rename to src/crawlee/storage_backends/_base/py.typed diff --git a/src/crawlee/storage_backends/_file_system/__init__.py b/src/crawlee/storage_backends/_file_system/__init__.py new file mode 100644 index 0000000000..9cd363992c --- /dev/null +++ b/src/crawlee/storage_backends/_file_system/__init__.py @@ -0,0 +1,11 @@ +from ._dataset_backend import FileSystemDatasetBackend +from ._key_value_store_backend import FileSystemKeyValueStoreBackend +from ._request_queue_backend import FileSystemRequestQueueBackend +from ._storage_backend import FileSystemStorageBackend + +__all__ = [ + 'FileSystemDatasetBackend', + 'FileSystemKeyValueStoreBackend', + 'FileSystemRequestQueueBackend', + 'FileSystemStorageBackend', +] diff --git a/src/crawlee/storage_clients/_file_system/_dataset_client.py b/src/crawlee/storage_backends/_file_system/_dataset_backend.py similarity index 93% rename from src/crawlee/storage_clients/_file_system/_dataset_client.py rename to src/crawlee/storage_backends/_file_system/_dataset_backend.py index 904979de91..5e39430a94 100644 --- a/src/crawlee/storage_clients/_file_system/_dataset_client.py +++ b/src/crawlee/storage_backends/_file_system/_dataset_backend.py @@ -17,8 +17,8 @@ from crawlee._utils.crypto import crypto_random_object_id from crawlee._utils.file import atomic_write, json_dumps, validate_subdirectory from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs -from crawlee.storage_clients._base import DatasetClient -from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata +from crawlee.storage_backends._base import DatasetBackend +from crawlee.storage_backends.models import DatasetItemsListPage, DatasetMetadata if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -28,10 +28,10 @@ logger = getLogger(__name__) -class FileSystemDatasetClient(DatasetClient): - """File system implementation of the dataset client. +class FileSystemDatasetBackend(DatasetBackend): + """File system implementation of the dataset backend. - This client persists dataset items to the file system as individual JSON files within a structured + This backend persists dataset items to the file system as individual JSON files within a structured directory hierarchy following the pattern: ``` @@ -64,7 +64,7 @@ def __init__( ) -> None: """Initialize a new instance. - Preferably use the `FileSystemDatasetClient.open` class method to create a new instance. + Preferably use the `FileSystemDatasetBackend.open` class method to create a new instance. """ self._metadata = metadata @@ -97,7 +97,7 @@ async def open( alias: str | None, configuration: Configuration, ) -> Self: - """Open or create a file system dataset client. + """Open or create a file system dataset backend. This method attempts to open an existing dataset from the file system. If a dataset with the specified ID or name exists, it loads the metadata from the stored files. If no existing dataset is found, a new one @@ -110,7 +110,7 @@ async def open( configuration: The configuration object containing storage directory settings. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. Raises: ValueError: If a dataset with the specified ID is not found, if metadata is invalid, @@ -141,12 +141,12 @@ async def open( file_content = json.load(file) metadata = DatasetMetadata(**file_content) if metadata.id == id: - client = cls( + backend = cls( metadata=metadata, path_to_dataset=dataset_base_path / dataset_dir, lock=asyncio.Lock(), ) - await client._update_metadata(update_accessed_at=True) + await backend._update_metadata(update_accessed_at=True) found = True break finally: @@ -162,7 +162,7 @@ async def open( path_to_dataset = validate_subdirectory(dataset_base_path, name or alias or 'default') path_to_metadata = path_to_dataset / METADATA_FILENAME - # If the dataset directory exists, reconstruct the client from the metadata file. + # If the dataset directory exists, reconstruct the backend from the metadata file. if path_to_dataset.exists() and path_to_metadata.exists(): file = await asyncio.to_thread(path_to_metadata.open, mode='r', encoding='utf-8') try: @@ -174,15 +174,15 @@ async def open( except ValidationError as exc: raise ValueError(f'Invalid metadata file for dataset "{name or alias}"') from exc - client = cls( + backend = cls( metadata=metadata, path_to_dataset=path_to_dataset, lock=asyncio.Lock(), ) - await client._update_metadata(update_accessed_at=True) + await backend._update_metadata(update_accessed_at=True) - # Otherwise, create a new dataset client. + # Otherwise, create a new dataset backend. else: now = datetime.now(UTC) metadata = DatasetMetadata( @@ -193,14 +193,14 @@ async def open( modified_at=now, item_count=0, ) - client = cls( + backend = cls( metadata=metadata, path_to_dataset=path_to_dataset, lock=asyncio.Lock(), ) - await client._update_metadata() + await backend._update_metadata() - return client + return backend @override async def drop(self) -> None: @@ -266,8 +266,7 @@ async def get_data( if unsupported: logger.warning( - f'The arguments {list(unsupported.keys())} of get_data are not supported by the ' - f'{self.__class__.__name__} client.' + f'The arguments {list(unsupported.keys())} of get_data are not supported by {self.__class__.__name__}.' ) # If the dataset directory does not exist, log a warning and return an empty page. @@ -361,8 +360,7 @@ async def iterate_items( if unsupported: logger.warning( - f'The arguments {list(unsupported.keys())} of iterate are not supported ' - f'by the {self.__class__.__name__} client.' + f'The arguments {list(unsupported.keys())} of iterate are not supported by {self.__class__.__name__}.' ) # If the dataset directory does not exist, log a warning and return immediately. diff --git a/src/crawlee/storage_clients/_file_system/_key_value_store_client.py b/src/crawlee/storage_backends/_file_system/_key_value_store_backend.py similarity index 93% rename from src/crawlee/storage_clients/_file_system/_key_value_store_client.py rename to src/crawlee/storage_backends/_file_system/_key_value_store_backend.py index c9c740c26d..ef08968d9e 100644 --- a/src/crawlee/storage_clients/_file_system/_key_value_store_client.py +++ b/src/crawlee/storage_backends/_file_system/_key_value_store_backend.py @@ -17,8 +17,8 @@ from crawlee._utils.crypto import crypto_random_object_id from crawlee._utils.file import atomic_write, infer_mime_type, json_dumps, validate_subdirectory from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs -from crawlee.storage_clients._base import KeyValueStoreClient -from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata +from crawlee.storage_backends._base import KeyValueStoreBackend +from crawlee.storage_backends.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -29,10 +29,10 @@ logger = getLogger(__name__) -class FileSystemKeyValueStoreClient(KeyValueStoreClient): - """File system implementation of the key-value store client. +class FileSystemKeyValueStoreBackend(KeyValueStoreBackend): + """File system implementation of the key-value store backend. - This client persists data to the file system, making it suitable for scenarios where data needs to + This backend persists data to the file system, making it suitable for scenarios where data needs to survive process restarts. Keys are mapped to file paths in a directory structure following the pattern: ``` @@ -62,7 +62,7 @@ def __init__( ) -> None: """Initialize a new instance. - Preferably use the `FileSystemKeyValueStoreClient.open` class method to create a new instance. + Preferably use the `FileSystemKeyValueStoreBackend.open` class method to create a new instance. """ self._metadata = metadata @@ -95,7 +95,7 @@ async def open( alias: str | None, configuration: Configuration, ) -> Self: - """Open or create a file system key-value store client. + """Open or create a file system key-value store backend. This method attempts to open an existing key-value store from the file system. If a KVS with the specified ID or name exists, it loads the metadata from the stored files. If no existing store is found, a new one @@ -108,7 +108,7 @@ async def open( configuration: The configuration object containing storage directory settings. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. Raises: ValueError: If a store with the specified ID is not found, if metadata is invalid, @@ -139,12 +139,12 @@ async def open( file_content = json.load(file) metadata = KeyValueStoreMetadata(**file_content) if metadata.id == id: - client = cls( + backend = cls( metadata=metadata, path_to_kvs=kvs_base_path / kvs_dir, lock=asyncio.Lock(), ) - await client._update_metadata(update_accessed_at=True) + await backend._update_metadata(update_accessed_at=True) found = True break finally: @@ -160,7 +160,7 @@ async def open( path_to_kvs = validate_subdirectory(kvs_base_path, name or alias or 'default') path_to_metadata = path_to_kvs / METADATA_FILENAME - # If the key-value store directory exists, reconstruct the client from the metadata file. + # If the key-value store directory exists, reconstruct the backend from the metadata file. if path_to_kvs.exists() and path_to_metadata.exists(): file = await asyncio.to_thread(path_to_metadata.open, mode='r', encoding='utf-8') try: @@ -172,15 +172,15 @@ async def open( except ValidationError as exc: raise ValueError(f'Invalid metadata file for key-value store "{name or alias}"') from exc - client = cls( + backend = cls( metadata=metadata, path_to_kvs=path_to_kvs, lock=asyncio.Lock(), ) - await client._update_metadata(update_accessed_at=True) + await backend._update_metadata(update_accessed_at=True) - # Otherwise, create a new key-value store client. + # Otherwise, create a new key-value store backend. else: now = datetime.now(UTC) metadata = KeyValueStoreMetadata( @@ -190,18 +190,18 @@ async def open( accessed_at=now, modified_at=now, ) - client = cls( + backend = cls( metadata=metadata, path_to_kvs=path_to_kvs, lock=asyncio.Lock(), ) - await client._update_metadata() + await backend._update_metadata() - return client + return backend @override async def drop(self) -> None: - # If the client directory exists, remove it recursively. + # If the storage directory exists, remove it recursively. if self.path_to_kvs.exists(): async with self._lock: await asyncio.to_thread(shutil.rmtree, self.path_to_kvs) diff --git a/src/crawlee/storage_clients/_file_system/_request_queue_client.py b/src/crawlee/storage_backends/_file_system/_request_queue_backend.py similarity index 95% rename from src/crawlee/storage_clients/_file_system/_request_queue_client.py rename to src/crawlee/storage_backends/_file_system/_request_queue_backend.py index bb846627cf..0e876fa8f2 100644 --- a/src/crawlee/storage_clients/_file_system/_request_queue_client.py +++ b/src/crawlee/storage_backends/_file_system/_request_queue_backend.py @@ -19,8 +19,8 @@ from crawlee._utils.file import atomic_write, json_dumps, validate_subdirectory from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs from crawlee._utils.recoverable_state import RecoverableState -from crawlee.storage_clients._base import RequestQueueClient -from crawlee.storage_clients.models import ( +from crawlee.storage_backends._base import RequestQueueBackend +from crawlee.storage_backends.models import ( AddRequestsResponse, ProcessedRequest, RequestQueueMetadata, @@ -37,7 +37,7 @@ class RequestQueueState(BaseModel): - """State model for the `FileSystemRequestQueueClient`.""" + """State model for the `FileSystemRequestQueueBackend`.""" sequence_counter: int = 0 """Counter for regular request ordering.""" @@ -58,10 +58,10 @@ class RequestQueueState(BaseModel): """Set of request unique keys that have been handled.""" -class FileSystemRequestQueueClient(RequestQueueClient): - """A file system implementation of the request queue client. +class FileSystemRequestQueueBackend(RequestQueueBackend): + """A file system implementation of the request queue backend. - This client persists requests to the file system as individual JSON files, making it suitable for scenarios + This backend persists requests to the file system as individual JSON files, making it suitable for scenarios where data needs to survive process restarts. Each request is stored as a separate file in a directory structure following the pattern: @@ -97,7 +97,7 @@ def __init__( ) -> None: """Initialize a new instance. - Preferably use the `FileSystemRequestQueueClient.open` class method to create a new instance. + Preferably use the `FileSystemRequestQueueBackend.open` class method to create a new instance. """ self._metadata = metadata @@ -136,10 +136,10 @@ def path_to_metadata(self) -> Path: @classmethod async def _create_recoverable_state(cls, id: str, configuration: Configuration) -> RecoverableState: async def kvs_factory() -> KeyValueStore: - from crawlee.storage_clients import FileSystemStorageClient # noqa: PLC0415 avoid circular import + from crawlee.storage_backends import FileSystemStorageBackend # noqa: PLC0415 avoid circular import from crawlee.storages import KeyValueStore # noqa: PLC0415 avoid circular import - return await KeyValueStore.open(storage_client=FileSystemStorageClient(), configuration=configuration) + return await KeyValueStore.open(storage_backend=FileSystemStorageBackend(), configuration=configuration) return RecoverableState[RequestQueueState]( default_state=RequestQueueState(), @@ -158,7 +158,7 @@ async def open( alias: str | None, configuration: Configuration, ) -> Self: - """Open or create a file system request queue client. + """Open or create a file system request queue backend. This method attempts to open an existing request queue from the file system. If a queue with the specified ID or name exists, it loads the metadata and state from the stored files. If no existing queue is found, @@ -171,7 +171,7 @@ async def open( configuration: The configuration object containing storage directory settings. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. Raises: ValueError: If a queue with the specified ID is not found, if metadata is invalid, @@ -203,7 +203,7 @@ async def open( metadata = RequestQueueMetadata(**file_content) if metadata.id == id: - client = cls( + backend = cls( metadata=metadata, path_to_rq=rq_base_path / rq_dir, lock=asyncio.Lock(), @@ -211,9 +211,9 @@ async def open( id=id, configuration=configuration ), ) - await client._state.initialize() - await client._discover_existing_requests() - await client._update_metadata(update_accessed_at=True) + await backend._state.initialize() + await backend._discover_existing_requests() + await backend._update_metadata(update_accessed_at=True) found = True break finally: @@ -229,7 +229,7 @@ async def open( path_to_rq = validate_subdirectory(rq_base_path, name or alias or 'default') path_to_metadata = path_to_rq / METADATA_FILENAME - # If the RQ directory exists, reconstruct the client from the metadata file. + # If the RQ directory exists, reconstruct the backend from the metadata file. if path_to_rq.exists() and path_to_metadata.exists(): file = await asyncio.to_thread(path_to_metadata.open, encoding='utf-8') try: @@ -241,18 +241,18 @@ async def open( except ValidationError as exc: raise ValueError(f'Invalid metadata file for request queue "{name or alias}"') from exc - client = cls( + backend = cls( metadata=metadata, path_to_rq=path_to_rq, lock=asyncio.Lock(), recoverable_state=await cls._create_recoverable_state(id=metadata.id, configuration=configuration), ) - await client._state.initialize() - await client._discover_existing_requests() - await client._update_metadata(update_accessed_at=True) + await backend._state.initialize() + await backend._discover_existing_requests() + await backend._update_metadata(update_accessed_at=True) - # Otherwise, create a new dataset client. + # Otherwise, create a new request queue backend. else: now = datetime.now(UTC) metadata = RequestQueueMetadata( @@ -266,16 +266,16 @@ async def open( pending_request_count=0, total_request_count=0, ) - client = cls( + backend = cls( metadata=metadata, path_to_rq=path_to_rq, lock=asyncio.Lock(), recoverable_state=await cls._create_recoverable_state(id=metadata.id, configuration=configuration), ) - await client._state.initialize() - await client._update_metadata() + await backend._state.initialize() + await backend._update_metadata() - return client + return backend @override async def drop(self) -> None: @@ -490,7 +490,7 @@ async def fetch_next_request(self) -> Request | None: self._is_empty_cache = None return next_request - # Yield to the event loop like the other clients do, so a caller polling an empty queue can't starve the + # Yield to the event loop like the other backends do, so a caller polling an empty queue can't starve the # tasks that would fill it. await asyncio.sleep(0) return None diff --git a/src/crawlee/storage_backends/_file_system/_storage_backend.py b/src/crawlee/storage_backends/_file_system/_storage_backend.py new file mode 100644 index 0000000000..36c1ce1058 --- /dev/null +++ b/src/crawlee/storage_backends/_file_system/_storage_backend.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from typing_extensions import override + +from crawlee._utils.docs import docs_group +from crawlee.configuration import Configuration +from crawlee.storage_backends._base import StorageBackend + +from ._dataset_backend import FileSystemDatasetBackend +from ._key_value_store_backend import FileSystemKeyValueStoreBackend +from ._request_queue_backend import FileSystemRequestQueueBackend + +if TYPE_CHECKING: + from collections.abc import Hashable + + +@docs_group('Storage backends') +class FileSystemStorageBackend(StorageBackend): + """File system implementation of the storage backend. + + This storage backend provides access to datasets, key-value stores, and request queues that persist data + to the local file system. Each storage type is implemented with its own specific file system backend + that stores data in a structured directory hierarchy. + + Data is stored in JSON format in predictable file paths, making it easy to inspect and manipulate + the stored data outside of the Crawlee application if needed. + + All data persists between program runs but is limited to access from the local machine + where the files are stored. + + Warning: This storage backend is not safe for concurrent access from multiple crawler processes. + Use it only when running a single crawler process at a time. + """ + + @override + def get_storage_backend_cache_key(self, configuration: Configuration) -> Hashable: + # Even different backend instances should return same storage if the storage_dir is the same. + return super().get_storage_backend_cache_key(configuration), configuration.storage_dir + + @override + async def create_dataset_backend( + self, + *, + id: str | None = None, + name: str | None = None, + alias: str | None = None, + configuration: Configuration | None = None, + ) -> FileSystemDatasetBackend: + configuration = configuration or Configuration.get_global_configuration() + backend = await FileSystemDatasetBackend.open(id=id, name=name, alias=alias, configuration=configuration) + await self._purge_if_needed(backend, configuration) + return backend + + @override + async def create_kvs_backend( + self, + *, + id: str | None = None, + name: str | None = None, + alias: str | None = None, + configuration: Configuration | None = None, + ) -> FileSystemKeyValueStoreBackend: + configuration = configuration or Configuration.get_global_configuration() + backend = await FileSystemKeyValueStoreBackend.open(id=id, name=name, alias=alias, configuration=configuration) + await self._purge_if_needed(backend, configuration) + return backend + + @override + async def create_rq_backend( + self, + *, + id: str | None = None, + name: str | None = None, + alias: str | None = None, + configuration: Configuration | None = None, + ) -> FileSystemRequestQueueBackend: + configuration = configuration or Configuration.get_global_configuration() + backend = await FileSystemRequestQueueBackend.open(id=id, name=name, alias=alias, configuration=configuration) + await self._purge_if_needed(backend, configuration) + return backend diff --git a/src/crawlee/storage_clients/_file_system/_utils.py b/src/crawlee/storage_backends/_file_system/_utils.py similarity index 100% rename from src/crawlee/storage_clients/_file_system/_utils.py rename to src/crawlee/storage_backends/_file_system/_utils.py diff --git a/src/crawlee/storage_clients/_file_system/py.typed b/src/crawlee/storage_backends/_file_system/py.typed similarity index 100% rename from src/crawlee/storage_clients/_file_system/py.typed rename to src/crawlee/storage_backends/_file_system/py.typed diff --git a/src/crawlee/storage_backends/_memory/__init__.py b/src/crawlee/storage_backends/_memory/__init__.py new file mode 100644 index 0000000000..67f0528db6 --- /dev/null +++ b/src/crawlee/storage_backends/_memory/__init__.py @@ -0,0 +1,11 @@ +from ._dataset_backend import MemoryDatasetBackend +from ._key_value_store_backend import MemoryKeyValueStoreBackend +from ._request_queue_backend import MemoryRequestQueueBackend +from ._storage_backend import MemoryStorageBackend + +__all__ = [ + 'MemoryDatasetBackend', + 'MemoryKeyValueStoreBackend', + 'MemoryRequestQueueBackend', + 'MemoryStorageBackend', +] diff --git a/src/crawlee/storage_clients/_memory/_dataset_client.py b/src/crawlee/storage_backends/_memory/_dataset_backend.py similarity index 90% rename from src/crawlee/storage_clients/_memory/_dataset_client.py rename to src/crawlee/storage_backends/_memory/_dataset_backend.py index 17e6412816..6fac4eeec9 100644 --- a/src/crawlee/storage_clients/_memory/_dataset_client.py +++ b/src/crawlee/storage_backends/_memory/_dataset_backend.py @@ -10,8 +10,8 @@ from crawlee._types import JsonSerializable from crawlee._utils.crypto import crypto_random_object_id from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs -from crawlee.storage_clients._base import DatasetClient -from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata +from crawlee.storage_backends._base import DatasetBackend +from crawlee.storage_backends.models import DatasetItemsListPage, DatasetMetadata if TYPE_CHECKING: from collections.abc import AsyncIterator @@ -20,10 +20,10 @@ logger = getLogger(__name__) -class MemoryDatasetClient(DatasetClient): - """Memory implementation of the dataset client. +class MemoryDatasetBackend(DatasetBackend): + """Memory implementation of the dataset backend. - This client stores dataset items in memory using Python lists and dictionaries. No data is persisted + This backend stores dataset items in memory using Python lists and dictionaries. No data is persisted between process runs, meaning all stored data is lost when the program terminates. This implementation is primarily useful for testing, development, and short-lived crawler operations where persistent storage is not required. @@ -40,7 +40,7 @@ def __init__( ) -> None: """Initialize a new instance. - Preferably use the `MemoryDatasetClient.open` class method to create a new instance. + Preferably use the `MemoryDatasetBackend.open` class method to create a new instance. """ self._metadata = metadata @@ -59,13 +59,13 @@ async def open( name: str | None, alias: str | None, ) -> Self: - """Open or create a new memory dataset client. + """Open or create a new memory dataset backend. This method creates a new in-memory dataset instance. Unlike persistent storage implementations, memory datasets don't check for existing datasets with the same name or ID since all data exists only in memory and is lost when the process terminates. - Alias does not have any effect on the memory storage client implementation, because unnamed storages + Alias does not have any effect on the memory storage backend implementation, because unnamed storages are supported by default, since data are not persisted. Args: @@ -74,7 +74,7 @@ async def open( alias: The alias of the dataset for unnamed (run scope) storages. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. Raises: ValueError: If both name and alias are provided, or if neither id, name, nor alias is provided. @@ -161,8 +161,7 @@ async def get_data( if unsupported: logger.warning( - f'The arguments {list(unsupported.keys())} of get_data are not supported ' - f'by the {self.__class__.__name__} client.' + f'The arguments {list(unsupported.keys())} of get_data are not supported by {self.__class__.__name__}.' ) total = len(self._records) @@ -216,8 +215,7 @@ async def iterate_items( if unsupported: logger.warning( - f'The arguments {list(unsupported.keys())} of iterate are not supported ' - f'by the {self.__class__.__name__} client.' + f'The arguments {list(unsupported.keys())} of iterate are not supported by {self.__class__.__name__}.' ) items = self._records.copy() diff --git a/src/crawlee/storage_clients/_memory/_key_value_store_client.py b/src/crawlee/storage_backends/_memory/_key_value_store_backend.py similarity index 88% rename from src/crawlee/storage_clients/_memory/_key_value_store_client.py rename to src/crawlee/storage_backends/_memory/_key_value_store_backend.py index aeb4141a8f..592d589a9c 100644 --- a/src/crawlee/storage_clients/_memory/_key_value_store_client.py +++ b/src/crawlee/storage_backends/_memory/_key_value_store_backend.py @@ -9,17 +9,17 @@ from crawlee._utils.crypto import crypto_random_object_id from crawlee._utils.file import infer_mime_type from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs -from crawlee.storage_clients._base import KeyValueStoreClient -from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata +from crawlee.storage_backends._base import KeyValueStoreBackend +from crawlee.storage_backends.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata if TYPE_CHECKING: from collections.abc import AsyncIterator -class MemoryKeyValueStoreClient(KeyValueStoreClient): - """Memory implementation of the key-value store client. +class MemoryKeyValueStoreBackend(KeyValueStoreBackend): + """Memory implementation of the key-value store backend. - This client stores data in memory as Python dictionaries. No data is persisted between + This backend stores data in memory as Python dictionaries. No data is persisted between process runs, meaning all stored data is lost when the program terminates. This implementation is primarily useful for testing, development, and short-lived crawler operations where persistence is not required. @@ -35,7 +35,7 @@ def __init__( ) -> None: """Initialize a new instance. - Preferably use the `MemoryKeyValueStoreClient.open` class method to create a new instance. + Preferably use the `MemoryKeyValueStoreBackend.open` class method to create a new instance. """ self._metadata = metadata @@ -54,13 +54,13 @@ async def open( name: str | None, alias: str | None, ) -> Self: - """Open or create a new memory key-value store client. + """Open or create a new memory key-value store backend. This method creates a new in-memory key-value store instance. Unlike persistent storage implementations, memory KVS don't check for existing stores with the same name or ID since all data exists only in memory and is lost when the process terminates. - Alias does not have any effect on the memory storage client implementation, because unnamed storages + Alias does not have any effect on the memory storage backend implementation, because unnamed storages are supported by default, since data are not persisted. Args: @@ -69,7 +69,7 @@ async def open( alias: The alias of the key-value store for unnamed (run scope) storages. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. Raises: ValueError: If both name and alias are provided. diff --git a/src/crawlee/storage_clients/_memory/_request_queue_client.py b/src/crawlee/storage_backends/_memory/_request_queue_backend.py similarity index 93% rename from src/crawlee/storage_clients/_memory/_request_queue_client.py rename to src/crawlee/storage_backends/_memory/_request_queue_backend.py index 640bf43ce1..10c6e2d7e3 100644 --- a/src/crawlee/storage_clients/_memory/_request_queue_client.py +++ b/src/crawlee/storage_backends/_memory/_request_queue_backend.py @@ -11,8 +11,8 @@ from crawlee import Request from crawlee._utils.crypto import crypto_random_object_id from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs -from crawlee.storage_clients._base import RequestQueueClient -from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata +from crawlee.storage_backends._base import RequestQueueBackend +from crawlee.storage_backends.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata if TYPE_CHECKING: from collections.abc import Sequence @@ -20,14 +20,14 @@ logger = getLogger(__name__) -class MemoryRequestQueueClient(RequestQueueClient): - """Memory implementation of the request queue client. +class MemoryRequestQueueBackend(RequestQueueBackend): + """Memory implementation of the request queue backend. No data is persisted between process runs, which means all requests are lost when the program terminates. This implementation is primarily useful for testing, development, and short-lived crawler runs where persistence is not required. - This client provides fast access to request data but is limited by available memory and does not support + This backend provides fast access to request data but is limited by available memory and does not support data sharing across different processes. """ @@ -38,7 +38,7 @@ def __init__( ) -> None: """Initialize a new instance. - Preferably use the `MemoryRequestQueueClient.open` class method to create a new instance. + Preferably use the `MemoryRequestQueueBackend.open` class method to create a new instance. """ self._metadata = metadata @@ -70,13 +70,13 @@ async def open( name: str | None, alias: str | None, ) -> Self: - """Open or create a new memory request queue client. + """Open or create a new memory request queue backend. This method creates a new in-memory request queue instance. Unlike persistent storage implementations, memory queues don't check for existing queues with the same name or ID since all data exists only in memory and is lost when the process terminates. - Alias does not have any effect on the memory storage client implementation, because unnamed storages + Alias does not have any effect on the memory storage backend implementation, because unnamed storages are supported by default, since data are not persisted. Args: @@ -85,7 +85,7 @@ async def open( alias: The alias of the request queue for unnamed (run scope) storages. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. Raises: ValueError: If both name and alias are provided. @@ -213,7 +213,7 @@ async def add_batch_of_requests( @override async def fetch_next_request(self) -> Request | None: if not self._pending_requests: - # Yield to the event loop like the other clients do, so a caller polling an empty queue can't starve the + # Yield to the event loop like the other backends do, so a caller polling an empty queue can't starve the # tasks that would fill it. await asyncio.sleep(0) return None diff --git a/src/crawlee/storage_clients/_memory/_storage_client.py b/src/crawlee/storage_backends/_memory/_storage_backend.py similarity index 53% rename from src/crawlee/storage_clients/_memory/_storage_client.py rename to src/crawlee/storage_backends/_memory/_storage_backend.py index fa8ff3589e..ef988eda74 100644 --- a/src/crawlee/storage_clients/_memory/_storage_client.py +++ b/src/crawlee/storage_backends/_memory/_storage_backend.py @@ -4,18 +4,18 @@ from crawlee._utils.docs import docs_group from crawlee.configuration import Configuration -from crawlee.storage_clients._base import StorageClient +from crawlee.storage_backends._base import StorageBackend -from ._dataset_client import MemoryDatasetClient -from ._key_value_store_client import MemoryKeyValueStoreClient -from ._request_queue_client import MemoryRequestQueueClient +from ._dataset_backend import MemoryDatasetBackend +from ._key_value_store_backend import MemoryKeyValueStoreBackend +from ._request_queue_backend import MemoryRequestQueueBackend -@docs_group('Storage clients') -class MemoryStorageClient(StorageClient): - """Memory implementation of the storage client. +@docs_group('Storage backends') +class MemoryStorageBackend(StorageBackend): + """Memory implementation of the storage backend. - This storage client provides access to datasets, key-value stores, and request queues that store all data + This storage backend provides access to datasets, key-value stores, and request queues that store all data in memory using Python data structures (lists and dictionaries). No data is persisted between process runs, meaning all stored data is lost when the program terminates. @@ -23,48 +23,48 @@ class MemoryStorageClient(StorageClient): support data sharing across different processes. All storage operations happen entirely in memory with no disk operations. - The memory storage client is useful for testing and development environments, or short-lived crawler + The memory storage backend is useful for testing and development environments, or short-lived crawler operations where persistence is not required. """ @override - async def create_dataset_client( + async def create_dataset_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> MemoryDatasetClient: + ) -> MemoryDatasetBackend: configuration = configuration or Configuration.get_global_configuration() - client = await MemoryDatasetClient.open(id=id, name=name, alias=alias) - await self._purge_if_needed(client, configuration) - return client + backend = await MemoryDatasetBackend.open(id=id, name=name, alias=alias) + await self._purge_if_needed(backend, configuration) + return backend @override - async def create_kvs_client( + async def create_kvs_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> MemoryKeyValueStoreClient: + ) -> MemoryKeyValueStoreBackend: configuration = configuration or Configuration.get_global_configuration() - client = await MemoryKeyValueStoreClient.open(id=id, name=name, alias=alias) - await self._purge_if_needed(client, configuration) - return client + backend = await MemoryKeyValueStoreBackend.open(id=id, name=name, alias=alias) + await self._purge_if_needed(backend, configuration) + return backend @override - async def create_rq_client( + async def create_rq_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> MemoryRequestQueueClient: + ) -> MemoryRequestQueueBackend: configuration = configuration or Configuration.get_global_configuration() - client = await MemoryRequestQueueClient.open(id=id, name=name, alias=alias) - await self._purge_if_needed(client, configuration) - return client + backend = await MemoryRequestQueueBackend.open(id=id, name=name, alias=alias) + await self._purge_if_needed(backend, configuration) + return backend diff --git a/src/crawlee/storage_clients/_memory/py.typed b/src/crawlee/storage_backends/_memory/py.typed similarity index 100% rename from src/crawlee/storage_clients/_memory/py.typed rename to src/crawlee/storage_backends/_memory/py.typed diff --git a/src/crawlee/storage_backends/_redis/__init__.py b/src/crawlee/storage_backends/_redis/__init__.py new file mode 100644 index 0000000000..7a256b120d --- /dev/null +++ b/src/crawlee/storage_backends/_redis/__init__.py @@ -0,0 +1,6 @@ +from ._dataset_backend import RedisDatasetBackend +from ._key_value_store_backend import RedisKeyValueStoreBackend +from ._request_queue_backend import RedisRequestQueueBackend +from ._storage_backend import RedisStorageBackend + +__all__ = ['RedisDatasetBackend', 'RedisKeyValueStoreBackend', 'RedisRequestQueueBackend', 'RedisStorageBackend'] diff --git a/src/crawlee/storage_clients/_redis/_client_mixin.py b/src/crawlee/storage_backends/_redis/_backend_mixin.py similarity index 87% rename from src/crawlee/storage_clients/_redis/_client_mixin.py rename to src/crawlee/storage_backends/_redis/_backend_mixin.py index cd7dc67b0a..4fe2c59434 100644 --- a/src/crawlee/storage_clients/_redis/_client_mixin.py +++ b/src/crawlee/storage_backends/_redis/_backend_mixin.py @@ -18,7 +18,7 @@ from redis.asyncio.client import Pipeline from redis.commands.core import AsyncScript - from crawlee.storage_clients.models import DatasetMetadata, KeyValueStoreMetadata, RequestQueueMetadata + from crawlee.storage_backends.models import DatasetMetadata, KeyValueStoreMetadata, RequestQueueMetadata logger = getLogger(__name__) @@ -31,10 +31,10 @@ class MetadataUpdateParams(TypedDict, total=False): update_modified_at: NotRequired[bool] -class RedisClientMixin: - """Mixin class for Redis clients. +class RedisBackendMixin: + """Mixin class for Redis storage backends. - This mixin provides common Redis operations and basic methods for Redis storage clients. + This mixin provides common Redis operations and basic methods for Redis storage backends. """ _DEFAULT_NAME = 'default' @@ -43,8 +43,8 @@ class RedisClientMixin: _MAIN_KEY: ClassVar[str] """Main Redis key prefix for this storage type.""" - _CLIENT_TYPE: ClassVar[str] - """Human-readable client type for error messages.""" + _STORAGE_TYPE: ClassVar[str] + """Human-readable storage type for error messages.""" def __init__(self, storage_name: str, storage_id: str, redis: Redis) -> None: self._storage_name = storage_name @@ -111,7 +111,7 @@ async def _open( extra_metadata_fields: dict[str, Any], instance_kwargs: dict[str, Any], ) -> Self: - """Open or create a new Redis storage client. + """Open or create a new Redis storage backend. Args: id: The ID of the storage. If not provided, a random ID will be generated. @@ -120,10 +120,10 @@ async def _open( redis: Redis client instance. metadata_model: Pydantic model for metadata validation. extra_metadata_fields: Storage-specific metadata fields. - instance_kwargs: Additional arguments for the client constructor. + instance_kwargs: Additional arguments for the backend constructor. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. """ internal_name = name or alias or cls._DEFAULT_NAME storage_id: str | None = None @@ -132,16 +132,16 @@ async def _open( storage_name = await cls._get_metadata_name_by_id(id=id, redis=redis) storage_id = id if storage_name is None: - raise ValueError(f'{cls._CLIENT_TYPE} with ID "{id}" does not exist.') + raise ValueError(f'{cls._STORAGE_TYPE} with ID "{id}" does not exist.') else: metadata_data = await cls._get_metadata_by_name(name=internal_name, redis=redis) storage_name = internal_name if metadata_data is not None else None storage_id = metadata_data['id'] if metadata_data is not None else None # If both storage_name and storage_id are found, open existing storage if storage_name and storage_id: - client = cls(storage_name=storage_name, storage_id=storage_id, redis=redis, **instance_kwargs) - async with client._get_pipeline() as pipe: - await client._update_metadata(pipe, update_accessed_at=True) + backend = cls(storage_name=storage_name, storage_id=storage_id, redis=redis, **instance_kwargs) + async with backend._get_pipeline() as pipe: + await backend._update_metadata(pipe, update_accessed_at=True) # Otherwise, create a new storage else: now = datetime.now(UTC) @@ -153,16 +153,16 @@ async def _open( modified_at=now, **extra_metadata_fields, ) - client = cls(storage_name=internal_name, storage_id=metadata.id, redis=redis, **instance_kwargs) - created = await client._create_metadata_and_storage(internal_name, metadata.model_dump()) - # The client was probably not created due to a race condition. Let's try to open it using the name. + backend = cls(storage_name=internal_name, storage_id=metadata.id, redis=redis, **instance_kwargs) + created = await backend._create_metadata_and_storage(internal_name, metadata.model_dump()) + # The storage was probably not created due to a race condition. Let's try to open it using the name. if not created: metadata_data = await cls._get_metadata_by_name(name=internal_name, redis=redis, with_wait=True) - client = cls(storage_name=internal_name, storage_id=metadata.id, redis=redis, **instance_kwargs) + backend = cls(storage_name=internal_name, storage_id=metadata.id, redis=redis, **instance_kwargs) # Ensure Lua scripts are loaded - await client._ensure_scripts_loaded() - return client + await backend._ensure_scripts_loaded() + return backend async def _load_scripts(self) -> None: """Load Lua scripts in Redis.""" @@ -243,10 +243,10 @@ async def _get_metadata(self, metadata_model: type[RequestQueueMetadata]) -> Req async def _get_metadata( self, metadata_model: type[DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata] ) -> DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata: - """Retrieve client metadata.""" + """Retrieve the storage metadata.""" metadata_dict = await self._get_metadata_by_name(name=self._storage_name, redis=self._redis) if metadata_dict is None: - raise ValueError(f'{self._CLIENT_TYPE} with name "{self._storage_name}" does not exist.') + raise ValueError(f'{self._STORAGE_TYPE} with name "{self._storage_name}" does not exist.') async with self._get_pipeline() as pipe: await self._update_metadata(pipe, update_accessed_at=True) diff --git a/src/crawlee/storage_clients/_redis/_dataset_client.py b/src/crawlee/storage_backends/_redis/_dataset_backend.py similarity index 92% rename from src/crawlee/storage_clients/_redis/_dataset_client.py rename to src/crawlee/storage_backends/_redis/_dataset_backend.py index 0aa7bec19d..384a7eb962 100644 --- a/src/crawlee/storage_clients/_redis/_dataset_client.py +++ b/src/crawlee/storage_backends/_redis/_dataset_backend.py @@ -8,10 +8,10 @@ from typing_extensions import override from crawlee._utils.retry import retry_on_error -from crawlee.storage_clients._base import DatasetClient -from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata +from crawlee.storage_backends._base import DatasetBackend +from crawlee.storage_backends.models import DatasetItemsListPage, DatasetMetadata -from ._client_mixin import MetadataUpdateParams, RedisClientMixin +from ._backend_mixin import MetadataUpdateParams, RedisBackendMixin from ._utils import await_redis_response if TYPE_CHECKING: @@ -33,10 +33,10 @@ class _DatasetMetadataUpdateParams(MetadataUpdateParams): delta_item_count: NotRequired[int] -class RedisDatasetClient(DatasetClient, RedisClientMixin): - """Redis implementation of the dataset client. +class RedisDatasetBackend(DatasetBackend, RedisBackendMixin): + """Redis implementation of the dataset backend. - This client persists dataset items to Redis using JSON arrays for efficient storage and retrieval. + This backend persists dataset items to Redis using JSON arrays for efficient storage and retrieval. Items are stored as JSON objects with automatic ordering preservation through Redis list operations. The dataset data is stored in Redis using the following key pattern: @@ -54,13 +54,13 @@ class RedisDatasetClient(DatasetClient, RedisClientMixin): _MAIN_KEY = 'datasets' """Main Redis key prefix for Dataset.""" - _CLIENT_TYPE = 'Dataset' - """Human-readable client type for error messages.""" + _STORAGE_TYPE = 'Dataset' + """Human-readable storage type for error messages.""" def __init__(self, storage_name: str, storage_id: str, redis: Redis) -> None: """Initialize a new instance. - Preferably use the `RedisDatasetClient.open` class method to create a new instance. + Preferably use the `RedisDatasetBackend.open` class method to create a new instance. Args: storage_name: Internal storage name used for Redis keys. @@ -82,8 +82,8 @@ async def open( name: str | None, alias: str | None, redis: Redis, - ) -> RedisDatasetClient: - """Open or create a new Redis dataset client. + ) -> RedisDatasetBackend: + """Open or create a new Redis dataset backend. This method attempts to open an existing dataset from the Redis database. If a dataset with the specified ID or name exists, it loads the metadata from the database. If no existing store is found, a new one @@ -96,7 +96,7 @@ async def open( redis: Redis client instance. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. """ return await cls._open( id=id, @@ -175,8 +175,7 @@ async def get_data( if unsupported: logger.warning( - f'The arguments {list(unsupported.keys())} of get_data are not supported ' - f'by the {self.__class__.__name__} client.' + f'The arguments {list(unsupported.keys())} of get_data are not supported by {self.__class__.__name__}.' ) metadata = await self.get_metadata() @@ -263,7 +262,7 @@ async def iterate_items( if unsupported: logger.warning( f'The arguments {list(unsupported.keys())} of iterate_items are not supported ' - f'by the {self.__class__.__name__} client.' + f'by {self.__class__.__name__}.' ) metadata = await self.get_metadata() diff --git a/src/crawlee/storage_clients/_redis/_key_value_store_client.py b/src/crawlee/storage_backends/_redis/_key_value_store_backend.py similarity index 92% rename from src/crawlee/storage_clients/_redis/_key_value_store_client.py rename to src/crawlee/storage_backends/_redis/_key_value_store_backend.py index 49afc81834..91c915410f 100644 --- a/src/crawlee/storage_clients/_redis/_key_value_store_client.py +++ b/src/crawlee/storage_backends/_redis/_key_value_store_backend.py @@ -9,11 +9,11 @@ from crawlee._utils.file import infer_mime_type from crawlee._utils.retry import retry_on_error -from crawlee.storage_clients._base import KeyValueStoreClient -from crawlee.storage_clients._utils import batch_records_by_size -from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata +from crawlee.storage_backends._base import KeyValueStoreBackend +from crawlee.storage_backends._utils import batch_records_by_size +from crawlee.storage_backends.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata -from ._client_mixin import MetadataUpdateParams, RedisClientMixin +from ._backend_mixin import MetadataUpdateParams, RedisBackendMixin from ._utils import await_redis_response, expect_bytes if TYPE_CHECKING: @@ -24,10 +24,10 @@ logger = getLogger(__name__) -class RedisKeyValueStoreClient(KeyValueStoreClient, RedisClientMixin): - """Redis implementation of the key-value store client. +class RedisKeyValueStoreBackend(KeyValueStoreBackend, RedisBackendMixin): + """Redis implementation of the key-value store backend. - This client persists key-value data to Redis using hash data structures for efficient storage and retrieval. + This backend persists key-value data to Redis using hash data structures for efficient storage and retrieval. Keys are mapped to values with automatic content type detection and size tracking for metadata management. The key-value store data is stored in Redis using the following key pattern: @@ -39,7 +39,7 @@ class RedisKeyValueStoreClient(KeyValueStoreClient, RedisClientMixin): text values as UTF-8 encoded strings, and binary data as-is. The implementation automatically handles content type detection and maintains metadata about each record including size and MIME type information. - All operations are atomic through Redis hash operations and pipeline transactions. The client supports + All operations are atomic through Redis hash operations and pipeline transactions. The backend supports concurrent access through Redis's built-in atomic operations for hash fields. Values are stored as raw bytes, so the Redis client must not decode responses. A Redis client created with @@ -52,8 +52,8 @@ class RedisKeyValueStoreClient(KeyValueStoreClient, RedisClientMixin): _MAIN_KEY = 'key_value_stores' """Main Redis key prefix for Key-Value Store.""" - _CLIENT_TYPE = 'Key-value store' - """Human-readable client type for error messages.""" + _STORAGE_TYPE = 'Key-value store' + """Human-readable storage type for error messages.""" _ITERATE_ENTRIES_BATCH_MAX_KEYS = 100 """Maximum number of records fetched with a single HMGET call in `iterate_entries`.""" @@ -67,7 +67,7 @@ class RedisKeyValueStoreClient(KeyValueStoreClient, RedisClientMixin): def __init__(self, storage_name: str, storage_id: str, redis: Redis) -> None: """Initialize a new instance. - Preferably use the `RedisKeyValueStoreClient.open` class method to create a new instance. + Preferably use the `RedisKeyValueStoreBackend.open` class method to create a new instance. """ super().__init__(storage_name=storage_name, storage_id=storage_id, redis=redis) @@ -89,8 +89,8 @@ async def open( name: str | None, alias: str | None, redis: Redis, - ) -> RedisKeyValueStoreClient: - """Open or create a new Redis key-value store client. + ) -> RedisKeyValueStoreBackend: + """Open or create a new Redis key-value store backend. This method attempts to open an existing key-value store from the Redis database. If a store with the specified ID or name exists, it loads the metadata from the database. If no existing store is found, a new one @@ -103,7 +103,7 @@ async def open( redis: Redis client instance. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. """ return await cls._open( id=id, diff --git a/src/crawlee/storage_clients/_redis/_request_queue_client.py b/src/crawlee/storage_backends/_redis/_request_queue_backend.py similarity index 96% rename from src/crawlee/storage_clients/_redis/_request_queue_client.py rename to src/crawlee/storage_backends/_redis/_request_queue_backend.py index 30958fd484..58e25395fd 100644 --- a/src/crawlee/storage_clients/_redis/_request_queue_client.py +++ b/src/crawlee/storage_backends/_redis/_request_queue_backend.py @@ -12,10 +12,10 @@ from crawlee import Request from crawlee._utils.crypto import crypto_random_object_id from crawlee._utils.retry import retry_on_error -from crawlee.storage_clients._base import RequestQueueClient -from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata +from crawlee.storage_backends._base import RequestQueueBackend +from crawlee.storage_backends.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata -from ._client_mixin import MetadataUpdateParams, RedisClientMixin +from ._backend_mixin import MetadataUpdateParams, RedisBackendMixin from ._utils import await_redis_response if TYPE_CHECKING: @@ -41,10 +41,10 @@ class _QueueMetadataUpdateParams(MetadataUpdateParams): update_had_multiple_clients: NotRequired[bool] -class RedisRequestQueueClient(RequestQueueClient, RedisClientMixin): - """Redis implementation of the request queue client. +class RedisRequestQueueBackend(RequestQueueBackend, RedisBackendMixin): + """Redis implementation of the request queue backend. - This client persists requests to Redis using multiple data structures for efficient queue operations, + This backend persists requests to Redis using multiple data structures for efficient queue operations, deduplication, and concurrent access safety. Requests are stored with FIFO ordering and support both regular and forefront (high-priority) insertion modes. @@ -74,8 +74,8 @@ class RedisRequestQueueClient(RequestQueueClient, RedisClientMixin): _MAIN_KEY = 'request_queues' """Main Redis key prefix for Request Queue.""" - _CLIENT_TYPE = 'Request queue' - """Human-readable client type for error messages.""" + _STORAGE_TYPE = 'Request queue' + """Human-readable storage type for error messages.""" _MAX_BATCH_FETCH_SIZE = 10 """Maximum number of requests to fetch in a single batch operation.""" @@ -96,7 +96,7 @@ def __init__( ) -> None: """Initialize a new instance. - Preferably use the `RedisRequestQueueClient.open` class method to create a new instance. + Preferably use the `RedisRequestQueueBackend.open` class method to create a new instance. """ super().__init__(storage_name=storage_name, storage_id=storage_id, redis=redis) @@ -172,8 +172,8 @@ async def open( redis: Redis, dedup_strategy: Literal['default', 'bloom'] = 'default', bloom_error_rate: float = 1e-7, - ) -> RedisRequestQueueClient: - """Open or create a new Redis request queue client. + ) -> RedisRequestQueueBackend: + """Open or create a new Redis request queue backend. This method attempts to open an existing request queue from the Redis database. If a queue with the specified ID or name exists, it loads the metadata from the database. If no existing queue is found, a new one @@ -192,7 +192,7 @@ async def open( `dedup_strategy` is set to 'bloom'. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. """ return await cls._open( id=id, @@ -255,7 +255,7 @@ async def add_batch_of_requests( forefront: bool = False, ) -> AddRequestsResponse: if self._add_requests_script is None: - raise RuntimeError('Scripts not loaded. Call _ensure_scripts_loaded() before using the client.') + raise RuntimeError('Scripts not loaded. Call _ensure_scripts_loaded() before using the backend.') processed_requests = [] @@ -361,7 +361,7 @@ async def fetch_next_request(self) -> Request | None: return self._pending_fetch_cache.popleft() if self._fetch_script is None: - raise RuntimeError('Scripts not loaded. Call _ensure_scripts_loaded() before using the client.') + raise RuntimeError('Scripts not loaded. Call _ensure_scripts_loaded() before using the backend.') blocked_until_timestamp = int(datetime.now(tz=UTC).timestamp() * 1000) + self._BLOCK_REQUEST_TIME @@ -549,7 +549,7 @@ async def _create_storage(self, pipeline: Pipeline) -> None: async def _reclaim_stale_requests(self) -> None: """Reclaim requests that have been in progress for too long.""" if self._reclaim_stale_script is None: - raise RuntimeError('Scripts not loaded. Call _ensure_scripts_loaded() before using the client.') + raise RuntimeError('Scripts not loaded. Call _ensure_scripts_loaded() before using the backend.') current_time = int(datetime.now(tz=UTC).timestamp() * 1000) diff --git a/src/crawlee/storage_clients/_redis/_storage_client.py b/src/crawlee/storage_backends/_redis/_storage_backend.py similarity index 75% rename from src/crawlee/storage_clients/_redis/_storage_client.py rename to src/crawlee/storage_backends/_redis/_storage_backend.py index bc8c8759e0..df0bfef2e7 100644 --- a/src/crawlee/storage_clients/_redis/_storage_client.py +++ b/src/crawlee/storage_backends/_redis/_storage_backend.py @@ -8,23 +8,23 @@ from crawlee._utils.docs import docs_group from crawlee.configuration import Configuration -from crawlee.storage_clients._base import StorageClient +from crawlee.storage_backends._base import StorageBackend -from ._dataset_client import RedisDatasetClient -from ._key_value_store_client import RedisKeyValueStoreClient -from ._request_queue_client import RedisRequestQueueClient +from ._dataset_backend import RedisDatasetBackend +from ._key_value_store_backend import RedisKeyValueStoreBackend +from ._request_queue_backend import RedisRequestQueueBackend -@docs_group('Storage clients') -class RedisStorageClient(StorageClient): - """Redis implementation of the storage client. +@docs_group('Storage backends') +class RedisStorageBackend(StorageBackend): + """Redis implementation of the storage backend. - This storage client provides access to datasets, key-value stores, and request queues that persist data + This storage backend provides access to datasets, key-value stores, and request queues that persist data to a Redis database v8.0+. Each storage type uses Redis-specific data structures and key patterns for efficient storage and retrieval. - The client accepts either a Redis connection string or a pre-configured Redis client instance. Exactly one of these - parameters must be provided during initialization. Key-value stores hold binary values, so their client needs + The backend accepts either a Redis connection string or a pre-configured Redis client instance. Exactly one of these + parameters must be provided during initialization. Key-value stores hold binary values, so their backend needs a Redis client that returns raw bytes, which is the default; a client created with `decode_responses=True` works for datasets and request queues only. @@ -46,7 +46,7 @@ def __init__( queue_dedup_strategy: Literal['default', 'bloom'] = 'default', queue_bloom_error_rate: float = 1e-7, ) -> None: - """Initialize the Redis storage client. + """Initialize the Redis storage backend. Args: connection_string: Redis connection string (e.g., "redis://localhost:6379"). @@ -78,7 +78,7 @@ def __init__( # Call the notification only once warnings.warn( ( - 'RedisStorageClient is experimental and its API, behavior, and key structure may change in future ' + 'RedisStorageBackend is experimental and its API, behavior, and key structure may change in future ' 'releases.' ), category=UserWarning, @@ -86,59 +86,59 @@ def __init__( ) @override - async def create_dataset_client( + async def create_dataset_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> RedisDatasetClient: + ) -> RedisDatasetBackend: configuration = configuration or Configuration.get_global_configuration() - client = await RedisDatasetClient.open( + backend = await RedisDatasetBackend.open( id=id, name=name, alias=alias, redis=self._redis, ) - await self._purge_if_needed(client, configuration) - return client + await self._purge_if_needed(backend, configuration) + return backend @override - async def create_kvs_client( + async def create_kvs_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> RedisKeyValueStoreClient: + ) -> RedisKeyValueStoreBackend: configuration = configuration or Configuration.get_global_configuration() - client = await RedisKeyValueStoreClient.open( + backend = await RedisKeyValueStoreBackend.open( id=id, name=name, alias=alias, redis=self._redis, ) - await self._purge_if_needed(client, configuration) - return client + await self._purge_if_needed(backend, configuration) + return backend @override - async def create_rq_client( + async def create_rq_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> RedisRequestQueueClient: + ) -> RedisRequestQueueBackend: configuration = configuration or Configuration.get_global_configuration() - client = await RedisRequestQueueClient.open( + backend = await RedisRequestQueueBackend.open( id=id, name=name, alias=alias, @@ -147,5 +147,5 @@ async def create_rq_client( bloom_error_rate=self._queue_bloom_error_rate, ) - await self._purge_if_needed(client, configuration) - return client + await self._purge_if_needed(backend, configuration) + return backend diff --git a/src/crawlee/storage_clients/_redis/_utils.py b/src/crawlee/storage_backends/_redis/_utils.py similarity index 89% rename from src/crawlee/storage_clients/_redis/_utils.py rename to src/crawlee/storage_backends/_redis/_utils.py index 5359191d74..95dc728dbf 100644 --- a/src/crawlee/storage_clients/_redis/_utils.py +++ b/src/crawlee/storage_backends/_redis/_utils.py @@ -22,7 +22,7 @@ def expect_bytes(value: bytes | str | None) -> bytes | None: """Narrow a Redis reply to raw bytes, rejecting a client that decodes responses. redis-py types every reply as `bytes | str | None`, because a client created with `decode_responses=True` returns - strings. The key-value store client stores binary values and needs the raw bytes back, so it rejects such + strings. The key-value store backend stores binary values and needs the raw bytes back, so it rejects such a client. Raises: @@ -30,7 +30,7 @@ def expect_bytes(value: bytes | str | None) -> bytes | None: """ if isinstance(value, str): raise TypeError( - 'The Redis client returned a decoded string instead of raw bytes. The Redis key-value store client ' + 'The Redis client returned a decoded string instead of raw bytes. The Redis key-value store backend ' 'requires a Redis client created without `decode_responses=True`.' ) return value diff --git a/src/crawlee/storage_clients/_redis/lua_scripts/atomic_bloom_add_requests.lua b/src/crawlee/storage_backends/_redis/lua_scripts/atomic_bloom_add_requests.lua similarity index 100% rename from src/crawlee/storage_clients/_redis/lua_scripts/atomic_bloom_add_requests.lua rename to src/crawlee/storage_backends/_redis/lua_scripts/atomic_bloom_add_requests.lua diff --git a/src/crawlee/storage_clients/_redis/lua_scripts/atomic_fetch_request.lua b/src/crawlee/storage_backends/_redis/lua_scripts/atomic_fetch_request.lua similarity index 100% rename from src/crawlee/storage_clients/_redis/lua_scripts/atomic_fetch_request.lua rename to src/crawlee/storage_backends/_redis/lua_scripts/atomic_fetch_request.lua diff --git a/src/crawlee/storage_clients/_redis/lua_scripts/atomic_set_add_requests.lua b/src/crawlee/storage_backends/_redis/lua_scripts/atomic_set_add_requests.lua similarity index 100% rename from src/crawlee/storage_clients/_redis/lua_scripts/atomic_set_add_requests.lua rename to src/crawlee/storage_backends/_redis/lua_scripts/atomic_set_add_requests.lua diff --git a/src/crawlee/storage_clients/_redis/lua_scripts/reclaim_stale_requests.lua b/src/crawlee/storage_backends/_redis/lua_scripts/reclaim_stale_requests.lua similarity index 100% rename from src/crawlee/storage_clients/_redis/lua_scripts/reclaim_stale_requests.lua rename to src/crawlee/storage_backends/_redis/lua_scripts/reclaim_stale_requests.lua diff --git a/src/crawlee/storage_clients/_redis/py.typed b/src/crawlee/storage_backends/_redis/py.typed similarity index 100% rename from src/crawlee/storage_clients/_redis/py.typed rename to src/crawlee/storage_backends/_redis/py.typed diff --git a/src/crawlee/storage_backends/_sql/__init__.py b/src/crawlee/storage_backends/_sql/__init__.py new file mode 100644 index 0000000000..6c55a16b8a --- /dev/null +++ b/src/crawlee/storage_backends/_sql/__init__.py @@ -0,0 +1,6 @@ +from ._dataset_backend import SqlDatasetBackend +from ._key_value_store_backend import SqlKeyValueStoreBackend +from ._request_queue_backend import SqlRequestQueueBackend +from ._storage_backend import SqlStorageBackend + +__all__ = ['SqlDatasetBackend', 'SqlKeyValueStoreBackend', 'SqlRequestQueueBackend', 'SqlStorageBackend'] diff --git a/src/crawlee/storage_clients/_sql/_client_mixin.py b/src/crawlee/storage_backends/_sql/_backend_mixin.py similarity index 91% rename from src/crawlee/storage_clients/_sql/_client_mixin.py rename to src/crawlee/storage_backends/_sql/_backend_mixin.py index 56a7f1ba60..4f3c561f7a 100644 --- a/src/crawlee/storage_clients/_sql/_client_mixin.py +++ b/src/crawlee/storage_backends/_sql/_backend_mixin.py @@ -23,7 +23,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import DeclarativeBase - from crawlee.storage_clients.models import DatasetMetadata, KeyValueStoreMetadata, RequestQueueMetadata + from crawlee.storage_backends.models import DatasetMetadata, KeyValueStoreMetadata, RequestQueueMetadata from ._db_models import ( DatasetItemDb, @@ -36,7 +36,7 @@ RequestQueueMetadataBufferDb, RequestQueueMetadataDb, ) - from ._storage_client import SqlStorageClient + from ._storage_backend import SqlStorageBackend logger = getLogger(__name__) @@ -49,10 +49,10 @@ class MetadataUpdateParams(TypedDict, total=False): modified_at: NotRequired[datetime] -class SqlClientMixin(ABC): - """Mixin class for SQL clients. +class SqlBackendMixin(ABC): + """Mixin class for SQL storage backends. - This mixin provides common SQL operations and basic methods for SQL storage clients. + This mixin provides common SQL operations and basic methods for SQL storage backends. """ _DEFAULT_NAME: ClassVar[str] @@ -69,15 +69,15 @@ class SqlClientMixin(ABC): _ITEM_TABLE: ClassVar[type[DatasetItemDb | KeyValueStoreRecordDb | RequestDb]] """SQLAlchemy model for items.""" - _CLIENT_TYPE: ClassVar[str] - """Human-readable client type for error messages.""" + _STORAGE_TYPE: ClassVar[str] + """Human-readable storage type for error messages.""" _BLOCK_BUFFER_TIME = timedelta(seconds=1) """Time interval that blocks buffer reading to update metadata.""" - def __init__(self, *, id: str, storage_client: SqlStorageClient) -> None: + def __init__(self, *, id: str, storage_backend: SqlStorageBackend) -> None: self._id = id - self._storage_client = storage_client + self._storage_backend = storage_backend @classmethod async def _open( @@ -86,7 +86,7 @@ async def _open( id: str | None, name: str | None, internal_name: str, - storage_client: SqlStorageClient, + storage_backend: SqlStorageBackend, metadata_model: type[DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata], session: AsyncSession, extra_metadata_fields: dict[str, Any], @@ -99,7 +99,7 @@ async def _open( id: Storage ID to open (takes precedence over name). name: The name of the storage. internal_name: The database name for the storage based on name or alias. - storage_client: SQL storage client instance. + storage_backend: SQL storage backend instance. metadata_model: Pydantic model for metadata validation. session: Active database session. extra_metadata_fields: Storage-specific metadata fields. @@ -108,17 +108,17 @@ async def _open( if id: orm_metadata = await session.get(cls._METADATA_TABLE, id) if not orm_metadata: - raise ValueError(f'{cls._CLIENT_TYPE} with ID "{id}" not found.') + raise ValueError(f'{cls._STORAGE_TYPE} with ID "{id}" not found.') else: stmt = select(cls._METADATA_TABLE).where(cls._METADATA_TABLE.internal_name == internal_name) result = await session.execute(stmt) orm_metadata = result.scalar_one_or_none() if orm_metadata: - client = cls(id=orm_metadata.id, storage_client=storage_client) - await client._add_buffer_record(session) + backend = cls(id=orm_metadata.id, storage_backend=storage_backend) + await backend._add_buffer_record(session) # Ensure any pending buffer updates are processed - await client._process_buffers() + await backend._process_buffers() else: now = datetime.now(UTC) metadata = metadata_model( @@ -129,10 +129,10 @@ async def _open( modified_at=now, **extra_metadata_fields, ) - client = cls(id=metadata.id, storage_client=storage_client) + backend = cls(id=metadata.id, storage_backend=storage_backend) session.add(cls._METADATA_TABLE(**metadata.model_dump(), internal_name=internal_name)) - return client + return backend @classmethod async def _safely_open( @@ -141,7 +141,7 @@ async def _safely_open( id: str | None, name: str | None, alias: str | None = None, - storage_client: SqlStorageClient, + storage_backend: SqlStorageBackend, metadata_model: type[DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata], extra_metadata_fields: dict[str, Any], ) -> Self: @@ -151,7 +151,7 @@ async def _safely_open( id: Storage ID to open (takes precedence over name). name: The name of the storage for named (global scope) storages. alias: The alias of the storage for unnamed (run scope) storages. - storage_client: SQL storage client instance. + storage_backend: SQL storage backend instance. metadata_model: Pydantic model for metadata validation. extra_metadata_fields: Storage-specific metadata fields. """ @@ -162,13 +162,13 @@ async def _safely_open( internal_name = name or alias or cls._DEFAULT_NAME - async with storage_client.create_session() as session: + async with storage_backend.create_session() as session: try: - client = await cls._open( + backend = await cls._open( id=id, name=name, internal_name=internal_name, - storage_client=storage_client, + storage_backend=storage_backend, metadata_model=metadata_model, session=session, extra_metadata_fields=extra_metadata_fields, @@ -186,16 +186,16 @@ async def _safely_open( ) if not orm_metadata: - raise ValueError(f'{cls._CLIENT_TYPE} with Name "{internal_name}" not found.') from None + raise ValueError(f'{cls._STORAGE_TYPE} with Name "{internal_name}" not found.') from None - client = cls(id=orm_metadata.id, storage_client=storage_client) + backend = cls(id=orm_metadata.id, storage_backend=storage_backend) - return client + return backend @asynccontextmanager async def get_session(self, *, with_simple_commit: bool = False) -> AsyncGenerator[AsyncSession]: """Create a new SQLAlchemy session for this storage.""" - async with self._storage_client.create_session() as session: + async with self._storage_backend.create_session() as session: # For operations where a final commit is mandatory and does not require specific processing conditions if with_simple_commit: try: @@ -220,7 +220,7 @@ def _build_insert_stmt_with_ignore( if isinstance(insert_values, dict): insert_values = [insert_values] - dialect = self._storage_client.get_dialect_name() + dialect = self._storage_backend.get_dialect_name() if dialect == 'postgresql': return pg_insert(table_model).values(insert_values).on_conflict_do_nothing() @@ -252,7 +252,7 @@ def _build_upsert_stmt( if isinstance(insert_values, dict): insert_values = [insert_values] - dialect = self._storage_client.get_dialect_name() + dialect = self._storage_backend.get_dialect_name() if dialect == 'postgresql': pg_stmt = pg_insert(table_model).values(insert_values) @@ -295,7 +295,7 @@ async def _drop(self) -> None: buffer_stmt = delete(self._BUFFER_TABLE).where(self._BUFFER_TABLE.storage_id == self._id) async with self.get_session(with_simple_commit=True) as session: - if self._storage_client.get_dialect_name() == 'sqlite': + if self._storage_backend.get_dialect_name() == 'sqlite': # foreign_keys=ON is set at the connection level. Required for cascade deletion. await session.execute(text('PRAGMA foreign_keys=ON')) await session.execute(stmt) @@ -311,14 +311,14 @@ async def _get_metadata(self, metadata_model: type[RequestQueueMetadata]) -> Req async def _get_metadata( self, metadata_model: type[DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata] ) -> DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata: - """Retrieve client metadata.""" + """Retrieve the storage metadata.""" # Process any pending buffer updates first await self._process_buffers() async with self.get_session() as session: orm_metadata = await session.get(self._METADATA_TABLE, self._id) if not orm_metadata: - raise ValueError(f'{self._CLIENT_TYPE} with ID "{self._id}" not found.') + raise ValueError(f'{self._STORAGE_TYPE} with ID "{self._id}" not found.') return metadata_model.model_validate(orm_metadata) @@ -414,7 +414,7 @@ async def _try_acquire_buffer_lock(self, session: AsyncSession) -> bool: capture_error_code = 1020 # MariaDB error code for "Record has changed since last read" now = datetime.now(UTC) lock_until = now + self._BLOCK_BUFFER_TIME - dialect = self._storage_client.get_dialect_name() + dialect = self._storage_backend.get_dialect_name() if dialect in {'postgresql', 'mysql', 'mariadb'}: select_stmt = ( diff --git a/src/crawlee/storage_clients/_sql/_dataset_client.py b/src/crawlee/storage_backends/_sql/_dataset_backend.py similarity index 91% rename from src/crawlee/storage_clients/_sql/_dataset_client.py rename to src/crawlee/storage_backends/_sql/_dataset_backend.py index d414618a18..3c7f1edcf8 100644 --- a/src/crawlee/storage_clients/_sql/_dataset_client.py +++ b/src/crawlee/storage_backends/_sql/_dataset_backend.py @@ -11,10 +11,10 @@ from typing_extensions import override from crawlee._utils.retry import retry_on_error -from crawlee.storage_clients._base import DatasetClient -from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata +from crawlee.storage_backends._base import DatasetBackend +from crawlee.storage_backends.models import DatasetItemsListPage, DatasetMetadata -from ._client_mixin import MetadataUpdateParams, SqlClientMixin +from ._backend_mixin import MetadataUpdateParams, SqlBackendMixin from ._db_models import DatasetItemDb, DatasetMetadataBufferDb, DatasetMetadataDb if TYPE_CHECKING: @@ -26,7 +26,7 @@ from crawlee._types import JsonSerializable - from ._storage_client import SqlStorageClient + from ._storage_backend import SqlStorageBackend logger = getLogger(__name__) @@ -39,10 +39,10 @@ class _DatasetMetadataUpdateParams(MetadataUpdateParams): delta_item_count: NotRequired[int] -class SqlDatasetClient(DatasetClient, SqlClientMixin): - """SQL implementation of the dataset client. +class SqlDatasetBackend(DatasetBackend, SqlBackendMixin): + """SQL implementation of the dataset backend. - This client persists dataset items to a SQL database using two tables for storage + This backend persists dataset items to a SQL database using two tables for storage and retrieval. Items are stored as JSON with automatic ordering preservation. The dataset data is stored in SQL database tables following the pattern: @@ -64,8 +64,8 @@ class SqlDatasetClient(DatasetClient, SqlClientMixin): _ITEM_TABLE: ClassVar[type[DatasetItemDb]] = DatasetItemDb """SQLAlchemy model for dataset items.""" - _CLIENT_TYPE = 'Dataset' - """Human-readable client type for error messages.""" + _STORAGE_TYPE = 'Dataset' + """Human-readable storage type for error messages.""" _BUFFER_TABLE: ClassVar[type[DatasetMetadataBufferDb]] = DatasetMetadataBufferDb """SQLAlchemy model for metadata buffer.""" @@ -74,13 +74,13 @@ def __init__( self, *, id: str, - storage_client: SqlStorageClient, + storage_backend: SqlStorageBackend, ) -> None: """Initialize a new instance. - Preferably use the `SqlDatasetClient.open` class method to create a new instance. + Preferably use the `SqlDatasetBackend.open` class method to create a new instance. """ - super().__init__(id=id, storage_client=storage_client) + super().__init__(id=id, storage_backend=storage_backend) @classmethod async def open( @@ -89,7 +89,7 @@ async def open( id: str | None, name: str | None, alias: str | None, - storage_client: SqlStorageClient, + storage_backend: SqlStorageBackend, ) -> Self: """Open an existing dataset or create a new one. @@ -97,10 +97,10 @@ async def open( id: The ID of the dataset to open. If provided, searches for existing dataset by ID. name: The name of the dataset for named (global scope) storages. alias: The alias of the dataset for unnamed (run scope) storages. - storage_client: The SQL storage client instance. + storage_backend: The SQL storage backend instance. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. Raises: ValueError: If a dataset with the specified ID is not found. @@ -109,7 +109,7 @@ async def open( id=id, name=name, alias=alias, - storage_client=storage_client, + storage_backend=storage_backend, metadata_model=DatasetMetadata, extra_metadata_fields={'item_count': 0}, ) @@ -269,8 +269,7 @@ def _prepare_get_stmt( if unsupported: logger.warning( - f'The arguments {list(unsupported.keys())} of get_data are not supported by the ' - f'{self.__class__.__name__} client.' + f'The arguments {list(unsupported.keys())} of get_data are not supported by {self.__class__.__name__}.' ) stmt = select(self._ITEM_TABLE).where(self._ITEM_TABLE.dataset_id == self._id) diff --git a/src/crawlee/storage_clients/_sql/_db_models.py b/src/crawlee/storage_backends/_sql/_db_models.py similarity index 96% rename from src/crawlee/storage_clients/_sql/_db_models.py rename to src/crawlee/storage_backends/_sql/_db_models.py index 522ae16793..4aa880384e 100644 --- a/src/crawlee/storage_clients/_sql/_db_models.py +++ b/src/crawlee/storage_backends/_sql/_db_models.py @@ -171,7 +171,7 @@ class KeyValueStoreRecordDb(Base): kvs: Mapped[KeyValueStoreMetadataDb] = relationship(back_populates='records') storage_id = synonym('key_value_store_id') - """Alias for key_value_store_id to match SqlClientMixin expectations.""" + """Alias for key_value_store_id to match SqlBackendMixin expectations.""" class DatasetItemDb(Base): @@ -196,7 +196,7 @@ class DatasetItemDb(Base): dataset: Mapped[DatasetMetadataDb] = relationship(back_populates='items') storage_id = synonym('dataset_id') - """Alias for dataset_id to match SqlClientMixin expectations.""" + """Alias for dataset_id to match SqlBackendMixin expectations.""" class RequestDb(Base): @@ -245,7 +245,7 @@ class RequestDb(Base): queue: Mapped[RequestQueueMetadataDb] = relationship(back_populates='requests') storage_id = synonym('request_queue_id') - """Alias for request_queue_id to match SqlClientMixin expectations.""" + """Alias for request_queue_id to match SqlBackendMixin expectations.""" class RequestQueueStateDb(Base): @@ -300,7 +300,7 @@ class KeyValueStoreMetadataBufferDb(MetadataBufferDb, Base): """ID of the key-value store being updated.""" storage_id = synonym('key_value_store_id') - """Alias for key_value_store_id to match SqlClientMixin expectations.""" + """Alias for key_value_store_id to match SqlBackendMixin expectations.""" class DatasetMetadataBufferDb(MetadataBufferDb, Base): @@ -317,7 +317,7 @@ class DatasetMetadataBufferDb(MetadataBufferDb, Base): """Delta for dataset item_count.""" storage_id = synonym('dataset_id') - """Alias for dataset_id to match SqlClientMixin expectations.""" + """Alias for dataset_id to match SqlBackendMixin expectations.""" class RequestQueueMetadataBufferDb(MetadataBufferDb, Base): @@ -348,4 +348,4 @@ class RequestQueueMetadataBufferDb(MetadataBufferDb, Base): """Flag indicating that counters need recalculation from actual data.""" storage_id = synonym('request_queue_id') - """Alias for request_queue_id to match SqlClientMixin expectations.""" + """Alias for request_queue_id to match SqlBackendMixin expectations.""" diff --git a/src/crawlee/storage_clients/_sql/_key_value_store_client.py b/src/crawlee/storage_backends/_sql/_key_value_store_backend.py similarity index 93% rename from src/crawlee/storage_clients/_sql/_key_value_store_client.py rename to src/crawlee/storage_backends/_sql/_key_value_store_backend.py index 58de99f1ad..9315a721bd 100644 --- a/src/crawlee/storage_clients/_sql/_key_value_store_client.py +++ b/src/crawlee/storage_backends/_sql/_key_value_store_backend.py @@ -12,15 +12,15 @@ from crawlee._utils.file import infer_mime_type from crawlee._utils.retry import retry_on_error -from crawlee.storage_clients._base import KeyValueStoreClient -from crawlee.storage_clients._utils import batch_records_by_size -from crawlee.storage_clients.models import ( +from crawlee.storage_backends._base import KeyValueStoreBackend +from crawlee.storage_backends._utils import batch_records_by_size +from crawlee.storage_backends.models import ( KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata, ) -from ._client_mixin import MetadataUpdateParams, SqlClientMixin +from ._backend_mixin import MetadataUpdateParams, SqlBackendMixin from ._db_models import KeyValueStoreMetadataBufferDb, KeyValueStoreMetadataDb, KeyValueStoreRecordDb if TYPE_CHECKING: @@ -28,16 +28,16 @@ from sqlalchemy.ext.asyncio import AsyncSession - from ._storage_client import SqlStorageClient + from ._storage_backend import SqlStorageBackend logger = getLogger(__name__) -class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin): - """SQL implementation of the key-value store client. +class SqlKeyValueStoreBackend(KeyValueStoreBackend, SqlBackendMixin): + """SQL implementation of the key-value store backend. - This client persists key-value data to a SQL database with transaction support and + This backend persists key-value data to a SQL database with transaction support and concurrent access safety. Keys are mapped to rows in database tables with proper indexing for efficient retrieval. @@ -53,7 +53,7 @@ class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin): about each record including size and MIME type information. All database operations are wrapped in transactions with proper error handling and rollback - mechanisms. The client supports atomic upsert operations and handles race conditions when + mechanisms. The backend supports atomic upsert operations and handles race conditions when multiple clients access the same store using composite primary keys (key_value_store_id, key). """ @@ -75,8 +75,8 @@ class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin): _ITEM_TABLE: ClassVar[type[KeyValueStoreRecordDb]] = KeyValueStoreRecordDb """SQLAlchemy model for key-value store items.""" - _CLIENT_TYPE = 'Key-value store' - """Human-readable client type for error messages.""" + _STORAGE_TYPE = 'Key-value store' + """Human-readable storage type for error messages.""" _BUFFER_TABLE: ClassVar[type[KeyValueStoreMetadataBufferDb]] = KeyValueStoreMetadataBufferDb """SQLAlchemy model for metadata buffer.""" @@ -84,14 +84,14 @@ class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin): def __init__( self, *, - storage_client: SqlStorageClient, + storage_backend: SqlStorageBackend, id: str, ) -> None: """Initialize a new instance. - Preferably use the `SqlKeyValueStoreClient.open` class method to create a new instance. + Preferably use the `SqlKeyValueStoreBackend.open` class method to create a new instance. """ - super().__init__(id=id, storage_client=storage_client) + super().__init__(id=id, storage_backend=storage_backend) @classmethod async def open( @@ -100,9 +100,9 @@ async def open( id: str | None, name: str | None, alias: str | None, - storage_client: SqlStorageClient, + storage_backend: SqlStorageBackend, ) -> Self: - """Open or create a SQL key-value store client. + """Open or create a SQL key-value store backend. This method attempts to open an existing key-value store from the SQL database. If a KVS with the specified ID or name exists, it loads the metadata from the database. If no existing store is found, a new one @@ -112,10 +112,10 @@ async def open( id: The ID of the key-value store to open. If provided, searches for existing store by ID. name: The name of the key-value store for named (global scope) storages. alias: The alias of the key-value store for unnamed (run scope) storages. - storage_client: The SQL storage client used to access the database. + storage_backend: The SQL storage backend used to access the database. Returns: - An instance for the opened or created storage client. + An instance for the opened or created storage backend. Raises: ValueError: If a store with the specified ID is not found, or if metadata is invalid. @@ -124,7 +124,7 @@ async def open( id=id, name=name, alias=alias, - storage_client=storage_client, + storage_backend=storage_backend, metadata_model=KeyValueStoreMetadata, extra_metadata_fields={}, ) diff --git a/src/crawlee/storage_clients/_sql/_request_queue_client.py b/src/crawlee/storage_backends/_sql/_request_queue_backend.py similarity index 96% rename from src/crawlee/storage_clients/_sql/_request_queue_client.py rename to src/crawlee/storage_backends/_sql/_request_queue_backend.py index e258cffc76..705104ee2a 100644 --- a/src/crawlee/storage_clients/_sql/_request_queue_client.py +++ b/src/crawlee/storage_backends/_sql/_request_queue_backend.py @@ -15,15 +15,15 @@ from crawlee import Request from crawlee._utils.crypto import crypto_random_object_id from crawlee._utils.retry import retry_on_error -from crawlee.storage_clients._base import RequestQueueClient -from crawlee.storage_clients.models import ( +from crawlee.storage_backends._base import RequestQueueBackend +from crawlee.storage_backends.models import ( AddRequestsResponse, ProcessedRequest, RequestQueueMetadata, UnprocessedRequest, ) -from ._client_mixin import MetadataUpdateParams, SqlClientMixin +from ._backend_mixin import MetadataUpdateParams, SqlBackendMixin from ._db_models import RequestDb, RequestQueueMetadataBufferDb, RequestQueueMetadataDb, RequestQueueStateDb if TYPE_CHECKING: @@ -32,7 +32,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.sql import ColumnElement - from ._storage_client import SqlStorageClient + from ._storage_backend import SqlStorageBackend logger = getLogger(__name__) @@ -51,10 +51,10 @@ class _QueueMetadataUpdateParams(MetadataUpdateParams): update_had_multiple_clients: NotRequired[bool] -class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin): - """SQL implementation of the request queue client. +class SqlRequestQueueBackend(RequestQueueBackend, SqlBackendMixin): + """SQL implementation of the request queue backend. - This client persists requests to a SQL database with transaction handling and + This backend persists requests to a SQL database with transaction handling and concurrent access safety. Requests are stored with sequence-based ordering and efficient querying capabilities. @@ -90,8 +90,8 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin): _ITEM_TABLE: ClassVar[type[RequestDb]] = RequestDb """SQLAlchemy model for request items.""" - _CLIENT_TYPE = 'Request queue' - """Human-readable client type for error messages.""" + _STORAGE_TYPE = 'Request queue' + """Human-readable storage type for error messages.""" _BLOCK_REQUEST_TIME = 300 """Number of seconds for which a request is considered blocked in the database after being fetched for processing. @@ -104,13 +104,13 @@ def __init__( self, *, id: str, - storage_client: SqlStorageClient, + storage_backend: SqlStorageBackend, ) -> None: """Initialize a new instance. - Preferably use the `SqlRequestQueueClient.open` class method to create a new instance. + Preferably use the `SqlRequestQueueBackend.open` class method to create a new instance. """ - super().__init__(id=id, storage_client=storage_client) + super().__init__(id=id, storage_backend=storage_backend) self._pending_fetch_cache: deque[Request] = deque() """Cache for requests: ordered by sequence number.""" @@ -128,19 +128,19 @@ async def open( id: str | None, name: str | None, alias: str | None, - storage_client: SqlStorageClient, + storage_backend: SqlStorageBackend, ) -> Self: """Open an existing request queue or create a new one. This method first tries to find an existing queue by ID or name. - If found, it returns a client for that queue. If not found, it creates + If found, it returns a backend for that queue. If not found, it creates a new queue with the specified parameters. Args: id: The ID of the request queue to open. Takes precedence over name. name: The name of the request queue for named (global scope) storages. alias: The alias of the request queue for unnamed (run scope) storages. - storage_client: The SQL storage client used to access the database. + storage_backend: The SQL storage backend used to access the database. Returns: An instance for the opened or created request queue. @@ -152,7 +152,7 @@ async def open( id=id, name=name, alias=alias, - storage_client=storage_client, + storage_backend=storage_backend, metadata_model=RequestQueueMetadata, extra_metadata_fields={ 'had_multiple_clients': False, @@ -422,7 +422,7 @@ async def fetch_next_request(self) -> Request | None: now = datetime.now(UTC) block_until = now + timedelta(seconds=self._BLOCK_REQUEST_TIME) - dialect = self._storage_client.get_dialect_name() + dialect = self._storage_backend.get_dialect_name() # Get available requests not blocked by another client stmt = ( @@ -828,7 +828,7 @@ async def _apply_buffer_updates(self, session: AsyncSession, max_buffer_id: int) if not self._had_multiple_clients: aggregations.append(func.count(func.distinct(self._BUFFER_TABLE.client_id)).label('unique_clients_count')) - if self._storage_client.get_dialect_name() == 'postgresql': + if self._storage_backend.get_dialect_name() == 'postgresql': aggregations.append(func.bool_or(self._BUFFER_TABLE.need_recalc).label('need_recalc')) else: aggregations.append(func.max(self._BUFFER_TABLE.need_recalc).label('need_recalc')) diff --git a/src/crawlee/storage_clients/_sql/_storage_client.py b/src/crawlee/storage_backends/_sql/_storage_backend.py similarity index 86% rename from src/crawlee/storage_clients/_sql/_storage_client.py rename to src/crawlee/storage_backends/_sql/_storage_backend.py index a5ff864e3c..2c6def7b8b 100644 --- a/src/crawlee/storage_clients/_sql/_storage_client.py +++ b/src/crawlee/storage_backends/_sql/_storage_backend.py @@ -13,12 +13,12 @@ from crawlee._utils.docs import docs_group from crawlee.configuration import Configuration -from crawlee.storage_clients._base import StorageClient +from crawlee.storage_backends._base import StorageBackend -from ._dataset_client import SqlDatasetClient +from ._dataset_backend import SqlDatasetBackend from ._db_models import Base, VersionDb -from ._key_value_store_client import SqlKeyValueStoreClient -from ._request_queue_client import SqlRequestQueueClient +from ._key_value_store_backend import SqlKeyValueStoreBackend +from ._request_queue_backend import SqlRequestQueueBackend if TYPE_CHECKING: from types import TracebackType @@ -31,15 +31,15 @@ logger = getLogger(__name__) -@docs_group('Storage clients') -class SqlStorageClient(StorageClient): - """SQL implementation of the storage client. +@docs_group('Storage backends') +class SqlStorageBackend(StorageBackend): + """SQL implementation of the storage backend. - This storage client provides access to datasets, key-value stores, and request queues that persist data + This storage backend provides access to datasets, key-value stores, and request queues that persist data to a SQL database using SQLAlchemy 2+. Each storage type uses two tables: one for metadata and one for records. - The client accepts either a database connection string or a pre-configured AsyncEngine. If neither is + The backend accepts either a database connection string or a pre-configured AsyncEngine. If neither is provided, it creates a default SQLite database 'crawlee.db' in the storage directory. Database schema is automatically created during initialization. SQLite databases receive performance @@ -60,7 +60,7 @@ def __init__( connection_string: str | None = None, engine: AsyncEngine | None = None, ) -> None: - """Initialize the SQL storage client. + """Initialize the SQL storage backend. Args: connection_string: Database connection string (e.g., "sqlite+aiosqlite:///crawlee.db"). @@ -80,12 +80,12 @@ def __init__( # Call the notification only once warnings.warn( - 'The SqlStorageClient is experimental and may change or be removed in future releases.', + 'The SqlStorageBackend is experimental and may change or be removed in future releases.', category=UserWarning, stacklevel=2, ) - async def __aenter__(self) -> SqlStorageClient: + async def __aenter__(self) -> SqlStorageBackend: """Async context manager entry.""" return self @@ -113,7 +113,7 @@ async def initialize(self, configuration: Configuration) -> None: """Initialize the database schema. This method creates all necessary tables if they don't exist. - Should be called before using the storage client. + Should be called before using the storage backend. """ if not self._initialized: engine = self._get_or_create_engine(configuration) @@ -175,70 +175,70 @@ def create_session(self) -> AsyncSession: return self.session_maker() @override - async def create_dataset_client( + async def create_dataset_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> SqlDatasetClient: + ) -> SqlDatasetBackend: configuration = configuration or Configuration.get_global_configuration() await self.initialize(configuration) - client = await SqlDatasetClient.open( + backend = await SqlDatasetBackend.open( id=id, name=name, alias=alias, - storage_client=self, + storage_backend=self, ) - await self._purge_if_needed(client, configuration) - return client + await self._purge_if_needed(backend, configuration) + return backend @override - async def create_kvs_client( + async def create_kvs_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> SqlKeyValueStoreClient: + ) -> SqlKeyValueStoreBackend: configuration = configuration or Configuration.get_global_configuration() await self.initialize(configuration) - client = await SqlKeyValueStoreClient.open( + backend = await SqlKeyValueStoreBackend.open( id=id, name=name, alias=alias, - storage_client=self, + storage_backend=self, ) - await self._purge_if_needed(client, configuration) - return client + await self._purge_if_needed(backend, configuration) + return backend @override - async def create_rq_client( + async def create_rq_backend( self, *, id: str | None = None, name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - ) -> SqlRequestQueueClient: + ) -> SqlRequestQueueBackend: configuration = configuration or Configuration.get_global_configuration() await self.initialize(configuration) - client = await SqlRequestQueueClient.open( + backend = await SqlRequestQueueBackend.open( id=id, name=name, alias=alias, - storage_client=self, + storage_backend=self, ) - await self._purge_if_needed(client, configuration) - return client + await self._purge_if_needed(backend, configuration) + return backend def _get_or_create_engine(self, configuration: Configuration) -> AsyncEngine: """Get or create the database engine based on configuration.""" diff --git a/src/crawlee/storage_clients/_sql/py.typed b/src/crawlee/storage_backends/_sql/py.typed similarity index 100% rename from src/crawlee/storage_clients/_sql/py.typed rename to src/crawlee/storage_backends/_sql/py.typed diff --git a/src/crawlee/storage_clients/_utils.py b/src/crawlee/storage_backends/_utils.py similarity index 93% rename from src/crawlee/storage_clients/_utils.py rename to src/crawlee/storage_backends/_utils.py index 9fb65ba9ad..6600c900e2 100644 --- a/src/crawlee/storage_clients/_utils.py +++ b/src/crawlee/storage_backends/_utils.py @@ -5,7 +5,7 @@ if TYPE_CHECKING: from collections.abc import Iterable, Iterator - from crawlee.storage_clients.models import KeyValueStoreRecordMetadata + from crawlee.storage_backends.models import KeyValueStoreRecordMetadata def batch_records_by_size( diff --git a/src/crawlee/storage_clients/models.py b/src/crawlee/storage_backends/models.py similarity index 98% rename from src/crawlee/storage_clients/models.py rename to src/crawlee/storage_backends/models.py index 827d7bd4f6..f53e2f9a5a 100644 --- a/src/crawlee/storage_clients/models.py +++ b/src/crawlee/storage_backends/models.py @@ -161,7 +161,7 @@ class ProcessedRequest(BaseModel): ) id: Annotated[str | None, Field(alias='requestId')] = None - """Internal representation of the request by the storage client. Only some clients use id.""" + """Internal representation of the request by the storage backend. Only some backends use id.""" unique_key: str was_already_present: bool diff --git a/src/crawlee/storage_clients/py.typed b/src/crawlee/storage_backends/py.typed similarity index 100% rename from src/crawlee/storage_clients/py.typed rename to src/crawlee/storage_backends/py.typed diff --git a/src/crawlee/storage_clients/__init__.py b/src/crawlee/storage_clients/__init__.py deleted file mode 100644 index bfb229853d..0000000000 --- a/src/crawlee/storage_clients/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -from crawlee._utils.try_import import install_import_hook as _install_import_hook -from crawlee._utils.try_import import try_import as _try_import - -# These imports have only mandatory dependencies, so they are imported directly. -from ._base import StorageClient -from ._file_system import FileSystemStorageClient -from ._memory import MemoryStorageClient - -_install_import_hook(__name__) - -# The following imports are wrapped in try_import to handle optional dependencies, -# ensuring the module can still function even if these dependencies are missing. -with _try_import(__name__, 'SqlStorageClient', extra_name=['sql_sqlite', 'sql_postgres', 'sql_mysql']): - from ._sql import SqlStorageClient - -with _try_import(__name__, 'RedisStorageClient', extra_name='redis'): - from ._redis import RedisStorageClient - -__all__ = [ - 'FileSystemStorageClient', - 'MemoryStorageClient', - 'RedisStorageClient', - 'SqlStorageClient', - 'StorageClient', -] diff --git a/src/crawlee/storage_clients/_base/__init__.py b/src/crawlee/storage_clients/_base/__init__.py deleted file mode 100644 index 73298560da..0000000000 --- a/src/crawlee/storage_clients/_base/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from ._dataset_client import DatasetClient -from ._key_value_store_client import KeyValueStoreClient -from ._request_queue_client import RequestQueueClient -from ._storage_client import StorageClient - -__all__ = [ - 'DatasetClient', - 'KeyValueStoreClient', - 'RequestQueueClient', - 'StorageClient', -] diff --git a/src/crawlee/storage_clients/_file_system/__init__.py b/src/crawlee/storage_clients/_file_system/__init__.py deleted file mode 100644 index 2169896d86..0000000000 --- a/src/crawlee/storage_clients/_file_system/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from ._dataset_client import FileSystemDatasetClient -from ._key_value_store_client import FileSystemKeyValueStoreClient -from ._request_queue_client import FileSystemRequestQueueClient -from ._storage_client import FileSystemStorageClient - -__all__ = [ - 'FileSystemDatasetClient', - 'FileSystemKeyValueStoreClient', - 'FileSystemRequestQueueClient', - 'FileSystemStorageClient', -] diff --git a/src/crawlee/storage_clients/_file_system/_storage_client.py b/src/crawlee/storage_clients/_file_system/_storage_client.py deleted file mode 100644 index bc5ce31592..0000000000 --- a/src/crawlee/storage_clients/_file_system/_storage_client.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from typing_extensions import override - -from crawlee._utils.docs import docs_group -from crawlee.configuration import Configuration -from crawlee.storage_clients._base import StorageClient - -from ._dataset_client import FileSystemDatasetClient -from ._key_value_store_client import FileSystemKeyValueStoreClient -from ._request_queue_client import FileSystemRequestQueueClient - -if TYPE_CHECKING: - from collections.abc import Hashable - - -@docs_group('Storage clients') -class FileSystemStorageClient(StorageClient): - """File system implementation of the storage client. - - This storage client provides access to datasets, key-value stores, and request queues that persist data - to the local file system. Each storage type is implemented with its own specific file system client - that stores data in a structured directory hierarchy. - - Data is stored in JSON format in predictable file paths, making it easy to inspect and manipulate - the stored data outside of the Crawlee application if needed. - - All data persists between program runs but is limited to access from the local machine - where the files are stored. - - Warning: This storage client is not safe for concurrent access from multiple crawler processes. - Use it only when running a single crawler process at a time. - """ - - @override - def get_storage_client_cache_key(self, configuration: Configuration) -> Hashable: - # Even different client instances should return same storage if the storage_dir is the same. - return super().get_storage_client_cache_key(configuration), configuration.storage_dir - - @override - async def create_dataset_client( - self, - *, - id: str | None = None, - name: str | None = None, - alias: str | None = None, - configuration: Configuration | None = None, - ) -> FileSystemDatasetClient: - configuration = configuration or Configuration.get_global_configuration() - client = await FileSystemDatasetClient.open(id=id, name=name, alias=alias, configuration=configuration) - await self._purge_if_needed(client, configuration) - return client - - @override - async def create_kvs_client( - self, - *, - id: str | None = None, - name: str | None = None, - alias: str | None = None, - configuration: Configuration | None = None, - ) -> FileSystemKeyValueStoreClient: - configuration = configuration or Configuration.get_global_configuration() - client = await FileSystemKeyValueStoreClient.open(id=id, name=name, alias=alias, configuration=configuration) - await self._purge_if_needed(client, configuration) - return client - - @override - async def create_rq_client( - self, - *, - id: str | None = None, - name: str | None = None, - alias: str | None = None, - configuration: Configuration | None = None, - ) -> FileSystemRequestQueueClient: - configuration = configuration or Configuration.get_global_configuration() - client = await FileSystemRequestQueueClient.open(id=id, name=name, alias=alias, configuration=configuration) - await self._purge_if_needed(client, configuration) - return client diff --git a/src/crawlee/storage_clients/_memory/__init__.py b/src/crawlee/storage_clients/_memory/__init__.py deleted file mode 100644 index 3746907b4f..0000000000 --- a/src/crawlee/storage_clients/_memory/__init__.py +++ /dev/null @@ -1,11 +0,0 @@ -from ._dataset_client import MemoryDatasetClient -from ._key_value_store_client import MemoryKeyValueStoreClient -from ._request_queue_client import MemoryRequestQueueClient -from ._storage_client import MemoryStorageClient - -__all__ = [ - 'MemoryDatasetClient', - 'MemoryKeyValueStoreClient', - 'MemoryRequestQueueClient', - 'MemoryStorageClient', -] diff --git a/src/crawlee/storage_clients/_redis/__init__.py b/src/crawlee/storage_clients/_redis/__init__.py deleted file mode 100644 index 889023c137..0000000000 --- a/src/crawlee/storage_clients/_redis/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from ._dataset_client import RedisDatasetClient -from ._key_value_store_client import RedisKeyValueStoreClient -from ._request_queue_client import RedisRequestQueueClient -from ._storage_client import RedisStorageClient - -__all__ = ['RedisDatasetClient', 'RedisKeyValueStoreClient', 'RedisRequestQueueClient', 'RedisStorageClient'] diff --git a/src/crawlee/storage_clients/_sql/__init__.py b/src/crawlee/storage_clients/_sql/__init__.py deleted file mode 100644 index 56a6a2c717..0000000000 --- a/src/crawlee/storage_clients/_sql/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from ._dataset_client import SqlDatasetClient -from ._key_value_store_client import SqlKeyValueStoreClient -from ._request_queue_client import SqlRequestQueueClient -from ._storage_client import SqlStorageClient - -__all__ = ['SqlDatasetClient', 'SqlKeyValueStoreClient', 'SqlRequestQueueClient', 'SqlStorageClient'] diff --git a/src/crawlee/storages/_base.py b/src/crawlee/storages/_base.py index cd17007904..512ce0688a 100644 --- a/src/crawlee/storages/_base.py +++ b/src/crawlee/storages/_base.py @@ -7,8 +7,8 @@ if TYPE_CHECKING: from crawlee.configuration import Configuration - from crawlee.storage_clients._base import StorageClient - from crawlee.storage_clients.models import DatasetMetadata, KeyValueStoreMetadata, RequestQueueMetadata + from crawlee.storage_backends._base import StorageBackend + from crawlee.storage_backends.models import DatasetMetadata, KeyValueStoreMetadata, RequestQueueMetadata @docs_group('Storages') @@ -38,7 +38,7 @@ async def open( name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - storage_client: StorageClient | None = None, + storage_backend: StorageBackend | None = None, ) -> Storage: """Open a storage, either restore existing or create a new one. @@ -49,17 +49,17 @@ async def open( (e.g. "my-value-1"). alias: The storage alias (run scope, creates unnamed storage). configuration: Configuration object used during the storage creation or restoration process. - storage_client: Underlying storage client to use. If not provided, the default global storage client + storage_backend: Underlying storage backend to use. If not provided, the default global storage backend from the service locator will be used. """ @abstractmethod async def drop(self) -> None: - """Drop the storage, removing it from the underlying storage client and clearing the cache.""" + """Drop the storage, removing it from the underlying storage backend and clearing the cache.""" @abstractmethod async def purge(self) -> None: - """Purge the storage, removing all items from the underlying storage client. + """Purge the storage, removing all items from the underlying storage backend. This method does not remove the storage itself, e.g. don't remove the metadata, but clears all items within it. diff --git a/src/crawlee/storages/_dataset.py b/src/crawlee/storages/_dataset.py index 7d8e0f9d3c..3d3ad1565e 100644 --- a/src/crawlee/storages/_dataset.py +++ b/src/crawlee/storages/_dataset.py @@ -20,9 +20,9 @@ from crawlee._types import ExportDataCsvKwargs, ExportDataJsonKwargs, JsonSerializable from crawlee.configuration import Configuration - from crawlee.storage_clients import StorageClient - from crawlee.storage_clients._base import DatasetClient - from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata + from crawlee.storage_backends import StorageBackend + from crawlee.storage_backends._base import DatasetBackend + from crawlee.storage_backends.models import DatasetItemsListPage, DatasetMetadata logger = logging.getLogger(__name__) @@ -43,7 +43,7 @@ class Dataset(Storage): The class provides methods for adding data, retrieving data with various filtering options, and exporting data to different formats. You can create a dataset using the `open` class method, specifying either a name or ID. The underlying storage implementation is determined by - the configured storage client. + the configured storage backend. ### Usage @@ -64,19 +64,19 @@ class Dataset(Storage): ``` """ - def __init__(self, client: DatasetClient, id: str, name: str | None) -> None: + def __init__(self, backend: DatasetBackend, id: str, name: str | None) -> None: """Initialize a new instance. Preferably use the `Dataset.open` constructor to create a new instance. Args: - client: An instance of a storage client. + backend: The dataset backend to use. id: The unique identifier of the storage. name: The name of the storage, if available. """ validate_storage_name(name) - self._client = client + self._backend = backend self._id = id self._name = name @@ -92,7 +92,7 @@ def name(self) -> str | None: @override async def get_metadata(self) -> DatasetMetadata: - return await self._client.get_metadata() + return await self._backend.get_metadata() @override @classmethod @@ -103,39 +103,39 @@ async def open( name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - storage_client: StorageClient | None = None, + storage_backend: StorageBackend | None = None, ) -> Dataset: configuration = service_locator.get_configuration() if configuration is None else configuration - storage_client = service_locator.get_storage_client() if storage_client is None else storage_client + storage_backend = service_locator.get_storage_backend() if storage_backend is None else storage_backend - client_opener_coro = storage_client.create_dataset_client( + backend_opener_coro = storage_backend.create_dataset_backend( id=id, name=name, alias=alias, configuration=configuration ) - storage_client_cache_key = storage_client.get_storage_client_cache_key(configuration=configuration) + storage_backend_cache_key = storage_backend.get_storage_backend_cache_key(configuration=configuration) return await service_locator.storage_instance_manager.open_storage_instance( cls, id=id, name=name, alias=alias, - client_opener_coro=client_opener_coro, - storage_client_cache_key=storage_client_cache_key, + backend_opener_coro=backend_opener_coro, + storage_backend_cache_key=storage_backend_cache_key, ) @override async def drop(self) -> None: storage_instance_manager = service_locator.storage_instance_manager storage_instance_manager.remove_from_cache(self) - await self._client.drop() + await self._backend.drop() @override async def purge(self) -> None: - await self._client.purge() + await self._backend.purge() async def push_data(self, data: Sequence[Mapping[str, JsonSerializable]] | Mapping[str, JsonSerializable]) -> None: """Store an object or an array of objects to the dataset. - Local storage clients impose no size limit on the pushed data. Some remote storage clients do, + Local storage backends impose no size limit on the pushed data. Some remote storage backends do, however. For example, when running on the Apify platform, the receiving API only accepts objects whose JSON representation is smaller than 9MB, and each item of a passed array is subject to the same per-item limit (the array itself may be of any size). @@ -143,7 +143,7 @@ async def push_data(self, data: Sequence[Mapping[str, JsonSerializable]] | Mappi Args: data: A JSON serializable data structure to be stored in the dataset. """ - await self._client.push_data(data=data) + await self._backend.push_data(data=data) async def get_data( self, @@ -182,7 +182,7 @@ async def get_data( Returns: An object with filtered, sorted, and paginated dataset items plus pagination details. """ - return await self._client.get_data( + return await self._backend.get_data( offset=offset, limit=limit, clean=clean, @@ -231,7 +231,7 @@ async def iterate_items( An asynchronous iterator of dictionary objects, each representing a dataset item after applying the specified filters and transformations. """ - async for item in self._client.iterate_items( + async for item in self._backend.iterate_items( offset=offset, limit=limit, clean=clean, @@ -317,7 +317,7 @@ async def export_to( content_type: Literal['json'], to_kvs_id: str | None = None, to_kvs_name: str | None = None, - to_kvs_storage_client: StorageClient | None = None, + to_kvs_storage_backend: StorageBackend | None = None, to_kvs_configuration: Configuration | None = None, **kwargs: Unpack[ExportDataJsonKwargs], ) -> None: ... @@ -330,7 +330,7 @@ async def export_to( content_type: Literal['csv'], to_kvs_id: str | None = None, to_kvs_name: str | None = None, - to_kvs_storage_client: StorageClient | None = None, + to_kvs_storage_backend: StorageBackend | None = None, to_kvs_configuration: Configuration | None = None, collect_all_keys: bool = False, **kwargs: Unpack[ExportDataCsvKwargs], @@ -343,7 +343,7 @@ async def export_to( content_type: Literal['json', 'csv'] = 'json', to_kvs_id: str | None = None, to_kvs_name: str | None = None, - to_kvs_storage_client: StorageClient | None = None, + to_kvs_storage_backend: StorageBackend | None = None, to_kvs_configuration: Configuration | None = None, **kwargs: Any, ) -> None: @@ -361,7 +361,7 @@ async def export_to( Specify only one of ID or name. to_kvs_name: Name of the key-value store to save the exported file. Specify only one of ID or name. - to_kvs_storage_client: Storage client to use for the key-value store. + to_kvs_storage_backend: Storage backend to use for the key-value store. to_kvs_configuration: Configuration for the key-value store. collect_all_keys: CSV only. When True, the columns are the keys of all items combined, so no value is dropped. When False, the columns are the keys of the first non-empty item, and keys introduced by @@ -372,7 +372,7 @@ async def export_to( id=to_kvs_id, name=to_kvs_name, configuration=to_kvs_configuration, - storage_client=to_kvs_storage_client, + storage_backend=to_kvs_storage_backend, ) dst = StringIO() diff --git a/src/crawlee/storages/_key_value_store.py b/src/crawlee/storages/_key_value_store.py index bb06642f67..8f5883bd10 100644 --- a/src/crawlee/storages/_key_value_store.py +++ b/src/crawlee/storages/_key_value_store.py @@ -12,7 +12,7 @@ from crawlee._types import JsonSerializable # noqa: TC001 from crawlee._utils.docs import docs_group from crawlee._utils.recoverable_state import RecoverableState -from crawlee.storage_clients.models import KeyValueStoreMetadata +from crawlee.storage_backends.models import KeyValueStoreMetadata from ._base import Storage from ._utils import validate_storage_name @@ -21,9 +21,9 @@ from collections.abc import AsyncIterator from crawlee.configuration import Configuration - from crawlee.storage_clients import StorageClient - from crawlee.storage_clients._base import KeyValueStoreClient - from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecordMetadata + from crawlee.storage_backends import StorageBackend + from crawlee.storage_backends._base import KeyValueStoreBackend + from crawlee.storage_backends.models import KeyValueStoreMetadata, KeyValueStoreRecordMetadata else: from crawlee._utils.recoverable_state import RecoverableState @@ -51,7 +51,7 @@ class KeyValueStore(Storage): You can instantiate a key-value store using the `open` class method, which will create a store with the specified name or id. The underlying storage implementation is determined by the configured - storage client. + storage backend. ### Usage @@ -75,19 +75,19 @@ class KeyValueStore(Storage): ] = {} """Cache for recoverable (auto-saved) values.""" - def __init__(self, client: KeyValueStoreClient, id: str, name: str | None) -> None: + def __init__(self, backend: KeyValueStoreBackend, id: str, name: str | None) -> None: """Initialize a new instance. Preferably use the `KeyValueStore.open` constructor to create a new instance. Args: - client: An instance of a storage client. + backend: The key-value store backend to use. id: The unique identifier of the storage. name: The name of the storage, if available. """ validate_storage_name(name) - self._client = client + self._backend = backend self._id = id self._name = name @@ -106,7 +106,7 @@ def name(self) -> str | None: @override async def get_metadata(self) -> KeyValueStoreMetadata: - return await self._client.get_metadata() + return await self._backend.get_metadata() @override @classmethod @@ -117,23 +117,23 @@ async def open( name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - storage_client: StorageClient | None = None, + storage_backend: StorageBackend | None = None, ) -> KeyValueStore: configuration = service_locator.get_configuration() if configuration is None else configuration - storage_client = service_locator.get_storage_client() if storage_client is None else storage_client + storage_backend = service_locator.get_storage_backend() if storage_backend is None else storage_backend - client_opener_coro = storage_client.create_kvs_client( + backend_opener_coro = storage_backend.create_kvs_backend( id=id, name=name, alias=alias, configuration=configuration ) - additional_cache_key = storage_client.get_storage_client_cache_key(configuration=configuration) + additional_cache_key = storage_backend.get_storage_backend_cache_key(configuration=configuration) return await service_locator.storage_instance_manager.open_storage_instance( cls, id=id, name=name, alias=alias, - client_opener_coro=client_opener_coro, - storage_client_cache_key=additional_cache_key, + backend_opener_coro=backend_opener_coro, + storage_backend_cache_key=additional_cache_key, ) @override @@ -142,11 +142,11 @@ async def drop(self) -> None: storage_instance_manager.remove_from_cache(self) await self._clear_cache() # Clear cache with persistent values. - await self._client.drop() + await self._backend.drop() @override async def purge(self) -> None: - await self._client.purge() + await self._backend.purge() @overload async def get_value(self, key: str) -> Any: ... @@ -167,7 +167,7 @@ async def get_value(self, key: str, default_value: T | None = None) -> T | None: Returns: The value associated with the given key. `default_value` is used in case the record does not exist. """ - record = await self._client.get_value(key=key) + record = await self._backend.get_value(key=key) return record.value if record else default_value async def set_value( @@ -184,7 +184,7 @@ async def set_value( value: Value to set. content_type: The MIME content type string. """ - await self._client.set_value(key=key, value=value, content_type=content_type) + await self._backend.set_value(key=key, value=value, content_type=content_type) async def delete_value(self, key: str) -> None: """Delete a value from the KVS. @@ -192,7 +192,7 @@ async def delete_value(self, key: str) -> None: Args: key: Key of the record to delete. """ - await self._client.delete_value(key=key) + await self._backend.delete_value(key=key) async def iterate_keys( self, @@ -209,7 +209,7 @@ async def iterate_keys( Yields: Information about the key. """ - async for item in self._client.iterate_keys( + async for item in self._backend.iterate_keys( exclusive_start_key=exclusive_start_key, limit=limit, ): @@ -225,7 +225,7 @@ async def iterate_values( The records are fetched lazily as the iteration advances, so only a bounded number of values is held in memory at a time. By default this means one request per record on top of the key listing, unless the storage - client reads the values in bounded batches instead. + backend reads the values in bounded batches instead. Args: exclusive_start_key: Key to start the iteration from. @@ -247,7 +247,7 @@ async def iterate_entries( The records are fetched lazily as the iteration advances, so only a bounded number of values is held in memory at a time. By default this means one request per record on top of the key listing, unless the storage - client reads the values in bounded batches instead. A record deleted while the iteration is in + backend reads the values in bounded batches instead. A record deleted while the iteration is in progress may or may not be yielded, depending on whether its value was already read. Args: @@ -257,7 +257,7 @@ async def iterate_entries( Yields: A `(key, value)` tuple for each record. """ - async for record in self._client.iterate_entries(exclusive_start_key=exclusive_start_key, limit=limit): + async for record in self._backend.iterate_entries(exclusive_start_key=exclusive_start_key, limit=limit): yield record.key, record.value def __aiter__(self) -> AsyncIterator[str]: @@ -284,7 +284,7 @@ async def list_keys( ) -> list[KeyValueStoreRecordMetadata]: """List all the existing keys in the KVS. - It uses client's `iterate_keys` method to get the keys. + It uses the backend's `iterate_keys` method to get the keys. Args: exclusive_start_key: Key to start the iteration from. @@ -295,7 +295,7 @@ async def list_keys( """ return [ key - async for key in self._client.iterate_keys( + async for key in self._backend.iterate_keys( exclusive_start_key=exclusive_start_key, limit=limit, ) @@ -310,7 +310,7 @@ async def record_exists(self, key: str) -> bool: Returns: True if a record with the given key exists, False otherwise. """ - return await self._client.record_exists(key=key) + return await self._backend.record_exists(key=key) async def get_public_url(self, key: str) -> str: """Get the public URL for the given key. @@ -321,7 +321,7 @@ async def get_public_url(self, key: str) -> str: Returns: The public URL for the given key. """ - return await self._client.get_public_url(key=key) + return await self._backend.get_public_url(key=key) async def get_auto_saved_value( self, diff --git a/src/crawlee/storages/_request_queue.py b/src/crawlee/storages/_request_queue.py index cb13f79a5e..a2f4ca6ad6 100644 --- a/src/crawlee/storages/_request_queue.py +++ b/src/crawlee/storages/_request_queue.py @@ -11,7 +11,7 @@ from crawlee._utils.docs import docs_group from crawlee._utils.wait import wait_for_all_tasks_to_finish from crawlee.request_loaders import FinishedStatus, ReadyStatus, RequestManager, WaitingStatus -from crawlee.storage_clients.models import AddRequestsResponse +from crawlee.storage_backends.models import AddRequestsResponse from ._base import Storage from ._utils import validate_storage_name @@ -22,9 +22,9 @@ from crawlee import Request from crawlee.configuration import Configuration from crawlee.request_loaders import RequestLoaderStatus - from crawlee.storage_clients import StorageClient - from crawlee.storage_clients._base import RequestQueueClient - from crawlee.storage_clients.models import ProcessedRequest, RequestQueueMetadata + from crawlee.storage_backends import StorageBackend + from crawlee.storage_backends._base import RequestQueueBackend + from crawlee.storage_backends.models import ProcessedRequest, RequestQueueMetadata logger = getLogger(__name__) @@ -48,7 +48,7 @@ class RequestQueue(Storage, RequestManager): processing fails. You can open a request queue using the `open` class method, specifying either a name or ID to identify - the queue. The underlying storage implementation is determined by the configured storage client. + the queue. The underlying storage implementation is determined by the configured storage backend. ### Usage @@ -73,19 +73,19 @@ class RequestQueue(Storage, RequestManager): ``` """ - def __init__(self, client: RequestQueueClient, id: str, name: str | None) -> None: + def __init__(self, backend: RequestQueueBackend, id: str, name: str | None) -> None: """Initialize a new instance. Preferably use the `RequestQueue.open` constructor to create a new instance. Args: - client: An instance of a storage client. + backend: The request queue backend to use. id: The unique identifier of the storage. name: The name of the storage, if available. """ validate_storage_name(name) - self._client = client + self._backend = backend self._id = id self._name = name @@ -104,16 +104,16 @@ def name(self) -> str | None: @override async def get_metadata(self) -> RequestQueueMetadata: - return await self._client.get_metadata() + return await self._backend.get_metadata() @override async def get_handled_count(self) -> int: - metadata = await self._client.get_metadata() + metadata = await self._backend.get_metadata() return metadata.handled_request_count @override async def get_total_count(self) -> int: - metadata = await self._client.get_metadata() + metadata = await self._backend.get_metadata() return metadata.total_request_count @override @@ -125,21 +125,23 @@ async def open( name: str | None = None, alias: str | None = None, configuration: Configuration | None = None, - storage_client: StorageClient | None = None, + storage_backend: StorageBackend | None = None, ) -> RequestQueue: configuration = service_locator.get_configuration() if configuration is None else configuration - storage_client = service_locator.get_storage_client() if storage_client is None else storage_client + storage_backend = service_locator.get_storage_backend() if storage_backend is None else storage_backend - client_opener_coro = storage_client.create_rq_client(id=id, name=name, alias=alias, configuration=configuration) - additional_cache_key = storage_client.get_storage_client_cache_key(configuration=configuration) + backend_opener_coro = storage_backend.create_rq_backend( + id=id, name=name, alias=alias, configuration=configuration + ) + additional_cache_key = storage_backend.get_storage_backend_cache_key(configuration=configuration) return await service_locator.storage_instance_manager.open_storage_instance( cls, id=id, name=name, alias=alias, - client_opener_coro=client_opener_coro, - storage_client_cache_key=additional_cache_key, + backend_opener_coro=backend_opener_coro, + storage_backend_cache_key=additional_cache_key, ) @override @@ -148,11 +150,11 @@ async def drop(self) -> None: storage_instance_manager = service_locator.storage_instance_manager storage_instance_manager.remove_from_cache(self) - await self._client.drop() + await self._backend.drop() @override async def purge(self) -> None: - await self._client.purge() + await self._backend.purge() @override async def add_request( @@ -172,8 +174,8 @@ async def add_request( # (neither processed nor unprocessed) is unexpected here. if not response.unprocessed_requests: logger.warning( - f'Request {request.url} was not processed by storage client ' - f'"{self._client.__class__.__name__}" (received an empty response).' + f'Request {request.url} was not processed by storage backend ' + f'"{self._backend.__class__.__name__}" (received an empty response).' ) return None @@ -243,7 +245,7 @@ async def fetch_next_request(self) -> Request | None: Returns: The next request to process, or `None` if there are no more pending requests. """ - return await self._client.fetch_next_request() + return await self._backend.fetch_next_request() async def get_request(self, unique_key: str) -> Request | None: """Retrieve a specific request from the queue by its ID. @@ -254,7 +256,7 @@ async def get_request(self, unique_key: str) -> Request | None: Returns: The request with the specified ID, or `None` if no such request exists. """ - return await self._client.get_request(unique_key) + return await self._backend.get_request(unique_key) async def mark_request_as_handled(self, request: Request) -> ProcessedRequest | None: """Mark a request as handled after successful processing. @@ -269,7 +271,7 @@ async def mark_request_as_handled(self, request: Request) -> ProcessedRequest | Returns: Information about the queue operation. """ - return await self._client.mark_request_as_handled(request) + return await self._backend.mark_request_as_handled(request) async def reclaim_request( self, @@ -291,7 +293,7 @@ async def reclaim_request( Returns: Information about the queue operation. """ - return await self._client.reclaim_request(request, forefront=forefront) + return await self._backend.reclaim_request(request, forefront=forefront) @override async def check_readiness(self) -> RequestLoaderStatus: @@ -301,13 +303,13 @@ async def check_readiness(self) -> RequestLoaderStatus: still being added in the background. Neither has a clock, so `WaitingStatus.ready_at` stays `None`. """ # A fetchable request outranks everything below, so this is the only backend call a ready queue needs. - if not await self._client.is_empty(): + if not await self._backend.is_empty(): return ReadyStatus() if self._add_requests_tasks: return WaitingStatus() - if await self._client.is_finished(): + if await self._backend.is_finished(): return FinishedStatus() return WaitingStatus() @@ -327,7 +329,7 @@ async def _process_batch( retries are exhausted. """ max_attempts = 5 - response = await self._client.add_batch_of_requests(batch, forefront=forefront) + response = await self._backend.add_batch_of_requests(batch, forefront=forefront) request_count = len(batch) - len(response.unprocessed_requests) if request_count: diff --git a/src/crawlee/storages/_storage_instance_manager.py b/src/crawlee/storages/_storage_instance_manager.py index 05b6eeb081..e7a7b70708 100644 --- a/src/crawlee/storages/_storage_instance_manager.py +++ b/src/crawlee/storages/_storage_instance_manager.py @@ -8,7 +8,7 @@ from weakref import WeakValueDictionary from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs -from crawlee.storage_clients._base import DatasetClient, KeyValueStoreClient, RequestQueueClient +from crawlee.storage_backends._base import DatasetBackend, KeyValueStoreBackend, RequestQueueBackend from ._utils import validate_storage_alias, validate_storage_name @@ -62,8 +62,8 @@ def remove_from_cache(self, storage_instance: Storage) -> None: del self.by_alias[storage_type][alias_key][additional_key] -ClientOpenerCoro = Coroutine[None, None, DatasetClient | KeyValueStoreClient | RequestQueueClient] -"""Type alias for the client opener function.""" +BackendOpenerCoro = Coroutine[None, None, DatasetBackend | KeyValueStoreBackend | RequestQueueBackend] +"""Type alias for the backend opener function.""" class StorageInstanceManager: @@ -87,8 +87,8 @@ async def open_storage_instance( id: str | None, name: str | None, alias: str | None, - client_opener_coro: ClientOpenerCoro, - storage_client_cache_key: Hashable = '', + backend_opener_coro: BackendOpenerCoro, + storage_backend_cache_key: Hashable = '', ) -> T: """Open a storage instance with caching support. @@ -99,8 +99,8 @@ async def open_storage_instance( the digits "0" through "9", and the hyphen ("-") but only in the middle of the string (e.g. "my-value-1"). alias: Storage alias (run scope, creates unnamed storage). - client_opener_coro: Coroutine to open the storage client when storage instance not found in cache. - storage_client_cache_key: Additional optional key from storage client to differentiate cache entries. + backend_opener_coro: Coroutine to open the dataset, KVS or RQ backend when the instance is not cached. + storage_backend_cache_key: Additional optional key from storage backend to differentiate cache entries. Returns: The storage instance. @@ -127,7 +127,7 @@ async def open_storage_instance( id=id, name=name, alias=alias, - storage_client_cache_key=storage_client_cache_key, + storage_backend_cache_key=storage_backend_cache_key, ): return cached_instance @@ -140,7 +140,7 @@ async def open_storage_instance( validate_storage_alias(alias) # Acquire lock for this opener - opener_lock_key = (cls, str(id or name or alias), storage_client_cache_key) + opener_lock_key = (cls, str(id or name or alias), storage_backend_cache_key) if not (lock := self._opener_locks.get(opener_lock_key)): lock = Lock() self._opener_locks[opener_lock_key] = lock @@ -153,7 +153,7 @@ async def open_storage_instance( id=id, name=name, alias=alias, - storage_client_cache_key=storage_client_cache_key, + storage_backend_cache_key=storage_backend_cache_key, ): return cached_instance @@ -162,16 +162,16 @@ async def open_storage_instance( cls, name=name, alias=alias, - storage_client_cache_key=storage_client_cache_key, + storage_backend_cache_key=storage_backend_cache_key, ) # Create new instance - client: KeyValueStoreClient | DatasetClient | RequestQueueClient - client = await client_opener_coro + backend: KeyValueStoreBackend | DatasetBackend | RequestQueueBackend + backend = await backend_opener_coro - metadata = await client.get_metadata() + metadata = await backend.get_metadata() - instance = cls(client, metadata.id, metadata.name) # ty: ignore[too-many-positional-arguments] + instance = cls(backend, metadata.id, metadata.name) # ty: ignore[too-many-positional-arguments] instance_name = getattr(instance, 'name', None) # Cache the instance. @@ -179,22 +179,22 @@ async def open_storage_instance( # atomically to ensure pre-checks outside the lock see consistent state. # Always cache by id. - self._cache.by_id[cls][instance.id][storage_client_cache_key] = instance + self._cache.by_id[cls][instance.id][storage_backend_cache_key] = instance # Cache named storage. if instance_name is not None: - self._cache.by_name[cls][instance_name][storage_client_cache_key] = instance + self._cache.by_name[cls][instance_name][storage_backend_cache_key] = instance # Cache unnamed storage. if alias is not None: - self._cache.by_alias[cls][alias][storage_client_cache_key] = instance + self._cache.by_alias[cls][alias][storage_backend_cache_key] = instance return instance finally: - # Make sure the client opener is closed. + # Make sure the backend opener is closed. # If it was awaited, then closing is no operation, if it was not awaited, this is the cleanup. - client_opener_coro.close() + backend_opener_coro.close() def remove_from_cache(self, storage_instance: Storage) -> None: """Remove a storage instance from the cache. @@ -215,20 +215,20 @@ def _get_from_cache( id: str | None = None, name: str | None = None, alias: str | None = None, - storage_client_cache_key: Hashable = '', + storage_backend_cache_key: Hashable = '', ) -> T | None: """Get a storage instance from the cache.""" - if id is not None and (cached_instance := self._cache.by_id[cls][id].get(storage_client_cache_key)): + if id is not None and (cached_instance := self._cache.by_id[cls][id].get(storage_backend_cache_key)): if isinstance(cached_instance, cls): return cached_instance raise RuntimeError('Cached instance type mismatch.') - if name is not None and (cached_instance := self._cache.by_name[cls][name].get(storage_client_cache_key)): + if name is not None and (cached_instance := self._cache.by_name[cls][name].get(storage_backend_cache_key)): if isinstance(cached_instance, cls): return cached_instance raise RuntimeError('Cached instance type mismatch.') - if alias is not None and (cached_instance := self._cache.by_alias[cls][alias].get(storage_client_cache_key)): + if alias is not None and (cached_instance := self._cache.by_alias[cls][alias].get(storage_backend_cache_key)): if isinstance(cached_instance, cls): return cached_instance raise RuntimeError('Cached instance type mismatch.') @@ -241,16 +241,16 @@ def _check_name_alias_conflict( *, name: str | None = None, alias: str | None = None, - storage_client_cache_key: Hashable = '', + storage_backend_cache_key: Hashable = '', ) -> None: """Check for conflicts between named and alias storages.""" - if alias and (self._cache.by_name[cls][alias].get(storage_client_cache_key)): + if alias and (self._cache.by_name[cls][alias].get(storage_backend_cache_key)): raise ValueError( f'Cannot create alias storage "{alias}" because a named storage with the same name already exists. ' f'Use a different alias or drop the existing named storage first.' ) - if name and (self._cache.by_alias[cls][name].get(storage_client_cache_key)): + if name and (self._cache.by_alias[cls][name].get(storage_backend_cache_key)): raise ValueError( f'Cannot create named storage "{name}" because an alias storage with the same name already exists. ' f'Use a different name or drop the existing alias storage first.' diff --git a/tests/unit/_utils/test_file.py b/tests/unit/_utils/test_file.py index 54c31d2a32..244fca87d5 100644 --- a/tests/unit/_utils/test_file.py +++ b/tests/unit/_utils/test_file.py @@ -153,7 +153,7 @@ async def test_export_csv_to_stream_warns_about_dropped_keys(caplog: pytest.LogC async def test_export_csv_to_stream_warns_about_dropped_keys_that_are_not_strings( caplog: pytest.LogCaptureFixture, ) -> None: - """Dropped keys are reported even when they are not strings, which a storage client may hand over as pushed.""" + """Dropped keys are reported even when they are not strings, which a storage backend may hand over as pushed.""" dst = StringIO() items = cast('list[Mapping[str, JsonSerializable]]', [{'name': 'Alice'}, {1: 'x', 'city': 'NYC'}]) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 4179cb0f4a..d14fbf4217 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -36,7 +36,7 @@ async def suppress_user_warning() -> AsyncGenerator[None, None]: """Suppress user warnings during tests. - Mostly to suppress warnings about the experimental status of the SqlStorageClient. + Mostly to suppress warnings about the experimental status of the SqlStorageBackend. """ with warnings.catch_warnings(): warnings.simplefilter('ignore', UserWarning) @@ -69,7 +69,7 @@ def _prepare_test_env() -> None: # Reset the services in the service locator. service_locator._configuration = None service_locator._event_manager = None - service_locator._storage_client = None + service_locator._storage_backend = None service_locator.storage_instance_manager.clear_cache() # Verify that the test environment was set up correctly. diff --git a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py index 6e2b212123..b1d843ed27 100644 --- a/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py +++ b/tests/unit/crawlers/_adaptive_playwright/test_adaptive_playwright_crawler.py @@ -35,7 +35,7 @@ from crawlee.http_clients import HttpClient, HttpCrawlingResult from crawlee.sessions import SessionPool from crawlee.statistics import Statistics -from crawlee.storage_clients import SqlStorageClient +from crawlee.storage_backends import SqlStorageBackend from crawlee.storages import KeyValueStore, RequestQueue from tests.unit.utils import open_throttler_held_until_reclaim @@ -923,12 +923,12 @@ async def request_handler(context: AdaptivePlaywrightCrawlingContext) -> None: async def test_adaptive_playwright_crawler_with_sql_storage(test_urls: list[str], tmp_path: Path) -> None: - """Tests that AdaptivePlaywrightCrawler can be initialized with SqlStorageClient.""" + """Tests that AdaptivePlaywrightCrawler can be initialized with SqlStorageBackend.""" storage_dir = tmp_path / 'test_table.db' - async with SqlStorageClient(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_client: + async with SqlStorageBackend(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_backend: crawler = AdaptivePlaywrightCrawler.with_beautifulsoup_static_parser( - storage_client=storage_client, + storage_backend=storage_backend, ) mocked_handler = Mock() diff --git a/tests/unit/crawlers/_basic/test_basic_crawler.py b/tests/unit/crawlers/_basic/test_basic_crawler.py index 3b3e2d5fa6..67981ee2f7 100644 --- a/tests/unit/crawlers/_basic/test_basic_crawler.py +++ b/tests/unit/crawlers/_basic/test_basic_crawler.py @@ -48,7 +48,7 @@ ) from crawlee.sessions import Session, SessionPool from crawlee.statistics import FinalStatistics, Statistics, StatisticsState -from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient +from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend from crawlee.storages import Dataset, KeyValueStore, RequestQueue from tests.unit.utils import open_throttler_held_until_reclaim, poll_until_condition @@ -559,9 +559,9 @@ async def failed_request_handler(context: BasicCrawlingContext, error: Exception async def test_handlers_use_context_helpers(tmp_path: Path, handler: str) -> None: """Test that context helpers used in `failed_request_handler` and in `error_handler` have effect.""" # Prepare crawler - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() crawler = BasicCrawler( - max_request_retries=1, storage_client=storage_client, configuration=Configuration(storage_dir=str(tmp_path)) + max_request_retries=1, storage_backend=storage_backend, configuration=Configuration(storage_dir=str(tmp_path)) ) # Test data rq_alias = 'other' @@ -586,9 +586,9 @@ async def handler_implementation(context: BasicCrawlingContext, error: Exception await crawler.run(['https://b.placeholder.com']) # Verify that the context helpers used in handlers had effect on used storages - dataset = await Dataset.open(storage_client=storage_client) - kvs = await KeyValueStore.open(storage_client=storage_client) - rq = await RequestQueue.open(alias=rq_alias, storage_client=storage_client) + dataset = await Dataset.open(storage_backend=storage_backend) + kvs = await KeyValueStore.open(storage_backend=storage_backend) + rq = await RequestQueue.open(alias=rq_alias, storage_backend=storage_backend) assert test_value == await kvs.get_value(test_key) assert [test_data] == (await dataset.get_data()).items @@ -1900,17 +1900,17 @@ async def test_stop_before_run_starts() -> None: async def test_services_no_side_effect_on_crawler_init() -> None: custom_configuration = Configuration() custom_event_manager = LocalEventManager.from_config(custom_configuration) - custom_storage_client = MemoryStorageClient() + custom_storage_backend = MemoryStorageBackend() _ = BasicCrawler( configuration=custom_configuration, event_manager=custom_event_manager, - storage_client=custom_storage_client, + storage_backend=custom_storage_backend, ) assert service_locator.get_configuration() is not custom_configuration assert service_locator.get_event_manager() is not custom_event_manager - assert service_locator.get_storage_client() is not custom_storage_client + assert service_locator.get_storage_backend() is not custom_storage_backend async def test_crawler_uses_default_services() -> None: @@ -1920,35 +1920,35 @@ async def test_crawler_uses_default_services() -> None: custom_event_manager = LocalEventManager.from_config(custom_configuration) service_locator.set_event_manager(custom_event_manager) - custom_storage_client = MemoryStorageClient() - service_locator.set_storage_client(custom_storage_client) + custom_storage_backend = MemoryStorageBackend() + service_locator.set_storage_backend(custom_storage_backend) basic_crawler = BasicCrawler() assert basic_crawler._service_locator.get_configuration() is custom_configuration assert basic_crawler._service_locator.get_event_manager() is custom_event_manager - assert basic_crawler._service_locator.get_storage_client() is custom_storage_client + assert basic_crawler._service_locator.get_storage_backend() is custom_storage_backend async def test_services_crawlers_can_use_different_services() -> None: custom_configuration_1 = Configuration() custom_event_manager_1 = LocalEventManager.from_config(custom_configuration_1) - custom_storage_client_1 = MemoryStorageClient() + custom_storage_backend_1 = MemoryStorageBackend() custom_configuration_2 = Configuration() custom_event_manager_2 = LocalEventManager.from_config(custom_configuration_2) - custom_storage_client_2 = MemoryStorageClient() + custom_storage_backend_2 = MemoryStorageBackend() _ = BasicCrawler( configuration=custom_configuration_1, event_manager=custom_event_manager_1, - storage_client=custom_storage_client_1, + storage_backend=custom_storage_backend_1, ) _ = BasicCrawler( configuration=custom_configuration_2, event_manager=custom_event_manager_2, - storage_client=custom_storage_client_2, + storage_backend=custom_storage_backend_2, ) @@ -1981,7 +1981,7 @@ async def test_crawler_can_use_other_storages(tmp_path: Path) -> None: kvs = await KeyValueStore.open() rq = await RequestQueue.open() - crawler = BasicCrawler(storage_client=MemoryStorageClient()) + crawler = BasicCrawler(storage_backend=MemoryStorageBackend()) assert dataset is not await crawler.get_dataset() assert kvs is not await crawler.get_key_value_store() @@ -2010,14 +2010,14 @@ async def test_crawler_can_use_other_storages_of_same_type(tmp_path: Path) -> No # Set global configuration service_locator.set_configuration(configuration_a) - service_locator.set_storage_client(FileSystemStorageClient()) + service_locator.set_storage_backend(FileSystemStorageBackend()) # Create storages based on the global services dataset = await Dataset.open() kvs = await KeyValueStore.open() rq = await RequestQueue.open() - # Set the crawler to use different storage client - crawler = BasicCrawler(storage_client=FileSystemStorageClient(), configuration=configuration_b) + # Set the crawler to use different storage backend + crawler = BasicCrawler(storage_backend=FileSystemStorageBackend(), configuration=configuration_b) # Assert that the storages are different assert dataset is not await crawler.get_dataset() @@ -2029,23 +2029,23 @@ async def test_crawler_can_use_other_storages_of_same_type(tmp_path: Path) -> No assert path.is_dir() -async def test_allows_storage_client_overwrite_before_run(monkeypatch: pytest.MonkeyPatch) -> None: - custom_storage_client = MemoryStorageClient() +async def test_allows_storage_backend_overwrite_before_run(monkeypatch: pytest.MonkeyPatch) -> None: + custom_storage_backend = MemoryStorageBackend() crawler = BasicCrawler( - storage_client=custom_storage_client, + storage_backend=custom_storage_backend, ) @crawler.router.default_handler async def handler(context: BasicCrawlingContext) -> None: await context.push_data({'foo': 'bar'}) - other_storage_client = MemoryStorageClient() - service_locator.set_storage_client(other_storage_client) + other_storage_backend = MemoryStorageBackend() + service_locator.set_storage_backend(other_storage_backend) with monkeypatch.context() as monkey: - spy = Mock(wraps=service_locator.get_storage_client) - monkey.setattr(service_locator, 'get_storage_client', spy) + spy = Mock(wraps=service_locator.get_storage_backend) + monkey.setattr(service_locator, 'get_storage_backend', spy) await crawler.run(['https://does-not-matter.com']) assert spy.call_count >= 1 @@ -2105,7 +2105,7 @@ async def test_timeout_in_handler(sleep_type: str) -> None: crawler = BasicCrawler( request_handler_timeout=handler_timeout, max_request_retries=max_request_retries, - storage_client=MemoryStorageClient(), + storage_backend=MemoryStorageBackend(), ) mocked_handler_before_sleep = Mock() @@ -2155,7 +2155,7 @@ async def test_keep_alive( max_requests_per_crawl=max_requests_per_crawl, # If more request can run in parallel, then max_requests_per_crawl is not deterministic. concurrency_settings=ConcurrencySettings(desired_concurrency=1, max_concurrency=1), - storage_client=MemoryStorageClient(), + storage_backend=MemoryStorageBackend(), ) mocked_handler = Mock() @@ -2600,7 +2600,7 @@ async def test_crawler_state_persistence(tmp_path: Path) -> None: This test simulates starting the crawler process twice, and checks that the statistics include first run.""" state_kvs = await KeyValueStore.open( - storage_client=FileSystemStorageClient(), configuration=Configuration(storage_dir=str(tmp_path)) + storage_backend=FileSystemStorageBackend(), configuration=Configuration(storage_dir=str(tmp_path)) ) with ProcessPoolExecutor(mp_context=get_context('spawn')) as executor: @@ -2649,7 +2649,7 @@ async def test_crawler_state_persistence_2_crawlers_with_migration(tmp_path: Pat This test simulates starting the crawler process twice, and checks that the statistics include first run. Each time two distinct crawlers are running, and they should keep using their own statistics and state.""" state_kvs = await KeyValueStore.open( - storage_client=FileSystemStorageClient(), configuration=Configuration(storage_dir=str(tmp_path)) + storage_backend=FileSystemStorageBackend(), configuration=Configuration(storage_dir=str(tmp_path)) ) with ProcessPoolExecutor(mp_context=get_context('spawn')) as executor: @@ -2920,7 +2920,7 @@ async def test_warn_no_throttling_manager_once_on_429(caplog: pytest.LogCaptureF async def test_warn_unconfigured_throttle_domain_once_per_domain(caplog: pytest.LogCaptureFixture) -> None: """A 429 from a domain not in `ThrottlingRequestManager.domains` logs once per domain; repeats are suppressed.""" - inner = await RequestQueue.open(name='test-inner-warn', storage_client=MemoryStorageClient()) + inner = await RequestQueue.open(name='test-inner-warn', storage_backend=MemoryStorageBackend()) throttler = ThrottlingRequestManager( inner, domains=['only-this.test'], @@ -2945,10 +2945,10 @@ async def test_warn_unconfigured_throttle_domain_once_per_domain(caplog: pytest. async def test_crawl_waits_out_backoff() -> None: """Test that a domain in a cooldown reads as waiting, yet its queued requests are still crawled.""" - storage_client = MemoryStorageClient() - # The throttler opens its sub-managers through the global service locator, so point that at the same client. - service_locator.set_storage_client(storage_client) - inner = await RequestQueue.open(name='test-inner-backoff', storage_client=storage_client) + storage_backend = MemoryStorageBackend() + # The throttler opens its sub-managers through the global service locator, so point that at the same backend. + service_locator.set_storage_backend(storage_backend) + inner = await RequestQueue.open(name='test-inner-backoff', storage_backend=storage_backend) throttler = ThrottlingRequestManager( inner, domains=['throttled.placeholder.com'], diff --git a/tests/unit/storage_clients/_file_system/test_fs_dataset_client.py b/tests/unit/storage_backends/_file_system/test_fs_dataset_backend.py similarity index 59% rename from tests/unit/storage_clients/_file_system/test_fs_dataset_client.py rename to tests/unit/storage_backends/_file_system/test_fs_dataset_backend.py index 7e96518019..476a5fdf4d 100644 --- a/tests/unit/storage_clients/_file_system/test_fs_dataset_client.py +++ b/tests/unit/storage_backends/_file_system/test_fs_dataset_backend.py @@ -9,12 +9,12 @@ from crawlee._consts import METADATA_FILENAME from crawlee.configuration import Configuration -from crawlee.storage_clients import FileSystemStorageClient +from crawlee.storage_backends import FileSystemStorageBackend if TYPE_CHECKING: from collections.abc import AsyncGenerator - from crawlee.storage_clients._file_system import FileSystemDatasetClient + from crawlee.storage_backends._file_system import FileSystemDatasetBackend @pytest.fixture @@ -25,30 +25,30 @@ def configuration(tmp_path: Path) -> Configuration: @pytest.fixture -async def dataset_client(configuration: Configuration) -> AsyncGenerator[FileSystemDatasetClient, None]: - """A fixture for a file system dataset client.""" - client = await FileSystemStorageClient().create_dataset_client(name='test-dataset', configuration=configuration) - yield client - await client.drop() +async def dataset_backend(configuration: Configuration) -> AsyncGenerator[FileSystemDatasetBackend, None]: + """A fixture for a file system dataset backend.""" + backend = await FileSystemStorageBackend().create_dataset_backend(name='test-dataset', configuration=configuration) + yield backend + await backend.drop() async def test_file_and_directory_creation(configuration: Configuration) -> None: """Test that file system dataset creates proper files and directories.""" - client = await FileSystemStorageClient().create_dataset_client(name='new-dataset', configuration=configuration) + backend = await FileSystemStorageBackend().create_dataset_backend(name='new-dataset', configuration=configuration) # Verify files were created - assert client.path_to_dataset.exists() - assert client.path_to_metadata.exists() + assert backend.path_to_dataset.exists() + assert backend.path_to_metadata.exists() # Verify metadata file structure - with client.path_to_metadata.open() as f: + with backend.path_to_metadata.open() as f: metadata = json.load(f) - client_metadata = await client.get_metadata() - assert metadata['id'] == client_metadata.id + backend_metadata = await backend.get_metadata() + assert metadata['id'] == backend_metadata.id assert metadata['name'] == 'new-dataset' assert metadata['itemCount'] == 0 - await client.drop() + await backend.drop() @pytest.mark.parametrize( @@ -62,29 +62,29 @@ async def test_file_and_directory_creation(configuration: Configuration) -> None async def test_open_rejects_invalid_name_or_alias( configuration: Configuration, tmp_path: Path, invalid_value: str ) -> None: - """The low-level client must reject names/aliases that resolve outside the storage directory. + """The low-level backend must reject names/aliases that resolve outside the storage directory. - This covers direct usage of the storage client, which bypasses the high-level validation. + This covers direct usage of the storage backend, which bypasses the high-level validation. """ - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() with pytest.raises(ValueError, match='Invalid storage name or alias'): - await storage_client.create_dataset_client(alias=invalid_value, configuration=configuration) + await storage_backend.create_dataset_backend(alias=invalid_value, configuration=configuration) with pytest.raises(ValueError, match='Invalid storage name or alias'): - await storage_client.create_dataset_client(name=invalid_value, configuration=configuration) + await storage_backend.create_dataset_backend(name=invalid_value, configuration=configuration) # Nothing should have been written outside the configured storage directory. assert not (tmp_path / 'outside').exists() -async def test_file_persistence_and_content_verification(dataset_client: FileSystemDatasetClient) -> None: +async def test_file_persistence_and_content_verification(dataset_backend: FileSystemDatasetBackend) -> None: """Test that data is properly persisted to files with correct content.""" item = {'key': 'value', 'number': 42} - await dataset_client.push_data(item) + await dataset_backend.push_data(item) # Verify files are created on disk - all_files = list(dataset_client.path_to_dataset.glob('*.json')) + all_files = list(dataset_backend.path_to_dataset.glob('*.json')) assert len(all_files) == 2 # 1 data file + 1 metadata file # Verify actual file content @@ -97,31 +97,31 @@ async def test_file_persistence_and_content_verification(dataset_client: FileSys # Test multiple items file creation items = [{'id': 1, 'name': 'Item 1'}, {'id': 2, 'name': 'Item 2'}, {'id': 3, 'name': 'Item 3'}] - await dataset_client.push_data(items) + await dataset_backend.push_data(items) - all_files = list(dataset_client.path_to_dataset.glob('*.json')) + all_files = list(dataset_backend.path_to_dataset.glob('*.json')) assert len(all_files) == 5 # 4 data files + 1 metadata file data_files = [f for f in all_files if f.name != METADATA_FILENAME] assert len(data_files) == 4 # Original item + 3 new items -async def test_drop_removes_files_from_disk(dataset_client: FileSystemDatasetClient) -> None: +async def test_drop_removes_files_from_disk(dataset_backend: FileSystemDatasetBackend) -> None: """Test that dropping a dataset removes the entire dataset directory from disk.""" - await dataset_client.push_data({'test': 'data'}) + await dataset_backend.push_data({'test': 'data'}) - assert dataset_client.path_to_dataset.exists() + assert dataset_backend.path_to_dataset.exists() # Drop the dataset - await dataset_client.drop() + await dataset_backend.drop() - assert not dataset_client.path_to_dataset.exists() + assert not dataset_backend.path_to_dataset.exists() -async def test_metadata_file_updates(dataset_client: FileSystemDatasetClient) -> None: +async def test_metadata_file_updates(dataset_backend: FileSystemDatasetBackend) -> None: """Test that metadata file is updated correctly after operations.""" # Record initial timestamps - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -130,10 +130,10 @@ async def test_metadata_file_updates(dataset_client: FileSystemDatasetClient) -> await asyncio.sleep(0.01) # Perform an operation that updates accessed_at - await dataset_client.get_data() + await dataset_backend.get_data() # Verify timestamps - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -144,37 +144,37 @@ async def test_metadata_file_updates(dataset_client: FileSystemDatasetClient) -> await asyncio.sleep(0.01) # Perform an operation that updates modified_at - await dataset_client.push_data({'new': 'item'}) + await dataset_backend.push_data({'new': 'item'}) # Verify timestamps again - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_get # Verify metadata file is updated on disk - with dataset_client.path_to_metadata.open() as f: + with dataset_backend.path_to_metadata.open() as f: metadata_json = json.load(f) assert metadata_json['itemCount'] == 1 async def test_data_persistence_across_reopens() -> None: """Test that data persists correctly when reopening the same dataset.""" - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() # Create dataset and add data - original_client = await storage_client.create_dataset_client(name='persistence-test') + original_backend = await storage_backend.create_dataset_backend(name='persistence-test') test_data = {'test_item': 'test_value', 'id': 123} - await original_client.push_data(test_data) + await original_backend.push_data(test_data) - dataset_id = (await original_client.get_metadata()).id + dataset_id = (await original_backend.get_metadata()).id # Reopen by ID and verify data persists - reopened_client = await storage_client.create_dataset_client(id=dataset_id) + reopened_backend = await storage_backend.create_dataset_backend(id=dataset_id) - data = await reopened_client.get_data() + data = await reopened_backend.get_data() assert len(data.items) == 1 assert data.items[0] == test_data - await reopened_client.drop() + await reopened_backend.drop() diff --git a/tests/unit/storage_clients/_file_system/test_fs_kvs_client.py b/tests/unit/storage_backends/_file_system/test_fs_kvs_backend.py similarity index 54% rename from tests/unit/storage_clients/_file_system/test_fs_kvs_client.py rename to tests/unit/storage_backends/_file_system/test_fs_kvs_backend.py index f3662302d7..2c03efba8c 100644 --- a/tests/unit/storage_clients/_file_system/test_fs_kvs_client.py +++ b/tests/unit/storage_backends/_file_system/test_fs_kvs_backend.py @@ -8,14 +8,14 @@ from crawlee._consts import METADATA_FILENAME from crawlee.configuration import Configuration -from crawlee.storage_clients import FileSystemStorageClient -from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecordMetadata +from crawlee.storage_backends import FileSystemStorageBackend +from crawlee.storage_backends.models import KeyValueStoreMetadata, KeyValueStoreRecordMetadata if TYPE_CHECKING: from collections.abc import AsyncGenerator from pathlib import Path - from crawlee.storage_clients._file_system import FileSystemKeyValueStoreClient + from crawlee.storage_backends._file_system import FileSystemKeyValueStoreBackend @pytest.fixture @@ -26,28 +26,28 @@ def configuration(tmp_path: Path) -> Configuration: @pytest.fixture -async def kvs_client(configuration: Configuration) -> AsyncGenerator[FileSystemKeyValueStoreClient, None]: - """A fixture for a file system key-value store client.""" - client = await FileSystemStorageClient().create_kvs_client(name='test-kvs', configuration=configuration) - yield client - await client.drop() +async def kvs_backend(configuration: Configuration) -> AsyncGenerator[FileSystemKeyValueStoreBackend, None]: + """A fixture for a file system key-value store backend.""" + backend = await FileSystemStorageBackend().create_kvs_backend(name='test-kvs', configuration=configuration) + yield backend + await backend.drop() async def test_file_and_directory_creation(configuration: Configuration) -> None: """Test that file system KVS creates proper files and directories.""" - client = await FileSystemStorageClient().create_kvs_client(name='new-kvs', configuration=configuration) + backend = await FileSystemStorageBackend().create_kvs_backend(name='new-kvs', configuration=configuration) # Verify files were created - assert client.path_to_kvs.exists() - assert client.path_to_metadata.exists() + assert backend.path_to_kvs.exists() + assert backend.path_to_metadata.exists() # Verify metadata file structure - with client.path_to_metadata.open() as f: + with backend.path_to_metadata.open() as f: metadata = json.load(f) - assert metadata['id'] == (await client.get_metadata()).id + assert metadata['id'] == (await backend.get_metadata()).id assert metadata['name'] == 'new-kvs' - await client.drop() + await backend.drop() @pytest.mark.parametrize( @@ -61,31 +61,31 @@ async def test_file_and_directory_creation(configuration: Configuration) -> None async def test_open_rejects_invalid_name_or_alias( configuration: Configuration, tmp_path: Path, invalid_value: str ) -> None: - """The low-level client must reject names/aliases that resolve outside the storage directory. + """The low-level backend must reject names/aliases that resolve outside the storage directory. - This covers direct usage of the storage client, which bypasses the high-level validation. + This covers direct usage of the storage backend, which bypasses the high-level validation. """ - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() with pytest.raises(ValueError, match='Invalid storage name or alias'): - await storage_client.create_kvs_client(alias=invalid_value, configuration=configuration) + await storage_backend.create_kvs_backend(alias=invalid_value, configuration=configuration) with pytest.raises(ValueError, match='Invalid storage name or alias'): - await storage_client.create_kvs_client(name=invalid_value, configuration=configuration) + await storage_backend.create_kvs_backend(name=invalid_value, configuration=configuration) # Nothing should have been written outside the configured storage directory. assert not (tmp_path / 'outside').exists() -async def test_value_file_creation_and_content(kvs_client: FileSystemKeyValueStoreClient) -> None: +async def test_value_file_creation_and_content(kvs_backend: FileSystemKeyValueStoreBackend) -> None: """Test that values are properly persisted to files with correct content and metadata.""" test_key = 'test-key' test_value = 'Hello, world!' - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) # Check if the files were created - key_path = kvs_client.path_to_kvs / test_key - key_metadata_path = kvs_client.path_to_kvs / f'{test_key}.{METADATA_FILENAME}' + key_path = kvs_backend.path_to_kvs / test_key + key_metadata_path = kvs_backend.path_to_kvs / f'{test_key}.{METADATA_FILENAME}' assert key_path.exists() assert key_metadata_path.exists() @@ -101,14 +101,14 @@ async def test_value_file_creation_and_content(kvs_client: FileSystemKeyValueSto assert metadata['size'] == len(test_value.encode('utf-8')) -async def test_binary_data_persistence(kvs_client: FileSystemKeyValueStoreClient) -> None: +async def test_binary_data_persistence(kvs_backend: FileSystemKeyValueStoreBackend) -> None: """Test that binary data is stored correctly without corruption.""" test_key = 'test-binary' test_value = b'\x00\x01\x02\x03\x04' - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) # Verify binary file exists - key_path = kvs_client.path_to_kvs / test_key + key_path = kvs_backend.path_to_kvs / test_key assert key_path.exists() # Verify binary content is preserved @@ -116,63 +116,63 @@ async def test_binary_data_persistence(kvs_client: FileSystemKeyValueStoreClient assert content == test_value # Verify retrieval works correctly - record = await kvs_client.get_value(key=test_key) + record = await kvs_backend.get_value(key=test_key) assert record is not None assert record.value == test_value assert record.content_type == 'application/octet-stream' -async def test_json_serialization_to_file(kvs_client: FileSystemKeyValueStoreClient) -> None: +async def test_json_serialization_to_file(kvs_backend: FileSystemKeyValueStoreBackend) -> None: """Test that JSON objects are properly serialized to files.""" test_key = 'test-json' test_value = {'name': 'John', 'age': 30, 'items': [1, 2, 3]} - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) # Check if file content is valid JSON - key_path = kvs_client.path_to_kvs / test_key + key_path = kvs_backend.path_to_kvs / test_key with key_path.open() as f: file_content = json.load(f) assert file_content == test_value -async def test_file_deletion_on_value_delete(kvs_client: FileSystemKeyValueStoreClient) -> None: +async def test_file_deletion_on_value_delete(kvs_backend: FileSystemKeyValueStoreBackend) -> None: """Test that deleting a value removes its files from disk.""" test_key = 'test-delete' test_value = 'Delete me' # Set a value - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) # Verify files exist - key_path = kvs_client.path_to_kvs / test_key - metadata_path = kvs_client.path_to_kvs / f'{test_key}.{METADATA_FILENAME}' + key_path = kvs_backend.path_to_kvs / test_key + metadata_path = kvs_backend.path_to_kvs / f'{test_key}.{METADATA_FILENAME}' assert key_path.exists() assert metadata_path.exists() # Delete the value - await kvs_client.delete_value(key=test_key) + await kvs_backend.delete_value(key=test_key) # Verify files were deleted assert not key_path.exists() assert not metadata_path.exists() -async def test_drop_removes_directory(kvs_client: FileSystemKeyValueStoreClient) -> None: +async def test_drop_removes_directory(kvs_backend: FileSystemKeyValueStoreBackend) -> None: """Test that drop removes the entire store directory from disk.""" - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') - assert kvs_client.path_to_kvs.exists() + assert kvs_backend.path_to_kvs.exists() # Drop the store - await kvs_client.drop() + await kvs_backend.drop() - assert not kvs_client.path_to_kvs.exists() + assert not kvs_backend.path_to_kvs.exists() -async def test_metadata_file_updates(kvs_client: FileSystemKeyValueStoreClient) -> None: +async def test_metadata_file_updates(kvs_backend: FileSystemKeyValueStoreBackend) -> None: """Test that read/write operations properly update metadata file timestamps.""" # Record initial timestamps - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -181,10 +181,10 @@ async def test_metadata_file_updates(kvs_client: FileSystemKeyValueStoreClient) await asyncio.sleep(0.01) # Perform a read operation - await kvs_client.get_value(key='nonexistent') + await kvs_backend.get_value(key='nonexistent') # Verify accessed timestamp was updated - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -195,10 +195,10 @@ async def test_metadata_file_updates(kvs_client: FileSystemKeyValueStoreClient) await asyncio.sleep(0.01) # Perform a write operation - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') # Verify modified timestamp was updated - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_read @@ -206,49 +206,49 @@ async def test_metadata_file_updates(kvs_client: FileSystemKeyValueStoreClient) async def test_data_persistence_across_reopens(configuration: Configuration) -> None: """Test that data persists correctly when reopening the same KVS.""" - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() # Create KVS and add data - original_client = await storage_client.create_kvs_client(name='persistence-test', configuration=configuration) + original_backend = await storage_backend.create_kvs_backend(name='persistence-test', configuration=configuration) test_key = 'persistent-key' test_value = 'persistent-value' - await original_client.set_value(key=test_key, value=test_value) + await original_backend.set_value(key=test_key, value=test_value) - kvs_id = (await original_client.get_metadata()).id + kvs_id = (await original_backend.get_metadata()).id # Reopen by ID and verify data persists - reopened_client = await storage_client.create_kvs_client( + reopened_backend = await storage_backend.create_kvs_backend( id=kvs_id, ) - record = await reopened_client.get_value(key=test_key) + record = await reopened_backend.get_value(key=test_key) assert record is not None assert record.value == test_value - await reopened_client.drop() + await reopened_backend.drop() async def test_snake_case_files_from_v1_are_still_read(configuration: Configuration) -> None: """Store and record metadata files with snake_case keys, as Crawlee v1 wrote them, still load.""" - storage_client = FileSystemStorageClient() - client = await storage_client.create_kvs_client(name='snake-case-kvs', configuration=configuration) - await client.set_value(key='key', value='value') - kvs_id = (await client.get_metadata()).id + storage_backend = FileSystemStorageBackend() + backend = await storage_backend.create_kvs_backend(name='snake-case-kvs', configuration=configuration) + await backend.set_value(key='key', value='value') + kvs_id = (await backend.get_metadata()).id # Rewrite every metadata file the way v1 serialized it, without aliases. - for path in client.path_to_kvs.glob(f'*{METADATA_FILENAME}'): - model = KeyValueStoreMetadata if path == client.path_to_metadata else KeyValueStoreRecordMetadata + for path in backend.path_to_kvs.glob(f'*{METADATA_FILENAME}'): + model = KeyValueStoreMetadata if path == backend.path_to_metadata else KeyValueStoreRecordMetadata snake_case_data = model.model_validate_json(path.read_text()).model_dump(mode='json', by_alias=False) path.write_text(json.dumps(snake_case_data)) - reopened_client = await storage_client.create_kvs_client(id=kvs_id, configuration=configuration) - assert (await reopened_client.get_metadata()).id == kvs_id + reopened_backend = await storage_backend.create_kvs_backend(id=kvs_id, configuration=configuration) + assert (await reopened_backend.get_metadata()).id == kvs_id - record = await reopened_client.get_value(key='key') + record = await reopened_backend.get_value(key='key') assert record is not None assert record.value == 'value' assert record.content_type == 'text/plain; charset=utf-8' - assert [key.key async for key in reopened_client.iterate_keys()] == ['key'] + assert [key.key async for key in reopened_backend.iterate_keys()] == ['key'] - await reopened_client.drop() + await reopened_backend.drop() diff --git a/tests/unit/storage_clients/_file_system/test_fs_rq_client.py b/tests/unit/storage_backends/_file_system/test_fs_rq_backend.py similarity index 57% rename from tests/unit/storage_clients/_file_system/test_fs_rq_client.py rename to tests/unit/storage_backends/_file_system/test_fs_rq_backend.py index eb8541385e..f77d9c4e18 100644 --- a/tests/unit/storage_clients/_file_system/test_fs_rq_client.py +++ b/tests/unit/storage_backends/_file_system/test_fs_rq_backend.py @@ -9,14 +9,14 @@ from crawlee import Request, service_locator from crawlee.configuration import Configuration -from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient -from crawlee.storage_clients.models import RequestQueueMetadata +from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend +from crawlee.storage_backends.models import RequestQueueMetadata if TYPE_CHECKING: from collections.abc import AsyncGenerator from pathlib import Path - from crawlee.storage_clients._file_system import FileSystemRequestQueueClient + from crawlee.storage_backends._file_system import FileSystemRequestQueueBackend @pytest.fixture @@ -27,30 +27,30 @@ def configuration(tmp_path: Path) -> Configuration: @pytest.fixture -async def rq_client() -> AsyncGenerator[FileSystemRequestQueueClient, None]: - """A fixture for a file system request queue client.""" - client = await FileSystemStorageClient().create_rq_client( +async def rq_backend() -> AsyncGenerator[FileSystemRequestQueueBackend, None]: + """A fixture for a file system request queue backend.""" + backend = await FileSystemStorageBackend().create_rq_backend( name='test-request-queue', ) - yield client - await client.drop() + yield backend + await backend.drop() async def test_file_and_directory_creation() -> None: """Test that file system RQ creates proper files and directories.""" - client = await FileSystemStorageClient().create_rq_client(name='new-request-queue') + backend = await FileSystemStorageBackend().create_rq_backend(name='new-request-queue') # Verify files were created - assert client.path_to_rq.exists() - assert client.path_to_metadata.exists() + assert backend.path_to_rq.exists() + assert backend.path_to_metadata.exists() # Verify metadata file structure - with client.path_to_metadata.open() as f: + with backend.path_to_metadata.open() as f: metadata = json.load(f) - assert metadata['id'] == (await client.get_metadata()).id + assert metadata['id'] == (await backend.get_metadata()).id assert metadata['name'] == 'new-request-queue' - await client.drop() + await backend.drop() @pytest.mark.parametrize( @@ -64,23 +64,23 @@ async def test_file_and_directory_creation() -> None: async def test_open_rejects_invalid_name_or_alias( configuration: Configuration, tmp_path: Path, invalid_value: str ) -> None: - """The low-level client must reject names/aliases that resolve outside the storage directory. + """The low-level backend must reject names/aliases that resolve outside the storage directory. - This covers direct usage of the storage client, which bypasses the high-level validation. + This covers direct usage of the storage backend, which bypasses the high-level validation. """ - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() with pytest.raises(ValueError, match='Invalid storage name or alias'): - await storage_client.create_rq_client(alias=invalid_value, configuration=configuration) + await storage_backend.create_rq_backend(alias=invalid_value, configuration=configuration) with pytest.raises(ValueError, match='Invalid storage name or alias'): - await storage_client.create_rq_client(name=invalid_value, configuration=configuration) + await storage_backend.create_rq_backend(name=invalid_value, configuration=configuration) # Nothing should have been written outside the configured storage directory. assert not (tmp_path / 'outside').exists() -async def test_request_file_persistence(rq_client: FileSystemRequestQueueClient) -> None: +async def test_request_file_persistence(rq_backend: FileSystemRequestQueueBackend) -> None: """Test that requests are properly persisted to files.""" requests = [ Request.from_url('https://example.com/1'), @@ -88,16 +88,16 @@ async def test_request_file_persistence(rq_client: FileSystemRequestQueueClient) Request.from_url('https://example.com/3'), ] - await rq_client.add_batch_of_requests(requests) + await rq_backend.add_batch_of_requests(requests) # Verify request files are created - request_files = list(rq_client.path_to_rq.glob('*.json')) + request_files = list(rq_backend.path_to_rq.glob('*.json')) # Should have 3 request files + 1 metadata file assert len(request_files) == 4 - assert rq_client.path_to_metadata in request_files + assert rq_backend.path_to_metadata in request_files # Verify actual request file content - data_files = [f for f in request_files if f != rq_client.path_to_metadata] + data_files = [f for f in request_files if f != rq_backend.path_to_metadata] assert len(data_files) == 3 for req_file in data_files: @@ -109,30 +109,30 @@ async def test_request_file_persistence(rq_client: FileSystemRequestQueueClient) async def test_opening_rq_does_not_have_side_effect_on_service_locator(configuration: Configuration) -> None: - """Opening request queue client should cause setting storage client in the global service locator.""" - await FileSystemStorageClient().create_rq_client(name='test_request_queue', configuration=configuration) + """Opening request queue backend should cause setting storage backend in the global service locator.""" + await FileSystemStorageBackend().create_rq_backend(name='test_request_queue', configuration=configuration) - # Set some specific storage client in the service locator. There should be no `ServiceConflictError`. - service_locator.set_storage_client(MemoryStorageClient()) + # Set some specific storage backend in the service locator. There should be no `ServiceConflictError`. + service_locator.set_storage_backend(MemoryStorageBackend()) -async def test_drop_removes_directory(rq_client: FileSystemRequestQueueClient) -> None: +async def test_drop_removes_directory(rq_backend: FileSystemRequestQueueBackend) -> None: """Test that drop removes the entire RQ directory from disk.""" - await rq_client.add_batch_of_requests([Request.from_url('https://example.com')]) + await rq_backend.add_batch_of_requests([Request.from_url('https://example.com')]) - rq_path = rq_client.path_to_rq + rq_path = rq_backend.path_to_rq assert rq_path.exists() # Drop the request queue - await rq_client.drop() + await rq_backend.drop() assert not rq_path.exists() -async def test_metadata_file_updates(rq_client: FileSystemRequestQueueClient) -> None: +async def test_metadata_file_updates(rq_backend: FileSystemRequestQueueBackend) -> None: """Test that metadata file is updated correctly after operations.""" # Record initial timestamps - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -141,10 +141,10 @@ async def test_metadata_file_updates(rq_client: FileSystemRequestQueueClient) -> await asyncio.sleep(0.01) # Perform a read operation - await rq_client.is_empty() + await rq_backend.is_empty() # Verify accessed timestamp was updated - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -155,26 +155,26 @@ async def test_metadata_file_updates(rq_client: FileSystemRequestQueueClient) -> await asyncio.sleep(0.01) # Perform a write operation - await rq_client.add_batch_of_requests([Request.from_url('https://example.com')]) + await rq_backend.add_batch_of_requests([Request.from_url('https://example.com')]) # Verify modified timestamp was updated - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_read # Verify metadata file is updated on disk - with rq_client.path_to_metadata.open() as f: + with rq_backend.path_to_metadata.open() as f: metadata_json = json.load(f) assert metadata_json['totalRequestCount'] == 1 async def test_data_persistence_across_reopens() -> None: """Test that requests persist correctly when reopening the same RQ.""" - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() # Create RQ and add requests - original_client = await storage_client.create_rq_client( + original_backend = await storage_backend.create_rq_backend( name='persistence-test', ) @@ -182,55 +182,55 @@ async def test_data_persistence_across_reopens() -> None: Request.from_url('https://example.com/1'), Request.from_url('https://example.com/2'), ] - await original_client.add_batch_of_requests(test_requests) + await original_backend.add_batch_of_requests(test_requests) - rq_id = (await original_client.get_metadata()).id + rq_id = (await original_backend.get_metadata()).id # Reopen by ID and verify requests persist - reopened_client = await storage_client.create_rq_client( + reopened_backend = await storage_backend.create_rq_backend( id=rq_id, ) - metadata = await reopened_client.get_metadata() + metadata = await reopened_backend.get_metadata() assert metadata.total_request_count == 2 # Fetch requests to verify they're still there - request1 = await reopened_client.fetch_next_request() - request2 = await reopened_client.fetch_next_request() + request1 = await reopened_backend.fetch_next_request() + request2 = await reopened_backend.fetch_next_request() assert request1 is not None assert request2 is not None assert {request1.url, request2.url} == {'https://example.com/1', 'https://example.com/2'} - await reopened_client.drop() + await reopened_backend.drop() async def test_snake_case_files_from_v1_are_still_read() -> None: """Metadata and request files with snake_case keys, as Crawlee v1 wrote them, still load.""" - storage_client = FileSystemStorageClient() - client = await storage_client.create_rq_client(name='snake-case-rq') + storage_backend = FileSystemStorageBackend() + backend = await storage_backend.create_rq_backend(name='snake-case-rq') request = Request.from_url('https://example.com/1', label='detail') request.crawl_depth = 2 - await client.add_batch_of_requests([request]) - rq_id = (await client.get_metadata()).id + await backend.add_batch_of_requests([request]) + rq_id = (await backend.get_metadata()).id # Rewrite every file the way v1 serialized it, without aliases. - for path in client.path_to_rq.glob('*.json'): + for path in backend.path_to_rq.glob('*.json'): data = json.loads(path.read_text()) - model = RequestQueueMetadata if path == client.path_to_metadata else Request + model = RequestQueueMetadata if path == backend.path_to_metadata else Request snake_case_data = model.model_validate(data).model_dump(mode='json', by_alias=False) path.write_text(json.dumps(snake_case_data)) - reopened_client = await storage_client.create_rq_client(id=rq_id) - assert (await reopened_client.get_metadata()).total_request_count == 1 + reopened_backend = await storage_backend.create_rq_backend(id=rq_id) + assert (await reopened_backend.get_metadata()).total_request_count == 1 - fetched = await reopened_client.fetch_next_request() + fetched = await reopened_backend.fetch_next_request() assert fetched is not None assert fetched.unique_key == request.unique_key assert fetched.label == 'detail' assert fetched.crawl_depth == 2 - await reopened_client.drop() + await reopened_backend.drop() async def test_in_progress_requests_recovered_after_crash() -> None: @@ -239,70 +239,70 @@ async def test_in_progress_requests_recovered_after_crash() -> None: Simulates a crash: requests are added, one is fetched (in-progress), state is persisted, then the queue is reopened. The in-progress request should be available for fetching again. """ - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() # Create RQ and add requests. - original_client = await storage_client.create_rq_client(name='crash-recovery-test') + original_backend = await storage_backend.create_rq_backend(name='crash-recovery-test') test_requests = [ Request.from_url('https://example.com/1'), Request.from_url('https://example.com/2'), Request.from_url('https://example.com/3'), ] - await original_client.add_batch_of_requests(test_requests) + await original_backend.add_batch_of_requests(test_requests) # Fetch one request, putting it in-progress (simulating work before crash). - fetched = await original_client.fetch_next_request() + fetched = await original_backend.fetch_next_request() assert fetched is not None # Persist state explicitly (simulating what happens periodically or at crash boundary). - await original_client._state.persist_state() + await original_backend._state.persist_state() - rq_id = (await original_client.get_metadata()).id + rq_id = (await original_backend.get_metadata()).id # Simulate crash: reopen the queue without calling mark_request_as_handled or reclaim_request. - reopened_client = await storage_client.create_rq_client(id=rq_id) + reopened_backend = await storage_backend.create_rq_backend(id=rq_id) # All 3 requests should be fetchable (the in-progress one should have been reclaimed). fetched_urls = set() for _ in range(3): - req = await reopened_client.fetch_next_request() + req = await reopened_backend.fetch_next_request() assert req is not None, f'Expected 3 fetchable requests, only got {len(fetched_urls)}' fetched_urls.add(req.url) assert fetched_urls == {'https://example.com/1', 'https://example.com/2', 'https://example.com/3'} # No more requests should be available. - assert await reopened_client.fetch_next_request() is None + assert await reopened_backend.fetch_next_request() is None - await reopened_client.drop() + await reopened_backend.drop() -async def test_get_request_does_not_mark_in_progress(rq_client: FileSystemRequestQueueClient) -> None: +async def test_get_request_does_not_mark_in_progress(rq_backend: FileSystemRequestQueueBackend) -> None: """Test that get_request does not block a request from being fetched.""" request = Request.from_url('https://example.com/blocked') - await rq_client.add_batch_of_requests([request]) + await rq_backend.add_batch_of_requests([request]) - fetched = await rq_client.get_request(request.unique_key) + fetched = await rq_backend.get_request(request.unique_key) assert fetched is not None assert fetched.unique_key == request.unique_key - next_request = await rq_client.fetch_next_request() + next_request = await rq_backend.fetch_next_request() assert next_request is not None assert next_request.unique_key == request.unique_key async def test_forefront_add_fetch_handle_parses_linear_number_of_files( - rq_client: FileSystemRequestQueueClient, + rq_backend: FileSystemRequestQueueBackend, ) -> None: """Test that per-request forefront adds (the `RequestManagerTandem` pattern) do not rescan all request files.""" - with patch.object(rq_client, '_parse_request_file', wraps=rq_client._parse_request_file) as parse_spy: + with patch.object(rq_backend, '_parse_request_file', wraps=rq_backend._parse_request_file) as parse_spy: n = 25 for i in range(n): - await rq_client.add_batch_of_requests([Request.from_url(f'https://example.com/{i}')], forefront=True) - request = await rq_client.fetch_next_request() + await rq_backend.add_batch_of_requests([Request.from_url(f'https://example.com/{i}')], forefront=True) + request = await rq_backend.fetch_next_request() assert request is not None - await rq_client.mark_request_as_handled(request) + await rq_backend.mark_request_as_handled(request) # Before the fix, each forefront add invalidated the cache and each fetch re-parsed every request file # ever written, giving n * (n + 1) / 2 parses in total. With the fix, new forefront requests go straight @@ -311,38 +311,38 @@ async def test_forefront_add_fetch_handle_parses_linear_number_of_files( async def test_cache_refresh_skips_handled_request_files( - rq_client: FileSystemRequestQueueClient, + rq_backend: FileSystemRequestQueueBackend, ) -> None: """Test that a cache refresh reads only pending request files, not files of already handled requests.""" - await rq_client.add_batch_of_requests([Request.from_url(f'https://example.com/{i}') for i in range(5)]) + await rq_backend.add_batch_of_requests([Request.from_url(f'https://example.com/{i}') for i in range(5)]) for _ in range(5): - request = await rq_client.fetch_next_request() + request = await rq_backend.fetch_next_request() assert request is not None - await rq_client.mark_request_as_handled(request) + await rq_backend.mark_request_as_handled(request) # The 5 handled request files stay on disk. Add 5 new requests; fetching the next one refreshes # the cache, which must not parse the handled files again. - await rq_client.add_batch_of_requests([Request.from_url(f'https://example.com/new/{i}') for i in range(5)]) + await rq_backend.add_batch_of_requests([Request.from_url(f'https://example.com/new/{i}') for i in range(5)]) - with patch.object(rq_client, '_parse_request_file', wraps=rq_client._parse_request_file) as parse_spy: - request = await rq_client.fetch_next_request() + with patch.object(rq_backend, '_parse_request_file', wraps=rq_backend._parse_request_file) as parse_spy: + request = await rq_backend.fetch_next_request() assert request is not None assert parse_spy.call_count == 5 -async def test_handled_requests_pruned_from_pending_state(rq_client: FileSystemRequestQueueClient) -> None: +async def test_handled_requests_pruned_from_pending_state(rq_backend: FileSystemRequestQueueBackend) -> None: """Test that handling a request removes it from the pending state mappings so the state stays bounded.""" - await rq_client.add_batch_of_requests([Request.from_url(f'https://example.com/{i}') for i in range(3)]) + await rq_backend.add_batch_of_requests([Request.from_url(f'https://example.com/{i}') for i in range(3)]) handled_keys = set() for _ in range(3): - request = await rq_client.fetch_next_request() + request = await rq_backend.fetch_next_request() assert request is not None - await rq_client.mark_request_as_handled(request) + await rq_backend.mark_request_as_handled(request) handled_keys.add(request.unique_key) - state = rq_client._state.current_value + state = rq_backend._state.current_value assert not state.regular_requests assert not state.forefront_requests assert not state.in_progress_requests @@ -351,18 +351,18 @@ async def test_handled_requests_pruned_from_pending_state(rq_client: FileSystemR async def test_handled_requests_deduplicated_after_reopen() -> None: """Test that requests handled before a reopen are not served again and still deduplicate re-adds.""" - storage_client = FileSystemStorageClient() - client = await storage_client.create_rq_client(name='handled-reopen-test') + storage_backend = FileSystemStorageBackend() + backend = await storage_backend.create_rq_backend(name='handled-reopen-test') request = Request.from_url('https://example.com/handled') - await client.add_batch_of_requests([request]) - fetched = await client.fetch_next_request() + await backend.add_batch_of_requests([request]) + fetched = await backend.fetch_next_request() assert fetched is not None - await client.mark_request_as_handled(fetched) - await client._state.persist_state() + await backend.mark_request_as_handled(fetched) + await backend._state.persist_state() - rq_id = (await client.get_metadata()).id - reopened = await storage_client.create_rq_client(id=rq_id) + rq_id = (await backend.get_metadata()).id + reopened = await storage_backend.create_rq_backend(id=rq_id) response = await reopened.add_batch_of_requests([request]) assert response.processed_requests[0].was_already_handled is True diff --git a/tests/unit/storage_clients/_memory/test_memory_dataset_client.py b/tests/unit/storage_backends/_memory/test_memory_dataset_backend.py similarity index 58% rename from tests/unit/storage_clients/_memory/test_memory_dataset_client.py rename to tests/unit/storage_backends/_memory/test_memory_dataset_backend.py index ccb29dec32..6368bae9e6 100644 --- a/tests/unit/storage_clients/_memory/test_memory_dataset_client.py +++ b/tests/unit/storage_backends/_memory/test_memory_dataset_backend.py @@ -5,48 +5,48 @@ import pytest -from crawlee.storage_clients import MemoryStorageClient +from crawlee.storage_backends import MemoryStorageBackend if TYPE_CHECKING: from collections.abc import AsyncGenerator - from crawlee.storage_clients._memory import MemoryDatasetClient + from crawlee.storage_backends._memory import MemoryDatasetBackend @pytest.fixture -async def dataset_client() -> AsyncGenerator[MemoryDatasetClient, None]: - """Fixture that provides a fresh memory dataset client for each test.""" - client = await MemoryStorageClient().create_dataset_client(name='test-dataset') - yield client - await client.drop() +async def dataset_backend() -> AsyncGenerator[MemoryDatasetBackend, None]: + """Fixture that provides a fresh memory dataset backend for each test.""" + backend = await MemoryStorageBackend().create_dataset_backend(name='test-dataset') + yield backend + await backend.drop() async def test_memory_specific_purge_behavior() -> None: """Test memory-specific purge behavior and in-memory storage characteristics.""" # Create dataset and add data - dataset_client1 = await MemoryStorageClient().create_dataset_client( + dataset_backend1 = await MemoryStorageBackend().create_dataset_backend( name='test-purge-dataset', ) - await dataset_client1.push_data({'item': 'initial data'}) + await dataset_backend1.push_data({'item': 'initial data'}) # Verify data was added - items = await dataset_client1.get_data() + items = await dataset_backend1.get_data() assert len(items.items) == 1 - # Reopen with same storage client instance - dataset_client2 = await MemoryStorageClient().create_dataset_client( + # Reopen with same storage backend instance + dataset_backend2 = await MemoryStorageBackend().create_dataset_backend( name='test-purge-dataset', ) # Verify data was purged (memory storage specific behavior) - items = await dataset_client2.get_data() + items = await dataset_backend2.get_data() assert len(items.items) == 0 -async def test_memory_metadata_updates(dataset_client: MemoryDatasetClient) -> None: +async def test_memory_metadata_updates(dataset_backend: MemoryDatasetBackend) -> None: """Test that metadata timestamps are updated correctly in memory storage.""" # Record initial timestamps - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -55,10 +55,10 @@ async def test_memory_metadata_updates(dataset_client: MemoryDatasetClient) -> N await asyncio.sleep(0.01) # Perform a read operation - await dataset_client.get_data() + await dataset_backend.get_data() # Verify timestamps (memory-specific behavior) - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -69,10 +69,10 @@ async def test_memory_metadata_updates(dataset_client: MemoryDatasetClient) -> N await asyncio.sleep(0.01) # Perform a write operation - await dataset_client.push_data({'new': 'item'}) + await dataset_backend.push_data({'new': 'item'}) # Verify timestamps were updated - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_read diff --git a/tests/unit/storage_clients/_memory/test_memory_kvs_client.py b/tests/unit/storage_backends/_memory/test_memory_kvs_backend.py similarity index 56% rename from tests/unit/storage_clients/_memory/test_memory_kvs_client.py rename to tests/unit/storage_backends/_memory/test_memory_kvs_backend.py index 4c7fce8058..7c099ec820 100644 --- a/tests/unit/storage_clients/_memory/test_memory_kvs_client.py +++ b/tests/unit/storage_backends/_memory/test_memory_kvs_backend.py @@ -5,50 +5,50 @@ import pytest -from crawlee.storage_clients import MemoryStorageClient +from crawlee.storage_backends import MemoryStorageBackend if TYPE_CHECKING: from collections.abc import AsyncGenerator - from crawlee.storage_clients._memory import MemoryKeyValueStoreClient + from crawlee.storage_backends._memory import MemoryKeyValueStoreBackend @pytest.fixture -async def kvs_client() -> AsyncGenerator[MemoryKeyValueStoreClient, None]: - """Fixture that provides a fresh memory key-value store client for each test.""" - client = await MemoryStorageClient().create_kvs_client(name='test-kvs') - yield client - await client.drop() +async def kvs_backend() -> AsyncGenerator[MemoryKeyValueStoreBackend, None]: + """Fixture that provides a fresh memory key-value store backend for each test.""" + backend = await MemoryStorageBackend().create_kvs_backend(name='test-kvs') + yield backend + await backend.drop() async def test_memory_specific_purge_behavior() -> None: """Test memory-specific purge behavior and in-memory storage characteristics.""" # Create KVS and add data - kvs_client1 = await MemoryStorageClient().create_kvs_client( + kvs_backend1 = await MemoryStorageBackend().create_kvs_backend( name='test-purge-kvs', ) - await kvs_client1.set_value(key='test-key', value='initial value') + await kvs_backend1.set_value(key='test-key', value='initial value') # Verify value was set - record = await kvs_client1.get_value(key='test-key') + record = await kvs_backend1.get_value(key='test-key') assert record is not None assert record.value == 'initial value' - # Reopen with same storage client instance - kvs_client2 = await MemoryStorageClient().create_kvs_client( + # Reopen with same storage backend instance + kvs_backend2 = await MemoryStorageBackend().create_kvs_backend( name='test-purge-kvs', ) # Verify value was purged (memory storage specific behavior) - record = await kvs_client2.get_value(key='test-key') + record = await kvs_backend2.get_value(key='test-key') assert record is None -async def test_memory_metadata_updates(kvs_client: MemoryKeyValueStoreClient) -> None: +async def test_memory_metadata_updates(kvs_backend: MemoryKeyValueStoreBackend) -> None: """Test that metadata timestamps are updated correctly in memory storage.""" # Record initial timestamps - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -57,10 +57,10 @@ async def test_memory_metadata_updates(kvs_client: MemoryKeyValueStoreClient) -> await asyncio.sleep(0.01) # Perform a read operation - await kvs_client.get_value(key='nonexistent') + await kvs_backend.get_value(key='nonexistent') # Verify timestamps (memory-specific behavior) - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -71,24 +71,24 @@ async def test_memory_metadata_updates(kvs_client: MemoryKeyValueStoreClient) -> await asyncio.sleep(0.01) # Perform a write operation - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') # Verify timestamps were updated - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_read -async def test_iterate_keys_skips_record_deleted_during_iteration(kvs_client: MemoryKeyValueStoreClient) -> None: +async def test_iterate_keys_skips_record_deleted_during_iteration(kvs_backend: MemoryKeyValueStoreBackend) -> None: """Test that a record deleted while `iterate_keys` is suspended is skipped.""" - await kvs_client.set_value(key='key1', value='a') - await kvs_client.set_value(key='key2', value='b') + await kvs_backend.set_value(key='key1', value='a') + await kvs_backend.set_value(key='key2', value='b') collected_keys = [] - async for metadata in kvs_client.iterate_keys(): + async for metadata in kvs_backend.iterate_keys(): if metadata.key == 'key1': - await kvs_client.delete_value(key='key2') + await kvs_backend.delete_value(key='key2') collected_keys.append(metadata.key) assert collected_keys == ['key1'] diff --git a/tests/unit/storage_clients/_memory/test_memory_rq_client.py b/tests/unit/storage_backends/_memory/test_memory_rq_backend.py similarity index 60% rename from tests/unit/storage_clients/_memory/test_memory_rq_client.py rename to tests/unit/storage_backends/_memory/test_memory_rq_backend.py index b3440afe77..d52a12d6f3 100644 --- a/tests/unit/storage_clients/_memory/test_memory_rq_client.py +++ b/tests/unit/storage_backends/_memory/test_memory_rq_backend.py @@ -7,52 +7,52 @@ import pytest from crawlee import Request -from crawlee.storage_clients import MemoryStorageClient +from crawlee.storage_backends import MemoryStorageBackend if TYPE_CHECKING: from collections.abc import AsyncGenerator - from crawlee.storage_clients._memory import MemoryRequestQueueClient + from crawlee.storage_backends._memory import MemoryRequestQueueBackend @pytest.fixture -async def rq_client() -> AsyncGenerator[MemoryRequestQueueClient, None]: - """Fixture that provides a fresh memory request queue client for each test.""" - client = await MemoryStorageClient().create_rq_client(name='test-rq') - yield client - await client.drop() +async def rq_backend() -> AsyncGenerator[MemoryRequestQueueBackend, None]: + """Fixture that provides a fresh memory request queue backend for each test.""" + backend = await MemoryStorageBackend().create_rq_backend(name='test-rq') + yield backend + await backend.drop() async def test_memory_specific_purge_behavior() -> None: """Test memory-specific purge behavior and in-memory storage characteristics.""" # Create RQ and add data - rq_client1 = await MemoryStorageClient().create_rq_client( + rq_backend1 = await MemoryStorageBackend().create_rq_backend( name='test-purge-rq', ) request = Request.from_url(url='https://example.com/initial') - await rq_client1.add_batch_of_requests([request]) + await rq_backend1.add_batch_of_requests([request]) # Verify request was added - assert await rq_client1.is_empty() is False + assert await rq_backend1.is_empty() is False - # Reopen with same storage client instance - rq_client2 = await MemoryStorageClient().create_rq_client( + # Reopen with same storage backend instance + rq_backend2 = await MemoryStorageBackend().create_rq_backend( name='test-purge-rq', ) # Verify queue was purged (memory storage specific behavior) - assert await rq_client2.is_empty() is True + assert await rq_backend2.is_empty() is True async def test_add_existing_pending_request_returns_single_processed_request( - rq_client: MemoryRequestQueueClient, + rq_backend: MemoryRequestQueueBackend, ) -> None: """Test that re-adding a pending (not handled) request yields exactly one `ProcessedRequest` entry.""" request = Request.from_url('https://example.com') - await rq_client.add_batch_of_requests([request]) + await rq_backend.add_batch_of_requests([request]) # Re-add the same request while it is still pending (not handled, not in progress). - response = await rq_client.add_batch_of_requests([request]) + response = await rq_backend.add_batch_of_requests([request]) assert len(response.processed_requests) == 1 processed_request = response.processed_requests[0] @@ -61,10 +61,10 @@ async def test_add_existing_pending_request_returns_single_processed_request( assert processed_request.was_already_handled is False -async def test_memory_metadata_updates(rq_client: MemoryRequestQueueClient) -> None: +async def test_memory_metadata_updates(rq_backend: MemoryRequestQueueBackend) -> None: """Test that metadata timestamps are updated correctly in memory storage.""" # Record initial timestamps - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -73,10 +73,10 @@ async def test_memory_metadata_updates(rq_client: MemoryRequestQueueClient) -> N await asyncio.sleep(0.01) # Perform a read operation - await rq_client.is_empty() + await rq_backend.is_empty() # Verify timestamps (memory-specific behavior) - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -87,19 +87,19 @@ async def test_memory_metadata_updates(rq_client: MemoryRequestQueueClient) -> N await asyncio.sleep(0.01) # Perform a write operation - await rq_client.add_batch_of_requests([Request.from_url('https://example.com')]) + await rq_backend.add_batch_of_requests([Request.from_url('https://example.com')]) # Verify timestamps were updated - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_read -async def test_readd_of_pending_requests_does_not_create_duplicates(rq_client: MemoryRequestQueueClient) -> None: +async def test_readd_of_pending_requests_does_not_create_duplicates(rq_backend: MemoryRequestQueueBackend) -> None: """Test that mixing regular and forefront re-adds of pending requests leaves no stale duplicates behind.""" urls = [f'https://example.com/{i}' for i in range(3)] - await rq_client.add_batch_of_requests([Request.from_url(url) for url in urls]) + await rq_backend.add_batch_of_requests([Request.from_url(url) for url in urls]) # Re-add the same URLs as freshly built objects with a differing payload, the way the higher-level API does # when it rebuilds requests discovered on another page. @@ -110,37 +110,37 @@ async def test_readd_of_pending_requests_does_not_create_duplicates(rq_client: M duplicate.user_data['version'] = version duplicates.append(duplicate) - await rq_client.add_batch_of_requests(duplicates) - await rq_client.add_batch_of_requests(duplicates, forefront=True) + await rq_backend.add_batch_of_requests(duplicates) + await rq_backend.add_batch_of_requests(duplicates, forefront=True) fetched = [] - while (request := await rq_client.fetch_next_request()) is not None: + while (request := await rq_backend.fetch_next_request()) is not None: fetched.append(request) - await rq_client.mark_request_as_handled(request) + await rq_backend.mark_request_as_handled(request) assert len(fetched) == len(urls) assert {request.url for request in fetched} == set(urls) - assert await rq_client.is_finished() is True + assert await rq_backend.is_finished() is True - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.total_request_count == len(urls) assert metadata.pending_request_count == 0 assert metadata.handled_request_count == len(urls) -async def test_forefront_readd_preserves_order_and_dedup(rq_client: MemoryRequestQueueClient) -> None: +async def test_forefront_readd_preserves_order_and_dedup(rq_backend: MemoryRequestQueueBackend) -> None: """Test that repositioning already-pending requests to the forefront keeps LIFO order and dedup.""" requests = [Request.from_url(f'https://example.com/{i}') for i in range(3)] - await rq_client.add_batch_of_requests(requests) + await rq_backend.add_batch_of_requests(requests) # Re-add a subset (0 and 1) to the forefront while still pending. Request 1 is added last, so it must # end up at the very front, followed by request 0, then the untouched regular request 2. - await rq_client.add_batch_of_requests(requests[:2], forefront=True) + await rq_backend.add_batch_of_requests(requests[:2], forefront=True) fetched_urls = [] - while (request := await rq_client.fetch_next_request()) is not None: + while (request := await rq_backend.fetch_next_request()) is not None: fetched_urls.append(request.url) - await rq_client.mark_request_as_handled(request) + await rq_backend.mark_request_as_handled(request) assert fetched_urls == [ 'https://example.com/1', @@ -149,92 +149,92 @@ async def test_forefront_readd_preserves_order_and_dedup(rq_client: MemoryReques ] # No stale duplicates should linger after all live requests are drained. - assert await rq_client.is_empty() is True - assert await rq_client.is_finished() is True + assert await rq_backend.is_empty() is True + assert await rq_backend.is_finished() is True -async def test_forefront_readd_of_in_progress_request_is_a_no_op(rq_client: MemoryRequestQueueClient) -> None: +async def test_forefront_readd_of_in_progress_request_is_a_no_op(rq_backend: MemoryRequestQueueBackend) -> None: """Test that a forefront re-add of an in-progress request does not raise and does not re-enqueue it.""" requests = [Request.from_url(f'https://example.com/{i}') for i in range(3)] - await rq_client.add_batch_of_requests(requests) + await rq_backend.add_batch_of_requests(requests) - in_progress = await rq_client.fetch_next_request() + in_progress = await rq_backend.fetch_next_request() assert in_progress is not None - response = await rq_client.add_batch_of_requests([in_progress], forefront=True) + response = await rq_backend.add_batch_of_requests([in_progress], forefront=True) assert len(response.processed_requests) == 1 assert response.processed_requests[0].was_already_present is True assert response.processed_requests[0].was_already_handled is False # The request must not have been queued again ahead of the requests still waiting behind it. - await rq_client.mark_request_as_handled(in_progress) + await rq_backend.mark_request_as_handled(in_progress) fetched_urls = [] - while (request := await rq_client.fetch_next_request()) is not None: + while (request := await rq_backend.fetch_next_request()) is not None: fetched_urls.append(request.url) - await rq_client.mark_request_as_handled(request) + await rq_backend.mark_request_as_handled(request) assert fetched_urls == [ 'https://example.com/1', 'https://example.com/2', ] - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.total_request_count == 3 assert metadata.pending_request_count == 0 assert metadata.handled_request_count == 3 -async def test_forefront_readd_of_handled_request_is_a_no_op(rq_client: MemoryRequestQueueClient) -> None: +async def test_forefront_readd_of_handled_request_is_a_no_op(rq_backend: MemoryRequestQueueBackend) -> None: """Test that a forefront re-add of an already handled request does not raise and does not re-enqueue it.""" - await rq_client.add_batch_of_requests([Request.from_url('https://example.com/page')]) + await rq_backend.add_batch_of_requests([Request.from_url('https://example.com/page')]) - fetched = await rq_client.fetch_next_request() + fetched = await rq_backend.fetch_next_request() assert fetched is not None - await rq_client.mark_request_as_handled(fetched) + await rq_backend.mark_request_as_handled(fetched) - response = await rq_client.add_batch_of_requests([Request.from_url('https://example.com/page')], forefront=True) + response = await rq_backend.add_batch_of_requests([Request.from_url('https://example.com/page')], forefront=True) assert len(response.processed_requests) == 1 assert response.processed_requests[0].was_already_present is True assert response.processed_requests[0].was_already_handled is True - assert await rq_client.fetch_next_request() is None - assert await rq_client.is_empty() is True - assert await rq_client.is_finished() is True + assert await rq_backend.fetch_next_request() is None + assert await rq_backend.is_empty() is True + assert await rq_backend.is_finished() is True - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.total_request_count == 1 assert metadata.pending_request_count == 0 assert metadata.handled_request_count == 1 -async def test_readd_keeps_the_originally_enqueued_request(rq_client: MemoryRequestQueueClient) -> None: +async def test_readd_keeps_the_originally_enqueued_request(rq_backend: MemoryRequestQueueBackend) -> None: """Test that re-adding a still-pending request does not replace it with the incoming duplicate.""" original = Request.from_url('https://example.com/page') original.user_data['version'] = 1 - await rq_client.add_batch_of_requests([original]) + await rq_backend.add_batch_of_requests([original]) # Re-add the same URL while still pending, as a distinct object (as the higher-level API does when it # rebuilds requests). Neither a regular nor a forefront re-add may overwrite the enqueued request. duplicate = Request.from_url('https://example.com/page') duplicate.user_data['version'] = 2 assert duplicate.unique_key == original.unique_key - await rq_client.add_batch_of_requests([duplicate]) - await rq_client.add_batch_of_requests([duplicate], forefront=True) + await rq_backend.add_batch_of_requests([duplicate]) + await rq_backend.add_batch_of_requests([duplicate], forefront=True) - assert await rq_client.get_request(original.unique_key) is original - assert await rq_client.is_empty() is False + assert await rq_backend.get_request(original.unique_key) is original + assert await rq_backend.is_empty() is False - fetched = await rq_client.fetch_next_request() + fetched = await rq_backend.fetch_next_request() assert fetched is original assert fetched.user_data['version'] == 1 - await rq_client.mark_request_as_handled(fetched) + await rq_backend.mark_request_as_handled(fetched) - assert await rq_client.fetch_next_request() is None - assert await rq_client.is_empty() is True - assert await rq_client.is_finished() is True + assert await rq_backend.fetch_next_request() is None + assert await rq_backend.is_empty() is True + assert await rq_backend.is_finished() is True - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.total_request_count == 1 assert metadata.pending_request_count == 0 assert metadata.handled_request_count == 1 @@ -248,39 +248,39 @@ async def test_readd_keeps_the_originally_enqueued_request(rq_client: MemoryRequ ], ) async def test_readd_does_not_reset_retry_count_of_reclaimed_request( - rq_client: MemoryRequestQueueClient, + rq_backend: MemoryRequestQueueBackend, *, forefront: bool, ) -> None: """Test that a duplicate enqueued while a failed request awaits a retry does not reset its `retry_count`.""" request = Request.from_url('https://example.com/page') - await rq_client.add_batch_of_requests([request]) + await rq_backend.add_batch_of_requests([request]) - fetched = await rq_client.fetch_next_request() + fetched = await rq_backend.fetch_next_request() assert fetched is not None fetched.retry_count += 1 - await rq_client.reclaim_request(fetched) + await rq_backend.reclaim_request(fetched) # A handler running in parallel discovers the same URL and enqueues a freshly built request for it. - await rq_client.add_batch_of_requests([Request.from_url('https://example.com/page')], forefront=forefront) + await rq_backend.add_batch_of_requests([Request.from_url('https://example.com/page')], forefront=forefront) - retried = await rq_client.fetch_next_request() + retried = await rq_backend.fetch_next_request() assert retried is not None assert retried.retry_count == 1 -async def test_regular_readd_does_not_reorder_pending_queue(rq_client: MemoryRequestQueueClient) -> None: +async def test_regular_readd_does_not_reorder_pending_queue(rq_backend: MemoryRequestQueueBackend) -> None: """Test that a regular re-add of an already-pending request leaves the FIFO order untouched.""" requests = [Request.from_url(f'https://example.com/{i}') for i in range(3)] - await rq_client.add_batch_of_requests(requests) + await rq_backend.add_batch_of_requests(requests) # Re-add the first request (still pending) without `forefront`; it must stay in its original position. - await rq_client.add_batch_of_requests([requests[0]]) + await rq_backend.add_batch_of_requests([requests[0]]) fetched_urls = [] - while (request := await rq_client.fetch_next_request()) is not None: + while (request := await rq_backend.fetch_next_request()) is not None: fetched_urls.append(request.url) - await rq_client.mark_request_as_handled(request) + await rq_backend.mark_request_as_handled(request) assert fetched_urls == [ 'https://example.com/0', @@ -289,52 +289,52 @@ async def test_regular_readd_does_not_reorder_pending_queue(rq_client: MemoryReq ] -async def test_forefront_readd_does_not_compare_pending_requests(rq_client: MemoryRequestQueueClient) -> None: +async def test_forefront_readd_does_not_compare_pending_requests(rq_backend: MemoryRequestQueueBackend) -> None: """Test that a forefront re-add repositions pending requests without comparing them pair by pair.""" requests = [Request.from_url(f'https://example.com/{i}') for i in range(20)] - await rq_client.add_batch_of_requests(requests) + await rq_backend.add_batch_of_requests(requests) # Locating a pending request by equality means scanning the whole queue, which is what makes a batch of K # forefront re-adds against N pending requests cost O(K*N). Repositioning must be a keyed lookup instead. with patch.object(Request, '__eq__', autospec=True, side_effect=Request.__eq__) as eq_counter: - await rq_client.add_batch_of_requests(requests, forefront=True) + await rq_backend.add_batch_of_requests(requests, forefront=True) eq_counter.assert_not_called() -async def test_reclaim_stores_the_modified_request(rq_client: MemoryRequestQueueClient) -> None: +async def test_reclaim_stores_the_modified_request(rq_backend: MemoryRequestQueueBackend) -> None: """Test that reclaiming a modified request updates both the queue and the lookup by unique key.""" request = Request.from_url('https://example.com/page') - await rq_client.add_batch_of_requests([request]) + await rq_backend.add_batch_of_requests([request]) - fetched = await rq_client.fetch_next_request() + fetched = await rq_backend.fetch_next_request() assert fetched is request modified = request.model_copy(deep=True) modified.user_data['reclaimed'] = True - await rq_client.reclaim_request(modified) + await rq_backend.reclaim_request(modified) - assert await rq_client.get_request(request.unique_key) is modified - assert await rq_client.is_empty() is False + assert await rq_backend.get_request(request.unique_key) is modified + assert await rq_backend.is_empty() is False - reclaimed = await rq_client.fetch_next_request() + reclaimed = await rq_backend.fetch_next_request() assert reclaimed is modified assert reclaimed.user_data['reclaimed'] is True -async def test_reclaim_to_forefront_moves_request_to_the_front(rq_client: MemoryRequestQueueClient) -> None: +async def test_reclaim_to_forefront_moves_request_to_the_front(rq_backend: MemoryRequestQueueBackend) -> None: """Test that a request reclaimed with `forefront` is fetched again ahead of the requests waiting behind it.""" requests = [Request.from_url(f'https://example.com/{i}') for i in range(3)] - await rq_client.add_batch_of_requests(requests) + await rq_backend.add_batch_of_requests(requests) - first = await rq_client.fetch_next_request() + first = await rq_backend.fetch_next_request() assert first is not None - await rq_client.reclaim_request(first, forefront=True) + await rq_backend.reclaim_request(first, forefront=True) fetched_urls = [] - while (request := await rq_client.fetch_next_request()) is not None: + while (request := await rq_backend.fetch_next_request()) is not None: fetched_urls.append(request.url) - await rq_client.mark_request_as_handled(request) + await rq_backend.mark_request_as_handled(request) assert fetched_urls == [ 'https://example.com/0', diff --git a/tests/unit/storage_clients/_redis/test_redis_dataset_client.py b/tests/unit/storage_backends/_redis/test_redis_dataset_backend.py similarity index 59% rename from tests/unit/storage_clients/_redis/test_redis_dataset_client.py rename to tests/unit/storage_backends/_redis/test_redis_dataset_backend.py index 28d182f0f6..9533a0d10a 100644 --- a/tests/unit/storage_clients/_redis/test_redis_dataset_client.py +++ b/tests/unit/storage_backends/_redis/test_redis_dataset_backend.py @@ -7,68 +7,68 @@ import pytest from redis.exceptions import RedisError -from crawlee.storage_clients import RedisStorageClient -from crawlee.storage_clients._redis._utils import await_redis_response +from crawlee.storage_backends import RedisStorageBackend +from crawlee.storage_backends._redis._utils import await_redis_response if TYPE_CHECKING: from collections.abc import AsyncGenerator from fakeredis import FakeAsyncRedis - from crawlee.storage_clients._redis import RedisDatasetClient + from crawlee.storage_backends._redis import RedisDatasetBackend @pytest.fixture -async def dataset_client( +async def dataset_backend( redis_client: FakeAsyncRedis, suppress_user_warning: None, # noqa: ARG001 -) -> AsyncGenerator[RedisDatasetClient, None]: - """A fixture for a Redis dataset client.""" - client = await RedisStorageClient(redis=redis_client).create_dataset_client( +) -> AsyncGenerator[RedisDatasetBackend, None]: + """A fixture for a Redis dataset backend.""" + backend = await RedisStorageBackend(redis=redis_client).create_dataset_backend( name='test_dataset', ) - yield client - await client.drop() + yield backend + await backend.drop() -async def test_base_keys_creation(dataset_client: RedisDatasetClient) -> None: - """Test that Redis dataset client creates proper keys.""" - metadata = await dataset_client.get_metadata() - name = await await_redis_response(dataset_client.redis.hget('datasets:id_to_name', metadata.id)) +async def test_base_keys_creation(dataset_backend: RedisDatasetBackend) -> None: + """Test that Redis dataset backend creates proper keys.""" + metadata = await dataset_backend.get_metadata() + name = await await_redis_response(dataset_backend.redis.hget('datasets:id_to_name', metadata.id)) assert name is not None assert (name.decode() if isinstance(name, bytes) else name) == 'test_dataset' - dataset_id = await await_redis_response(dataset_client.redis.hget('datasets:name_to_id', 'test_dataset')) + dataset_id = await await_redis_response(dataset_backend.redis.hget('datasets:name_to_id', 'test_dataset')) assert dataset_id is not None assert (dataset_id.decode() if isinstance(dataset_id, bytes) else dataset_id) == metadata.id - items = await await_redis_response(dataset_client.redis.json().get('datasets:test_dataset:items', '$')) + items = await await_redis_response(dataset_backend.redis.json().get('datasets:test_dataset:items', '$')) assert items is not None assert isinstance(items, list) assert len(items) == 0 - metadata_data = await await_redis_response(dataset_client.redis.json().get('datasets:test_dataset:metadata')) + metadata_data = await await_redis_response(dataset_backend.redis.json().get('datasets:test_dataset:metadata')) assert isinstance(metadata_data, dict) assert metadata_data['id'] == metadata.id -async def test_record_and_content_verification(dataset_client: RedisDatasetClient) -> None: +async def test_record_and_content_verification(dataset_backend: RedisDatasetBackend) -> None: """Test that data is properly persisted to Redis with correct content.""" item = {'key': 'value', 'number': 42} - await dataset_client.push_data(item) + await dataset_backend.push_data(item) # Verify metadata record - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() assert metadata.item_count == 1 assert metadata.created_at is not None assert metadata.modified_at is not None assert metadata.accessed_at is not None # Verify records in Redis - all_items = await await_redis_response(dataset_client.redis.json().get('datasets:test_dataset:items', '$')) + all_items = await await_redis_response(dataset_backend.redis.json().get('datasets:test_dataset:items', '$')) assert all_items is not None assert isinstance(all_items, list) @@ -79,22 +79,22 @@ async def test_record_and_content_verification(dataset_client: RedisDatasetClien # Test multiple records items = [{'id': 1, 'name': 'Item 1'}, {'id': 2, 'name': 'Item 2'}, {'id': 3, 'name': 'Item 3'}] - await dataset_client.push_data(items) + await dataset_backend.push_data(items) - all_items = await await_redis_response(dataset_client.redis.json().get('datasets:test_dataset:items', '$')) + all_items = await await_redis_response(dataset_backend.redis.json().get('datasets:test_dataset:items', '$')) assert all_items is not None assert isinstance(all_items, list) assert len(all_items) == 4 -async def test_drop_removes_records(dataset_client: RedisDatasetClient) -> None: +async def test_drop_removes_records(dataset_backend: RedisDatasetBackend) -> None: """Test that dropping a dataset removes all records from Redis.""" - await dataset_client.push_data({'test': 'data'}) + await dataset_backend.push_data({'test': 'data'}) - metadata = await dataset_client.get_metadata() - name = await await_redis_response(dataset_client.redis.hget('datasets:id_to_name', metadata.id)) - dataset_id = await await_redis_response(dataset_client.redis.hget('datasets:name_to_id', 'test_dataset')) - items = await await_redis_response(dataset_client.redis.json().get('datasets:test_dataset:items', '$')) + metadata = await dataset_backend.get_metadata() + name = await await_redis_response(dataset_backend.redis.hget('datasets:id_to_name', metadata.id)) + dataset_id = await await_redis_response(dataset_backend.redis.hget('datasets:name_to_id', 'test_dataset')) + items = await await_redis_response(dataset_backend.redis.json().get('datasets:test_dataset:items', '$')) assert name is not None assert (name.decode() if isinstance(name, bytes) else name) == 'test_dataset' @@ -105,12 +105,14 @@ async def test_drop_removes_records(dataset_client: RedisDatasetClient) -> None: assert len(items) == 1 # Drop the dataset - await dataset_client.drop() + await dataset_backend.drop() # Verify removal of all records - name_after_drop = await await_redis_response(dataset_client.redis.hget('datasets:id_to_name', metadata.id)) - dataset_id_after_drop = await await_redis_response(dataset_client.redis.hget('datasets:name_to_id', 'test_dataset')) - items_after_drop = await await_redis_response(dataset_client.redis.json().get('datasets:test_dataset:items', '$')) + name_after_drop = await await_redis_response(dataset_backend.redis.hget('datasets:id_to_name', metadata.id)) + dataset_id_after_drop = await await_redis_response( + dataset_backend.redis.hget('datasets:name_to_id', 'test_dataset') + ) + items_after_drop = await await_redis_response(dataset_backend.redis.json().get('datasets:test_dataset:items', '$')) assert name_after_drop is None assert dataset_id_after_drop is None @@ -118,15 +120,15 @@ async def test_drop_removes_records(dataset_client: RedisDatasetClient) -> None: async def test_drop_preserves_sibling_index_entries( - dataset_client: RedisDatasetClient, + dataset_backend: RedisDatasetBackend, redis_client: FakeAsyncRedis, ) -> None: """Test that dropping a dataset does not wipe the shared index entries of other datasets.""" - storage_client = RedisStorageClient(redis=redis_client) - sibling_client = await storage_client.create_dataset_client(name='sibling_dataset') - sibling_metadata = await sibling_client.get_metadata() + storage_backend = RedisStorageBackend(redis=redis_client) + sibling_backend = await storage_backend.create_dataset_backend(name='sibling_dataset') + sibling_metadata = await sibling_backend.get_metadata() - await dataset_client.drop() + await dataset_backend.drop() # The sibling's entries in the shared id_to_name and name_to_id hashes must remain intact. sibling_name = await await_redis_response(redis_client.hget('datasets:id_to_name', sibling_metadata.id)) @@ -138,14 +140,14 @@ async def test_drop_preserves_sibling_index_entries( assert (sibling_id.decode() if isinstance(sibling_id, bytes) else sibling_id) == sibling_metadata.id # Opening the sibling by ID must still work. - reopened = await storage_client.create_dataset_client(id=sibling_metadata.id) + reopened = await storage_backend.create_dataset_backend(id=sibling_metadata.id) assert (await reopened.get_metadata()).id == sibling_metadata.id -async def test_metadata_record_updates(dataset_client: RedisDatasetClient) -> None: +async def test_metadata_record_updates(dataset_backend: RedisDatasetBackend) -> None: """Test that metadata record is updated correctly after operations.""" # Record initial timestamps - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -154,10 +156,10 @@ async def test_metadata_record_updates(dataset_client: RedisDatasetClient) -> No await asyncio.sleep(0.01) # Perform an operation that updates accessed_at - await dataset_client.get_data() + await dataset_backend.get_data() # Verify timestamps - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -168,16 +170,16 @@ async def test_metadata_record_updates(dataset_client: RedisDatasetClient) -> No await asyncio.sleep(0.01) # Perform an operation that updates modified_at - await dataset_client.push_data({'new': 'item'}) + await dataset_backend.push_data({'new': 'item'}) # Verify timestamps again - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_get -async def test_error_handling_on_push_failure(dataset_client: RedisDatasetClient) -> None: +async def test_error_handling_on_push_failure(dataset_backend: RedisDatasetBackend) -> None: """Test that push_data properly handles Redis errors and retries.""" mock_pipe = MagicMock() mock_pipe.execute = AsyncMock(side_effect=RedisError('connection lost')) @@ -188,16 +190,16 @@ async def test_error_handling_on_push_failure(dataset_client: RedisDatasetClient with ( patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, - patch.object(dataset_client.redis, 'pipeline', return_value=mock_pipeline_ctx), + patch.object(dataset_backend.redis, 'pipeline', return_value=mock_pipeline_ctx), pytest.raises(RedisError), ): - await dataset_client.push_data({'test': 'data'}) + await dataset_backend.push_data({'test': 'data'}) # Verify that retry logic was attempted assert mock_sleep.call_count == 2 # Assuming default max_attempts=3 -async def test_push_data_does_not_retry_on_unexpected_exception(dataset_client: RedisDatasetClient) -> None: +async def test_push_data_does_not_retry_on_unexpected_exception(dataset_backend: RedisDatasetBackend) -> None: """Test that push_data does not retry on unexpected exceptions.""" mock_pipe = MagicMock() mock_pipe.execute = AsyncMock(side_effect=ValueError('unexpected error')) @@ -208,10 +210,10 @@ async def test_push_data_does_not_retry_on_unexpected_exception(dataset_client: with ( patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, - patch.object(dataset_client.redis, 'pipeline', return_value=mock_pipeline_ctx), + patch.object(dataset_backend.redis, 'pipeline', return_value=mock_pipeline_ctx), pytest.raises(ValueError, match='unexpected error'), ): - await dataset_client.push_data({'test': 'data'}) + await dataset_backend.push_data({'test': 'data'}) # Verify that retry logic was not attempted assert mock_sleep.call_count == 0 diff --git a/tests/unit/storage_clients/_redis/test_redis_kvs_client.py b/tests/unit/storage_backends/_redis/test_redis_kvs_backend.py similarity index 53% rename from tests/unit/storage_clients/_redis/test_redis_kvs_client.py rename to tests/unit/storage_backends/_redis/test_redis_kvs_backend.py index 6018c37511..80d413f3e1 100644 --- a/tests/unit/storage_clients/_redis/test_redis_kvs_client.py +++ b/tests/unit/storage_backends/_redis/test_redis_kvs_backend.py @@ -9,68 +9,68 @@ from fakeredis import FakeAsyncRedis from redis.exceptions import RedisError -from crawlee.storage_clients import RedisStorageClient -from crawlee.storage_clients._redis._utils import await_redis_response +from crawlee.storage_backends import RedisStorageBackend +from crawlee.storage_backends._redis._utils import await_redis_response if TYPE_CHECKING: from collections.abc import AsyncGenerator, Iterator - from crawlee.storage_clients._redis import RedisKeyValueStoreClient + from crawlee.storage_backends._redis import RedisKeyValueStoreBackend @pytest.fixture -async def kvs_client( +async def kvs_backend( redis_client: FakeAsyncRedis, suppress_user_warning: None, # noqa: ARG001 -) -> AsyncGenerator[RedisKeyValueStoreClient, None]: - """A fixture for a Redis KVS client.""" - client = await RedisStorageClient(redis=redis_client).create_kvs_client( +) -> AsyncGenerator[RedisKeyValueStoreBackend, None]: + """A fixture for a Redis KVS backend.""" + backend = await RedisStorageBackend(redis=redis_client).create_kvs_backend( name='test_kvs', ) - yield client - await client.drop() + yield backend + await backend.drop() -async def test_base_keys_creation(kvs_client: RedisKeyValueStoreClient) -> None: - """Test that Redis KVS client creates proper keys.""" - metadata = await kvs_client.get_metadata() - name = await await_redis_response(kvs_client.redis.hget('key_value_stores:id_to_name', metadata.id)) +async def test_base_keys_creation(kvs_backend: RedisKeyValueStoreBackend) -> None: + """Test that Redis KVS backend creates proper keys.""" + metadata = await kvs_backend.get_metadata() + name = await await_redis_response(kvs_backend.redis.hget('key_value_stores:id_to_name', metadata.id)) assert name is not None assert (name.decode() if isinstance(name, bytes) else name) == 'test_kvs' - kvs_id = await await_redis_response(kvs_client.redis.hget('key_value_stores:name_to_id', 'test_kvs')) + kvs_id = await await_redis_response(kvs_backend.redis.hget('key_value_stores:name_to_id', 'test_kvs')) assert kvs_id is not None assert (kvs_id.decode() if isinstance(kvs_id, bytes) else kvs_id) == metadata.id - metadata_data = await await_redis_response(kvs_client.redis.json().get('key_value_stores:test_kvs:metadata')) + metadata_data = await await_redis_response(kvs_backend.redis.json().get('key_value_stores:test_kvs:metadata')) assert isinstance(metadata_data, dict) assert metadata_data['id'] == metadata.id -async def test_value_record_creation_and_content(kvs_client: RedisKeyValueStoreClient) -> None: +async def test_value_record_creation_and_content(kvs_backend: RedisKeyValueStoreBackend) -> None: """Test that values are properly persisted to records with correct content and metadata.""" test_key = 'test-key' test_value = 'Hello, world!' - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) # Check if the records were created records_key = 'key_value_stores:test_kvs:items' records_items_metadata = 'key_value_stores:test_kvs:metadata_items' - record_exists = await await_redis_response(kvs_client.redis.hexists(records_key, test_key)) - metadata_exists = await await_redis_response(kvs_client.redis.hexists(records_items_metadata, test_key)) + record_exists = await await_redis_response(kvs_backend.redis.hexists(records_key, test_key)) + metadata_exists = await await_redis_response(kvs_backend.redis.hexists(records_items_metadata, test_key)) assert record_exists is True assert metadata_exists is True # Check record content - content = await await_redis_response(kvs_client.redis.hget(records_key, test_key)) + content = await await_redis_response(kvs_backend.redis.hget(records_key, test_key)) content = content.decode() if isinstance(content, bytes) else content assert content == test_value # Check record metadata - record_metadata = await await_redis_response(kvs_client.redis.hget(records_items_metadata, test_key)) + record_metadata = await await_redis_response(kvs_backend.redis.hget(records_items_metadata, test_key)) assert record_metadata is not None assert isinstance(record_metadata, (str, bytes)) metadata = json.loads(record_metadata) @@ -81,51 +81,51 @@ async def test_value_record_creation_and_content(kvs_client: RedisKeyValueStoreC assert metadata['size'] == len(test_value.encode('utf-8')) # Verify retrieval works correctly - check_value = await kvs_client.get_value(key=test_key) + check_value = await kvs_backend.get_value(key=test_key) assert check_value is not None assert check_value.value == test_value -async def test_binary_data_persistence(kvs_client: RedisKeyValueStoreClient) -> None: +async def test_binary_data_persistence(kvs_backend: RedisKeyValueStoreBackend) -> None: """Test that binary data is stored correctly without corruption.""" test_key = 'test-binary' test_value = b'\x00\x01\x02\x03\x04' records_key = 'key_value_stores:test_kvs:items' records_items_metadata = 'key_value_stores:test_kvs:metadata_items' - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) # Verify binary file exists - record_exists = await await_redis_response(kvs_client.redis.hexists(records_key, test_key)) - metadata_exists = await await_redis_response(kvs_client.redis.hexists(records_items_metadata, test_key)) + record_exists = await await_redis_response(kvs_backend.redis.hexists(records_key, test_key)) + metadata_exists = await await_redis_response(kvs_backend.redis.hexists(records_items_metadata, test_key)) assert record_exists is True assert metadata_exists is True # Verify binary content is preserved - content = await await_redis_response(kvs_client.redis.hget(records_key, test_key)) + content = await await_redis_response(kvs_backend.redis.hget(records_key, test_key)) assert content == test_value # Verify retrieval works correctly - record = await kvs_client.get_value(key=test_key) + record = await kvs_backend.get_value(key=test_key) assert record is not None assert record.value == test_value assert record.content_type == 'application/octet-stream' -async def test_json_serialization_to_record(kvs_client: RedisKeyValueStoreClient) -> None: +async def test_json_serialization_to_record(kvs_backend: RedisKeyValueStoreBackend) -> None: """Test that JSON objects are properly serialized to records.""" test_key = 'test-json' test_value = {'name': 'John', 'age': 30, 'items': [1, 2, 3]} - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) # Check if record content is valid JSON records_key = 'key_value_stores:test_kvs:items' - record = await await_redis_response(kvs_client.redis.hget(records_key, test_key)) + record = await await_redis_response(kvs_backend.redis.hget(records_key, test_key)) assert record is not None assert isinstance(record, (str, bytes)) assert json.loads(record) == test_value -async def test_records_deletion_on_value_delete(kvs_client: RedisKeyValueStoreClient) -> None: +async def test_records_deletion_on_value_delete(kvs_backend: RedisKeyValueStoreBackend) -> None: """Test that deleting a value removes its records from Redis.""" test_key = 'test-delete' test_value = 'Delete me' @@ -133,33 +133,33 @@ async def test_records_deletion_on_value_delete(kvs_client: RedisKeyValueStoreCl records_items_metadata = 'key_value_stores:test_kvs:metadata_items' # Set a value - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) # Verify records exist - record_exists = await await_redis_response(kvs_client.redis.hexists(records_key, test_key)) - metadata_exists = await await_redis_response(kvs_client.redis.hexists(records_items_metadata, test_key)) + record_exists = await await_redis_response(kvs_backend.redis.hexists(records_key, test_key)) + metadata_exists = await await_redis_response(kvs_backend.redis.hexists(records_items_metadata, test_key)) assert record_exists is True assert metadata_exists is True # Delete the value - await kvs_client.delete_value(key=test_key) + await kvs_backend.delete_value(key=test_key) # Verify files were deleted - record_exists = await await_redis_response(kvs_client.redis.hexists(records_key, test_key)) - metadata_exists = await await_redis_response(kvs_client.redis.hexists(records_items_metadata, test_key)) + record_exists = await await_redis_response(kvs_backend.redis.hexists(records_key, test_key)) + metadata_exists = await await_redis_response(kvs_backend.redis.hexists(records_items_metadata, test_key)) assert record_exists is False assert metadata_exists is False -async def test_drop_removes_keys(kvs_client: RedisKeyValueStoreClient) -> None: +async def test_drop_removes_keys(kvs_backend: RedisKeyValueStoreBackend) -> None: """Test that drop removes the entire store directory from disk.""" - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') - metadata = await kvs_client.get_metadata() - name = await await_redis_response(kvs_client.redis.hget('key_value_stores:id_to_name', metadata.id)) - kvs_id = await await_redis_response(kvs_client.redis.hget('key_value_stores:name_to_id', 'test_kvs')) - items = await await_redis_response(kvs_client.redis.hgetall('key_value_stores:test_kvs:items')) - metadata_items = await await_redis_response(kvs_client.redis.hgetall('key_value_stores:test_kvs:metadata_items')) + metadata = await kvs_backend.get_metadata() + name = await await_redis_response(kvs_backend.redis.hget('key_value_stores:id_to_name', metadata.id)) + kvs_id = await await_redis_response(kvs_backend.redis.hget('key_value_stores:name_to_id', 'test_kvs')) + items = await await_redis_response(kvs_backend.redis.hgetall('key_value_stores:test_kvs:items')) + metadata_items = await await_redis_response(kvs_backend.redis.hgetall('key_value_stores:test_kvs:metadata_items')) assert name is not None assert (name.decode() if isinstance(name, bytes) else name) == 'test_kvs' @@ -171,22 +171,22 @@ async def test_drop_removes_keys(kvs_client: RedisKeyValueStoreClient) -> None: assert metadata_items != {} # Drop the store - await kvs_client.drop() + await kvs_backend.drop() - name = await await_redis_response(kvs_client.redis.hget('key_value_stores:id_to_name', metadata.id)) - kvs_id = await await_redis_response(kvs_client.redis.hget('key_value_stores:name_to_id', 'test_kvs')) - items = await await_redis_response(kvs_client.redis.hgetall('key_value_stores:test_kvs:items')) - metadata_items = await await_redis_response(kvs_client.redis.hgetall('key_value_stores:test_kvs:metadata_items')) + name = await await_redis_response(kvs_backend.redis.hget('key_value_stores:id_to_name', metadata.id)) + kvs_id = await await_redis_response(kvs_backend.redis.hget('key_value_stores:name_to_id', 'test_kvs')) + items = await await_redis_response(kvs_backend.redis.hgetall('key_value_stores:test_kvs:items')) + metadata_items = await await_redis_response(kvs_backend.redis.hgetall('key_value_stores:test_kvs:metadata_items')) assert name is None assert kvs_id is None assert items == {} assert metadata_items == {} -async def test_metadata_record_updates(kvs_client: RedisKeyValueStoreClient) -> None: +async def test_metadata_record_updates(kvs_backend: RedisKeyValueStoreBackend) -> None: """Test that read/write operations properly update metadata file timestamps.""" # Record initial timestamps - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -195,10 +195,10 @@ async def test_metadata_record_updates(kvs_client: RedisKeyValueStoreClient) -> await asyncio.sleep(0.01) # Perform a read operation - await kvs_client.get_value(key='nonexistent') + await kvs_backend.get_value(key='nonexistent') # Verify accessed timestamp was updated - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -209,16 +209,16 @@ async def test_metadata_record_updates(kvs_client: RedisKeyValueStoreClient) -> await asyncio.sleep(0.01) # Perform a write operation - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') # Verify modified timestamp was updated - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_read -async def test_error_handling_on_set_failure(kvs_client: RedisKeyValueStoreClient) -> None: +async def test_error_handling_on_set_failure(kvs_backend: RedisKeyValueStoreBackend) -> None: """Test that set_value properly handles Redis errors and retries.""" mock_pipe = MagicMock() mock_pipe.execute = AsyncMock(side_effect=RedisError('connection lost')) @@ -229,16 +229,16 @@ async def test_error_handling_on_set_failure(kvs_client: RedisKeyValueStoreClien with ( patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, - patch.object(kvs_client.redis, 'pipeline', return_value=mock_pipeline_ctx), + patch.object(kvs_backend.redis, 'pipeline', return_value=mock_pipeline_ctx), pytest.raises(RedisError), ): - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') # Verify that retry logic was attempted assert mock_sleep.call_count == 2 # Assuming default max_attempts=3 -async def test_set_value_does_not_retry_on_unexpected_exception(kvs_client: RedisKeyValueStoreClient) -> None: +async def test_set_value_does_not_retry_on_unexpected_exception(kvs_backend: RedisKeyValueStoreBackend) -> None: """Test that set_value does not retry on unexpected exceptions.""" mock_pipe = MagicMock() mock_pipe.execute = AsyncMock(side_effect=ValueError('unexpected error')) @@ -249,19 +249,19 @@ async def test_set_value_does_not_retry_on_unexpected_exception(kvs_client: Redi with ( patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, - patch.object(kvs_client.redis, 'pipeline', return_value=mock_pipeline_ctx), + patch.object(kvs_backend.redis, 'pipeline', return_value=mock_pipeline_ctx), pytest.raises(ValueError, match='unexpected error'), ): - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') # Verify that retry logic was not attempted assert mock_sleep.call_count == 0 @pytest.fixture -def hmget(kvs_client: RedisKeyValueStoreClient) -> Iterator[MagicMock]: - """Wrap the Redis `hmget` of the client in a mock that records the calls while still performing them.""" - with patch.object(kvs_client.redis, 'hmget', wraps=kvs_client.redis.hmget) as mock: +def hmget(kvs_backend: RedisKeyValueStoreBackend) -> Iterator[MagicMock]: + """Wrap the Redis `hmget` of the backend in a mock that records the calls while still performing them.""" + with patch.object(kvs_backend.redis, 'hmget', wraps=kvs_backend.redis.hmget) as mock: yield mock @@ -269,15 +269,17 @@ def hmget_keys(hmget: MagicMock) -> list[list[str]]: return [list(call.args[1]) for call in hmget.call_args_list] -async def test_iterate_entries_reads_values_in_batches(kvs_client: RedisKeyValueStoreClient, hmget: MagicMock) -> None: +async def test_iterate_entries_reads_values_in_batches( + kvs_backend: RedisKeyValueStoreBackend, hmget: MagicMock +) -> None: """Test that `iterate_entries` fetches values with batched HMGET calls instead of `get_value` per key.""" - await kvs_client.set_value(key='a-json', value={'nested': [1, 2]}) - await kvs_client.set_value(key='b-text', value='plain text') - await kvs_client.set_value(key='c-bytes', value=b'\x00\x01binary', content_type='application/octet-stream') - await kvs_client.set_value(key='d-none', value=None) + await kvs_backend.set_value(key='a-json', value={'nested': [1, 2]}) + await kvs_backend.set_value(key='b-text', value='plain text') + await kvs_backend.set_value(key='c-bytes', value=b'\x00\x01binary', content_type='application/octet-stream') + await kvs_backend.set_value(key='d-none', value=None) - with patch.object(kvs_client, 'get_value', side_effect=AssertionError('get_value must not be called')): - records = [record async for record in kvs_client.iterate_entries()] + with patch.object(kvs_backend, 'get_value', side_effect=AssertionError('get_value must not be called')): + records = [record async for record in kvs_backend.iterate_entries()] assert [record.key for record in records] == ['a-json', 'b-text', 'c-bytes', 'd-none'] assert [record.value for record in records] == [{'nested': [1, 2]}, 'plain text', b'\x00\x01binary', None] @@ -290,45 +292,45 @@ async def test_iterate_entries_reads_values_in_batches(kvs_client: RedisKeyValue async def test_iterate_entries_batches_are_bounded_by_key_count( - kvs_client: RedisKeyValueStoreClient, hmget: MagicMock + kvs_backend: RedisKeyValueStoreBackend, hmget: MagicMock ) -> None: """Test that `iterate_entries` splits the HMGET calls when a batch reaches the maximum number of keys.""" for i in range(5): - await kvs_client.set_value(key=f'key{i}', value=f'value{i}') + await kvs_backend.set_value(key=f'key{i}', value=f'value{i}') - with patch.object(type(kvs_client), '_ITERATE_ENTRIES_BATCH_MAX_KEYS', 2): - records = [record async for record in kvs_client.iterate_entries()] + with patch.object(type(kvs_backend), '_ITERATE_ENTRIES_BATCH_MAX_KEYS', 2): + records = [record async for record in kvs_backend.iterate_entries()] assert [(record.key, record.value) for record in records] == [(f'key{i}', f'value{i}') for i in range(5)] assert hmget_keys(hmget) == [['key0', 'key1'], ['key2', 'key3'], ['key4']] async def test_iterate_entries_batches_are_bounded_by_size( - kvs_client: RedisKeyValueStoreClient, hmget: MagicMock + kvs_backend: RedisKeyValueStoreBackend, hmget: MagicMock ) -> None: """Test that `iterate_entries` splits the HMGET calls by the record sizes known from the metadata. A record larger than the limit is still fetched, but alone in its batch. """ - await kvs_client.set_value(key='small1', value='ab') - await kvs_client.set_value(key='small2', value='cd') - await kvs_client.set_value(key='large', value='x' * 100) - await kvs_client.set_value(key='small3', value='ef') + await kvs_backend.set_value(key='small1', value='ab') + await kvs_backend.set_value(key='small2', value='cd') + await kvs_backend.set_value(key='large', value='x' * 100) + await kvs_backend.set_value(key='small3', value='ef') - with patch.object(type(kvs_client), '_ITERATE_ENTRIES_BATCH_MAX_BYTES', 10): - records = [record async for record in kvs_client.iterate_entries()] + with patch.object(type(kvs_backend), '_ITERATE_ENTRIES_BATCH_MAX_BYTES', 10): + records = [record async for record in kvs_backend.iterate_entries()] assert [record.key for record in records] == ['large', 'small1', 'small2', 'small3'] assert hmget_keys(hmget) == [['large'], ['small1', 'small2', 'small3']] -async def test_iterate_entries_with_exclusive_start_key_and_limit(kvs_client: RedisKeyValueStoreClient) -> None: +async def test_iterate_entries_with_exclusive_start_key_and_limit(kvs_backend: RedisKeyValueStoreBackend) -> None: """Test that `iterate_entries` applies `exclusive_start_key` and `limit`.""" for i in range(6): - await kvs_client.set_value(key=f'key{i}', value=f'value{i}') + await kvs_backend.set_value(key=f'key{i}', value=f'value{i}') - with patch.object(kvs_client, 'get_value', side_effect=AssertionError('get_value must not be called')): - records = [record async for record in kvs_client.iterate_entries(exclusive_start_key='key1', limit=3)] + with patch.object(kvs_backend, 'get_value', side_effect=AssertionError('get_value must not be called')): + records = [record async for record in kvs_backend.iterate_entries(exclusive_start_key='key1', limit=3)] assert [(record.key, record.value) for record in records] == [ ('key2', 'value2'), @@ -338,35 +340,35 @@ async def test_iterate_entries_with_exclusive_start_key_and_limit(kvs_client: Re async def test_iterate_entries_skips_value_deleted_after_listing( - kvs_client: RedisKeyValueStoreClient, caplog: pytest.LogCaptureFixture + kvs_backend: RedisKeyValueStoreBackend, caplog: pytest.LogCaptureFixture ) -> None: """Test that `iterate_entries` silently skips a record whose value is gone by the time its batch is fetched.""" - await kvs_client.set_value(key='kept', value='a') - await kvs_client.set_value(key='removed', value='b') - await await_redis_response(kvs_client.redis.hdel(kvs_client._items_key, 'removed')) + await kvs_backend.set_value(key='kept', value='a') + await kvs_backend.set_value(key='removed', value='b') + await await_redis_response(kvs_backend.redis.hdel(kvs_backend._items_key, 'removed')) with caplog.at_level('WARNING'): - records = [record async for record in kvs_client.iterate_entries()] + records = [record async for record in kvs_backend.iterate_entries()] assert [(record.key, record.value) for record in records] == [('kept', 'a')] assert 'missing' not in caplog.text -async def test_iterate_entries_empty_store(kvs_client: RedisKeyValueStoreClient) -> None: +async def test_iterate_entries_empty_store(kvs_backend: RedisKeyValueStoreBackend) -> None: """Test that `iterate_entries` on an empty store yields nothing.""" - records = [record async for record in kvs_client.iterate_entries()] + records = [record async for record in kvs_backend.iterate_entries()] assert records == [] async def test_decoding_redis_client_is_rejected(suppress_user_warning: None) -> None: # noqa: ARG001 """Test that reading values through a Redis client created with `decode_responses=True` raises a clear error.""" - storage_client = RedisStorageClient(redis=FakeAsyncRedis(decode_responses=True)) - kvs_client = await storage_client.create_kvs_client(name='decoding_kvs') - await kvs_client.set_value(key='key', value='value') + storage_backend = RedisStorageBackend(redis=FakeAsyncRedis(decode_responses=True)) + kvs_backend = await storage_backend.create_kvs_backend(name='decoding_kvs') + await kvs_backend.set_value(key='key', value='value') with pytest.raises(TypeError, match='decode_responses'): - await kvs_client.get_value(key='key') + await kvs_backend.get_value(key='key') with pytest.raises(TypeError, match='decode_responses'): - _ = [record async for record in kvs_client.iterate_entries()] + _ = [record async for record in kvs_backend.iterate_entries()] diff --git a/tests/unit/storage_clients/_redis/test_redis_rq_client.py b/tests/unit/storage_backends/_redis/test_redis_rq_backend.py similarity index 58% rename from tests/unit/storage_clients/_redis/test_redis_rq_client.py rename to tests/unit/storage_backends/_redis/test_redis_rq_backend.py index f90860de48..b728e26822 100644 --- a/tests/unit/storage_clients/_redis/test_redis_rq_client.py +++ b/tests/unit/storage_backends/_redis/test_redis_rq_backend.py @@ -11,68 +11,70 @@ from redis.exceptions import RedisError from crawlee import Request -from crawlee.storage_clients import RedisStorageClient -from crawlee.storage_clients._redis import _client_mixin -from crawlee.storage_clients._redis._utils import await_redis_response +from crawlee.storage_backends import RedisStorageBackend +from crawlee.storage_backends._redis import _backend_mixin +from crawlee.storage_backends._redis._utils import await_redis_response if TYPE_CHECKING: from collections.abc import AsyncGenerator from fakeredis import FakeAsyncRedis - from crawlee.storage_clients._redis import RedisRequestQueueClient + from crawlee.storage_backends._redis import RedisRequestQueueBackend -# The Redis RQ client runs Lua scripts, which `fakeredis` executes only with `lupa` installed. +# The Redis RQ backend runs Lua scripts, which `fakeredis` executes only with `lupa` installed. pytestmark = pytest.mark.skipif(find_spec('lupa') is None, reason='`lupa` is not installed') @pytest.fixture(params=['default', 'bloom']) -async def rq_client( +async def rq_backend( redis_client: FakeAsyncRedis, request: pytest.FixtureRequest, suppress_user_warning: None, # noqa: ARG001 -) -> AsyncGenerator[RedisRequestQueueClient, None]: - """A fixture for a Redis RQ client.""" - client = await RedisStorageClient(redis=redis_client, queue_dedup_strategy=request.param).create_rq_client( +) -> AsyncGenerator[RedisRequestQueueBackend, None]: + """A fixture for a Redis RQ backend.""" + backend = await RedisStorageBackend(redis=redis_client, queue_dedup_strategy=request.param).create_rq_backend( name='test_request_queue' ) - yield client - await client.drop() + yield backend + await backend.drop() -async def test_base_keys_creation(rq_client: RedisRequestQueueClient) -> None: - """Test that Redis RQ client creates proper keys.""" +async def test_base_keys_creation(rq_backend: RedisRequestQueueBackend) -> None: + """Test that Redis RQ backend creates proper keys.""" - metadata = await rq_client.get_metadata() - name = await await_redis_response(rq_client.redis.hget('request_queues:id_to_name', metadata.id)) + metadata = await rq_backend.get_metadata() + name = await await_redis_response(rq_backend.redis.hget('request_queues:id_to_name', metadata.id)) assert name is not None assert (name.decode() if isinstance(name, bytes) else name) == 'test_request_queue' - kvs_id = await await_redis_response(rq_client.redis.hget('request_queues:name_to_id', 'test_request_queue')) + kvs_id = await await_redis_response(rq_backend.redis.hget('request_queues:name_to_id', 'test_request_queue')) assert kvs_id is not None assert (kvs_id.decode() if isinstance(kvs_id, bytes) else kvs_id) == metadata.id - if rq_client._dedup_strategy == 'bloom': + if rq_backend._dedup_strategy == 'bloom': added_bf = await await_redis_response( - rq_client.redis.exists('request_queues:test_request_queue:added_bloom_filter') + rq_backend.redis.exists('request_queues:test_request_queue:added_bloom_filter') ) assert added_bf == 1 handled_bf = await await_redis_response( - rq_client.redis.exists('request_queues:test_request_queue:handled_bloom_filter') + rq_backend.redis.exists('request_queues:test_request_queue:handled_bloom_filter') ) assert handled_bf == 1 - metadata_data = await await_redis_response(rq_client.redis.json().get('request_queues:test_request_queue:metadata')) + metadata_data = await await_redis_response( + rq_backend.redis.json().get('request_queues:test_request_queue:metadata') + ) assert isinstance(metadata_data, dict) assert metadata_data['id'] == metadata.id -async def test_request_records_persistence(rq_client: RedisRequestQueueClient) -> None: +async def test_request_records_persistence(rq_backend: RedisRequestQueueBackend) -> None: """Test that requests are properly persisted to Redis.""" requests = [ Request.from_url('https://example.com/1'), @@ -80,11 +82,11 @@ async def test_request_records_persistence(rq_client: RedisRequestQueueClient) - Request.from_url('https://example.com/3'), ] - await rq_client.add_batch_of_requests(requests) + await rq_backend.add_batch_of_requests(requests) # Verify request records are created request_queue_response = await await_redis_response( - rq_client.redis.lmpop(1, 'request_queues:test_request_queue:queue', direction='left', count=10) + rq_backend.redis.lmpop(1, 'request_queues:test_request_queue:queue', direction='left', count=10) ) assert request_queue_response is not None assert isinstance(request_queue_response, list) @@ -94,7 +96,7 @@ async def test_request_records_persistence(rq_client: RedisRequestQueueClient) - # Verify actual request file content requests_records_data = await await_redis_response( - rq_client.redis.hgetall('request_queues:test_request_queue:data') + rq_backend.redis.hgetall('request_queues:test_request_queue:data') ) assert isinstance(requests_records_data, dict) @@ -104,9 +106,9 @@ async def test_request_records_persistence(rq_client: RedisRequestQueueClient) - assert request_data['url'].startswith('https://example.com/') -async def test_drop_removes_records(rq_client: RedisRequestQueueClient) -> None: +async def test_drop_removes_records(rq_backend: RedisRequestQueueBackend) -> None: """Test that drop removes all request records from Redis.""" - await rq_client.add_batch_of_requests([Request.from_url('https://example.com')]) + await rq_backend.add_batch_of_requests([Request.from_url('https://example.com')]) rq_queue = 'request_queues:test_request_queue:queue' rq_data = 'request_queues:test_request_queue:data' @@ -116,95 +118,95 @@ async def test_drop_removes_records(rq_client: RedisRequestQueueClient) -> None: handled_set = 'request_queues:test_request_queue:handled_set' metadata_key = 'request_queues:test_request_queue:metadata' - metadata = await rq_client.get_metadata() - name = await await_redis_response(rq_client.redis.hget('request_queues:id_to_name', metadata.id)) + metadata = await rq_backend.get_metadata() + name = await await_redis_response(rq_backend.redis.hget('request_queues:id_to_name', metadata.id)) assert name is not None assert (name.decode() if isinstance(name, bytes) else name) == 'test_request_queue' - rq_id = await await_redis_response(rq_client.redis.hget('request_queues:name_to_id', 'test_request_queue')) + rq_id = await await_redis_response(rq_backend.redis.hget('request_queues:name_to_id', 'test_request_queue')) assert rq_id is not None assert rq_id.decode() if isinstance(rq_id, bytes) else rq_id - rq_queue_exists = await await_redis_response(rq_client.redis.exists(rq_queue)) - rq_data_exists = await await_redis_response(rq_client.redis.exists(rq_data)) - metadata_exists = await await_redis_response(rq_client.redis.exists(metadata_key)) + rq_queue_exists = await await_redis_response(rq_backend.redis.exists(rq_queue)) + rq_data_exists = await await_redis_response(rq_backend.redis.exists(rq_data)) + metadata_exists = await await_redis_response(rq_backend.redis.exists(metadata_key)) assert rq_queue_exists == 1 assert rq_data_exists == 1 assert metadata_exists == 1 - if rq_client._dedup_strategy == 'bloom': - added_bf_exists = await await_redis_response(rq_client.redis.exists(added_bf)) - handled_bf_exists = await await_redis_response(rq_client.redis.exists(handled_bf)) + if rq_backend._dedup_strategy == 'bloom': + added_bf_exists = await await_redis_response(rq_backend.redis.exists(added_bf)) + handled_bf_exists = await await_redis_response(rq_backend.redis.exists(handled_bf)) assert added_bf_exists == 1 assert handled_bf_exists == 1 - elif rq_client._dedup_strategy == 'default': - pending_set_exists = await await_redis_response(rq_client.redis.exists(pending_set)) - handled_set_exists = await await_redis_response(rq_client.redis.exists(handled_set)) + elif rq_backend._dedup_strategy == 'default': + pending_set_exists = await await_redis_response(rq_backend.redis.exists(pending_set)) + handled_set_exists = await await_redis_response(rq_backend.redis.exists(handled_set)) assert pending_set_exists == 1 # No requests marked as handled assert handled_set_exists == 0 # Drop the request queue - await rq_client.drop() + await rq_backend.drop() # Verify removal of all records - name_after_drop = await await_redis_response(rq_client.redis.hget('request_queues:id_to_name', metadata.id)) + name_after_drop = await await_redis_response(rq_backend.redis.hget('request_queues:id_to_name', metadata.id)) rq_id_after_drop = await await_redis_response( - rq_client.redis.hget('request_queues:name_to_id', 'test_request_queue') + rq_backend.redis.hget('request_queues:name_to_id', 'test_request_queue') ) - rq_queue_exists = await await_redis_response(rq_client.redis.exists(rq_queue)) - rq_data_exists = await await_redis_response(rq_client.redis.exists(rq_data)) - metadata_exists = await await_redis_response(rq_client.redis.exists(metadata_key)) + rq_queue_exists = await await_redis_response(rq_backend.redis.exists(rq_queue)) + rq_data_exists = await await_redis_response(rq_backend.redis.exists(rq_data)) + metadata_exists = await await_redis_response(rq_backend.redis.exists(metadata_key)) assert name_after_drop is None assert rq_id_after_drop is None assert rq_queue_exists == 0 assert rq_data_exists == 0 assert metadata_exists == 0 - if rq_client._dedup_strategy == 'bloom': - added_bf_exists = await await_redis_response(rq_client.redis.exists(added_bf)) - handled_bf_exists = await await_redis_response(rq_client.redis.exists(handled_bf)) + if rq_backend._dedup_strategy == 'bloom': + added_bf_exists = await await_redis_response(rq_backend.redis.exists(added_bf)) + handled_bf_exists = await await_redis_response(rq_backend.redis.exists(handled_bf)) assert added_bf_exists == 0 assert handled_bf_exists == 0 - elif rq_client._dedup_strategy == 'default': - pending_set_exists = await await_redis_response(rq_client.redis.exists(pending_set)) - handled_set_exists = await await_redis_response(rq_client.redis.exists(handled_set)) + elif rq_backend._dedup_strategy == 'default': + pending_set_exists = await await_redis_response(rq_backend.redis.exists(pending_set)) + handled_set_exists = await await_redis_response(rq_backend.redis.exists(handled_set)) assert pending_set_exists == 0 assert handled_set_exists == 0 -async def test_metadata_record_updates(rq_client: RedisRequestQueueClient, monkeypatch: pytest.MonkeyPatch) -> None: +async def test_metadata_record_updates(rq_backend: RedisRequestQueueBackend, monkeypatch: pytest.MonkeyPatch) -> None: """Fetching bumps only `accessed_at`, adding requests bumps `modified_at` too, and `created_at` never changes.""" - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() initial_created = metadata.created_at initial_modified = metadata.modified_at # Stamp the metadata updates from a controlled clock, so the checks don't depend on the wall clock resolution. clock = Mock() - monkeypatch.setattr(_client_mixin, 'datetime', clock) + monkeypatch.setattr(_backend_mixin, 'datetime', clock) read_time = metadata.accessed_at + timedelta(seconds=1) clock.now.return_value = read_time - await rq_client.fetch_next_request() + await rq_backend.fetch_next_request() # `get_metadata()` reads the record before stamping its own access, so it reports the fetch's stamp. - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at == read_time assert metadata.modified_at == initial_modified write_time = read_time + timedelta(seconds=1) clock.now.return_value = write_time - await rq_client.add_batch_of_requests([Request.from_url('https://example.com')]) + await rq_backend.add_batch_of_requests([Request.from_url('https://example.com')]) - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at == write_time assert metadata.modified_at == write_time -async def test_get_request(rq_client: RedisRequestQueueClient) -> None: +async def test_get_request(rq_backend: RedisRequestQueueBackend) -> None: """Test that get_request works correctly.""" requests = [ Request.from_url('https://example.com/1'), @@ -212,21 +214,21 @@ async def test_get_request(rq_client: RedisRequestQueueClient) -> None: Request.from_url('https://example.com/3'), ] - added_requests = await rq_client.add_batch_of_requests(requests) + added_requests = await rq_backend.add_batch_of_requests(requests) assert len(added_requests.processed_requests) == 3 for req in requests: - fetched_request = await rq_client.get_request(req.unique_key) + fetched_request = await rq_backend.get_request(req.unique_key) assert fetched_request is not None assert fetched_request.unique_key == req.unique_key assert fetched_request.url == req.url # Test fetching a non-existent request - non_existent = await rq_client.get_request('non-existent-id') + non_existent = await rq_backend.get_request('non-existent-id') assert non_existent is None -async def test_deduplication(rq_client: RedisRequestQueueClient) -> None: +async def test_deduplication(rq_backend: RedisRequestQueueBackend) -> None: """Test that request deduplication works correctly.""" requests = [ Request.from_url('https://example.com/1'), @@ -234,29 +236,29 @@ async def test_deduplication(rq_client: RedisRequestQueueClient) -> None: Request.from_url('https://example.com/3'), ] - await rq_client.add_batch_of_requests(requests) + await rq_backend.add_batch_of_requests(requests) # Verify only unique requests are added - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.pending_request_count == 2 assert metadata.total_request_count == 2 # Fetch requests and verify order - request1 = await rq_client.fetch_next_request() + request1 = await rq_backend.fetch_next_request() assert request1 is not None assert request1 == requests[0] # Fetch the next request, which should skip the duplicate - request2 = await rq_client.fetch_next_request() + request2 = await rq_backend.fetch_next_request() assert request2 is not None assert request2 == requests[2] # Verify no more requests are available - request3 = await rq_client.fetch_next_request() + request3 = await rq_backend.fetch_next_request() assert request3 is None -async def test_error_handling_on_get_metadata_failure(rq_client: RedisRequestQueueClient) -> None: +async def test_error_handling_on_get_metadata_failure(rq_backend: RedisRequestQueueBackend) -> None: """Test that get_metadata properly handles Redis errors and retries.""" mock_pipe = MagicMock() mock_pipe.execute = AsyncMock(side_effect=RedisError('connection lost')) @@ -267,16 +269,16 @@ async def test_error_handling_on_get_metadata_failure(rq_client: RedisRequestQue with ( patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, - patch.object(rq_client.redis, 'pipeline', return_value=mock_pipeline_ctx), + patch.object(rq_backend.redis, 'pipeline', return_value=mock_pipeline_ctx), pytest.raises(RedisError), ): - await rq_client.get_metadata() + await rq_backend.get_metadata() # Verify that retry logic was attempted assert mock_sleep.call_count == 2 # Assuming default max_attempts=3 -async def test_get_metadata_does_not_retry_on_unexpected_exception(rq_client: RedisRequestQueueClient) -> None: +async def test_get_metadata_does_not_retry_on_unexpected_exception(rq_backend: RedisRequestQueueBackend) -> None: """Test that get_metadata does not retry on unexpected exceptions.""" mock_pipe = MagicMock() mock_pipe.execute = AsyncMock(side_effect=ValueError('unexpected error')) @@ -287,49 +289,49 @@ async def test_get_metadata_does_not_retry_on_unexpected_exception(rq_client: Re with ( patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, - patch.object(rq_client.redis, 'pipeline', return_value=mock_pipeline_ctx), + patch.object(rq_backend.redis, 'pipeline', return_value=mock_pipeline_ctx), pytest.raises(ValueError, match='unexpected error'), ): - await rq_client.get_metadata() + await rq_backend.get_metadata() # Verify that retry logic was not attempted assert mock_sleep.call_count == 0 async def test_mark_request_as_handled_concurrent_no_double_decrement( - rq_client: RedisRequestQueueClient, + rq_backend: RedisRequestQueueBackend, ) -> None: """Test that concurrent calls to mark_request_as_handled decrement pending_request_count exactly once.""" request = Request.from_url('https://example.com/concurrent') - await rq_client.add_batch_of_requests([request]) + await rq_backend.add_batch_of_requests([request]) - fetched = await rq_client.fetch_next_request() + fetched = await rq_backend.fetch_next_request() assert fetched is not None results = await asyncio.gather( - rq_client.mark_request_as_handled(fetched), - rq_client.mark_request_as_handled(fetched), - rq_client.mark_request_as_handled(fetched), - rq_client.mark_request_as_handled(fetched), - rq_client.mark_request_as_handled(fetched), + rq_backend.mark_request_as_handled(fetched), + rq_backend.mark_request_as_handled(fetched), + rq_backend.mark_request_as_handled(fetched), + rq_backend.mark_request_as_handled(fetched), + rq_backend.mark_request_as_handled(fetched), ) successful = [result for result in results if result is not None] assert len(successful) == 1 - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.pending_request_count == 0 assert metadata.handled_request_count == 1 async def test_mark_request_as_handled_restores_in_progress_on_pipeline_failure( - rq_client: RedisRequestQueueClient, + rq_backend: RedisRequestQueueBackend, ) -> None: """Test that 'request' is restored to 'in_progress' when the pipeline fails after 'hdel'.""" request = Request.from_url('https://example.com/restore') - await rq_client.add_batch_of_requests([request]) + await rq_backend.add_batch_of_requests([request]) - fetched = await rq_client.fetch_next_request() + fetched = await rq_backend.fetch_next_request() assert fetched is not None mock_pipe = MagicMock() @@ -341,14 +343,14 @@ async def test_mark_request_as_handled_restores_in_progress_on_pipeline_failure( with ( patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock), - patch.object(rq_client.redis, 'pipeline', return_value=mock_pipeline_ctx), + patch.object(rq_backend.redis, 'pipeline', return_value=mock_pipeline_ctx), pytest.raises(RedisError), ): - await rq_client.mark_request_as_handled(fetched) + await rq_backend.mark_request_as_handled(fetched) - in_progress = await await_redis_response(rq_client.redis.hexists(rq_client._in_progress_key, fetched.unique_key)) + in_progress = await await_redis_response(rq_backend.redis.hexists(rq_backend._in_progress_key, fetched.unique_key)) assert in_progress - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.pending_request_count == 1 assert metadata.handled_request_count == 0 diff --git a/tests/unit/storage_clients/_redis/test_redis_storage_client.py b/tests/unit/storage_backends/_redis/test_redis_storage_backend.py similarity index 71% rename from tests/unit/storage_clients/_redis/test_redis_storage_client.py rename to tests/unit/storage_backends/_redis/test_redis_storage_backend.py index 38b2a773fc..5d5d5180a8 100644 --- a/tests/unit/storage_clients/_redis/test_redis_storage_client.py +++ b/tests/unit/storage_backends/_redis/test_redis_storage_backend.py @@ -10,7 +10,7 @@ def test_import_error_handled() -> None: blocked = {mod_name: None for mod_name in sys.modules if mod_name == 'redis' or mod_name.startswith('redis.')} with patch.dict('sys.modules', blocked): for mod_name in list(sys.modules): - if mod_name.startswith('crawlee.storage_clients._redis'): + if mod_name.startswith('crawlee.storage_backends._redis'): sys.modules.pop(mod_name, None) with pytest.raises(ImportError): - from crawlee.storage_clients._redis import RedisStorageClient # noqa: F401 PLC0415 + from crawlee.storage_backends._redis import RedisStorageBackend # noqa: F401 PLC0415 diff --git a/tests/unit/storage_clients/_sql/test_sql_dataset_client.py b/tests/unit/storage_backends/_sql/test_sql_dataset_backend.py similarity index 67% rename from tests/unit/storage_clients/_sql/test_sql_dataset_client.py rename to tests/unit/storage_backends/_sql/test_sql_dataset_backend.py index d756f02a61..6306c65fbb 100644 --- a/tests/unit/storage_clients/_sql/test_sql_dataset_client.py +++ b/tests/unit/storage_backends/_sql/test_sql_dataset_backend.py @@ -10,8 +10,8 @@ from sqlalchemy.ext.asyncio import create_async_engine from crawlee.configuration import Configuration -from crawlee.storage_clients import SqlStorageClient -from crawlee.storage_clients._sql._db_models import DatasetItemDb, DatasetMetadataDb +from crawlee.storage_backends import SqlStorageBackend +from crawlee.storage_backends._sql._db_models import DatasetItemDb, DatasetMetadataDb if TYPE_CHECKING: from collections.abc import AsyncGenerator @@ -19,7 +19,7 @@ from sqlalchemy import Connection - from crawlee.storage_clients._sql import SqlDatasetClient + from crawlee.storage_backends._sql import SqlDatasetBackend @pytest.fixture @@ -37,43 +37,43 @@ def get_tables(sync_conn: Connection) -> list[str]: @pytest.fixture -async def dataset_client( +async def dataset_backend( configuration: Configuration, -) -> AsyncGenerator[SqlDatasetClient, None]: - """A fixture for a SQL dataset client.""" - async with SqlStorageClient() as storage_client: - client = await storage_client.create_dataset_client( +) -> AsyncGenerator[SqlDatasetBackend, None]: + """A fixture for a SQL dataset backend.""" + async with SqlStorageBackend() as storage_backend: + backend = await storage_backend.create_dataset_backend( name='test-dataset', configuration=configuration, ) - yield client - await client.drop() + yield backend + await backend.drop() async def test_create_tables_with_connection_string(configuration: Configuration, tmp_path: Path) -> None: - """Test that SQL dataset client creates tables with a connection string.""" + """Test that SQL dataset backend creates tables with a connection string.""" storage_dir = tmp_path / 'test_table.db' - async with SqlStorageClient(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_client: - await storage_client.create_dataset_client( + async with SqlStorageBackend(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_backend: + await storage_backend.create_dataset_backend( name='new-dataset', configuration=configuration, ) - async with storage_client.engine.begin() as conn: + async with storage_backend.engine.begin() as conn: tables = await conn.run_sync(get_tables) assert 'dataset_records' in tables assert 'datasets' in tables async def test_create_tables_with_engine(configuration: Configuration, tmp_path: Path) -> None: - """Test that SQL dataset client creates tables with a pre-configured engine.""" + """Test that SQL dataset backend creates tables with a pre-configured engine.""" storage_dir = tmp_path / 'test_table.db' engine = create_async_engine(f'sqlite+aiosqlite:///{storage_dir}', future=True, echo=False) - async with SqlStorageClient(engine=engine) as storage_client: - await storage_client.create_dataset_client( + async with SqlStorageBackend(engine=engine) as storage_backend: + await storage_backend.create_dataset_backend( name='new-dataset', configuration=configuration, ) @@ -86,44 +86,44 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path: async def test_tables_and_metadata_record(configuration: Configuration) -> None: """Test that SQL dataset creates proper tables and metadata records.""" - async with SqlStorageClient() as storage_client: - client = await storage_client.create_dataset_client( + async with SqlStorageBackend() as storage_backend: + backend = await storage_backend.create_dataset_backend( name='new-dataset', configuration=configuration, ) - client_metadata = await client.get_metadata() + backend_metadata = await backend.get_metadata() - async with storage_client.engine.begin() as conn: + async with storage_backend.engine.begin() as conn: tables = await conn.run_sync(get_tables) assert 'dataset_records' in tables assert 'datasets' in tables - async with client.get_session() as session: + async with backend.get_session() as session: stmt = select(DatasetMetadataDb).where(DatasetMetadataDb.name == 'new-dataset') result = await session.execute(stmt) orm_metadata = result.scalar_one_or_none() assert orm_metadata is not None - assert orm_metadata.id == client_metadata.id + assert orm_metadata.id == backend_metadata.id assert orm_metadata.name == 'new-dataset' assert orm_metadata.item_count == 0 - await client.drop() + await backend.drop() -async def test_record_and_content_verification(dataset_client: SqlDatasetClient) -> None: - """Test that dataset client can push data and verify its content.""" +async def test_record_and_content_verification(dataset_backend: SqlDatasetBackend) -> None: + """Test that dataset backend can push data and verify its content.""" item = {'key': 'value', 'number': 42} - await dataset_client.push_data(item) + await dataset_backend.push_data(item) # Verify metadata record - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() assert metadata.item_count == 1 assert metadata.created_at is not None assert metadata.modified_at is not None assert metadata.accessed_at is not None - async with dataset_client.get_session() as session: + async with dataset_backend.get_session() as session: stmt = select(DatasetItemDb).where(DatasetItemDb.dataset_id == metadata.id) result = await session.execute(stmt) records = result.scalars().all() @@ -133,43 +133,43 @@ async def test_record_and_content_verification(dataset_client: SqlDatasetClient) # Test pushing multiple items and verify total count items = [{'id': 1, 'name': 'Item 1'}, {'id': 2, 'name': 'Item 2'}, {'id': 3, 'name': 'Item 3'}] - await dataset_client.push_data(items) + await dataset_backend.push_data(items) - async with dataset_client.get_session() as session: + async with dataset_backend.get_session() as session: stmt = select(DatasetItemDb).where(DatasetItemDb.dataset_id == metadata.id) result = await session.execute(stmt) records = result.scalars().all() assert len(records) == 4 -async def test_drop_removes_records(dataset_client: SqlDatasetClient) -> None: +async def test_drop_removes_records(dataset_backend: SqlDatasetBackend) -> None: """Test that dropping a dataset removes all records from the database.""" - await dataset_client.push_data({'test': 'data'}) + await dataset_backend.push_data({'test': 'data'}) - client_metadata = await dataset_client.get_metadata() + backend_metadata = await dataset_backend.get_metadata() - async with dataset_client.get_session() as session: - stmt = select(DatasetItemDb).where(DatasetItemDb.dataset_id == client_metadata.id) + async with dataset_backend.get_session() as session: + stmt = select(DatasetItemDb).where(DatasetItemDb.dataset_id == backend_metadata.id) result = await session.execute(stmt) records = result.scalars().all() assert len(records) == 1 # Drop the dataset - await dataset_client.drop() + await dataset_backend.drop() - async with dataset_client.get_session() as session: - stmt = select(DatasetItemDb).where(DatasetItemDb.dataset_id == client_metadata.id) + async with dataset_backend.get_session() as session: + stmt = select(DatasetItemDb).where(DatasetItemDb.dataset_id == backend_metadata.id) result = await session.execute(stmt) records = result.scalars().all() assert len(records) == 0 - metadata = await session.get(DatasetMetadataDb, client_metadata.id) + metadata = await session.get(DatasetMetadataDb, backend_metadata.id) assert metadata is None -async def test_metadata_record_updates(dataset_client: SqlDatasetClient) -> None: +async def test_metadata_record_updates(dataset_backend: SqlDatasetBackend) -> None: """Test that metadata record is updated correctly after operations.""" # Record initial timestamps - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -178,10 +178,10 @@ async def test_metadata_record_updates(dataset_client: SqlDatasetClient) -> None await asyncio.sleep(0.01) # Perform an operation that updates accessed_at - await dataset_client.get_data() + await dataset_backend.get_data() # Verify timestamps - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -192,16 +192,16 @@ async def test_metadata_record_updates(dataset_client: SqlDatasetClient) -> None await asyncio.sleep(0.01) # Perform an operation that updates modified_at - await dataset_client.push_data({'new': 'item'}) + await dataset_backend.push_data({'new': 'item'}) # Verify timestamps again - metadata = await dataset_client.get_metadata() + metadata = await dataset_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_get # Verify metadata record is updated in db - async with dataset_client.get_session() as session: + async with dataset_backend.get_session() as session: orm_metadata = await session.get(DatasetMetadataDb, metadata.id) assert orm_metadata is not None orm_metadata.item_count = 1 @@ -212,33 +212,33 @@ async def test_metadata_record_updates(dataset_client: SqlDatasetClient) -> None async def test_data_persistence_across_reopens(configuration: Configuration) -> None: """Test that data persists correctly when reopening the same dataset.""" - async with SqlStorageClient() as storage_client: - original_client = await storage_client.create_dataset_client( + async with SqlStorageBackend() as storage_backend: + original_backend = await storage_backend.create_dataset_backend( name='persistence-test', configuration=configuration, ) test_data = {'test_item': 'test_value', 'id': 123} - await original_client.push_data(test_data) + await original_backend.push_data(test_data) - dataset_id = (await original_client.get_metadata()).id + dataset_id = (await original_backend.get_metadata()).id - reopened_client = await storage_client.create_dataset_client( + reopened_backend = await storage_backend.create_dataset_backend( id=dataset_id, configuration=configuration, ) - data = await reopened_client.get_data() + data = await reopened_backend.get_data() assert len(data.items) == 1 assert data.items[0] == test_data - await reopened_client.drop() + await reopened_backend.drop() -async def test_error_handling_on_push_failure(dataset_client: SqlDatasetClient) -> None: +async def test_error_handling_on_push_failure(dataset_backend: SqlDatasetBackend) -> None: """Test that push_data properly handles SQL errors and retries.""" with patch( - 'crawlee.storage_clients._sql._dataset_client.SqlDatasetClient.get_session', + 'crawlee.storage_backends._sql._dataset_backend.SqlDatasetBackend.get_session', ) as mock_get_session: mock_session = AsyncMock() mock_session.__aenter__ = AsyncMock(return_value=mock_session) @@ -250,16 +250,16 @@ async def test_error_handling_on_push_failure(dataset_client: SqlDatasetClient) patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, pytest.raises(SQLAlchemyError), ): - await dataset_client.push_data({'test': 'data'}) + await dataset_backend.push_data({'test': 'data'}) # Verify that retry logic was attempted assert mock_sleep.call_count == 2 # Assuming default max_attempts=3 -async def test_push_data_does_not_retry_on_unexpected_exception(dataset_client: SqlDatasetClient) -> None: +async def test_push_data_does_not_retry_on_unexpected_exception(dataset_backend: SqlDatasetBackend) -> None: """Test that push_data does not retry on unexpected exceptions.""" with patch( - 'crawlee.storage_clients._sql._dataset_client.SqlDatasetClient.get_session', + 'crawlee.storage_backends._sql._dataset_backend.SqlDatasetBackend.get_session', ) as mock_get_session: mock_session = AsyncMock() mock_session.__aenter__ = AsyncMock(return_value=mock_session) @@ -271,7 +271,7 @@ async def test_push_data_does_not_retry_on_unexpected_exception(dataset_client: patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, pytest.raises(ValueError, match='unexpected error'), ): - await dataset_client.push_data({'test': 'data'}) + await dataset_backend.push_data({'test': 'data'}) # Verify that retry logic was not attempted assert mock_sleep.call_count == 0 diff --git a/tests/unit/storage_clients/_sql/test_sql_kvs_client.py b/tests/unit/storage_backends/_sql/test_sql_kvs_backend.py similarity index 62% rename from tests/unit/storage_clients/_sql/test_sql_kvs_client.py rename to tests/unit/storage_backends/_sql/test_sql_kvs_backend.py index 1b31165964..ad4230e962 100644 --- a/tests/unit/storage_clients/_sql/test_sql_kvs_client.py +++ b/tests/unit/storage_backends/_sql/test_sql_kvs_backend.py @@ -11,9 +11,9 @@ from sqlalchemy.ext.asyncio import create_async_engine from crawlee.configuration import Configuration -from crawlee.storage_clients import SqlStorageClient -from crawlee.storage_clients._sql._db_models import KeyValueStoreMetadataDb, KeyValueStoreRecordDb -from crawlee.storage_clients.models import KeyValueStoreMetadata +from crawlee.storage_backends import SqlStorageBackend +from crawlee.storage_backends._sql._db_models import KeyValueStoreMetadataDb, KeyValueStoreRecordDb +from crawlee.storage_backends.models import KeyValueStoreMetadata if TYPE_CHECKING: from collections.abc import AsyncGenerator, Iterator @@ -21,7 +21,7 @@ from sqlalchemy import Connection - from crawlee.storage_clients._sql import SqlKeyValueStoreClient + from crawlee.storage_backends._sql import SqlKeyValueStoreBackend @pytest.fixture @@ -33,17 +33,17 @@ def configuration(tmp_path: Path) -> Configuration: @pytest.fixture -async def kvs_client( +async def kvs_backend( configuration: Configuration, -) -> AsyncGenerator[SqlKeyValueStoreClient, None]: - """A fixture for a SQL key-value store client.""" - async with SqlStorageClient() as storage_client: - client = await storage_client.create_kvs_client( +) -> AsyncGenerator[SqlKeyValueStoreBackend, None]: + """A fixture for a SQL key-value store backend.""" + async with SqlStorageBackend() as storage_backend: + backend = await storage_backend.create_kvs_backend( name='test-kvs', configuration=configuration, ) - yield client - await client.drop() + yield backend + await backend.drop() # Helper function that allows you to use inspect with an asynchronous engine @@ -53,29 +53,29 @@ def get_tables(sync_conn: Connection) -> list[str]: async def test_create_tables_with_connection_string(configuration: Configuration, tmp_path: Path) -> None: - """Test that SQL key-value store client creates tables with a connection string.""" + """Test that SQL key-value store backend creates tables with a connection string.""" storage_dir = tmp_path / 'test_table.db' - async with SqlStorageClient(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_client: - await storage_client.create_kvs_client( + async with SqlStorageBackend(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_backend: + await storage_backend.create_kvs_backend( name='new-kvs', configuration=configuration, ) - async with storage_client.engine.begin() as conn: + async with storage_backend.engine.begin() as conn: tables = await conn.run_sync(get_tables) assert 'key_value_stores' in tables assert 'key_value_store_records' in tables async def test_create_tables_with_engine(configuration: Configuration, tmp_path: Path) -> None: - """Test that SQL key-value store client creates tables with a pre-configured engine.""" + """Test that SQL key-value store backend creates tables with a pre-configured engine.""" storage_dir = tmp_path / 'test_table.db' engine = create_async_engine(f'sqlite+aiosqlite:///{storage_dir}', future=True, echo=False) - async with SqlStorageClient(engine=engine) as storage_client: - await storage_client.create_kvs_client( + async with SqlStorageBackend(engine=engine) as storage_backend: + await storage_backend.create_kvs_backend( name='new-kvs', configuration=configuration, ) @@ -88,36 +88,36 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path: async def test_tables_and_metadata_record(configuration: Configuration) -> None: """Test that SQL key-value store creates proper tables and metadata records.""" - async with SqlStorageClient() as storage_client: - client = await storage_client.create_kvs_client( + async with SqlStorageBackend() as storage_backend: + backend = await storage_backend.create_kvs_backend( name='new-kvs', configuration=configuration, ) - client_metadata = await client.get_metadata() + backend_metadata = await backend.get_metadata() - async with storage_client.engine.begin() as conn: + async with storage_backend.engine.begin() as conn: tables = await conn.run_sync(get_tables) assert 'key_value_stores' in tables assert 'key_value_store_records' in tables - async with client.get_session() as session: + async with backend.get_session() as session: stmt = select(KeyValueStoreMetadataDb).where(KeyValueStoreMetadataDb.name == 'new-kvs') result = await session.execute(stmt) orm_metadata = result.scalar_one_or_none() metadata = KeyValueStoreMetadata.model_validate(orm_metadata) - assert metadata.id == client_metadata.id + assert metadata.id == backend_metadata.id assert metadata.name == 'new-kvs' - await client.drop() + await backend.drop() -async def test_value_record_creation(kvs_client: SqlKeyValueStoreClient) -> None: - """Test that SQL key-value store client can create a record.""" +async def test_value_record_creation(kvs_backend: SqlKeyValueStoreBackend) -> None: + """Test that SQL key-value store backend can create a record.""" test_key = 'test-key' test_value = 'Hello, world!' - await kvs_client.set_value(key=test_key, value=test_value) - async with kvs_client.get_session() as session: + await kvs_backend.set_value(key=test_key, value=test_value) + async with kvs_backend.get_session() as session: stmt = select(KeyValueStoreRecordDb).where(KeyValueStoreRecordDb.key == test_key) result = await session.execute(stmt) record = result.scalar_one_or_none() @@ -128,13 +128,13 @@ async def test_value_record_creation(kvs_client: SqlKeyValueStoreClient) -> None assert record.value == test_value.encode('utf-8') -async def test_binary_data_persistence(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_binary_data_persistence(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that binary data is stored correctly without corruption.""" test_key = 'test-binary' test_value = b'\x00\x01\x02\x03\x04' - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) - async with kvs_client.get_session() as session: + async with kvs_backend.get_session() as session: stmt = select(KeyValueStoreRecordDb).where(KeyValueStoreRecordDb.key == test_key) result = await session.execute(stmt) record = result.scalar_one_or_none() @@ -144,19 +144,19 @@ async def test_binary_data_persistence(kvs_client: SqlKeyValueStoreClient) -> No assert record.size == len(test_value) assert record.value == test_value - verify_record = await kvs_client.get_value(key=test_key) + verify_record = await kvs_backend.get_value(key=test_key) assert verify_record is not None assert verify_record.value == test_value assert verify_record.content_type == 'application/octet-stream' -async def test_json_serialization_to_record(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_json_serialization_to_record(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that JSON objects are properly serialized to records.""" test_key = 'test-json' test_value = {'name': 'John', 'age': 30, 'items': [1, 2, 3]} - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) - async with kvs_client.get_session() as session: + async with kvs_backend.get_session() as session: stmt = select(KeyValueStoreRecordDb).where(KeyValueStoreRecordDb.key == test_key) result = await session.execute(stmt) record = result.scalar_one_or_none() @@ -165,15 +165,15 @@ async def test_json_serialization_to_record(kvs_client: SqlKeyValueStoreClient) assert json.loads(record.value.decode('utf-8')) == test_value -async def test_record_deletion_on_value_delete(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_record_deletion_on_value_delete(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that deleting a value removes its record from the database.""" test_key = 'test-delete' test_value = 'Delete me' # Set a value - await kvs_client.set_value(key=test_key, value=test_value) + await kvs_backend.set_value(key=test_key, value=test_value) - async with kvs_client.get_session() as session: + async with kvs_backend.get_session() as session: stmt = select(KeyValueStoreRecordDb).where(KeyValueStoreRecordDb.key == test_key) result = await session.execute(stmt) record = result.scalar_one_or_none() @@ -182,44 +182,44 @@ async def test_record_deletion_on_value_delete(kvs_client: SqlKeyValueStoreClien assert record.value == test_value.encode('utf-8') # Delete the value - await kvs_client.delete_value(key=test_key) + await kvs_backend.delete_value(key=test_key) # Verify record was deleted - async with kvs_client.get_session() as session: + async with kvs_backend.get_session() as session: stmt = select(KeyValueStoreRecordDb).where(KeyValueStoreRecordDb.key == test_key) result = await session.execute(stmt) record = result.scalar_one_or_none() assert record is None -async def test_drop_removes_records(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_drop_removes_records(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that drop removes all records from the database.""" - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') - client_metadata = await kvs_client.get_metadata() + backend_metadata = await kvs_backend.get_metadata() - async with kvs_client.get_session() as session: + async with kvs_backend.get_session() as session: stmt = select(KeyValueStoreRecordDb).where(KeyValueStoreRecordDb.key == 'test') result = await session.execute(stmt) record = result.scalar_one_or_none() assert record is not None # Drop the store - await kvs_client.drop() + await kvs_backend.drop() - async with kvs_client.get_session() as session: + async with kvs_backend.get_session() as session: stmt = select(KeyValueStoreRecordDb).where(KeyValueStoreRecordDb.key == 'test') result = await session.execute(stmt) record = result.scalar_one_or_none() assert record is None - metadata = await session.get(KeyValueStoreMetadataDb, client_metadata.id) + metadata = await session.get(KeyValueStoreMetadataDb, backend_metadata.id) assert metadata is None -async def test_metadata_record_updates(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_metadata_record_updates(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that read/write operations properly update metadata record timestamps.""" # Record initial timestamps - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -228,10 +228,10 @@ async def test_metadata_record_updates(kvs_client: SqlKeyValueStoreClient) -> No await asyncio.sleep(0.01) # Perform a read operation - await kvs_client.get_value(key='nonexistent') + await kvs_backend.get_value(key='nonexistent') # Verify accessed timestamp was updated - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -242,15 +242,15 @@ async def test_metadata_record_updates(kvs_client: SqlKeyValueStoreClient) -> No await asyncio.sleep(0.01) # Perform a write operation - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') # Verify modified timestamp was updated - metadata = await kvs_client.get_metadata() + metadata = await kvs_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_read - async with kvs_client.get_session() as session: + async with kvs_backend.get_session() as session: orm_metadata = await session.get(KeyValueStoreMetadataDb, metadata.id) assert orm_metadata is not None assert orm_metadata.created_at == metadata.created_at @@ -260,35 +260,35 @@ async def test_metadata_record_updates(kvs_client: SqlKeyValueStoreClient) -> No async def test_data_persistence_across_reopens(configuration: Configuration) -> None: """Test that data persists correctly when reopening the same key-value store.""" - async with SqlStorageClient() as storage_client: - original_client = await storage_client.create_kvs_client( + async with SqlStorageBackend() as storage_backend: + original_backend = await storage_backend.create_kvs_backend( name='persistence-test', configuration=configuration, ) test_key = 'persistent-key' test_value = 'persistent-value' - await original_client.set_value(key=test_key, value=test_value) + await original_backend.set_value(key=test_key, value=test_value) - kvs_id = (await original_client.get_metadata()).id + kvs_id = (await original_backend.get_metadata()).id # Reopen by ID and verify data persists - reopened_client = await storage_client.create_kvs_client( + reopened_backend = await storage_backend.create_kvs_backend( id=kvs_id, configuration=configuration, ) - record = await reopened_client.get_value(key=test_key) + record = await reopened_backend.get_value(key=test_key) assert record is not None assert record.value == test_value - await reopened_client.drop() + await reopened_backend.drop() -async def test_error_handling_on_set_failure(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_error_handling_on_set_failure(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that set_value properly handles SQL errors and retries.""" with patch( - 'crawlee.storage_clients._sql._key_value_store_client.SqlKeyValueStoreClient.get_session', + 'crawlee.storage_backends._sql._key_value_store_backend.SqlKeyValueStoreBackend.get_session', ) as mock_get_session: mock_session = AsyncMock() mock_session.__aenter__ = AsyncMock(return_value=mock_session) @@ -300,16 +300,16 @@ async def test_error_handling_on_set_failure(kvs_client: SqlKeyValueStoreClient) patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, pytest.raises(SQLAlchemyError), ): - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') # Verify that retry logic was attempted assert mock_sleep.call_count == 2 # Assuming default max_attempts=3 -async def test_set_value_does_not_retry_on_unexpected_exception(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_set_value_does_not_retry_on_unexpected_exception(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that set_value does not retry on unexpected exceptions.""" with patch( - 'crawlee.storage_clients._sql._key_value_store_client.SqlKeyValueStoreClient.get_session', + 'crawlee.storage_backends._sql._key_value_store_backend.SqlKeyValueStoreBackend.get_session', ) as mock_get_session: mock_session = AsyncMock() mock_session.__aenter__ = AsyncMock(return_value=mock_session) @@ -321,16 +321,16 @@ async def test_set_value_does_not_retry_on_unexpected_exception(kvs_client: SqlK patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, pytest.raises(ValueError, match='unexpected error'), ): - await kvs_client.set_value(key='test', value='test-value') + await kvs_backend.set_value(key='test', value='test-value') # Verify that retry logic was not attempted assert mock_sleep.call_count == 0 @pytest.fixture -def fetch_records(kvs_client: SqlKeyValueStoreClient) -> Iterator[AsyncMock]: - """Wrap `_fetch_records` of the client in a mock that records the value batches while still fetching them.""" - with patch.object(kvs_client, '_fetch_records', wraps=kvs_client._fetch_records) as mock: +def fetch_records(kvs_backend: SqlKeyValueStoreBackend) -> Iterator[AsyncMock]: + """Wrap `_fetch_records` of the backend in a mock that records the value batches while still fetching them.""" + with patch.object(kvs_backend, '_fetch_records', wraps=kvs_backend._fetch_records) as mock: yield mock @@ -338,15 +338,15 @@ def fetched_keys(fetch_records: AsyncMock) -> list[list[str]]: return [[item.key for item in call.args[0]] for call in fetch_records.await_args_list] -async def test_iterate_entries_reads_values_in_batches(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_iterate_entries_reads_values_in_batches(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that `iterate_entries` reads values in batches instead of calling `get_value` per key.""" - await kvs_client.set_value(key='a-json', value={'nested': [1, 2]}) - await kvs_client.set_value(key='b-text', value='plain text') - await kvs_client.set_value(key='c-bytes', value=b'\x00\x01binary', content_type='application/octet-stream') - await kvs_client.set_value(key='d-none', value=None) + await kvs_backend.set_value(key='a-json', value={'nested': [1, 2]}) + await kvs_backend.set_value(key='b-text', value='plain text') + await kvs_backend.set_value(key='c-bytes', value=b'\x00\x01binary', content_type='application/octet-stream') + await kvs_backend.set_value(key='d-none', value=None) - with patch.object(kvs_client, 'get_value', side_effect=AssertionError('get_value must not be called')): - records = [record async for record in kvs_client.iterate_entries()] + with patch.object(kvs_backend, 'get_value', side_effect=AssertionError('get_value must not be called')): + records = [record async for record in kvs_backend.iterate_entries()] assert [record.key for record in records] == ['a-json', 'b-text', 'c-bytes', 'd-none'] assert [record.value for record in records] == [{'nested': [1, 2]}, 'plain text', b'\x00\x01binary', None] @@ -356,13 +356,13 @@ async def test_iterate_entries_reads_values_in_batches(kvs_client: SqlKeyValueSt assert all(record.size is not None for record in records) -async def test_iterate_entries_with_exclusive_start_key_and_limit(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_iterate_entries_with_exclusive_start_key_and_limit(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that `iterate_entries` applies `exclusive_start_key` and `limit` in the query.""" for i in range(6): - await kvs_client.set_value(key=f'key{i}', value=f'value{i}') + await kvs_backend.set_value(key=f'key{i}', value=f'value{i}') - with patch.object(kvs_client, 'get_value', side_effect=AssertionError('get_value must not be called')): - records = [record async for record in kvs_client.iterate_entries(exclusive_start_key='key1', limit=3)] + with patch.object(kvs_backend, 'get_value', side_effect=AssertionError('get_value must not be called')): + records = [record async for record in kvs_backend.iterate_entries(exclusive_start_key='key1', limit=3)] assert [(record.key, record.value) for record in records] == [ ('key2', 'value2'), @@ -372,66 +372,66 @@ async def test_iterate_entries_with_exclusive_start_key_and_limit(kvs_client: Sq async def test_iterate_entries_limit_spans_multiple_pages( - kvs_client: SqlKeyValueStoreClient, fetch_records: AsyncMock + kvs_backend: SqlKeyValueStoreBackend, fetch_records: AsyncMock ) -> None: """Test that `iterate_entries` stops at `limit` when it falls in the middle of a later metadata page.""" for i in range(6): - await kvs_client.set_value(key=f'key{i}', value=f'value{i}') + await kvs_backend.set_value(key=f'key{i}', value=f'value{i}') - with patch.object(type(kvs_client), '_ITERATE_ENTRIES_BATCH_MAX_KEYS', 2): - records = [record async for record in kvs_client.iterate_entries(limit=3)] + with patch.object(type(kvs_backend), '_ITERATE_ENTRIES_BATCH_MAX_KEYS', 2): + records = [record async for record in kvs_backend.iterate_entries(limit=3)] assert [(record.key, record.value) for record in records] == [(f'key{i}', f'value{i}') for i in range(3)] assert fetched_keys(fetch_records) == [['key0', 'key1'], ['key2']] -async def test_fetch_records_skips_record_deleted_after_listing(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_fetch_records_skips_record_deleted_after_listing(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that a record deleted between the metadata listing and its value fetch is skipped.""" - await kvs_client.set_value(key='kept', value='a') - await kvs_client.set_value(key='removed', value='b') - batch = [item async for item in kvs_client.iterate_keys()] - await kvs_client.delete_value(key='removed') + await kvs_backend.set_value(key='kept', value='a') + await kvs_backend.set_value(key='removed', value='b') + batch = [item async for item in kvs_backend.iterate_keys()] + await kvs_backend.delete_value(key='removed') - records = await kvs_client._fetch_records(batch) + records = await kvs_backend._fetch_records(batch) assert [(record.key, record.value) for record in records] == [('kept', 'a')] -async def test_iterate_entries_empty_store(kvs_client: SqlKeyValueStoreClient) -> None: +async def test_iterate_entries_empty_store(kvs_backend: SqlKeyValueStoreBackend) -> None: """Test that `iterate_entries` on an empty store yields nothing.""" - records = [record async for record in kvs_client.iterate_entries()] + records = [record async for record in kvs_backend.iterate_entries()] assert records == [] async def test_iterate_entries_batches_are_bounded_by_key_count( - kvs_client: SqlKeyValueStoreClient, fetch_records: AsyncMock + kvs_backend: SqlKeyValueStoreBackend, fetch_records: AsyncMock ) -> None: """Test that `iterate_entries` splits the value queries when a batch reaches the maximum number of keys.""" for i in range(5): - await kvs_client.set_value(key=f'key{i}', value=f'value{i}') + await kvs_backend.set_value(key=f'key{i}', value=f'value{i}') - with patch.object(type(kvs_client), '_ITERATE_ENTRIES_BATCH_MAX_KEYS', 2): - records = [record async for record in kvs_client.iterate_entries()] + with patch.object(type(kvs_backend), '_ITERATE_ENTRIES_BATCH_MAX_KEYS', 2): + records = [record async for record in kvs_backend.iterate_entries()] assert [(record.key, record.value) for record in records] == [(f'key{i}', f'value{i}') for i in range(5)] assert fetched_keys(fetch_records) == [['key0', 'key1'], ['key2', 'key3'], ['key4']] async def test_iterate_entries_batches_are_bounded_by_size( - kvs_client: SqlKeyValueStoreClient, fetch_records: AsyncMock + kvs_backend: SqlKeyValueStoreBackend, fetch_records: AsyncMock ) -> None: """Test that `iterate_entries` splits the value queries by the record sizes. A record larger than the limit is still read, but alone in its batch. """ - await kvs_client.set_value(key='small1', value='ab') - await kvs_client.set_value(key='small2', value='cd') - await kvs_client.set_value(key='large', value='x' * 100) - await kvs_client.set_value(key='small3', value='ef') + await kvs_backend.set_value(key='small1', value='ab') + await kvs_backend.set_value(key='small2', value='cd') + await kvs_backend.set_value(key='large', value='x' * 100) + await kvs_backend.set_value(key='small3', value='ef') - with patch.object(type(kvs_client), '_ITERATE_ENTRIES_BATCH_MAX_BYTES', 10): - records = [record async for record in kvs_client.iterate_entries()] + with patch.object(type(kvs_backend), '_ITERATE_ENTRIES_BATCH_MAX_BYTES', 10): + records = [record async for record in kvs_backend.iterate_entries()] assert [record.key for record in records] == ['large', 'small1', 'small2', 'small3'] assert fetched_keys(fetch_records) == [['large'], ['small1', 'small2', 'small3']] diff --git a/tests/unit/storage_clients/_sql/test_sql_rq_client.py b/tests/unit/storage_backends/_sql/test_sql_rq_backend.py similarity index 69% rename from tests/unit/storage_clients/_sql/test_sql_rq_client.py rename to tests/unit/storage_backends/_sql/test_sql_rq_backend.py index c785ebffaf..a48512d42e 100644 --- a/tests/unit/storage_clients/_sql/test_sql_rq_client.py +++ b/tests/unit/storage_backends/_sql/test_sql_rq_backend.py @@ -12,9 +12,9 @@ from crawlee import Request from crawlee.configuration import Configuration -from crawlee.storage_clients import SqlStorageClient -from crawlee.storage_clients._sql._db_models import RequestDb, RequestQueueMetadataDb -from crawlee.storage_clients.models import RequestQueueMetadata +from crawlee.storage_backends import SqlStorageBackend +from crawlee.storage_backends._sql._db_models import RequestDb, RequestQueueMetadataDb +from crawlee.storage_backends.models import RequestQueueMetadata if TYPE_CHECKING: from collections.abc import AsyncGenerator @@ -22,7 +22,7 @@ from sqlalchemy import Connection - from crawlee.storage_clients._sql import SqlRequestQueueClient + from crawlee.storage_backends._sql import SqlRequestQueueBackend @pytest.fixture @@ -34,17 +34,17 @@ def configuration(tmp_path: Path) -> Configuration: @pytest.fixture -async def rq_client( +async def rq_backend( configuration: Configuration, -) -> AsyncGenerator[SqlRequestQueueClient, None]: - """A fixture for a SQL request queue client.""" - async with SqlStorageClient() as storage_client: - client = await storage_client.create_rq_client( +) -> AsyncGenerator[SqlRequestQueueBackend, None]: + """A fixture for a SQL request queue backend.""" + async with SqlStorageBackend() as storage_backend: + backend = await storage_backend.create_rq_backend( name='test-request-queue', configuration=configuration, ) - yield client - await client.drop() + yield backend + await backend.drop() # Helper function that allows you to use inspect with an asynchronous engine @@ -54,16 +54,16 @@ def get_tables(sync_conn: Connection) -> list[str]: async def test_create_tables_with_connection_string(configuration: Configuration, tmp_path: Path) -> None: - """Test that SQL request queue client creates tables with a connection string.""" + """Test that SQL request queue backend creates tables with a connection string.""" storage_dir = tmp_path / 'test_table.db' - async with SqlStorageClient(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_client: - await storage_client.create_rq_client( + async with SqlStorageBackend(connection_string=f'sqlite+aiosqlite:///{storage_dir}') as storage_backend: + await storage_backend.create_rq_backend( name='test-request-queue', configuration=configuration, ) - async with storage_client.engine.begin() as conn: + async with storage_backend.engine.begin() as conn: tables = await conn.run_sync(get_tables) assert 'request_queues' in tables assert 'request_queue_records' in tables @@ -71,13 +71,13 @@ async def test_create_tables_with_connection_string(configuration: Configuration async def test_create_tables_with_engine(configuration: Configuration, tmp_path: Path) -> None: - """Test that SQL request queue client creates tables with a pre-configured engine.""" + """Test that SQL request queue backend creates tables with a pre-configured engine.""" storage_dir = tmp_path / 'test_table.db' engine = create_async_engine(f'sqlite+aiosqlite:///{storage_dir}', future=True, echo=False) - async with SqlStorageClient(engine=engine) as storage_client: - await storage_client.create_rq_client( + async with SqlStorageBackend(engine=engine) as storage_backend: + await storage_backend.create_rq_backend( name='test-request-queue', configuration=configuration, ) @@ -91,32 +91,32 @@ async def test_create_tables_with_engine(configuration: Configuration, tmp_path: async def test_tables_and_metadata_record(configuration: Configuration) -> None: """Test that SQL request queue creates proper tables and metadata records.""" - async with SqlStorageClient() as storage_client: - client = await storage_client.create_rq_client( + async with SqlStorageBackend() as storage_backend: + backend = await storage_backend.create_rq_backend( name='test-request-queue', configuration=configuration, ) - client_metadata = await client.get_metadata() + backend_metadata = await backend.get_metadata() - async with storage_client.engine.begin() as conn: + async with storage_backend.engine.begin() as conn: tables = await conn.run_sync(get_tables) assert 'request_queues' in tables assert 'request_queue_records' in tables assert 'request_queue_state' in tables - async with client.get_session() as session: + async with backend.get_session() as session: stmt = select(RequestQueueMetadataDb).where(RequestQueueMetadataDb.name == 'test-request-queue') result = await session.execute(stmt) orm_metadata = result.scalar_one_or_none() metadata = RequestQueueMetadata.model_validate(orm_metadata) - assert metadata.id == client_metadata.id + assert metadata.id == backend_metadata.id assert metadata.name == 'test-request-queue' - await client.drop() + await backend.drop() -async def test_request_records_persistence(rq_client: SqlRequestQueueClient) -> None: +async def test_request_records_persistence(rq_backend: SqlRequestQueueBackend) -> None: """Test that all added requests are persisted and can be retrieved from the database.""" requests = [ Request.from_url('https://example.com/1'), @@ -124,12 +124,12 @@ async def test_request_records_persistence(rq_client: SqlRequestQueueClient) -> Request.from_url('https://example.com/3'), ] - await rq_client.add_batch_of_requests(requests) + await rq_backend.add_batch_of_requests(requests) - metadata_client = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() - async with rq_client.get_session() as session: - stmt = select(RequestDb).where(RequestDb.request_queue_id == metadata_client.id) + async with rq_backend.get_session() as session: + stmt = select(RequestDb).where(RequestDb.request_queue_id == metadata.id) result = await session.execute(stmt) db_requests = result.scalars().all() assert len(db_requests) == 3 @@ -138,19 +138,19 @@ async def test_request_records_persistence(rq_client: SqlRequestQueueClient) -> assert request['url'] in ['https://example.com/1', 'https://example.com/2', 'https://example.com/3'] -async def test_drop_removes_records(rq_client: SqlRequestQueueClient) -> None: +async def test_drop_removes_records(rq_backend: SqlRequestQueueBackend) -> None: """Test that drop removes all records from the database.""" - await rq_client.add_batch_of_requests([Request.from_url('https://example.com')]) - metadata = await rq_client.get_metadata() - async with rq_client.get_session() as session: + await rq_backend.add_batch_of_requests([Request.from_url('https://example.com')]) + metadata = await rq_backend.get_metadata() + async with rq_backend.get_session() as session: stmt = select(RequestDb).where(RequestDb.request_queue_id == metadata.id) result = await session.execute(stmt) records = result.scalars().all() assert len(records) == 1 - await rq_client.drop() + await rq_backend.drop() - async with rq_client.get_session() as session: + async with rq_backend.get_session() as session: stmt = select(RequestDb).where(RequestDb.request_queue_id == metadata.id) result = await session.execute(stmt) records = result.scalars().all() @@ -159,10 +159,10 @@ async def test_drop_removes_records(rq_client: SqlRequestQueueClient) -> None: assert db_metadata is None -async def test_metadata_record_updates(rq_client: SqlRequestQueueClient) -> None: +async def test_metadata_record_updates(rq_backend: SqlRequestQueueBackend) -> None: """Test that metadata record updates correctly after operations.""" # Record initial timestamps - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() initial_created = metadata.created_at initial_accessed = metadata.accessed_at initial_modified = metadata.modified_at @@ -171,10 +171,10 @@ async def test_metadata_record_updates(rq_client: SqlRequestQueueClient) -> None await asyncio.sleep(0.01) # Perform a read operation - await rq_client.is_empty() + await rq_backend.is_empty() # Verify accessed timestamp was updated - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.accessed_at > initial_accessed assert metadata.modified_at == initial_modified @@ -185,15 +185,15 @@ async def test_metadata_record_updates(rq_client: SqlRequestQueueClient) -> None await asyncio.sleep(0.01) # Perform a write operation - await rq_client.add_batch_of_requests([Request.from_url('https://example.com')]) + await rq_backend.add_batch_of_requests([Request.from_url('https://example.com')]) # Verify modified timestamp was updated - metadata = await rq_client.get_metadata() + metadata = await rq_backend.get_metadata() assert metadata.created_at == initial_created assert metadata.modified_at > initial_modified assert metadata.accessed_at > accessed_after_read - async with rq_client.get_session() as session: + async with rq_backend.get_session() as session: orm_metadata = await session.get(RequestQueueMetadataDb, metadata.id) assert orm_metadata is not None assert orm_metadata.created_at == metadata.created_at @@ -203,8 +203,8 @@ async def test_metadata_record_updates(rq_client: SqlRequestQueueClient) -> None async def test_data_persistence_across_reopens(configuration: Configuration) -> None: """Test that data persists correctly when reopening the same request queue.""" - async with SqlStorageClient() as storage_client: - original_client = await storage_client.create_rq_client( + async with SqlStorageBackend() as storage_backend: + original_backend = await storage_backend.create_rq_backend( name='persistence-test', configuration=configuration, ) @@ -213,34 +213,34 @@ async def test_data_persistence_across_reopens(configuration: Configuration) -> Request.from_url('https://example.com/1'), Request.from_url('https://example.com/2'), ] - await original_client.add_batch_of_requests(test_requests) + await original_backend.add_batch_of_requests(test_requests) - rq_id = (await original_client.get_metadata()).id + rq_id = (await original_backend.get_metadata()).id # Reopen by ID and verify data persists - reopened_client = await storage_client.create_rq_client( + reopened_backend = await storage_backend.create_rq_backend( id=rq_id, configuration=configuration, ) - metadata = await reopened_client.get_metadata() + metadata = await reopened_backend.get_metadata() assert metadata.total_request_count == 2 # Fetch requests to verify they're still there - request1 = await reopened_client.fetch_next_request() - request2 = await reopened_client.fetch_next_request() + request1 = await reopened_backend.fetch_next_request() + request2 = await reopened_backend.fetch_next_request() assert request1 is not None assert request2 is not None assert {request1.url, request2.url} == {'https://example.com/1', 'https://example.com/2'} - await reopened_client.drop() + await reopened_backend.drop() -async def test_error_handling_on_get_metadata_failure(rq_client: SqlRequestQueueClient) -> None: +async def test_error_handling_on_get_metadata_failure(rq_backend: SqlRequestQueueBackend) -> None: """Test that get_metadata properly handles SQL errors and retries.""" with patch( - 'crawlee.storage_clients._sql._request_queue_client.SqlRequestQueueClient.get_session', + 'crawlee.storage_backends._sql._request_queue_backend.SqlRequestQueueBackend.get_session', ) as mock_get_session: mock_session = AsyncMock() mock_session.__aenter__ = AsyncMock(return_value=mock_session) @@ -252,16 +252,16 @@ async def test_error_handling_on_get_metadata_failure(rq_client: SqlRequestQueue patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, pytest.raises(SQLAlchemyError), ): - await rq_client.get_metadata() + await rq_backend.get_metadata() # Verify that retry logic was attempted assert mock_sleep.call_count == 2 # Assuming default max_attempts=3 -async def test_get_metadata_does_not_retry_on_unexpected_exception(rq_client: SqlRequestQueueClient) -> None: +async def test_get_metadata_does_not_retry_on_unexpected_exception(rq_backend: SqlRequestQueueBackend) -> None: """Test that get_metadata does not retry on unexpected exceptions.""" with patch( - 'crawlee.storage_clients._sql._request_queue_client.SqlRequestQueueClient.get_session', + 'crawlee.storage_backends._sql._request_queue_backend.SqlRequestQueueBackend.get_session', ) as mock_get_session: mock_session = AsyncMock() mock_session.__aenter__ = AsyncMock(return_value=mock_session) @@ -273,7 +273,7 @@ async def test_get_metadata_does_not_retry_on_unexpected_exception(rq_client: Sq patch('crawlee._utils.retry._retry_sleep', new_callable=AsyncMock) as mock_sleep, pytest.raises(ValueError, match='unexpected error'), ): - await rq_client.get_metadata() + await rq_backend.get_metadata() # Verify that retry logic was not attempted assert mock_sleep.call_count == 0 diff --git a/tests/unit/storage_clients/_sql/test_sql_storage_client.py b/tests/unit/storage_backends/_sql/test_sql_storage_backend.py similarity index 71% rename from tests/unit/storage_clients/_sql/test_sql_storage_client.py rename to tests/unit/storage_backends/_sql/test_sql_storage_backend.py index 1b43cc2169..a42c5001bc 100644 --- a/tests/unit/storage_clients/_sql/test_sql_storage_client.py +++ b/tests/unit/storage_backends/_sql/test_sql_storage_backend.py @@ -9,7 +9,7 @@ from sqlalchemy.ext.asyncio import create_async_engine from crawlee.configuration import Configuration -from crawlee.storage_clients import SqlStorageClient +from crawlee.storage_backends import SqlStorageBackend if TYPE_CHECKING: from pathlib import Path @@ -19,10 +19,10 @@ async def test_sqlite_wal_mode_with_default_connection(tmp_path: Path) -> None: """Test that WAL mode is applied for the default SQLite connection.""" configuration = Configuration(storage_dir=str(tmp_path)) - async with SqlStorageClient() as storage_client: - await storage_client.initialize(configuration) + async with SqlStorageBackend() as storage_backend: + await storage_backend.initialize(configuration) - async with storage_client.engine.begin() as conn: + async with storage_backend.engine.begin() as conn: result = await conn.execute(text('PRAGMA journal_mode')) assert result.scalar() == 'wal' @@ -32,10 +32,10 @@ async def test_sqlite_wal_mode_with_connection_string(tmp_path: Path) -> None: db_path = tmp_path / 'test.db' configuration = Configuration(storage_dir=str(tmp_path)) - async with SqlStorageClient(connection_string=f'sqlite+aiosqlite:///{db_path}') as storage_client: - await storage_client.initialize(configuration) + async with SqlStorageBackend(connection_string=f'sqlite+aiosqlite:///{db_path}') as storage_backend: + await storage_backend.initialize(configuration) - async with storage_client.engine.begin() as conn: + async with storage_backend.engine.begin() as conn: result = await conn.execute(text('PRAGMA journal_mode')) assert result.scalar() == 'wal' @@ -46,8 +46,8 @@ async def test_sqlite_wal_mode_not_applied_with_custom_engine(tmp_path: Path) -> configuration = Configuration(storage_dir=str(tmp_path)) engine = create_async_engine(f'sqlite+aiosqlite:///{db_path}', future=True) - async with SqlStorageClient(engine=engine) as storage_client: - await storage_client.initialize(configuration) + async with SqlStorageBackend(engine=engine) as storage_backend: + await storage_backend.initialize(configuration) async with engine.begin() as conn: result = await conn.execute(text('PRAGMA journal_mode')) @@ -60,7 +60,7 @@ def test_import_error_handled() -> None: } with patch.dict('sys.modules', blocked): for mod_name in list(sys.modules): - if mod_name.startswith('crawlee.storage_clients._sql'): + if mod_name.startswith('crawlee.storage_backends._sql'): sys.modules.pop(mod_name, None) with pytest.raises(ImportError): - from crawlee.storage_clients._sql import SqlStorageClient # noqa: F401 PLC0415 + from crawlee.storage_backends._sql import SqlStorageBackend # noqa: F401 PLC0415 diff --git a/tests/unit/storage_clients/test_batch_records_by_size.py b/tests/unit/storage_backends/test_batch_records_by_size.py similarity index 93% rename from tests/unit/storage_clients/test_batch_records_by_size.py rename to tests/unit/storage_backends/test_batch_records_by_size.py index 00f224ff13..698d2759fc 100644 --- a/tests/unit/storage_clients/test_batch_records_by_size.py +++ b/tests/unit/storage_backends/test_batch_records_by_size.py @@ -1,7 +1,7 @@ from __future__ import annotations -from crawlee.storage_clients._utils import batch_records_by_size -from crawlee.storage_clients.models import KeyValueStoreRecordMetadata +from crawlee.storage_backends._utils import batch_records_by_size +from crawlee.storage_backends.models import KeyValueStoreRecordMetadata def make_record(key: str, size: int | None) -> KeyValueStoreRecordMetadata: diff --git a/tests/unit/storages/conftest.py b/tests/unit/storages/conftest.py index 5ea6962ceb..406987e51a 100644 --- a/tests/unit/storages/conftest.py +++ b/tests/unit/storages/conftest.py @@ -6,12 +6,12 @@ import pytest from crawlee import service_locator -from crawlee.storage_clients import ( - FileSystemStorageClient, - MemoryStorageClient, - RedisStorageClient, - SqlStorageClient, - StorageClient, +from crawlee.storage_backends import ( + FileSystemStorageBackend, + MemoryStorageBackend, + RedisStorageBackend, + SqlStorageBackend, + StorageBackend, ) if TYPE_CHECKING: @@ -21,35 +21,35 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None: - """Skip the Redis request queue cases when `lupa` is missing, since the Redis RQ client runs Lua scripts.""" + """Skip the Redis request queue cases when `lupa` is missing, since the Redis RQ backend runs Lua scripts.""" if find_spec('lupa') is not None: return for item in items: callspec = getattr(item, 'callspec', None) - if item.path.name == 'test_request_queue.py' and callspec and callspec.params.get('storage_client') == 'redis': + if item.path.name == 'test_request_queue.py' and callspec and callspec.params.get('storage_backend') == 'redis': item.add_marker(pytest.mark.skip(reason='`lupa` is not installed')) @pytest.fixture(params=['memory', 'file_system', 'sql', 'redis']) -async def storage_client( +async def storage_backend( request: pytest.FixtureRequest, redis_client: FakeAsyncRedis, -) -> AsyncGenerator[StorageClient, None]: - """Parameterized fixture to test with different storage clients.""" - storage_client: StorageClient +) -> AsyncGenerator[StorageBackend, None]: + """Parameterized fixture to test with different storage backends.""" + storage_backend: StorageBackend storage_type = request.param if storage_type == 'memory': - storage_client = MemoryStorageClient() + storage_backend = MemoryStorageBackend() elif storage_type == 'sql': - storage_client = SqlStorageClient() + storage_backend = SqlStorageBackend() elif storage_type == 'redis': - storage_client = RedisStorageClient(redis=redis_client) + storage_backend = RedisStorageBackend(redis=redis_client) else: - storage_client = FileSystemStorageClient() - service_locator.set_storage_client(storage_client) - yield storage_client + storage_backend = FileSystemStorageBackend() + service_locator.set_storage_backend(storage_backend) + yield storage_backend - if isinstance(storage_client, SqlStorageClient): - await storage_client.close() + if isinstance(storage_backend, SqlStorageBackend): + await storage_backend.close() diff --git a/tests/unit/storages/test_dataset.py b/tests/unit/storages/test_dataset.py index 0002f31cc4..401925b815 100644 --- a/tests/unit/storages/test_dataset.py +++ b/tests/unit/storages/test_dataset.py @@ -7,7 +7,7 @@ from crawlee import service_locator from crawlee.configuration import Configuration -from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient +from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend from crawlee.storages import Dataset, KeyValueStore from crawlee.storages._storage_instance_manager import StorageInstanceManager @@ -16,16 +16,16 @@ from pathlib import Path from typing import Any - from crawlee.storage_clients import StorageClient + from crawlee.storage_backends import StorageBackend @pytest.fixture async def dataset( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> AsyncGenerator[Dataset, None]: """Fixture that provides a dataset instance for each test.""" dataset = await Dataset.open( - storage_client=storage_client, + storage_backend=storage_backend, ) yield dataset @@ -33,12 +33,12 @@ async def dataset( async def test_open_creates_new_dataset( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that open() creates a new dataset with proper metadata.""" dataset = await Dataset.open( name='new-dataset', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify dataset properties @@ -52,12 +52,12 @@ async def test_open_creates_new_dataset( async def test_reopen_default( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test reopening a dataset with default parameters.""" # Create a first dataset instance with default parameters dataset_1 = await Dataset.open( - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify default properties @@ -72,7 +72,7 @@ async def test_reopen_default( # Reopen the same dataset dataset_2 = await Dataset.open( - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify both instances reference the same dataset @@ -90,13 +90,13 @@ async def test_reopen_default( async def test_open_by_id( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test opening a dataset by its ID.""" # First create a dataset by name dataset1 = await Dataset.open( name='dataset-by-id-test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add some data to identify it @@ -106,7 +106,7 @@ async def test_open_by_id( # Open the dataset by ID dataset2 = await Dataset.open( id=dataset1.id, - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify it's the same dataset @@ -144,7 +144,7 @@ async def test_open_existing_dataset( async def test_open_with_id_and_name( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that open() raises an error when both id and name are provided.""" with pytest.raises( @@ -155,7 +155,7 @@ async def test_open_with_id_and_name( await Dataset.open( id='some-id', name='some-name', - storage_client=storage_client, + storage_backend=storage_backend, ) @@ -386,12 +386,12 @@ async def test_list_items_with_options(dataset: Dataset) -> None: async def test_drop( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test dropping a dataset removes it from cache and clears its data.""" dataset = await Dataset.open( name='drop-test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add some data @@ -403,7 +403,7 @@ async def test_drop( # Verify dataset is empty (by creating a new one with the same name) new_dataset = await Dataset.open( name='drop-test', - storage_client=storage_client, + storage_backend=storage_backend, ) result = await new_dataset.get_data() @@ -413,7 +413,7 @@ async def test_drop( async def test_export_to_json( dataset: Dataset, - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test exporting dataset to JSON format.""" # Create a key-value store for export @@ -434,7 +434,7 @@ async def test_export_to_json( key='dataset_export.json', content_type='json', to_kvs_name='export-kvs', - to_kvs_storage_client=storage_client, + to_kvs_storage_backend=storage_backend, ) # Retrieve the exported file @@ -451,13 +451,13 @@ async def test_export_to_json( async def test_export_to_csv( dataset: Dataset, - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test exporting dataset to CSV format.""" # Create a key-value store for export kvs = await KeyValueStore.open( name='export-kvs', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add some items to the dataset @@ -473,7 +473,7 @@ async def test_export_to_csv( key='dataset_export.csv', content_type='csv', to_kvs_name='export-kvs', - to_kvs_storage_client=storage_client, + to_kvs_storage_backend=storage_backend, ) # Retrieve the exported file @@ -491,12 +491,12 @@ async def test_export_to_csv( async def test_export_to_csv_columns( dataset: Dataset, - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test the CSV columns come from the first item by default, and from all items with `collect_all_keys`.""" kvs = await KeyValueStore.open( name='export-kvs', - storage_client=storage_client, + storage_backend=storage_backend, ) await dataset.push_data( @@ -511,14 +511,14 @@ async def test_export_to_csv_columns( key='first_item_keys.csv', content_type='csv', to_kvs_name='export-kvs', - to_kvs_storage_client=storage_client, + to_kvs_storage_backend=storage_backend, lineterminator='\n', ) await dataset.export_to( key='all_keys.csv', content_type='csv', to_kvs_name='export-kvs', - to_kvs_storage_client=storage_client, + to_kvs_storage_backend=storage_backend, collect_all_keys=True, lineterminator='\n', ) @@ -538,7 +538,7 @@ async def test_export_to_invalid_content_type(dataset: Dataset) -> None: async def test_export_with_multiple_kwargs(dataset: Dataset, tmp_path: Path) -> None: """Test exporting dataset using many optional arguments together.""" target_kvs_name = 'some-kvs' - target_storage_client = FileSystemStorageClient() + target_storage_backend = FileSystemStorageBackend() export_key = 'exported_dataset' data = {'some key': 'some data'} @@ -558,7 +558,7 @@ async def test_export_with_multiple_kwargs(dataset: Dataset, tmp_path: Path) -> key=export_key, content_type='json', to_kvs_name=target_kvs_name, - to_kvs_storage_client=target_storage_client, + to_kvs_storage_backend=target_storage_backend, to_kvs_configuration=target_configuration, ) @@ -566,7 +566,7 @@ async def test_export_with_multiple_kwargs(dataset: Dataset, tmp_path: Path) -> assert expected_kvs_dir.is_dir() # Verify that kvs contains the exported data kvs = await KeyValueStore.open( - name=target_kvs_name, storage_client=target_storage_client, configuration=target_configuration + name=target_kvs_name, storage_backend=target_storage_backend, configuration=target_configuration ) assert await kvs.get_value(key=export_key) == expected_exported_data @@ -591,13 +591,13 @@ async def test_large_dataset(dataset: Dataset) -> None: async def test_purge( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test purging a dataset removes all data but keeps the dataset itself.""" # First create a dataset dataset = await Dataset.open( name='purge-test-dataset', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add some data @@ -645,17 +645,17 @@ async def test_purge( async def test_open_with_alias( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test opening datasets with alias parameter for NDU functionality.""" # Create datasets with different aliases dataset_1 = await Dataset.open( alias='test_alias_1', - storage_client=storage_client, + storage_backend=storage_backend, ) dataset_2 = await Dataset.open( alias='test_alias_2', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify they have different IDs but no names (unnamed) @@ -682,19 +682,19 @@ async def test_open_with_alias( async def test_alias_caching( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that datasets with same alias return same instance (cached).""" # Open dataset with alias dataset_1 = await Dataset.open( alias='cache_test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Open again with same alias dataset_2 = await Dataset.open( alias='cache_test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Should be same instance @@ -706,7 +706,7 @@ async def test_alias_caching( async def test_alias_with_id_error( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that providing both alias and id raises error.""" with pytest.raises( @@ -717,12 +717,12 @@ async def test_alias_with_id_error( await Dataset.open( id='some-id', alias='some-alias', - storage_client=storage_client, + storage_backend=storage_backend, ) async def test_alias_with_name_error( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that providing both alias and name raises error.""" with pytest.raises( @@ -733,12 +733,12 @@ async def test_alias_with_name_error( await Dataset.open( name='some-name', alias='some-alias', - storage_client=storage_client, + storage_backend=storage_backend, ) async def test_alias_with_all_parameters_error( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that providing id, name, and alias raises error.""" with pytest.raises( @@ -750,12 +750,12 @@ async def test_alias_with_all_parameters_error( id='some-id', name='some-name', alias='some-alias', - storage_client=storage_client, + storage_backend=storage_backend, ) async def test_alias_with_special_characters( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test alias functionality with special characters.""" special_aliases = [ @@ -770,7 +770,7 @@ async def test_alias_with_special_characters( for alias in special_aliases: dataset = await Dataset.open( alias=alias, - storage_client=storage_client, + storage_backend=storage_backend, ) datasets.append(dataset) @@ -789,40 +789,40 @@ async def test_alias_with_special_characters( async def test_named_vs_alias_conflict_detection( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that conflicts between named and alias storages are detected.""" # Test 1: Create named storage first, then try alias with same name - named_dataset = await Dataset.open(name='conflict-test', storage_client=storage_client) + named_dataset = await Dataset.open(name='conflict-test', storage_backend=storage_backend) assert named_dataset.name == 'conflict-test' # Try to create alias with same name - should raise error with pytest.raises(ValueError, match=r'Cannot create alias storage "conflict-test".*already exists'): - await Dataset.open(alias='conflict-test', storage_client=storage_client) + await Dataset.open(alias='conflict-test', storage_backend=storage_backend) # Clean up await named_dataset.drop() # Test 2: Create alias first, then try named with same name - alias_dataset = await Dataset.open(alias='conflict-test2', storage_client=storage_client) + alias_dataset = await Dataset.open(alias='conflict-test2', storage_backend=storage_backend) assert alias_dataset.name is None # Alias storages have no name # Try to create named with same name - should raise error with pytest.raises(ValueError, match=r'Cannot create named storage "conflict-test2".*already exists'): - await Dataset.open(name='conflict-test2', storage_client=storage_client) + await Dataset.open(name='conflict-test2', storage_backend=storage_backend) # Clean up await alias_dataset.drop() async def test_alias_parameter( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test dataset creation and operations with alias parameter.""" # Create dataset with alias alias_dataset = await Dataset.open( alias='test_alias', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify alias dataset properties @@ -839,13 +839,13 @@ async def test_alias_parameter( async def test_alias_vs_named_isolation( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that alias and named datasets with same identifier are isolated.""" # Create named dataset named_dataset = await Dataset.open( name='test-identifier', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify named dataset @@ -858,7 +858,7 @@ async def test_alias_vs_named_isolation( # Now create alias dataset with same identifier (should work after cleanup) alias_dataset = await Dataset.open( alias='test_identifier', - storage_client=storage_client, + storage_backend=storage_backend, ) # Should be different instance @@ -873,17 +873,17 @@ async def test_alias_vs_named_isolation( async def test_default_vs_alias_default_equivalence( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that default dataset and alias='default' are equivalent.""" # Open default dataset default_dataset = await Dataset.open( - storage_client=storage_client, + storage_backend=storage_backend, ) alias_default_dataset = await Dataset.open( alias=StorageInstanceManager._DEFAULT_STORAGE_ALIAS, - storage_client=storage_client, + storage_backend=storage_backend, ) # Should be the same @@ -900,7 +900,7 @@ async def test_default_vs_alias_default_equivalence( async def test_multiple_alias_isolation( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that different aliases create separate datasets.""" datasets = [] @@ -908,7 +908,7 @@ async def test_multiple_alias_isolation( for i in range(3): dataset = await Dataset.open( alias=f'alias_{i}', - storage_client=storage_client, + storage_backend=storage_backend, ) await dataset.push_data({'alias': f'alias_{i}', 'index': i}) datasets.append(dataset) @@ -925,30 +925,30 @@ async def test_multiple_alias_isolation( await dataset.drop() -async def test_purge_on_start_enabled(storage_client: StorageClient) -> None: +async def test_purge_on_start_enabled(storage_backend: StorageBackend) -> None: """Test purge behavior when purge_on_start=True: named storages retain data, unnamed storages are purged.""" - # Skip this test for memory storage since it doesn't persist data between client instances. - if isinstance(storage_client, MemoryStorageClient): - raise pytest.skip.Exception('Memory storage does not persist data between client instances.') + # Skip this test for memory storage since it doesn't persist data between backend instances. + if isinstance(storage_backend, MemoryStorageBackend): + raise pytest.skip.Exception('Memory storage does not persist data between backend instances.') configuration = Configuration(purge_on_start=True) # First, create all storage types with purge enabled and add data. default_dataset = await Dataset.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_dataset = await Dataset.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_dataset = await Dataset.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -979,17 +979,17 @@ async def test_purge_on_start_enabled(storage_client: StorageClient) -> None: # Now "reopen" all storages default_dataset_2 = await Dataset.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_dataset_2 = await Dataset.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_dataset_2 = await Dataset.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1011,30 +1011,30 @@ async def test_purge_on_start_enabled(storage_client: StorageClient) -> None: await default_dataset_2.drop() -async def test_purge_on_start_disabled(storage_client: StorageClient) -> None: +async def test_purge_on_start_disabled(storage_backend: StorageBackend) -> None: """Test purge behavior when purge_on_start=False: all storages retain data regardless of type.""" - # Skip this test for memory storage since it doesn't persist data between client instances. - if isinstance(storage_client, MemoryStorageClient): - raise pytest.skip.Exception('Memory storage does not persist data between client instances.') + # Skip this test for memory storage since it doesn't persist data between backend instances. + if isinstance(storage_backend, MemoryStorageBackend): + raise pytest.skip.Exception('Memory storage does not persist data between backend instances.') configuration = Configuration(purge_on_start=False) # First, create all storage types with purge disabled and add data. default_dataset = await Dataset.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_dataset = await Dataset.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_dataset = await Dataset.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1065,17 +1065,17 @@ async def test_purge_on_start_disabled(storage_client: StorageClient) -> None: # Now "reopen" all storages default_dataset_2 = await Dataset.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_dataset_2 = await Dataset.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_dataset_2 = await Dataset.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1099,14 +1099,14 @@ async def test_purge_on_start_disabled(storage_client: StorageClient) -> None: await named_dataset_2.drop() -async def test_name_default_not_allowed(storage_client: StorageClient) -> None: +async def test_name_default_not_allowed(storage_backend: StorageBackend) -> None: """Test that storage can't have default alias as name, to prevent collisions with unnamed storage alias.""" with pytest.raises( ValueError, match=f'Storage name cannot be "{StorageInstanceManager._DEFAULT_STORAGE_ALIAS}" as ' f'it is reserved for default alias.', ): - await Dataset.open(name=StorageInstanceManager._DEFAULT_STORAGE_ALIAS, storage_client=storage_client) + await Dataset.open(name=StorageInstanceManager._DEFAULT_STORAGE_ALIAS, storage_backend=storage_backend) @pytest.mark.parametrize( @@ -1131,16 +1131,16 @@ async def test_name_default_not_allowed(storage_client: StorageClient) -> None: pytest.param('with/slash', False, id='slash'), ], ) -async def test_validate_name(storage_client: StorageClient, name: str, *, is_valid: bool) -> None: +async def test_validate_name(storage_backend: StorageBackend, name: str, *, is_valid: bool) -> None: """Test name validation logic.""" if is_valid: # Should not raise - dataset = await Dataset.open(name=name, storage_client=storage_client) + dataset = await Dataset.open(name=name, storage_backend=storage_backend) assert dataset.name == name await dataset.drop() else: with pytest.raises(ValueError, match=rf'Invalid storage name "{name}".*'): - await Dataset.open(name=name, storage_client=storage_client) + await Dataset.open(name=name, storage_backend=storage_backend) @pytest.mark.parametrize( @@ -1157,15 +1157,15 @@ async def test_validate_name(storage_client: StorageClient, name: str, *, is_val pytest.param('back\\slash', False, id='backslash'), ], ) -async def test_validate_alias(storage_client: StorageClient, alias: str, *, is_valid: bool) -> None: +async def test_validate_alias(storage_backend: StorageBackend, alias: str, *, is_valid: bool) -> None: """Test alias validation logic, including rejection of values that would resolve outside the storage dir.""" if is_valid: # Should not raise. - dataset = await Dataset.open(alias=alias, storage_client=storage_client) + dataset = await Dataset.open(alias=alias, storage_backend=storage_backend) await dataset.drop() else: with pytest.raises(ValueError, match=r'Invalid storage alias'): - await Dataset.open(alias=alias, storage_client=storage_client) + await Dataset.open(alias=alias, storage_backend=storage_backend) async def test_record_with_noascii_chars(dataset: Dataset) -> None: diff --git a/tests/unit/storages/test_key_value_store.py b/tests/unit/storages/test_key_value_store.py index 302c8eb30e..a62b569c72 100644 --- a/tests/unit/storages/test_key_value_store.py +++ b/tests/unit/storages/test_key_value_store.py @@ -8,9 +8,9 @@ from crawlee import service_locator from crawlee.configuration import Configuration -from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient, SqlStorageClient, StorageClient -from crawlee.storage_clients._memory import MemoryKeyValueStoreClient -from crawlee.storage_clients.models import KeyValueStoreRecord +from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend, SqlStorageBackend, StorageBackend +from crawlee.storage_backends._memory import MemoryKeyValueStoreBackend +from crawlee.storage_backends.models import KeyValueStoreRecord from crawlee.storages import KeyValueStore from crawlee.storages._storage_instance_manager import StorageInstanceManager @@ -21,11 +21,11 @@ @pytest.fixture async def kvs( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> AsyncGenerator[KeyValueStore, None]: """Fixture that provides a key-value store instance for each test.""" kvs = await KeyValueStore.open( - storage_client=storage_client, + storage_backend=storage_backend, ) yield kvs @@ -33,12 +33,12 @@ async def kvs( async def test_open_creates_new_kvs( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that open() creates a new key-value store with proper metadata.""" kvs = await KeyValueStore.open( name='new-kvs', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify key-value store properties @@ -50,13 +50,13 @@ async def test_open_creates_new_kvs( async def test_open_existing_kvs( kvs: KeyValueStore, - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that open() loads an existing key-value store correctly.""" # Open the same key-value store again reopened_kvs = await KeyValueStore.open( name=kvs.name, - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify key-value store properties @@ -68,7 +68,7 @@ async def test_open_existing_kvs( async def test_open_with_id_and_name( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that open() raises an error when both id and name are provided.""" with pytest.raises( @@ -79,18 +79,18 @@ async def test_open_with_id_and_name( await KeyValueStore.open( id='some-id', name='some-name', - storage_client=storage_client, + storage_backend=storage_backend, ) async def test_open_by_id( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test opening a key-value store by its ID.""" # First create a key-value store by name kvs1 = await KeyValueStore.open( name='kvs-by-id-test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add some data to identify it @@ -99,7 +99,7 @@ async def test_open_by_id( # Open the key-value store by ID kvs2 = await KeyValueStore.open( id=kvs1.id, - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify it's the same key-value store @@ -317,10 +317,10 @@ async def test_iterate_entries_empty_kvs(kvs: KeyValueStore) -> None: assert collected_entries == [] -async def test_iterate_entries_uses_storage_client_implementation() -> None: - """Test that `iterate_entries` and `iterate_values` delegate to the storage client's `iterate_entries`.""" +async def test_iterate_entries_uses_storage_backend_implementation() -> None: + """Test that `iterate_entries` and `iterate_values` delegate to the storage backend's `iterate_entries`.""" - class OptimizedKeyValueStoreClient(MemoryKeyValueStoreClient): + class OptimizedKeyValueStoreBackend(MemoryKeyValueStoreBackend): async def iterate_entries( self, *, @@ -339,10 +339,12 @@ async def iterate_entries( ) async def get_value(self, *, key: str) -> KeyValueStoreRecord | None: - raise AssertionError(f'get_value must not be called for {key!r} when the client implements iterate_entries') + raise AssertionError( + f'get_value must not be called for {key!r} when the backend implements iterate_entries' + ) - client = await OptimizedKeyValueStoreClient.open(id=None, name=None, alias=None) - kvs = KeyValueStore(client, id=(await client.get_metadata()).id, name=None) + backend = await OptimizedKeyValueStoreBackend.open(id=None, name=None, alias=None) + kvs = KeyValueStore(backend, id=(await backend.get_metadata()).id, name=None) await kvs.set_value('key1', 'value1') await kvs.set_value('key2', 'value2') await kvs.set_value('key3', 'value3') @@ -365,12 +367,12 @@ async def test_async_iteration(kvs: KeyValueStore) -> None: async def test_drop( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test dropping a key-value store removes it from cache and clears its data.""" kvs = await KeyValueStore.open( name='drop-test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add some data @@ -382,7 +384,7 @@ async def test_drop( # Verify key-value store is empty (by creating a new one with the same name) new_kvs = await KeyValueStore.open( name='drop-test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Attempt to get a previously stored value @@ -392,12 +394,12 @@ async def test_drop( async def test_reopen_default( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test reopening the default key-value store.""" # Open the default key-value store kvs1 = await KeyValueStore.open( - storage_client=storage_client, + storage_backend=storage_backend, ) # Set a value @@ -405,7 +407,7 @@ async def test_reopen_default( # Open the default key-value store again kvs2 = await KeyValueStore.open( - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify they are the same store @@ -504,13 +506,13 @@ async def test_data_persistence_on_reopen() -> None: async def test_purge( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test purging a key-value store removes all values but keeps the store itself.""" # First create a key-value store kvs = await KeyValueStore.open( name='purge-test-kvs', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add some values @@ -665,17 +667,17 @@ async def test_record_exists_after_purge(kvs: KeyValueStore) -> None: async def test_open_with_alias( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test opening key-value stores with alias parameter for NDU functionality.""" # Create key-value stores with different aliases kvs_1 = await KeyValueStore.open( alias='test_alias_1', - storage_client=storage_client, + storage_backend=storage_backend, ) kvs_2 = await KeyValueStore.open( alias='test_alias_2', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify they have different IDs but no names (unnamed) @@ -700,19 +702,19 @@ async def test_open_with_alias( async def test_alias_caching( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that key-value stores with same alias return same instance (cached).""" # Open kvs with alias kvs_1 = await KeyValueStore.open( alias='cache_test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Open again with same alias kvs_2 = await KeyValueStore.open( alias='cache_test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Should be same instance @@ -724,7 +726,7 @@ async def test_alias_caching( async def test_alias_with_id_error( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that providing both alias and id raises error.""" with pytest.raises( @@ -735,12 +737,12 @@ async def test_alias_with_id_error( await KeyValueStore.open( id='some-id', alias='some-alias', - storage_client=storage_client, + storage_backend=storage_backend, ) async def test_alias_with_name_error( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that providing both alias and name raises error.""" with pytest.raises( @@ -751,12 +753,12 @@ async def test_alias_with_name_error( await KeyValueStore.open( name='some-name', alias='some-alias', - storage_client=storage_client, + storage_backend=storage_backend, ) async def test_alias_with_special_characters( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test alias functionality with special characters.""" special_aliases = [ @@ -771,7 +773,7 @@ async def test_alias_with_special_characters( for alias in special_aliases: kvs = await KeyValueStore.open( alias=alias, - storage_client=storage_client, + storage_backend=storage_backend, ) stores.append(kvs) @@ -790,12 +792,12 @@ async def test_alias_with_special_characters( async def test_alias_key_operations( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that key operations work correctly with alias stores.""" kvs = await KeyValueStore.open( alias='key_ops_test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Test setting multiple keys @@ -839,34 +841,34 @@ async def test_alias_key_operations( async def test_named_vs_alias_conflict_detection( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that conflicts between named and alias storages are detected.""" # Test 1: Create named storage first, then try alias with same name - named_kvs = await KeyValueStore.open(name='conflict-test', storage_client=storage_client) + named_kvs = await KeyValueStore.open(name='conflict-test', storage_backend=storage_backend) assert named_kvs.name == 'conflict-test' # Try to create alias with same name - should raise error with pytest.raises(ValueError, match=r'Cannot create alias storage "conflict-test".*already exists'): - await KeyValueStore.open(alias='conflict-test', storage_client=storage_client) + await KeyValueStore.open(alias='conflict-test', storage_backend=storage_backend) # Clean up await named_kvs.drop() # Test 2: Create alias first, then try named with same name - alias_kvs = await KeyValueStore.open(alias='conflict-test2', storage_client=storage_client) + alias_kvs = await KeyValueStore.open(alias='conflict-test2', storage_backend=storage_backend) assert alias_kvs.name is None # Alias storages have no name # Try to create named with same name - should raise error with pytest.raises(ValueError, match=r'Cannot create named storage "conflict-test2".*already exists'): - await KeyValueStore.open(name='conflict-test2', storage_client=storage_client) + await KeyValueStore.open(name='conflict-test2', storage_backend=storage_backend) # Clean up await alias_kvs.drop() # Test 3: Different names should work fine - named_kvs_ok = await KeyValueStore.open(name='different-name', storage_client=storage_client) - alias_kvs_ok = await KeyValueStore.open(alias='different-alias', storage_client=storage_client) + named_kvs_ok = await KeyValueStore.open(name='different-name', storage_backend=storage_backend) + alias_kvs_ok = await KeyValueStore.open(alias='different-alias', storage_backend=storage_backend) assert named_kvs_ok.name == 'different-name' assert alias_kvs_ok.name is None @@ -877,13 +879,13 @@ async def test_named_vs_alias_conflict_detection( async def test_alias_parameter( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test key-value store creation and operations with alias parameter.""" # Create kvs with alias alias_kvs = await KeyValueStore.open( alias='test_alias', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify alias kvs properties @@ -899,13 +901,13 @@ async def test_alias_parameter( async def test_alias_vs_named_isolation( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that alias and named key-value stores with same identifier are isolated.""" # Create named kvs named_kvs = await KeyValueStore.open( name='test-identifier', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify named kvs @@ -918,7 +920,7 @@ async def test_alias_vs_named_isolation( # Now create alias kvs with same identifier (should work after cleanup) alias_kvs = await KeyValueStore.open( alias='test_identifier', - storage_client=storage_client, + storage_backend=storage_backend, ) # Should be different instance @@ -933,17 +935,17 @@ async def test_alias_vs_named_isolation( async def test_default_vs_alias_default_equivalence( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that default key-value store and alias='default' are equivalent.""" # Open default kvs default_kvs = await KeyValueStore.open( - storage_client=storage_client, + storage_backend=storage_backend, ) alias_default_kvs = await KeyValueStore.open( alias=StorageInstanceManager._DEFAULT_STORAGE_ALIAS, - storage_client=storage_client, + storage_backend=storage_backend, ) # Should be the same @@ -960,7 +962,7 @@ async def test_default_vs_alias_default_equivalence( async def test_multiple_alias_isolation( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that different aliases create separate key-value stores.""" kvs_stores = [] @@ -968,7 +970,7 @@ async def test_multiple_alias_isolation( for i in range(3): kvs = await KeyValueStore.open( alias=f'alias_{i}', - storage_client=storage_client, + storage_backend=storage_backend, ) await kvs.set_value('alias', f'alias_{i}') await kvs.set_value('index', i) @@ -989,30 +991,30 @@ async def test_multiple_alias_isolation( await kvs.drop() -async def test_purge_on_start_enabled(storage_client: StorageClient) -> None: +async def test_purge_on_start_enabled(storage_backend: StorageBackend) -> None: """Test purge behavior when purge_on_start=True: named storages retain data, unnamed storages are purged.""" - # Skip this test for memory storage since it doesn't persist data between client instances. - if isinstance(storage_client, MemoryStorageClient): - raise pytest.skip.Exception('Memory storage does not persist data between client instances.') + # Skip this test for memory storage since it doesn't persist data between backend instances. + if isinstance(storage_backend, MemoryStorageBackend): + raise pytest.skip.Exception('Memory storage does not persist data between backend instances.') configuration = Configuration(purge_on_start=True) # First, create all storage types with purge enabled and add data. default_kvs = await KeyValueStore.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_kvs = await KeyValueStore.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_kvs = await KeyValueStore.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1043,17 +1045,17 @@ async def test_purge_on_start_enabled(storage_client: StorageClient) -> None: # Now "reopen" all storages default_kvs_2 = await KeyValueStore.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_kvs_2 = await KeyValueStore.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_kvs_2 = await KeyValueStore.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1075,30 +1077,30 @@ async def test_purge_on_start_enabled(storage_client: StorageClient) -> None: await default_kvs_2.drop() -async def test_purge_on_start_disabled(storage_client: StorageClient) -> None: +async def test_purge_on_start_disabled(storage_backend: StorageBackend) -> None: """Test purge behavior when purge_on_start=False: all storages retain data regardless of type.""" - # Skip this test for memory storage since it doesn't persist data between client instances. - if isinstance(storage_client, MemoryStorageClient): - raise pytest.skip.Exception('Memory storage does not persist data between client instances.') + # Skip this test for memory storage since it doesn't persist data between backend instances. + if isinstance(storage_backend, MemoryStorageBackend): + raise pytest.skip.Exception('Memory storage does not persist data between backend instances.') configuration = Configuration(purge_on_start=False) # First, create all storage types with purge disabled and add data. default_kvs = await KeyValueStore.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_kvs = await KeyValueStore.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_kvs = await KeyValueStore.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1120,17 +1122,17 @@ async def test_purge_on_start_disabled(storage_client: StorageClient) -> None: # Now "reopen" all storages default_kvs_2 = await KeyValueStore.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_kvs_2 = await KeyValueStore.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_kvs_2 = await KeyValueStore.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1150,14 +1152,14 @@ async def test_purge_on_start_disabled(storage_client: StorageClient) -> None: await default_kvs_2.drop() -async def test_name_default_not_allowed(storage_client: StorageClient) -> None: +async def test_name_default_not_allowed(storage_backend: StorageBackend) -> None: """Test that storage can't have default alias as name, to prevent collisions with unnamed storage alias.""" with pytest.raises( ValueError, match=f'Storage name cannot be "{StorageInstanceManager._DEFAULT_STORAGE_ALIAS}" as ' f'it is reserved for default alias.', ): - await KeyValueStore.open(name=StorageInstanceManager._DEFAULT_STORAGE_ALIAS, storage_client=storage_client) + await KeyValueStore.open(name=StorageInstanceManager._DEFAULT_STORAGE_ALIAS, storage_backend=storage_backend) @pytest.mark.parametrize( @@ -1182,16 +1184,16 @@ async def test_name_default_not_allowed(storage_client: StorageClient) -> None: pytest.param('with/slash', False, id='slash'), ], ) -async def test_validate_name(storage_client: StorageClient, name: str, *, is_valid: bool) -> None: +async def test_validate_name(storage_backend: StorageBackend, name: str, *, is_valid: bool) -> None: """Test name validation logic.""" if is_valid: # Should not raise - dataset = await KeyValueStore.open(name=name, storage_client=storage_client) + dataset = await KeyValueStore.open(name=name, storage_backend=storage_backend) assert dataset.name == name await dataset.drop() else: with pytest.raises(ValueError, match=rf'Invalid storage name "{name}".*'): - await KeyValueStore.open(name=name, storage_client=storage_client) + await KeyValueStore.open(name=name, storage_backend=storage_backend) @pytest.mark.parametrize( @@ -1208,39 +1210,41 @@ async def test_validate_name(storage_client: StorageClient, name: str, *, is_val pytest.param('back\\slash', False, id='backslash'), ], ) -async def test_validate_alias(storage_client: StorageClient, alias: str, *, is_valid: bool) -> None: +async def test_validate_alias(storage_backend: StorageBackend, alias: str, *, is_valid: bool) -> None: """Test alias validation logic, including rejection of values that would resolve outside the storage dir.""" if is_valid: # Should not raise. - kvs = await KeyValueStore.open(alias=alias, storage_client=storage_client) + kvs = await KeyValueStore.open(alias=alias, storage_backend=storage_backend) await kvs.drop() else: with pytest.raises(ValueError, match=r'Invalid storage alias'): - await KeyValueStore.open(alias=alias, storage_client=storage_client) + await KeyValueStore.open(alias=alias, storage_backend=storage_backend) @pytest.mark.parametrize( - 'tested_storage_client_class', + 'tested_storage_backend_class', [ - pytest.param(MemoryStorageClient, id='tested=MemoryStorageClient'), - pytest.param(FileSystemStorageClient, id='tested=FileSystemStorageClient'), - pytest.param(SqlStorageClient, id='tested=SqlStorageClient'), + pytest.param(MemoryStorageBackend, id='tested=MemoryStorageBackend'), + pytest.param(FileSystemStorageBackend, id='tested=FileSystemStorageBackend'), + pytest.param(SqlStorageBackend, id='tested=SqlStorageBackend'), ], ) @pytest.mark.parametrize( - 'global_storage_client_class', + 'global_storage_backend_class', [ - pytest.param(MemoryStorageClient, id='global=MemoryStorageClient'), - pytest.param(FileSystemStorageClient, id='global=FileSystemStorageClient'), - pytest.param(SqlStorageClient, id='global=SqlStorageClient'), + pytest.param(MemoryStorageBackend, id='global=MemoryStorageBackend'), + pytest.param(FileSystemStorageBackend, id='global=FileSystemStorageBackend'), + pytest.param(SqlStorageBackend, id='global=SqlStorageBackend'), ], ) -async def test_get_auto_saved_value_various_global_clients( - tmp_path: Path, tested_storage_client_class: type[StorageClient], global_storage_client_class: type[StorageClient] +async def test_get_auto_saved_value_various_global_backends( + tmp_path: Path, + tested_storage_backend_class: type[StorageBackend], + global_storage_backend_class: type[StorageBackend], ) -> None: - """Ensure that persistence is working for all clients regardless of what is set in service locator.""" - tested_storage_client = tested_storage_client_class() - global_storage_client = global_storage_client_class() + """Ensure that persistence is working for all backends regardless of what is set in service locator.""" + tested_storage_backend = tested_storage_backend_class() + global_storage_backend = global_storage_backend_class() service_locator.set_configuration( Configuration( @@ -1248,9 +1252,9 @@ async def test_get_auto_saved_value_various_global_clients( purge_on_start=True, ) ) - service_locator.set_storage_client(global_storage_client) + service_locator.set_storage_backend(global_storage_backend) - kvs = await KeyValueStore.open(storage_client=tested_storage_client) + kvs = await KeyValueStore.open(storage_backend=tested_storage_backend) values_kvs = {'key': 'some_value'} test_key = 'test_key' diff --git a/tests/unit/storages/test_request_queue.py b/tests/unit/storages/test_request_queue.py index f3f692006c..6e0ec5bdf7 100644 --- a/tests/unit/storages/test_request_queue.py +++ b/tests/unit/storages/test_request_queue.py @@ -9,8 +9,8 @@ from crawlee import Request, service_locator from crawlee.configuration import Configuration -from crawlee.storage_clients import MemoryStorageClient, StorageClient -from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, UnprocessedRequest +from crawlee.storage_backends import MemoryStorageBackend, StorageBackend +from crawlee.storage_backends.models import AddRequestsResponse, ProcessedRequest, UnprocessedRequest from crawlee.storages import RequestQueue from crawlee.storages._storage_instance_manager import StorageInstanceManager from tests.unit.utils import poll_until_condition @@ -18,16 +18,16 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator, Sequence - from crawlee.storage_clients import StorageClient + from crawlee.storage_backends import StorageBackend @pytest.fixture async def rq( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> AsyncGenerator[RequestQueue, None]: """Fixture that provides a request queue instance for each test.""" rq = await RequestQueue.open( - storage_client=storage_client, + storage_backend=storage_backend, ) yield rq @@ -35,12 +35,12 @@ async def rq( async def test_open_creates_new_rq( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that open() creates a new request queue with proper metadata.""" rq = await RequestQueue.open( name='new-request-queue', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify request queue properties @@ -56,13 +56,13 @@ async def test_open_creates_new_rq( async def test_open_existing_rq( rq: RequestQueue, - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that open() loads an existing request queue correctly.""" # Open the same request queue again reopened_rq = await RequestQueue.open( name=rq.name, - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify request queue properties @@ -74,7 +74,7 @@ async def test_open_existing_rq( async def test_open_with_id_and_name( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that open() raises an error when both id and name are provided.""" with pytest.raises( @@ -85,18 +85,18 @@ async def test_open_with_id_and_name( await RequestQueue.open( id='some-id', name='some-name', - storage_client=storage_client, + storage_backend=storage_backend, ) async def test_open_by_id( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test opening a request queue by its ID.""" # First create a request queue by name rq1 = await RequestQueue.open( name='rq-by-id-test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add a request to identify it @@ -105,7 +105,7 @@ async def test_open_by_id( # Open the request queue by ID rq2 = await RequestQueue.open( id=rq1.id, - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify it's the same request queue @@ -270,7 +270,7 @@ async def test_add_requests_retry_preserves_forefront( ) -> None: """Regression test: when ``add_batch_of_requests`` returns unprocessed requests, the retry must preserve the original `forefront` value rather than silently falling back to the parameter default.""" - rq = await RequestQueue.open(storage_client=MemoryStorageClient()) + rq = await RequestQueue.open(storage_backend=MemoryStorageBackend()) forefront_calls: list[bool] = [] async def patched_add_batch( @@ -296,7 +296,7 @@ async def patched_add_batch( unprocessed_requests=[], ) - monkeypatch.setattr(rq._client, 'add_batch_of_requests', patched_add_batch) + monkeypatch.setattr(rq._backend, 'add_batch_of_requests', patched_add_batch) try: await rq.add_requests( @@ -318,7 +318,7 @@ async def _no_sleep(_seconds: float) -> None: async def test_add_request_retries_unprocessed(monkeypatch: pytest.MonkeyPatch) -> None: """`add_request` must retry an unprocessed request (like `add_requests`) instead of silently dropping it.""" - rq = await RequestQueue.open(storage_client=MemoryStorageClient()) + rq = await RequestQueue.open(storage_backend=MemoryStorageBackend()) monkeypatch.setattr('crawlee.storages._request_queue.asyncio.sleep', _no_sleep) calls = 0 @@ -343,7 +343,7 @@ async def patched_add_batch( unprocessed_requests=[], ) - monkeypatch.setattr(rq._client, 'add_batch_of_requests', patched_add_batch) + monkeypatch.setattr(rq._backend, 'add_batch_of_requests', patched_add_batch) try: result = await rq.add_request('https://example.com/retry') @@ -357,7 +357,7 @@ async def patched_add_batch( async def test_add_request_returns_none_after_exhausting_retries(monkeypatch: pytest.MonkeyPatch) -> None: """When a request stays unprocessed across all retries, `add_request` returns `None` rather than raising.""" - rq = await RequestQueue.open(storage_client=MemoryStorageClient()) + rq = await RequestQueue.open(storage_backend=MemoryStorageBackend()) monkeypatch.setattr('crawlee.storages._request_queue.asyncio.sleep', _no_sleep) calls = 0 @@ -373,7 +373,7 @@ async def patched_add_batch( unprocessed_requests=[UnprocessedRequest(unique_key=r.unique_key, url=r.url) for r in requests], ) - monkeypatch.setattr(rq._client, 'add_batch_of_requests', patched_add_batch) + monkeypatch.setattr(rq._backend, 'add_batch_of_requests', patched_add_batch) try: result = await rq.add_request('https://example.com/doomed') @@ -565,14 +565,14 @@ async def test_check_readiness(rq: RequestQueue) -> None: async def test_readiness_waits_for_background_adds(rq: RequestQueue) -> None: """Test that a drained queue waits while a batch is still being added in the background.""" release = asyncio.Event() - add_batch = rq._client.add_batch_of_requests + add_batch = rq._backend.add_batch_of_requests async def add_batch_after_release(requests: Sequence[Request], *, forefront: bool = False) -> AddRequestsResponse: if requests[0].url == 'https://example.com/2': await release.wait() return await add_batch(requests, forefront=forefront) - with patch.object(rq._client, 'add_batch_of_requests', side_effect=add_batch_after_release): + with patch.object(rq._backend, 'add_batch_of_requests', side_effect=add_batch_after_release): await rq.add_requests(['https://example.com/1', 'https://example.com/2'], batch_size=1) request = await rq.fetch_next_request() @@ -649,12 +649,12 @@ async def test_reclaim_non_existent_request(rq: RequestQueue) -> None: async def test_drop( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test dropping a request queue removes it from cache and clears its data.""" rq = await RequestQueue.open( name='drop-test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add a request @@ -666,7 +666,7 @@ async def test_drop( # Verify request queue is empty (by creating a new one with the same name) new_rq = await RequestQueue.open( name='drop-test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify the queue is empty @@ -678,7 +678,7 @@ async def test_drop( async def test_reopen_default( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test reopening the default request queue.""" # First clean up any storage instance caches @@ -687,7 +687,7 @@ async def test_reopen_default( # Open the default request queue rq1 = await RequestQueue.open( - storage_client=storage_client, + storage_backend=storage_backend, ) # If a request queue already exists (due to previous test run), purge it to start fresh @@ -697,7 +697,7 @@ async def test_reopen_default( # If purge fails, try dropping and recreating await rq1.drop() rq1 = await RequestQueue.open( - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify we're starting fresh @@ -713,7 +713,7 @@ async def test_reopen_default( # Open the default request queue again rq2 = await RequestQueue.open( - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify they are the same queue @@ -735,13 +735,13 @@ async def test_reopen_default( async def test_purge( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test purging a request queue removes all requests but keeps the queue itself.""" # First create a request queue rq = await RequestQueue.open( name='purge-test-queue', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add some requests @@ -794,16 +794,16 @@ async def test_purge( pytest.param('purge-unsupported-named-test', id='named'), ], ) -async def test_purge_unsupported_by_client_raises( - storage_client: StorageClient, +async def test_purge_unsupported_by_backend_raises( + storage_backend: StorageBackend, monkeypatch: pytest.MonkeyPatch, name: str | None, ) -> None: - """Test that `purge` raises when the storage client doesn't support it, leaving the queue intact.""" - rq = await RequestQueue.open(name=name, storage_client=storage_client) + """Test that `purge` raises when the storage backend doesn't support it, leaving the queue intact.""" + rq = await RequestQueue.open(name=name, storage_backend=storage_backend) original_id = rq.id await rq.add_requests(['https://example.com/1', 'https://example.com/2']) - monkeypatch.setattr(rq._client, 'purge', AsyncMock(side_effect=NotImplementedError('Purge is not supported.'))) + monkeypatch.setattr(rq._backend, 'purge', AsyncMock(side_effect=NotImplementedError('Purge is not supported.'))) with pytest.raises(NotImplementedError, match='Purge is not supported'): await rq.purge() @@ -816,17 +816,17 @@ async def test_purge_unsupported_by_client_raises( async def test_open_with_alias( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test opening request queues with alias parameter for NDU functionality.""" # Create request queues with different aliases rq_1 = await RequestQueue.open( alias='test_alias_1', - storage_client=storage_client, + storage_backend=storage_backend, ) rq_2 = await RequestQueue.open( alias='test_alias_2', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify they have different IDs but no names (unnamed) @@ -854,19 +854,19 @@ async def test_open_with_alias( async def test_alias_caching( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that request queues with same alias return same instance (cached).""" # Open rq with alias rq_1 = await RequestQueue.open( alias='cache_test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Open again with same alias rq_2 = await RequestQueue.open( alias='cache_test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Should be same instance @@ -878,7 +878,7 @@ async def test_alias_caching( async def test_alias_with_id_error( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that providing both alias and id raises error.""" with pytest.raises( @@ -889,12 +889,12 @@ async def test_alias_with_id_error( await RequestQueue.open( id='some-id', alias='some-alias', - storage_client=storage_client, + storage_backend=storage_backend, ) async def test_alias_with_name_error( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that providing both alias and name raises error.""" with pytest.raises( @@ -905,12 +905,12 @@ async def test_alias_with_name_error( await RequestQueue.open( name='some-name', alias='some-alias', - storage_client=storage_client, + storage_backend=storage_backend, ) async def test_alias_with_special_characters( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test alias functionality with special characters.""" special_aliases = [ @@ -925,7 +925,7 @@ async def test_alias_with_special_characters( for alias in special_aliases: rq = await RequestQueue.open( alias=alias, - storage_client=storage_client, + storage_backend=storage_backend, ) queues.append(rq) @@ -944,12 +944,12 @@ async def test_alias_with_special_characters( async def test_alias_request_operations( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that request operations work correctly with alias queues.""" rq = await RequestQueue.open( alias='request_ops_test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Test adding multiple requests @@ -993,12 +993,12 @@ async def test_alias_request_operations( async def test_alias_forefront_operations( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test forefront operations work correctly with alias queues.""" rq = await RequestQueue.open( alias='forefront_test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Add normal requests @@ -1023,12 +1023,12 @@ async def test_alias_forefront_operations( async def test_alias_batch_operations( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test batch operations work correctly with alias queues.""" rq = await RequestQueue.open( alias='batch_test', - storage_client=storage_client, + storage_backend=storage_backend, ) # Test batch adding @@ -1052,30 +1052,30 @@ async def test_alias_batch_operations( async def test_named_vs_alias_conflict_detection( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that conflicts between named and alias storages are detected.""" # Test 1: Create named storage first, then try alias with same name named_rq = await RequestQueue.open( name='conflict-test', - storage_client=storage_client, + storage_backend=storage_backend, ) assert named_rq.name == 'conflict-test' # Try to create alias with same name - should raise error with pytest.raises(ValueError, match=r'Cannot create alias storage "conflict-test".*already exists'): - await RequestQueue.open(alias='conflict-test', storage_client=storage_client) + await RequestQueue.open(alias='conflict-test', storage_backend=storage_backend) # Clean up await named_rq.drop() # Test 2: Create alias first, then try named with same name - alias_rq = await RequestQueue.open(alias='conflict-test2', storage_client=storage_client) + alias_rq = await RequestQueue.open(alias='conflict-test2', storage_backend=storage_backend) assert alias_rq.name is None # Alias storages have no name # Try to create named with same name - should raise error with pytest.raises(ValueError, match=r'Cannot create named storage "conflict-test2".*already exists'): - await RequestQueue.open(name='conflict-test2', storage_client=storage_client) + await RequestQueue.open(name='conflict-test2', storage_backend=storage_backend) # Clean up await alias_rq.drop() @@ -1093,13 +1093,13 @@ async def test_named_vs_alias_conflict_detection( async def test_alias_parameter( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test request queue creation and operations with alias parameter.""" # Create request queue with alias alias_rq = await RequestQueue.open( alias='test_alias', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify alias request queue properties @@ -1115,13 +1115,13 @@ async def test_alias_parameter( async def test_alias_vs_named_isolation( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that alias and named request queues with same identifier are isolated.""" # Create named request queue named_rq = await RequestQueue.open( name='test-identifier', - storage_client=storage_client, + storage_backend=storage_backend, ) # Verify named request queue @@ -1134,7 +1134,7 @@ async def test_alias_vs_named_isolation( # Now create alias request queue with same identifier (should work after cleanup) alias_rq = await RequestQueue.open( alias='test-identifier', - storage_client=storage_client, + storage_backend=storage_backend, ) # Should be different instance @@ -1150,17 +1150,17 @@ async def test_alias_vs_named_isolation( async def test_default_vs_alias_default_equivalence( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that default request queue and alias='default' are equivalent.""" # Open default request queue default_rq = await RequestQueue.open( - storage_client=storage_client, + storage_backend=storage_backend, ) alias_default_rq = await RequestQueue.open( alias=StorageInstanceManager._DEFAULT_STORAGE_ALIAS, - storage_client=storage_client, + storage_backend=storage_backend, ) # Should be the same @@ -1177,7 +1177,7 @@ async def test_default_vs_alias_default_equivalence( async def test_multiple_alias_isolation( - storage_client: StorageClient, + storage_backend: StorageBackend, ) -> None: """Test that different aliases create separate request queues.""" request_queues = [] @@ -1185,7 +1185,7 @@ async def test_multiple_alias_isolation( for i in range(3): rq = await RequestQueue.open( alias=f'alias_{i}', - storage_client=storage_client, + storage_backend=storage_backend, ) await rq.add_request(f'https://example.com/alias_{i}') request_queues.append(rq) @@ -1203,30 +1203,30 @@ async def test_multiple_alias_isolation( await rq.drop() -async def test_purge_on_start_enabled(storage_client: StorageClient) -> None: +async def test_purge_on_start_enabled(storage_backend: StorageBackend) -> None: """Test purge behavior when purge_on_start=True: named storages retain data, unnamed storages are purged.""" - # Skip this test for memory storage since it doesn't persist data between client instances. - if isinstance(storage_client, MemoryStorageClient): - raise pytest.skip.Exception('Memory storage does not persist data between client instances.') + # Skip this test for memory storage since it doesn't persist data between backend instances. + if isinstance(storage_backend, MemoryStorageBackend): + raise pytest.skip.Exception('Memory storage does not persist data between backend instances.') configuration = Configuration(purge_on_start=True) # First, create all storage types with purge enabled and add data. default_rq = await RequestQueue.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_rq = await RequestQueue.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_rq = await RequestQueue.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1291,17 +1291,17 @@ async def test_purge_on_start_enabled(storage_client: StorageClient) -> None: # Now "reopen" all storages default_rq_2 = await RequestQueue.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_rq_2 = await RequestQueue.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_rq_2 = await RequestQueue.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1329,30 +1329,30 @@ async def test_purge_on_start_enabled(storage_client: StorageClient) -> None: await default_rq_2.drop() -async def test_purge_on_start_disabled(storage_client: StorageClient) -> None: +async def test_purge_on_start_disabled(storage_backend: StorageBackend) -> None: """Test purge behavior when purge_on_start=False: all storages retain data regardless of type.""" - # Skip this test for memory storage since it doesn't persist data between client instances. - if isinstance(storage_client, MemoryStorageClient): - raise pytest.skip.Exception('Memory storage does not persist data between client instances.') + # Skip this test for memory storage since it doesn't persist data between backend instances. + if isinstance(storage_backend, MemoryStorageBackend): + raise pytest.skip.Exception('Memory storage does not persist data between backend instances.') configuration = Configuration(purge_on_start=False) # First, create all storage types with purge disabled and add data. default_rq = await RequestQueue.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_rq = await RequestQueue.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_rq = await RequestQueue.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1417,17 +1417,17 @@ async def test_purge_on_start_disabled(storage_client: StorageClient) -> None: # Now "reopen" all storages default_rq_2 = await RequestQueue.open( - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) alias_rq_2 = await RequestQueue.open( alias='purge-test-alias', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) named_rq_2 = await RequestQueue.open( name='purge-test-named', - storage_client=storage_client, + storage_backend=storage_backend, configuration=configuration, ) @@ -1455,14 +1455,14 @@ async def test_purge_on_start_disabled(storage_client: StorageClient) -> None: await default_rq_2.drop() -async def test_name_default_not_allowed(storage_client: StorageClient) -> None: +async def test_name_default_not_allowed(storage_backend: StorageBackend) -> None: """Test that storage can't have default alias as name, to prevent collisions with unnamed storage alias.""" with pytest.raises( ValueError, match=f'Storage name cannot be "{StorageInstanceManager._DEFAULT_STORAGE_ALIAS}" as ' f'it is reserved for default alias.', ): - await RequestQueue.open(name=StorageInstanceManager._DEFAULT_STORAGE_ALIAS, storage_client=storage_client) + await RequestQueue.open(name=StorageInstanceManager._DEFAULT_STORAGE_ALIAS, storage_backend=storage_backend) @pytest.mark.parametrize( @@ -1487,16 +1487,16 @@ async def test_name_default_not_allowed(storage_client: StorageClient) -> None: pytest.param('with/slash', False, id='slash'), ], ) -async def test_validate_name(storage_client: StorageClient, name: str, *, is_valid: bool) -> None: +async def test_validate_name(storage_backend: StorageBackend, name: str, *, is_valid: bool) -> None: """Test name validation logic.""" if is_valid: # Should not raise - dataset = await RequestQueue.open(name=name, storage_client=storage_client) + dataset = await RequestQueue.open(name=name, storage_backend=storage_backend) assert dataset.name == name await dataset.drop() else: with pytest.raises(ValueError, match=rf'Invalid storage name "{name}".*'): - await RequestQueue.open(name=name, storage_client=storage_client) + await RequestQueue.open(name=name, storage_backend=storage_backend) @pytest.mark.parametrize( @@ -1513,15 +1513,15 @@ async def test_validate_name(storage_client: StorageClient, name: str, *, is_val pytest.param('back\\slash', False, id='backslash'), ], ) -async def test_validate_alias(storage_client: StorageClient, alias: str, *, is_valid: bool) -> None: +async def test_validate_alias(storage_backend: StorageBackend, alias: str, *, is_valid: bool) -> None: """Test alias validation logic, including rejection of values that would resolve outside the storage dir.""" if is_valid: # Should not raise. - rq = await RequestQueue.open(alias=alias, storage_client=storage_client) + rq = await RequestQueue.open(alias=alias, storage_backend=storage_backend) await rq.drop() else: with pytest.raises(ValueError, match=r'Invalid storage alias'): - await RequestQueue.open(alias=alias, storage_client=storage_client) + await RequestQueue.open(alias=alias, storage_backend=storage_backend) async def test_reclaim_request_with_change_state(rq: RequestQueue) -> None: diff --git a/tests/unit/storages/test_storage_instance_manager.py b/tests/unit/storages/test_storage_instance_manager.py index b821e9b156..6a56f5d549 100644 --- a/tests/unit/storages/test_storage_instance_manager.py +++ b/tests/unit/storages/test_storage_instance_manager.py @@ -7,7 +7,7 @@ from crawlee import service_locator from crawlee.configuration import Configuration -from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient +from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend from crawlee.storages import Dataset, KeyValueStore, RequestQueue from crawlee.storages._base import Storage @@ -23,37 +23,37 @@ def storage_type(request: pytest.FixtureRequest) -> type[Storage]: return cast('type[Storage]', request.param) -async def test_unique_storage_by_storage_client(tmp_path: Path, storage_type: type[Storage]) -> None: +async def test_unique_storage_by_storage_backend(tmp_path: Path, storage_type: type[Storage]) -> None: config = Configuration(purge_on_start=True, storage_dir=str(tmp_path)) - storage_1 = await storage_type.open(storage_client=MemoryStorageClient(), configuration=config) - storage_2 = await storage_type.open(storage_client=FileSystemStorageClient(), configuration=config) + storage_1 = await storage_type.open(storage_backend=MemoryStorageBackend(), configuration=config) + storage_2 = await storage_type.open(storage_backend=FileSystemStorageBackend(), configuration=config) assert storage_1 is not storage_2 -async def test_same_storage_when_different_client(tmp_path: Path, storage_type: type[Storage]) -> None: +async def test_same_storage_when_different_backend(tmp_path: Path, storage_type: type[Storage]) -> None: config = Configuration(purge_on_start=True, storage_dir=str(tmp_path)) - storage_1 = await storage_type.open(storage_client=MemoryStorageClient(), configuration=config) - storage_2 = await storage_type.open(storage_client=MemoryStorageClient(), configuration=config) + storage_1 = await storage_type.open(storage_backend=MemoryStorageBackend(), configuration=config) + storage_2 = await storage_type.open(storage_backend=MemoryStorageBackend(), configuration=config) assert storage_1 is storage_2 async def test_unique_storage_by_storage_type(tmp_path: Path) -> None: config = Configuration(purge_on_start=True, storage_dir=str(tmp_path)) - storage_client = MemoryStorageClient() + storage_backend = MemoryStorageBackend() - kvs = await KeyValueStore.open(storage_client=storage_client, configuration=config) - dataset = await Dataset.open(storage_client=storage_client, configuration=config) + kvs = await KeyValueStore.open(storage_backend=storage_backend, configuration=config) + dataset = await Dataset.open(storage_backend=storage_backend, configuration=config) assert kvs is not dataset async def test_unique_storage_by_name(storage_type: type[Storage]) -> None: - """Test that StorageInstanceManager support different storage clients at the same time.""" - storage_client = MemoryStorageClient() + """Test that StorageInstanceManager support different storage backends at the same time.""" + storage_backend = MemoryStorageBackend() - storage_1 = await storage_type.open(storage_client=storage_client, name='kvs1') - storage_2 = await storage_type.open(storage_client=storage_client, name='kvs2') + storage_1 = await storage_type.open(storage_backend=storage_backend, name='kvs1') + storage_2 = await storage_type.open(storage_backend=storage_backend, name='kvs2') assert storage_1 is not storage_2 @@ -68,10 +68,10 @@ async def test_unique_storage_by_unique_cache_key_different_path(tmp_path: Path, config_2 = Configuration(storage_dir=str(path_2)) - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() - storage_1 = await storage_type.open(storage_client=storage_client, configuration=config_1) - storage_2 = await storage_type.open(storage_client=storage_client, configuration=config_2) + storage_1 = await storage_type.open(storage_backend=storage_backend, configuration=config_1) + storage_2 = await storage_type.open(storage_backend=storage_backend, configuration=config_2) assert storage_1 is not storage_2 @@ -82,24 +82,24 @@ async def test_unique_storage_by_unique_cache_key_same_path(tmp_path: Path, stor config_2 = Configuration(storage_dir=str(tmp_path)) - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() - storage_1 = await storage_type.open(storage_client=storage_client, configuration=config_1) - storage_2 = await storage_type.open(storage_client=storage_client, configuration=config_2) + storage_1 = await storage_type.open(storage_backend=storage_backend, configuration=config_1) + storage_2 = await storage_type.open(storage_backend=storage_backend, configuration=config_2) assert storage_1 is storage_2 async def test_identical_storage_default_config(storage_type: type[Storage]) -> None: - """Test that StorageInstanceManager correctly caches storage based on the storage client.""" - storage_client = MemoryStorageClient() + """Test that StorageInstanceManager correctly caches storage based on the storage backend.""" + storage_backend = MemoryStorageBackend() - storage_1 = await storage_type.open(storage_client=storage_client) - storage_2 = await storage_type.open(storage_client=storage_client) + storage_1 = await storage_type.open(storage_backend=storage_backend) + storage_2 = await storage_type.open(storage_backend=storage_backend) assert storage_1 is storage_2 async def test_identical_storage_default_storage(storage_type: type[Storage]) -> None: - """Test that StorageInstanceManager correctly caches storage based on the storage client.""" + """Test that StorageInstanceManager correctly caches storage based on the storage backend.""" storage_1 = await storage_type.open() storage_2 = await storage_type.open() assert storage_1 is storage_2 @@ -121,13 +121,13 @@ async def test_identical_storage_remove_from_cache(storage_type: type[Storage]) async def test_preexisting_unnamed_storage_open_by_id(storage_type: type[Storage]) -> None: """Test that persisted pre-existing unnamed storage can be opened by ID.""" - storage_client = FileSystemStorageClient() - storage_1 = await storage_type.open(alias='custom_name', storage_client=storage_client) + storage_backend = FileSystemStorageBackend() + storage_1 = await storage_type.open(alias='custom_name', storage_backend=storage_backend) # Make service_locator unaware of this storage service_locator.storage_instance_manager.clear_cache() - storage_1_again = await storage_type.open(id=storage_1.id, storage_client=storage_client) + storage_1_again = await storage_type.open(id=storage_1.id, storage_backend=storage_backend) assert storage_1.id == storage_1_again.id @@ -189,21 +189,21 @@ async def open_dataset(name: str | None, alias: str | None) -> None: async def test_drop_only_evicts_own_cache_entry( tmp_path: Path, storage_type: type[Storage], name: str | None, *, drop_first: bool ) -> None: - """Dropping a storage evicts only its own cache entry; siblings with the same name but a different client stay.""" + """Dropping a storage evicts only its own cache entry; siblings with the same name but a different backend stay.""" config = Configuration(purge_on_start=True, storage_dir=str(tmp_path)) - mem_client = MemoryStorageClient() - fs_client = FileSystemStorageClient() + mem_backend = MemoryStorageBackend() + fs_backend = FileSystemStorageBackend() - # Two different client types produce different cache keys under the same name. - storage_mem = await storage_type.open(name=name, storage_client=mem_client, configuration=config) - storage_fs = await storage_type.open(name=name, storage_client=fs_client, configuration=config) + # Two different backend types produce different cache keys under the same name. + storage_mem = await storage_type.open(name=name, storage_backend=mem_backend, configuration=config) + storage_fs = await storage_type.open(name=name, storage_backend=fs_backend, configuration=config) assert storage_mem is not storage_fs if drop_first: await storage_mem.drop() - assert await storage_type.open(name=name, storage_client=fs_client, configuration=config) is storage_fs - assert await storage_type.open(name=name, storage_client=mem_client, configuration=config) is not storage_mem + assert await storage_type.open(name=name, storage_backend=fs_backend, configuration=config) is storage_fs + assert await storage_type.open(name=name, storage_backend=mem_backend, configuration=config) is not storage_mem else: await storage_fs.drop() - assert await storage_type.open(name=name, storage_client=mem_client, configuration=config) is storage_mem - assert await storage_type.open(name=name, storage_client=fs_client, configuration=config) is not storage_fs + assert await storage_type.open(name=name, storage_backend=mem_backend, configuration=config) is storage_mem + assert await storage_type.open(name=name, storage_backend=fs_backend, configuration=config) is not storage_fs diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index e40943234b..cdb30e1a18 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -10,8 +10,8 @@ from crawlee.configuration import Configuration from crawlee.crawlers import HttpCrawler, HttpCrawlingContext from crawlee.statistics import Statistics -from crawlee.storage_clients import MemoryStorageClient -from crawlee.storage_clients._file_system._storage_client import FileSystemStorageClient +from crawlee.storage_backends import MemoryStorageBackend +from crawlee.storage_backends._file_system._storage_backend import FileSystemStorageBackend if TYPE_CHECKING: from pathlib import Path @@ -60,9 +60,9 @@ def test_global_configuration_works_reversed() -> None: async def test_storage_not_persisted_when_non_persistable_storage_used(tmp_path: Path, server_url: URL) -> None: - """Make the Crawler use MemoryStorageClient which can't persist state.""" + """Make the Crawler use MemoryStorageBackend which can't persist state.""" service_locator.set_configuration(Configuration(storage_dir=str(tmp_path))) - crawler = HttpCrawler(storage_client=MemoryStorageClient()) + crawler = HttpCrawler(storage_backend=MemoryStorageBackend()) @crawler.router.default_handler async def default_handler(context: HttpCrawlingContext) -> None: @@ -78,15 +78,15 @@ async def default_handler(context: HttpCrawlingContext) -> None: async def test_storage_persisted_with_explicit_statistics_with_persistable_storage( tmp_path: Path, server_url: URL ) -> None: - """Make the Crawler use MemoryStorageClient which can't persist state, - but pass explicit statistics to it which will use global FileSystemStorageClient() that can persist state.""" + """Make the Crawler use MemoryStorageBackend which can't persist state, + but pass explicit statistics to it which will use global FileSystemStorageBackend() that can persist state.""" configuration = Configuration(storage_dir=str(tmp_path)) service_locator.set_configuration(configuration) - service_locator.set_storage_client(FileSystemStorageClient()) + service_locator.set_storage_backend(FileSystemStorageBackend()) crawler = HttpCrawler( - storage_client=MemoryStorageClient(), statistics=Statistics.with_default_state(persistence_enabled=True) + storage_backend=MemoryStorageBackend(), statistics=Statistics.with_default_state(persistence_enabled=True) ) @crawler.router.default_handler @@ -105,11 +105,11 @@ async def test_storage_persisted_when_enabled(tmp_path: Path, server_url: URL) - storage_dir=str(tmp_path), ) - storage_client = FileSystemStorageClient() + storage_backend = FileSystemStorageBackend() crawler = HttpCrawler( configuration=configuration, - storage_client=storage_client, + storage_backend=storage_backend, ) @crawler.router.default_handler diff --git a/tests/unit/test_service_locator.py b/tests/unit/test_service_locator.py index b9623c699b..cdb4532f74 100644 --- a/tests/unit/test_service_locator.py +++ b/tests/unit/test_service_locator.py @@ -6,7 +6,7 @@ from crawlee.configuration import Configuration from crawlee.errors import ServiceConflictError from crawlee.events import LocalEventManager -from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient +from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend def test_default_configuration() -> None: @@ -68,30 +68,30 @@ def test_event_manager_conflict() -> None: service_locator.set_event_manager(custom_event_manager) -def test_default_storage_client() -> None: - default_storage_client = service_locator.get_storage_client() - assert isinstance(default_storage_client, FileSystemStorageClient) +def test_default_storage_backend() -> None: + default_storage_backend = service_locator.get_storage_backend() + assert isinstance(default_storage_backend, FileSystemStorageBackend) -def test_custom_storage_client() -> None: - custom_storage_client = MemoryStorageClient() - service_locator.set_storage_client(custom_storage_client) - storage_client = service_locator.get_storage_client() - assert storage_client is custom_storage_client +def test_custom_storage_backend() -> None: + custom_storage_backend = MemoryStorageBackend() + service_locator.set_storage_backend(custom_storage_backend) + storage_backend = service_locator.get_storage_backend() + assert storage_backend is custom_storage_backend -def test_storage_client_overwrite_not_possible() -> None: - custom_storage_client = MemoryStorageClient() - service_locator.set_storage_client(custom_storage_client) +def test_storage_backend_overwrite_not_possible() -> None: + custom_storage_backend = MemoryStorageBackend() + service_locator.set_storage_backend(custom_storage_backend) - another_custom_storage_client = MemoryStorageClient() + another_custom_storage_backend = MemoryStorageBackend() with pytest.raises(ServiceConflictError): - service_locator.set_storage_client(another_custom_storage_client) + service_locator.set_storage_backend(another_custom_storage_backend) -def test_storage_client_conflict() -> None: - service_locator.get_storage_client() - custom_storage_client = MemoryStorageClient() +def test_storage_backend_conflict() -> None: + service_locator.get_storage_backend() + custom_storage_backend = MemoryStorageBackend() - with pytest.raises(ServiceConflictError, match=r'StorageClient is already in use.'): - service_locator.set_storage_client(custom_storage_client) + with pytest.raises(ServiceConflictError, match=r'StorageBackend is already in use.'): + service_locator.set_storage_backend(custom_storage_backend) diff --git a/tests/unit/test_throttling_request_manager.py b/tests/unit/test_throttling_request_manager.py index a7fc38abd6..1ef7e072f7 100644 --- a/tests/unit/test_throttling_request_manager.py +++ b/tests/unit/test_throttling_request_manager.py @@ -17,7 +17,7 @@ from crawlee.configuration import Configuration from crawlee.request_loaders import WaitingStatus from crawlee.request_loaders._throttling_request_manager import ThrottlingRequestManager -from crawlee.storage_clients import FileSystemStorageClient, MemoryStorageClient +from crawlee.storage_backends import FileSystemStorageBackend, MemoryStorageBackend from crawlee.storages import RequestQueue if TYPE_CHECKING: @@ -33,21 +33,21 @@ @pytest.fixture -def memory_storage_client() -> MemoryStorageClient: - """Create a MemoryStorageClient for testing.""" - return MemoryStorageClient() +def memory_storage_backend() -> MemoryStorageBackend: + """Create a MemoryStorageBackend for testing.""" + return MemoryStorageBackend() @pytest.fixture -def service_locator(memory_storage_client: MemoryStorageClient) -> ServiceLocator: - """Create a ServiceLocator with MemoryStorageClient for testing.""" - return ServiceLocator(storage_client=memory_storage_client) +def service_locator(memory_storage_backend: MemoryStorageBackend) -> ServiceLocator: + """Create a ServiceLocator with MemoryStorageBackend for testing.""" + return ServiceLocator(storage_backend=memory_storage_backend) @pytest.fixture -async def inner_queue(memory_storage_client: MemoryStorageClient) -> RequestQueue: - """Create a real RequestQueue with MemoryStorageClient.""" - return await RequestQueue.open(name='test-inner', storage_client=memory_storage_client) +async def inner_queue(memory_storage_backend: MemoryStorageBackend) -> RequestQueue: + """Create a real RequestQueue with MemoryStorageBackend.""" + return await RequestQueue.open(name='test-inner', storage_backend=memory_storage_backend) @pytest.fixture @@ -64,7 +64,7 @@ async def manager(inner_queue: RequestQueue, service_locator: ServiceLocator) -> @pytest.fixture def fs_service_locator() -> ServiceLocator: """Create a ServiceLocator backed by the file system, so storages survive a simulated restart.""" - return ServiceLocator(configuration=Configuration(purge_on_start=False), storage_client=FileSystemStorageClient()) + return ServiceLocator(configuration=Configuration(purge_on_start=False), storage_backend=FileSystemStorageBackend()) @pytest.fixture @@ -98,7 +98,7 @@ async def _open_fs_manager(service_locator: ServiceLocator) -> ThrottlingRequest """Open a throttling manager over the persistent storage directory, as a fresh process would.""" inner_queue = await RequestQueue.open( name='persistent-inner', - storage_client=service_locator.get_storage_client(), + storage_backend=service_locator.get_storage_backend(), configuration=service_locator.get_configuration(), ) return ThrottlingRequestManager( @@ -1050,7 +1050,7 @@ async def test_default_purge_on_start_empties_persisted_sub_queues(fs_service_lo # Restart under the default configuration, which purges an aliased store as it opens. fs_service_locator.storage_instance_manager.clear_cache() - purging_locator = ServiceLocator(configuration=Configuration(), storage_client=FileSystemStorageClient()) + purging_locator = ServiceLocator(configuration=Configuration(), storage_backend=FileSystemStorageBackend()) purged = await _open_fs_manager(purging_locator) assert (await purged.check_readiness()).status == 'finished' @@ -1070,7 +1070,7 @@ async def test_purge_empties_sub_queues_from_a_previous_run(fs_service_locator: sub_queue = await RequestQueue.open( alias=f'throttled-{THROTTLED_DOMAIN}', - storage_client=fs_service_locator.get_storage_client(), + storage_backend=fs_service_locator.get_storage_backend(), configuration=fs_service_locator.get_configuration(), ) assert await sub_queue.get_total_count() == 0 diff --git a/tests/unit/utils.py b/tests/unit/utils.py index 71ce08763c..8a34e0720d 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -21,7 +21,7 @@ from yarl import URL from crawlee import Request - from crawlee.storage_clients.models import ProcessedRequest + from crawlee.storage_backends.models import ProcessedRequest T = TypeVar('T') diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 34d243e8a9..a10d073214 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -19,7 +19,7 @@ const GROUP_ORDER = [ 'Request loaders', 'Session management', 'Statistics', - 'Storage clients', + 'Storage backends', 'Storage data', 'Storages', 'Other',