From 911d477b4852cb4cb21ed363b8aef5eee3333479 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 19 Aug 2026 08:44:05 +0200 Subject: [PATCH 1/6] feat: Add HTTPX-based HTTP client --- README.md | 11 +- pyproject.toml | 1 + src/apify_client/http_clients/__init__.py | 39 +++- src/apify_client/http_clients/_httpx.py | 244 ++++++++++++++++++++++ tests/integration/conftest.py | 76 +++++-- tests/integration/test_apify_client.py | 4 + tests/integration/test_dataset.py | 2 + tests/integration/test_key_value_store.py | 2 + tests/integration/test_log.py | 4 + tests/unit/conftest.py | 23 +- tests/unit/test_client_headers.py | 73 ++++++- tests/unit/test_client_streaming.py | 4 +- tests/unit/test_client_timeouts.py | 77 +++++-- tests/unit/test_http_clients.py | 110 ++++++++++ tests/unit/test_logging.py | 4 +- tests/unit/test_pluggable_http_client.py | 46 ++++ uv.lock | 6 +- 17 files changed, 668 insertions(+), 58 deletions(-) create mode 100644 src/apify_client/http_clients/_httpx.py diff --git a/README.md b/README.md index ba5b9084..7627e5fe 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,15 @@ uv add "apify-client[brotli]" ``` + [Impit](https://github.com/apify/impit) is the default HTTP client and is installed automatically. To use the + built-in [HTTPX](https://www.python-httpx.org/) client instead, install its optional extra: + + ```bash + pip install "apify-client[httpx]" + # or + uv add "apify-client[httpx]" + ``` + - From [conda-forge](https://anaconda.org/conda-forge/apify-client), it can be installed with [conda](https://docs.conda.io/en/latest/): ```bash @@ -124,7 +133,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)). - **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)). - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)). -- **Pluggable HTTP layer** — swap the default [Impit](https://github.com/apify/impit)-based HTTP client for `httpx`, `requests`, `aiohttp`, or any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). +- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX](https://www.python-httpx.org/) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)). - **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more ([Logging](https://docs.apify.com/api/client/python/docs/concepts/logging)). diff --git a/pyproject.toml b/pyproject.toml index 01e15bc8..b8922597 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ [project.optional-dependencies] brotli = ["brotli>=1.0.9"] +httpx = ["httpx>=0.27.0,<1.0.0"] [project.urls] "Apify Homepage" = "https://apify.com" diff --git a/src/apify_client/http_clients/__init__.py b/src/apify_client/http_clients/__init__.py index 417a0705..d1e06c90 100644 --- a/src/apify_client/http_clients/__init__.py +++ b/src/apify_client/http_clients/__init__.py @@ -1,10 +1,35 @@ +from apify_client._utils.try_import import install_import_hook as _install_import_hook +from apify_client._utils.try_import import try_import as _try_import from apify_client.http_clients._base import HttpClient, HttpClientAsync, HttpResponse from apify_client.http_clients._impit import ImpitHttpClient, ImpitHttpClientAsync -__all__ = [ - 'HttpClient', - 'HttpClientAsync', - 'HttpResponse', - 'ImpitHttpClient', - 'ImpitHttpClientAsync', -] +_install_import_hook(__name__) + +# `httpx` is an optional extra, so it's wrapped in try_import. Accessing the HTTPX clients +# without the extra installed raises a clear ImportError instead of failing at package import time. +with _try_import( + __name__, + 'HttpxHttpClient', + 'HttpxHttpClientAsync', + dependency_name='httpx', +) as _httpx_import: + from apify_client.http_clients._httpx import HttpxHttpClient, HttpxHttpClientAsync + +if _httpx_import.available: + __all__ = [ + 'HttpClient', + 'HttpClientAsync', + 'HttpResponse', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', + 'ImpitHttpClient', + 'ImpitHttpClientAsync', + ] +else: + __all__ = [ + 'HttpClient', + 'HttpClientAsync', + 'HttpResponse', + 'ImpitHttpClient', + 'ImpitHttpClientAsync', + ] diff --git a/src/apify_client/http_clients/_httpx.py b/src/apify_client/http_clients/_httpx.py new file mode 100644 index 00000000..8e256492 --- /dev/null +++ b/src/apify_client/http_clients/_httpx.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import httpx +from typing_extensions import override + +from apify_client._consts import ( + DEFAULT_MAX_RETRIES, + DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + DEFAULT_TIMEOUT_LONG, + DEFAULT_TIMEOUT_MAX, + DEFAULT_TIMEOUT_MEDIUM, + DEFAULT_TIMEOUT_SHORT, +) +from apify_client._docs import docs_group +from apify_client.http_clients._base import HttpClient, HttpClientAsync + +if TYPE_CHECKING: + from datetime import timedelta + + from apify_client._statistics import ClientStatistics + from apify_client.http_compressors._base import HttpCompressor + + +_PERMANENT_ERRORS = ( + # A request HTTPX rejects before sending it, e.g. one carrying an invalid header value. + httpx.LocalProtocolError, + # A URL scheme HTTPX refuses to speak, which repeating the request cannot change. + httpx.UnsupportedProtocol, + # An over-long redirect chain is a routing loop, which repeating the request cannot break. + httpx.TooManyRedirects, + # Only `Response.raise_for_status()` raises this, and the client never calls it - the shared pipeline decides on + # status codes from the response itself. + httpx.HTTPStatusError, +) +"""HTTPX errors that a retry cannot fix. Everything else in the `httpx.HTTPError` tree counts as transient.""" + + +@docs_group('HTTP clients') +class HttpxHttpClient(HttpClient): + """Synchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/). + + This client wraps `httpx.Client` and adds automatic retries with exponential backoff for rate-limited + (HTTP 429) and server error (HTTP 5xx) responses. + + Requires the `httpx` extra: `pip install "apify-client[httpx]"`. + """ + + def __init__( + self, + *, + token: str | None = None, + timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT, + timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM, + timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, + timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, + max_retries: int = DEFAULT_MAX_RETRIES, + min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + statistics: ClientStatistics | None = None, + headers: dict[str, str] | None = None, + http_compressor: HttpCompressor | None = None, + ) -> None: + """Initialize the HTTPX-based synchronous HTTP client. + + Args: + token: Apify API token for authentication. + timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). + timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). + timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). + timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. + max_retries: Maximum number of retry attempts for failed requests. + min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). + statistics: Statistics tracker for API calls. Created automatically if not provided. + headers: Additional HTTP headers to include in all requests. + http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. + """ + super().__init__( + token=token, + timeout_short=timeout_short, + timeout_medium=timeout_medium, + timeout_long=timeout_long, + timeout_max=timeout_max, + max_retries=max_retries, + min_delay_between_retries=min_delay_between_retries, + statistics=statistics, + headers=headers, + http_compressor=http_compressor, + ) + + self._httpx_client = httpx.Client( + follow_redirects=True, + event_hooks={'response': [self._clear_response_cookies]}, + ) + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures + # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the + # response status code, not here. + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + + @override + def close(self) -> None: + """Close the underlying HTTPX connection pool.""" + self._httpx_client.close() + + def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_client.cookies.clear() + + @override + def send_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> httpx.Response: + request = self._httpx_client.build_request( + method=method, + url=url, + headers=headers, + content=content, + timeout=timeout, + ) + _restore_explicit_cookie_header(request, headers) + return self._httpx_client.send(request, stream=stream) + + +@docs_group('HTTP clients') +class HttpxHttpClientAsync(HttpClientAsync): + """Asynchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/). + + This client wraps `httpx.AsyncClient` and adds automatic retries with exponential backoff for rate-limited + (HTTP 429) and server error (HTTP 5xx) responses. + + Requires the `httpx` extra: `pip install "apify-client[httpx]"`. + """ + + def __init__( + self, + *, + token: str | None = None, + timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT, + timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM, + timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, + timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, + max_retries: int = DEFAULT_MAX_RETRIES, + min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + statistics: ClientStatistics | None = None, + headers: dict[str, str] | None = None, + http_compressor: HttpCompressor | None = None, + ) -> None: + """Initialize the HTTPX-based asynchronous HTTP client. + + Args: + token: Apify API token for authentication. + timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). + timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). + timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). + timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. + max_retries: Maximum number of retry attempts for failed requests. + min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). + statistics: Statistics tracker for API calls. Created automatically if not provided. + headers: Additional HTTP headers to include in all requests. + http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. + """ + super().__init__( + token=token, + timeout_short=timeout_short, + timeout_medium=timeout_medium, + timeout_long=timeout_long, + timeout_max=timeout_max, + max_retries=max_retries, + min_delay_between_retries=min_delay_between_retries, + statistics=statistics, + headers=headers, + http_compressor=http_compressor, + ) + + self._httpx_async_client = httpx.AsyncClient( + follow_redirects=True, + event_hooks={'response': [self._clear_response_cookies]}, + ) + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures + # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the + # response status code, not here. + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + + @override + async def aclose(self) -> None: + """Close the underlying asynchronous HTTPX connection pool.""" + await self._httpx_async_client.aclose() + + async def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_async_client.cookies.clear() + + @override + async def send_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> httpx.Response: + request = self._httpx_async_client.build_request( + method=method, + url=url, + headers=headers, + content=content, + timeout=timeout, + ) + _restore_explicit_cookie_header(request, headers) + return await self._httpx_async_client.send(request, stream=stream) + + +def _restore_explicit_cookie_header(request: httpx.Request, headers: dict[str, str]) -> None: + """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar.""" + explicit_cookie = next((value for key, value in headers.items() if key.lower() == 'cookie'), None) + if explicit_cookie is None: + request.headers.pop('cookie', None) + else: + request.headers['cookie'] = explicit_cookie diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1db53b71..e0c9bed6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -2,6 +2,7 @@ import json import os +from dataclasses import dataclass from typing import TYPE_CHECKING import pytest @@ -17,9 +18,35 @@ from apify_client import ApifyClient, ApifyClientAsync from apify_client._consts import DEFAULT_API_URL from apify_client._utils.crypto import create_hmac_signature, create_storage_content_signature +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import AsyncGenerator, Generator + + +@dataclass(frozen=True) +class HttpClientClasses: + """Synchronous and asynchronous variants of a built-in HTTP client.""" + + sync: type[HttpClient] + async_: type[HttpClientAsync] + + +DEFAULT_HTTP_CLIENT_CLASSES = HttpClientClasses(sync=ImpitHttpClient, async_=ImpitHttpClientAsync) +"""HTTP clients the live-API suite runs with unless a test asks for another transport.""" + +ALL_HTTP_CLIENT_CLASSES = [ + pytest.param(DEFAULT_HTTP_CLIENT_CLASSES, id='impit'), + pytest.param(HttpClientClasses(sync=HttpxHttpClient, async_=HttpxHttpClientAsync), id='httpx'), +] +"""Every built-in HTTP client, for tests that exercise transport behavior rather than an API resource.""" # ============================================================================ @@ -110,17 +137,17 @@ def test_kvs_of_another_user(api_token_2: str) -> Generator[KvsFixture]: @pytest.fixture -def apify_client(api_token: str) -> ApifyClient: - """Sync Apify client instance.""" - api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL - return ApifyClient(api_token, api_url=api_url) +def http_client_classes(request: pytest.FixtureRequest) -> HttpClientClasses: + """Return the sync and async classes of the HTTP client the test runs with. + Defaults to Impit so the live-API suite isn't multiplied by every transport. A transport-level test opts into + the full matrix with `@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True)`. + """ + if not hasattr(request, 'param'): + return DEFAULT_HTTP_CLIENT_CLASSES -@pytest.fixture -def apify_client_async(api_token: str) -> ApifyClientAsync: - """Async Apify client instance.""" - api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL - return ApifyClientAsync(api_token, api_url=api_url) + assert isinstance(request.param, HttpClientClasses) + return request.param @pytest.fixture(params=['sync', 'async']) @@ -130,13 +157,30 @@ def client_type(request: pytest.FixtureRequest) -> str: @pytest.fixture -def client( +async def client( client_type: str, - apify_client: ApifyClient, - apify_client_async: ApifyClientAsync, -) -> ApifyClient | ApifyClientAsync: - """Return sync or async client based on parametrization.""" - return apify_client if client_type == 'sync' else apify_client_async + api_token: str, + http_client_classes: HttpClientClasses, +) -> AsyncGenerator[ApifyClient | ApifyClientAsync]: + """Return each sync/async and HTTP client implementation combination.""" + api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL + if client_type == 'sync': + http_client = http_client_classes.sync() + yield ApifyClient.with_custom_http_client( + api_token, + api_url=api_url, + http_client=http_client, + ) + http_client.close() + return + + http_client_async = http_client_classes.async_() + yield ApifyClientAsync.with_custom_http_client( + api_token, + api_url=api_url, + http_client=http_client_async, + ) + await http_client_async.aclose() @pytest.fixture diff --git a/tests/integration/test_apify_client.py b/tests/integration/test_apify_client.py index 126f40b3..4c15eab8 100644 --- a/tests/integration/test_apify_client.py +++ b/tests/integration/test_apify_client.py @@ -4,13 +4,17 @@ from typing import TYPE_CHECKING +import pytest + from .._utils import maybe_await +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import UserPrivateInfo, UserPublicInfo if TYPE_CHECKING: from apify_client import ApifyClient, ApifyClientAsync +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_apify_client(client: ApifyClient | ApifyClientAsync) -> None: """Test basic apify client functionality.""" user_client = client.user('me') diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index 333c7229..b7acab4c 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -18,6 +18,7 @@ maybe_await, poll_until_condition, ) +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import Dataset, DatasetListItem, DatasetStatistics, ListOfDatasets from apify_client._resource_clients.dataset import DatasetItemsPage from apify_client.errors import ApifyApiError @@ -698,6 +699,7 @@ async def get_items() -> DatasetItemsPage: await maybe_await(dataset_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_dataset_stream_items(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming dataset items.""" dataset_name = get_random_resource_name('dataset') diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index 5d1d9238..fee82954 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -19,6 +19,7 @@ maybe_sleep, poll_until_condition, ) +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import KeyValueStore, KeyValueStoreKey, ListOfKeys, ListOfKeyValueStores from apify_client.errors import ApifyApiError from apify_client.http_clients import HttpResponse @@ -706,6 +707,7 @@ async def get_keys() -> ListOfKeys: await maybe_await(store_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_key_value_store_stream_record_own(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming a record from one's own key-value store (no signature).""" store_name = get_random_resource_name('kvs') diff --git a/tests/integration/test_log.py b/tests/integration/test_log.py index df687e2f..13db91f8 100644 --- a/tests/integration/test_log.py +++ b/tests/integration/test_log.py @@ -5,7 +5,10 @@ from contextlib import AbstractAsyncContextManager, AbstractContextManager from typing import TYPE_CHECKING +import pytest + from .._utils import maybe_await +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import ListOfBuilds, Run from apify_client.http_clients import HttpResponse @@ -72,6 +75,7 @@ async def test_log_get_as_bytes(client: ApifyClient | ApifyClientAsync) -> None: await maybe_await(run_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_log_stream_from_run(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming a run's log via the stream() context manager.""" actor = client.actor(HELLO_WORLD_ACTOR) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 124c12cd..98d48d70 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -7,7 +7,14 @@ from pytest_httpserver import HTTPServer from apify_client import ApifyClient, ApifyClientAsync -from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) if TYPE_CHECKING: from collections.abc import Iterable @@ -43,13 +50,23 @@ def async_client(httpserver: HTTPServer) -> ApifyClientAsync: return ApifyClientAsync(token='test', api_url=httpserver.url_for('/').removesuffix('/')) -@pytest.fixture(params=[pytest.param(ImpitHttpClient, id='impit')]) +@pytest.fixture( + params=[ + pytest.param(ImpitHttpClient, id='impit'), + pytest.param(HttpxHttpClient, id='httpx'), + ] +) def http_client_class(request: pytest.FixtureRequest) -> type[HttpClient]: """Return each built-in synchronous HTTP client class.""" return request.param -@pytest.fixture(params=[pytest.param(ImpitHttpClientAsync, id='impit')]) +@pytest.fixture( + params=[ + pytest.param(ImpitHttpClientAsync, id='impit'), + pytest.param(HttpxHttpClientAsync, id='httpx'), + ] +) def http_client_async_class(request: pytest.FixtureRequest) -> type[HttpClientAsync]: """Return each built-in asynchronous HTTP client class.""" return request.param diff --git a/tests/unit/test_client_headers.py b/tests/unit/test_client_headers.py index 981d6857..b8e0b259 100644 --- a/tests/unit/test_client_headers.py +++ b/tests/unit/test_client_headers.py @@ -6,8 +6,11 @@ from importlib import metadata from typing import TYPE_CHECKING +import httpx from werkzeug import Request, Response +from apify_client.http_clients import HttpxHttpClient, HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync + if TYPE_CHECKING: from pytest_httpserver import HTTPServer @@ -19,6 +22,18 @@ def _parse_accept_encoding(header: str) -> set[str]: return {enc.strip() for enc in header.split(',')} +def _transport_wire_headers( + client_class: type[HttpClient | HttpClientAsync], +) -> tuple[dict[str, str], set[str]]: + """Return the headers the transport adds on its own and the content encodings it advertises.""" + if issubclass(client_class, (ImpitHttpClient, ImpitHttpClientAsync)): + return {}, {'zstd', 'gzip', 'deflate', 'br'} + # HTTPX advertises whichever decoders happen to be installed alongside it, so read the set off the client + # itself rather than hard-coding it and breaking whenever the environment gains or loses a codec. + with httpx.Client() as probe: + return {'Connection': 'keep-alive'}, _parse_accept_encoding(probe.headers['accept-encoding']) + + def _header_handler(request: Request) -> Response: return Response( status=200, @@ -43,15 +58,17 @@ async def test_default_headers_async(httpserver: HTTPServer, http_client_async_c response = await client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_async_class) expected_headers = { 'User-Agent': _get_user_agent(), 'Accept': 'application/json, */*', 'Authorization': 'Bearer placeholder_token', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings def test_default_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient]) -> None: @@ -63,15 +80,17 @@ def test_default_headers_sync(httpserver: HTTPServer, http_client_class: type[Ht response = client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_class) expected_headers = { 'User-Agent': _get_user_agent(), 'Accept': 'application/json, */*', 'Authorization': 'Bearer placeholder_token', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings async def test_headers_async(httpserver: HTTPServer, http_client_async_class: type[HttpClientAsync]) -> None: @@ -86,6 +105,7 @@ async def test_headers_async(httpserver: HTTPServer, http_client_async_class: ty response = await client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_async_class) expected_headers = { 'Test-Header': 'blah', @@ -93,9 +113,10 @@ async def test_headers_async(httpserver: HTTPServer, http_client_async_class: ty 'Accept': 'application/json, */*', 'Authorization': 'strange_value', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient]) -> None: @@ -114,6 +135,7 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient response = client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_class) expected_headers = { 'Test-Header': 'blah', @@ -121,9 +143,10 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient 'Accept': 'application/json, */*', 'Authorization': 'strange_value', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings async def test_per_request_headers_override_defaults_async( @@ -158,3 +181,45 @@ def test_per_request_headers_override_defaults_sync( # WSGI joins duplicate headers into one comma-separated value, so exact equality # also proves the authorization header was sent only once. assert request_headers['Authorization'] == 'Bearer per-request' + + +def _echo_cookie_handler(request: Request) -> Response: + return Response(json.dumps({'cookie': request.headers.get('Cookie')}), content_type='application/json') + + +def test_httpx_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """A Set-Cookie response must not silently leak into a later API request through HTTPX's shared cookie jar.""" + httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + client.call(method='GET', url=httpserver.url_for('/set-cookie')) + response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +async def test_httpx_async_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """The asynchronous HTTPX pool also remains stateless between API calls.""" + httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + async with HttpxHttpClientAsync() as client: + await client.call(method='GET', url=httpserver.url_for('/set-cookie')) + response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +def test_httpx_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: + """Disabling the shared cookie jar must not remove a Cookie header explicitly supplied by the caller.""" + httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + response = client.call( + method='GET', + url=httpserver.url_for('/echo-explicit-cookie'), + headers={'Cookie': 'explicit=value'}, + ) + + assert response.json() == {'cookie': 'explicit=value'} diff --git a/tests/unit/test_client_streaming.py b/tests/unit/test_client_streaming.py index d369455e..9e5782e7 100644 --- a/tests/unit/test_client_streaming.py +++ b/tests/unit/test_client_streaming.py @@ -105,7 +105,7 @@ def test_protocol_check_leaves_stream_unread_sync( with client.dataset(DATASET_ID).stream_items(item_format='json') as response: assert isinstance(response, HttpResponse) - # `is_stream_consumed` is transport state, not part of the protocol, but the built-in client exposes it. + # `is_stream_consumed` is transport state, not part of the protocol, but both built-in clients expose it. raw: Any = response assert raw.is_stream_consumed is False @@ -124,6 +124,6 @@ async def test_protocol_check_leaves_stream_unread_async( async with client.dataset(DATASET_ID).stream_items(item_format='json') as response: assert isinstance(response, HttpResponse) - # `is_stream_consumed` is transport state, not part of the protocol, but the built-in client exposes it. + # `is_stream_consumed` is transport state, not part of the protocol, but both built-in clients expose it. raw: Any = response assert raw.is_stream_consumed is False diff --git a/tests/unit/test_client_timeouts.py b/tests/unit/test_client_timeouts.py index b4410acd..58fd03b2 100644 --- a/tests/unit/test_client_timeouts.py +++ b/tests/unit/test_client_timeouts.py @@ -5,16 +5,27 @@ from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock +import httpx import impit import pytest from apify_client._logging import LoggerOnce, logger_name -from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) from apify_client.http_clients import _base as http_client_base if TYPE_CHECKING: from _pytest.logging import LogCaptureFixture +UNSET_HTTPX_TIMEOUT = {'connect': None, 'read': None, 'write': None, 'pool': None} +"""What HTTPX stores on a request built with `timeout=None`: every sub-timeout unset, not the client default.""" + @pytest.fixture def fresh_logger_once(monkeypatch: pytest.MonkeyPatch) -> None: @@ -26,6 +37,12 @@ def successful_response() -> Mock: return Mock(status_code=200) +def retryable_error(client: HttpClient | HttpClientAsync) -> Exception: + if isinstance(client, (ImpitHttpClient, ImpitHttpClientAsync)): + return impit.TimeoutException('timeout') + return httpx.ReadTimeout('timeout', request=httpx.Request('GET', 'https://example.com')) + + @pytest.mark.parametrize( ('timeout', 'expected'), [ @@ -93,7 +110,7 @@ async def test_timeout_resolves_for_async_clients( def test_compute_timeout_with_timedelta(http_client_class: type[HttpClient]) -> None: - """Concrete timedeltas double per attempt, are capped at the maximum, and `no_timeout` stays unbounded.""" + """Concrete timedeltas double per attempt and are capped at the configured maximum.""" client = http_client_class(timeout_max=timedelta(seconds=20)) assert client._compute_timeout(timedelta(seconds=5), attempt=1) == 5.0 @@ -160,7 +177,7 @@ def test_dynamic_timeout_sync_client(http_client_class: type[HttpClient], monkey def send_request(*_args: Any, **kwargs: Any) -> Mock: timeouts.append(kwargs['timeout']) if len(timeouts) < 4: - raise impit.TimeoutException('timeout') + raise retryable_error(client) return successful_response() monkeypatch.setattr(client, 'send_request', send_request) @@ -185,7 +202,7 @@ async def test_dynamic_timeout_async_client( async def send_request(*_args: Any, **kwargs: Any) -> Mock: timeouts.append(kwargs['timeout']) if len(timeouts) < 4: - raise impit.TimeoutException('timeout') + raise retryable_error(client) return successful_response() monkeypatch.setattr(client, 'send_request', send_request) @@ -196,23 +213,39 @@ async def send_request(*_args: Any, **kwargs: Any) -> Mock: assert response.status_code == 200 -def test_no_timeout_mapping_for_sync_adapter() -> None: - """The synchronous adapter maps no-timeout to Impit's effectively unbounded value.""" - client = ImpitHttpClient() - client._impit_client = Mock(request=Mock(return_value=successful_response())) - - client.send_request(method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False) - - assert client._impit_client.request.call_args.kwargs['timeout'] == 86_400 - - -async def test_no_timeout_mapping_for_async_adapter() -> None: - """The asynchronous adapter maps no-timeout to Impit's effectively unbounded value.""" - client = ImpitHttpClientAsync() - client._impit_async_client = Mock(request=AsyncMock(return_value=successful_response())) - - await client.send_request( +def test_no_timeout_mapping_for_sync_adapters(monkeypatch: pytest.MonkeyPatch) -> None: + """Each synchronous adapter maps no-timeout to its underlying library semantics.""" + impit_client = ImpitHttpClient() + impit_client._impit_client = Mock(request=Mock(return_value=successful_response())) + impit_client.send_request( method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False ) - - assert client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 + assert impit_client._impit_client.request.call_args.kwargs['timeout'] == 86_400 + + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. + with HttpxHttpClient() as httpx_client: + send = Mock(return_value=successful_response()) + monkeypatch.setattr(httpx_client._httpx_client, 'send', send) + httpx_client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT + + +async def test_no_timeout_mapping_for_async_adapters(monkeypatch: pytest.MonkeyPatch) -> None: + """Each asynchronous adapter maps no-timeout to its underlying library semantics.""" + impit_client = ImpitHttpClientAsync() + impit_client._impit_async_client = Mock(request=AsyncMock(return_value=successful_response())) + await impit_client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + assert impit_client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 + + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. + async with HttpxHttpClientAsync() as httpx_client: + send = AsyncMock(return_value=successful_response()) + monkeypatch.setattr(httpx_client._httpx_async_client, 'send', send) + await httpx_client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index adbf9ce7..c7f426df 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, Mock import brotli +import httpx import impit import pytest @@ -21,6 +22,8 @@ HttpClient, HttpClientAsync, HttpResponse, + HttpxHttpClient, + HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -264,6 +267,24 @@ async def test_http_client_async_creates_async_impit_client() -> None: await client.aclose() +def test_http_client_creates_sync_httpx_client() -> None: + """The synchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" + client = HttpxHttpClient(token='test_token_123') + + assert isinstance(client._httpx_client, httpx.Client) + client.close() + assert client._httpx_client.is_closed + + +async def test_http_client_async_creates_async_httpx_client() -> None: + """The asynchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" + client = HttpxHttpClientAsync(token='test_token_123') + + assert isinstance(client._httpx_async_client, httpx.AsyncClient) + await client.aclose() + assert client._httpx_async_client.is_closed + + def test_parse_params_none() -> None: """Test _parse_params with None input.""" assert HttpClient._parse_params(None) is None @@ -392,6 +413,70 @@ async def test_async_http_client_classifies_timeout_errors() -> None: assert not client.is_timeout_error(ValueError('test')) +@pytest.mark.parametrize( + 'exc', + [ + # Even the generic base class is transient: HTTPX subclasses it for every failure mode, so an + # unclassified failure is safer to retry. + pytest.param(httpx.HTTPError('unclassified failure'), id='bare HTTPError'), + pytest.param(httpx.TimeoutException('timeout'), id='TimeoutException'), + pytest.param(httpx.NetworkError('network error'), id='NetworkError'), + pytest.param(httpx.RemoteProtocolError('remote protocol error'), id='RemoteProtocolError'), + pytest.param(httpx.DecodingError('decoding error'), id='DecodingError'), + # One `ProxyError` covers both a proxy rejecting the CONNECT tunnel and a 407, so a transient case cannot + # be told from a permanent one - retrying is the safer default. + pytest.param(httpx.ProxyError('proxy error'), id='ProxyError'), + ], +) +def test_httpx_is_retryable_transport_error(exc: Exception) -> None: + """A transient HTTPX transport failure is classified as retryable.""" + with HttpxHttpClient() as client: + assert client.is_retryable_transport_error(exc) + + +@pytest.mark.parametrize( + 'exc', + [ + pytest.param(httpx.LocalProtocolError('invalid header value'), id='LocalProtocolError'), + pytest.param(httpx.UnsupportedProtocol('unsupported scheme'), id='UnsupportedProtocol'), + pytest.param(httpx.TooManyRedirects('too many redirects'), id='TooManyRedirects'), + pytest.param( + httpx.HTTPStatusError( + 'status error', + request=httpx.Request('GET', 'https://example.com'), + response=httpx.Response(500), + ), + id='HTTPStatusError', + ), + # HTTPX reports a bad URL outside the `httpx.HTTPError` tree entirely. + pytest.param(httpx.InvalidURL('unsupported scheme'), id='InvalidURL'), + pytest.param(ValueError('value error'), id='ValueError'), + pytest.param(RuntimeError('runtime error'), id='RuntimeError'), + pytest.param(Exception('generic exception'), id='Exception'), + ], +) +def test_httpx_is_not_retryable_transport_error(exc: Exception) -> None: + """A transport failure a retry cannot fix, and anything outside HTTPX's hierarchy, is not retried.""" + with HttpxHttpClient() as client: + assert not client.is_retryable_transport_error(exc) + + +def test_sync_httpx_client_classifies_timeout_errors() -> None: + """The built-in synchronous HTTPX client exposes transport-neutral timeout classification.""" + with HttpxHttpClient() as client: + assert client.is_timeout_error(TimeoutError('test')) + assert client.is_timeout_error(httpx.TimeoutException('test')) + assert not client.is_timeout_error(ValueError('test')) + + +async def test_async_httpx_client_classifies_timeout_errors() -> None: + """The built-in asynchronous HTTPX client exposes transport-neutral timeout classification.""" + async with HttpxHttpClientAsync() as client: + assert client.is_timeout_error(TimeoutError('test')) + assert client.is_timeout_error(httpx.TimeoutException('test')) + assert not client.is_timeout_error(ValueError('test')) + + def test_permanent_transport_error_is_not_retried() -> None: """A transport error a retry cannot fix fails on the first attempt instead of burning the whole backoff.""" client = ImpitHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) @@ -429,6 +514,31 @@ def test_transient_transport_error_is_retried() -> None: assert request.call_count == 3 +def test_httpx_permanent_transport_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX adapter feeds the same fail-fast classification into the shared pipeline.""" + with HttpxHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) as client: + send_request = Mock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client, 'send_request', send_request) + + with pytest.raises(httpx.UnsupportedProtocol): + client.call(method='GET', url='https://api.test.com/endpoint') + + send_request.assert_called_once() + + +def test_httpx_transient_transport_error_is_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX adapter keeps a transient transport failure inside the shared retry loop.""" + with HttpxHttpClient(token='test_token', max_retries=2, min_delay_between_retries=timedelta(0)) as client: + send_request = Mock(side_effect=httpx.TimeoutException('timeout')) + monkeypatch.setattr(client, 'send_request', send_request) + + with pytest.raises(httpx.TimeoutException): + client.call(method='GET', url='https://api.test.com/endpoint') + + # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. + assert send_request.call_count == 3 + + def test_error_response_read_failure_is_retried_and_closed() -> None: """A failure while buffering a streamed error body is retried like a failed send, and the response is closed.""" client = ImpitHttpClient(token='test_token', max_retries=1, min_delay_between_retries=timedelta(0)) diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index 35b10e2e..4c2f2271 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -877,7 +877,7 @@ def test_streamed_log_sync_does_not_leak_exception_on_stream_timeout( http_client_class: type[HttpClient], monkeypatch: pytest.MonkeyPatch, ) -> None: - """The streaming thread ends quietly when the transport times out while reading the log stream.""" + """The streaming thread ends quietly when either transport times out while reading the log stream.""" monkeypatch.setattr(StreamedLog, '_stream_timeout', timedelta(seconds=1)) release_server = threading.Event() @@ -970,7 +970,7 @@ async def test_streamed_log_async_does_not_error_on_stream_timeout( http_client_async_class: type[HttpClientAsync], monkeypatch: pytest.MonkeyPatch, ) -> None: - """The async streaming task treats a transport stream timeout as an expected terminal condition.""" + """The async streaming task treats either transport's stream timeout as an expected terminal condition.""" monkeypatch.setattr(StreamedLogAsync, '_stream_timeout', timedelta(seconds=1)) release_server = threading.Event() diff --git a/tests/unit/test_pluggable_http_client.py b/tests/unit/test_pluggable_http_client.py index 2fd20899..779125b8 100644 --- a/tests/unit/test_pluggable_http_client.py +++ b/tests/unit/test_pluggable_http_client.py @@ -2,9 +2,12 @@ import asyncio import json as jsonlib +import subprocess +import sys from dataclasses import dataclass, field from datetime import timedelta from http.client import HTTPConnection +from textwrap import dedent from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock from urllib.parse import urlsplit @@ -353,6 +356,8 @@ def test_public_exports() -> None: 'HttpClient', 'HttpClientAsync', 'HttpResponse', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', 'ImpitHttpClient', 'ImpitHttpClientAsync', ): @@ -362,6 +367,47 @@ def test_public_exports() -> None: assert not hasattr(http_clients_module, 'HttpClientBase') +def test_httpx_clients_raise_clear_error_when_extra_missing() -> None: + """Missing HTTPX keeps normal and star imports usable while explicit HTTPX access raises a clear error.""" + script = dedent( + """ + import sys + + class BlockHttpx: + def find_spec(self, name, *_args): + if name == 'httpx' or name.startswith('httpx.'): + raise ModuleNotFoundError(f"No module named '{name}'", name='httpx') + return None + + sys.meta_path.insert(0, BlockHttpx()) + + import apify_client.http_clients as module + assert module.HttpClient is not None + assert module.ImpitHttpClient is not None + + namespace = {} + exec('from apify_client.http_clients import *', namespace) + assert namespace['HttpClient'] is module.HttpClient + assert 'HttpxHttpClient' not in namespace + + for name in ('HttpxHttpClient', 'HttpxHttpClientAsync'): + try: + getattr(module, name) + except ImportError as exc: + assert "No module named 'httpx'" in str(exc) + else: + raise AssertionError(f'{name} did not raise ImportError') + """ + ) + result = subprocess.run( # noqa: S603 + [sys.executable, '-c', script], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + def test_apify_client_http_client_property_returns_correct_type() -> None: """Test that http_client property returns the correct type.""" # With default diff --git a/uv.lock b/uv.lock index 0f531050..0e2383f7 100644 --- a/uv.lock +++ b/uv.lock @@ -54,6 +54,9 @@ dependencies = [ brotli = [ { name = "brotli" }, ] +httpx = [ + { name = "httpx" }, +] [package.dev-dependencies] dev = [ @@ -80,12 +83,13 @@ dev = [ requires-dist = [ { name = "brotli", marker = "extra == 'brotli'", specifier = ">=1.0.9" }, { name = "colorama", specifier = ">=0.4.0" }, + { name = "httpx", marker = "extra == 'httpx'", specifier = ">=0.27.0,<1.0.0" }, { name = "impit", specifier = "~=0.13.0" }, { name = "more-itertools", specifier = ">=10.0.0" }, { name = "pydantic", extras = ["email"], specifier = ">=2.11.0" }, { name = "typing-extensions", specifier = ">=4.6.0" }, ] -provides-extras = ["brotli"] +provides-extras = ["brotli", "httpx"] [package.metadata.requires-dev] dev = [ From fe6130b6344abff1fa21d72ebda1903d285585e9 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Wed, 19 Aug 2026 08:45:30 +0200 Subject: [PATCH 2/6] docs: Document the transport-hook contract and the built-in HTTPX client --- README.md | 4 +- docs/01_introduction/index.mdx | 19 +++ docs/02_concepts/10_custom_http_clients.mdx | 119 +++++++++++--- .../02_concepts/code/10_httpx_client_async.py | 17 ++ docs/02_concepts/code/10_httpx_client_sync.py | 11 ++ docs/02_concepts/code/10_plugging_in_async.py | 27 ++-- docs/02_concepts/code/10_plugging_in_sync.py | 27 ++-- docs/03_guides/05_custom_http_client.mdx | 61 ++++---- .../code/05_custom_http_client_async.py | 142 ++++++++++++----- .../code/05_custom_http_client_sync.py | 148 +++++++++++++----- pyproject.toml | 7 +- 11 files changed, 418 insertions(+), 164 deletions(-) create mode 100644 docs/02_concepts/code/10_httpx_client_async.py create mode 100644 docs/02_concepts/code/10_httpx_client_sync.py diff --git a/README.md b/README.md index 7627e5fe..904158ef 100644 --- a/README.md +++ b/README.md @@ -133,7 +133,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)). - **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)). - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)). -- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX](https://www.python-httpx.org/) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). +- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX](https://www.python-httpx.org/) client, or plug in any custom implementation ([HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)). - **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more ([Logging](https://docs.apify.com/api/client/python/docs/concepts/logging)). @@ -201,7 +201,7 @@ The full documentation lives at **[docs.apify.com/api/client/python](https://doc | [Introduction](https://docs.apify.com/api/client/python/docs) | Overview, prerequisites, and a tour of the client. | | [Quick start](https://docs.apify.com/api/client/python/docs/quick-start) | Authenticate, run an Actor, and fetch its results step by step. | | [Concepts](https://docs.apify.com/api/client/python/docs/concepts/asyncio-support) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts. | -| [Guides](https://docs.apify.com/api/client/python/docs/guides/passing-input-to-actor) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. | +| [Guides](https://docs.apify.com/api/client/python/docs/guides/passing-input-to-actor) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), build a custom HTTP client. | | [Upgrading](https://docs.apify.com/api/client/python/docs/upgrading/upgrading-to-v3) | Migrating between major versions. | | [API reference](https://docs.apify.com/api/client/python/reference) | Generated reference for every class, method, and model. | | [Changelog](https://docs.apify.com/api/client/python/docs/changelog) | Release history and breaking changes. | diff --git a/docs/01_introduction/index.mdx b/docs/01_introduction/index.mdx index 86cfdf15..ac49941b 100644 --- a/docs/01_introduction/index.mdx +++ b/docs/01_introduction/index.mdx @@ -63,6 +63,25 @@ For better request-body compression, opt in to `brotli`, which compresses better For details, see [HTTP compression](../02_concepts/13_http_compression.mdx). +The client uses [Impit](https://github.com/apify/impit) as its default HTTP transport. To use the built-in +[HTTPX](https://www.python-httpx.org/) transport, install its optional dependency: + + + + ```bash + pip install "apify-client[httpx]" + ``` + + + ```bash + conda install conda-forge::apify-client conda-forge::httpx + ``` + + + +See [HTTP clients](../02_concepts/10_custom_http_clients.mdx) for synchronous and asynchronous examples and details +about the shared architecture. + ## Quick example The following example shows how to run an Actor and retrieve its results: diff --git a/docs/02_concepts/10_custom_http_clients.mdx b/docs/02_concepts/10_custom_http_clients.mdx index 0d2e41c1..1015acf7 100644 --- a/docs/02_concepts/10_custom_http_clients.mdx +++ b/docs/02_concepts/10_custom_http_clients.mdx @@ -1,7 +1,7 @@ --- id: custom-http-clients -title: Custom HTTP clients -description: Replace the default HTTP client with a custom implementation. +title: HTTP clients +description: Understand the built-in HTTP clients and the custom client interface. --- import Tabs from '@theme/Tabs'; @@ -11,13 +11,17 @@ import ApiLink from '@theme/ApiLink'; import DefaultHttpClientAsyncExample from '!!raw-loader!./code/10_default_http_client_async.py'; import DefaultHttpClientSyncExample from '!!raw-loader!./code/10_default_http_client_sync.py'; +import HttpxHttpClientAsyncExample from '!!raw-loader!./code/10_httpx_client_async.py'; +import HttpxHttpClientSyncExample from '!!raw-loader!./code/10_httpx_client_sync.py'; import ArchitectureImportsExample from '!!raw-loader!./code/10_architecture_imports.py'; import PluggingInAsyncExample from '!!raw-loader!./code/10_plugging_in_async.py'; import PluggingInSyncExample from '!!raw-loader!./code/10_plugging_in_sync.py'; -The Apify API client uses a pluggable HTTP client architecture. By default, it ships with an [Impit](https://github.com/apify/impit)-based HTTP client that handles retries, timeouts, passing headers, and more. You can replace it with your own implementation for use cases like custom logging, proxying, request modification, or integrating with a different HTTP library. +The Apify API client uses a pluggable HTTP layer. It ships with an [Impit](https://github.com/apify/impit)-based default, +offers [HTTPX](https://www.python-httpx.org/) as an optional built-in alternative, and accepts fully custom synchronous +or asynchronous implementations. ## Default HTTP client @@ -25,8 +29,8 @@ When you create an `ApifyClient` or `ApifyClient` or `ApifyClientAsync` constructor: @@ -43,35 +47,95 @@ You can configure the default client through the +## Built-in HTTPX client + +The package also provides `HttpxHttpClient` and +`HttpxHttpClientAsync`. They use the same request preparation, +compression, retry policy, timeout tiers and growth, error handling, logging, and statistics as the default Impit clients, with +[HTTPX](https://www.python-httpx.org/) as the transport. + +HTTPX is an optional dependency. Install `apify-client[httpx]`, then pass the appropriate client to +`ApifyClient.with_custom_http_client`. Impit remains +the default even when the HTTPX extra is installed. + +```bash +pip install "apify-client[httpx]" +# or +uv add "apify-client[httpx]" +``` + + + + + {HttpxHttpClientAsyncExample} + + + + + {HttpxHttpClientSyncExample} + + + + +Configure retries, timeout tiers, default headers, and compression on the HTTPX client instance. The token passed to +`with_custom_http_client` is applied automatically unless the HTTP client already has an `Authorization` header. +The examples use the clients as context managers so their connection pools are closed deterministically. If a context +manager does not fit your application's lifecycle, call `close()` on `HttpxHttpClient` or `await aclose()` on +`HttpxHttpClientAsync` during shutdown. + +Timeout values are passed to the selected transport. Impit treats them as whole-request timeouts, while HTTPX applies +its connect, read, write, and pool timeout semantics. In particular, an HTTPX read timeout limits inactivity between +chunks rather than the total duration of a streamed response. The `no_timeout` option disables HTTPX's timeouts. + ## Architecture -The HTTP client system is built on two key abstractions: +Internally, the HTTP client hierarchy has three layers: + +- A common internal base contains configuration and utilities shared by synchronous and asynchronous clients, including + headers, request-body preparation, parameters, compression, and timeout tiers. It is not a public extension point. +- `HttpClient` and `HttpClientAsync` + add the synchronous or asynchronous request pipeline, retry loop, transport hooks, and lifecycle interface. +- The built-in Impit and HTTPX classes inherit directly from the corresponding sync or async class and adapt the + underlying transport. + +`HttpClient.is_timeout_error(exc)` and `HttpClientAsync.is_timeout_error(exc)` provide the public, transport-neutral way +to determine whether an exception is a timeout. Their shared implementation recognizes Python's `TimeoutError`; +transport adapters override it when their HTTP library defines additional timeout exception types. This lets +higher-level features such as streamed logs classify timeouts without depending on Impit, HTTPX, or private +implementation details. + +Responses use one separate abstraction: -- `HttpClient` / `HttpClientAsync` - Abstract base classes that define the interface. Extend one of these to create a custom HTTP client by implementing the `call` method. - `HttpResponse` - A [runtime-checkable protocol](https://docs.python.org/3/library/typing.html#typing.runtime_checkable) that defines the expected response shape. Any object with the required attributes and methods satisfies the protocol — no inheritance needed. To plug in your custom implementation, use the `ApifyClient.with_custom_http_client` class method. -All of these are available as top-level imports from the `apify_client` package: +The built-in Impit and HTTPX classes are thin transport adapters over the request implementation in `HttpClient` and +`HttpClientAsync`. Custom transport adapters implement the request, error-classification, and lifecycle hooks. They +inherit request construction, retries, timeout growth, API error conversion, logging, and statistics from the base. + +All of these are available from the `apify_client.http_clients` module: {ArchitectureImportsExample} -### The call method +### The transport contract -The `call` method receives all the information needed to make an HTTP request: +The public `call` method provides the shared request pipeline. A concrete transport implements these hooks: -- `method` - HTTP method (`GET`, `POST`, `PUT`, `DELETE`, etc.). -- `url` - Full URL to make the request to. -- `headers` - Additional headers to include. -- `params` - Query parameters to append to the URL. -- `data` - Raw request body (mutually exclusive with `json`). -- `json` - JSON-serializable request body (mutually exclusive with `data`). -- `stream` - Whether to stream the response body. -- `timeout` - Timeout for the request as a `timedelta`. +- `send_request(...)` sends one prepared request and returns an `HttpResponse`. The inherited `call` needs it, so + every transport adapter has to implement it. +- `is_retryable_transport_error(exc)` classifies transport failures for the shared retry loop. The default classifies + nothing as retryable, so a transport that skips it gives up on the first connection failure. +- `is_timeout_error(exc)` identifies transport-specific timeout exceptions for higher-level client features. The + default recognizes Python's `TimeoutError`. Timeout classification is independent of retryability, so a timeout + the retry loop should retry has to be listed in `is_retryable_transport_error` too. +- `close()` or `aclose()` closes resources owned by the transport. The default does nothing, which is correct for a + transport that owns no pool or session. -It must return an object satisfying the `HttpResponse` protocol. +The `@override` decorators in the built-in Impit and HTTPX adapters make these implementations explicit and allow type +checkers to catch misspelled or incompatible overrides. ### The HTTP response protocol @@ -93,6 +157,10 @@ It must return an object satisfying the `HttpRe :::note Many HTTP libraries, including our default [Impit](https://github.com/apify/impit) or for example [HTTPX](https://www.python-httpx.org/) already satisfy this protocol out of the box. + +For a streamed response, consume the body inside its context manager with `iter_bytes()` / `aiter_bytes()`, or call +`read()` / `aread()` before accessing `content`. Some transports, including HTTPX, intentionally reject `content` on +an unread streamed response. ::: ### Plugging it in @@ -115,18 +183,23 @@ Use the `ApifyClient.wit After that, all API calls made through the client will go through your custom HTTP client. :::warning -When using a custom HTTP client, you are responsible for constructing the request, handling retries, timeouts, and errors yourself. The default retry logic is not applied. +If you override `call` itself, your implementation becomes responsible for request preparation, retries, timeouts, API +error conversion, logging, and statistics. Implementing the transport hooks and inheriting `call` keeps the shared +behavior. ::: ## Use cases -Custom HTTP clients might be useful when you need to: +Custom HTTP clients might be useful when the built-in Impit and HTTPX clients do not cover your requirements, for +example when you need to: -- **Use a different HTTP library** - Swap Impit for [httpx](https://www.python-httpx.org/), [requests](https://requests.readthedocs.io/), or [aiohttp](https://docs.aiohttp.org/). +- **Use a different HTTP library** - Integrate [requests](https://requests.readthedocs.io/), [aiohttp](https://docs.aiohttp.org/), or another transport. - **Route through a proxy** - Add proxy support or request routing. - **Implement custom retry logic** - Use different backoff strategies or retry conditions. - **Log requests and responses** - Track API calls for debugging or auditing. - **Modify requests** - Add custom fields, modify the body, or change headers. - **Collect custom metrics** - Measure request latency, track error rates, or count API calls. -For a step-by-step walkthrough of building a custom HTTP client, see the [Using HTTPX as the HTTP client](/api/client/python/docs/guides/custom-http-client-httpx) guide. +For complete synchronous and asynchronous implementations over a transport with a different response API, see +[Build a custom HTTP client](../03_guides/05_custom_http_client.mdx). You can also refer to the +`HttpClient` API reference for the synchronous contract. diff --git a/docs/02_concepts/code/10_httpx_client_async.py b/docs/02_concepts/code/10_httpx_client_async.py new file mode 100644 index 00000000..8a7062ed --- /dev/null +++ b/docs/02_concepts/code/10_httpx_client_async.py @@ -0,0 +1,17 @@ +import asyncio + +from apify_client import ApifyClientAsync +from apify_client.http_clients import HttpxHttpClientAsync + + +async def main() -> None: + async with HttpxHttpClientAsync() as http_client: + client = ApifyClientAsync.with_custom_http_client( + token='MY-APIFY-TOKEN', + http_client=http_client, + ) + print(await client.actor('apify/hello-world').get()) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/docs/02_concepts/code/10_httpx_client_sync.py b/docs/02_concepts/code/10_httpx_client_sync.py new file mode 100644 index 00000000..f5de3cb3 --- /dev/null +++ b/docs/02_concepts/code/10_httpx_client_sync.py @@ -0,0 +1,11 @@ +from apify_client import ApifyClient +from apify_client.http_clients import HttpxHttpClient + + +def main() -> None: + with HttpxHttpClient() as http_client: + client = ApifyClient.with_custom_http_client( + token='MY-APIFY-TOKEN', + http_client=http_client, + ) + print(client.actor('apify/hello-world').get()) diff --git a/docs/02_concepts/code/10_plugging_in_async.py b/docs/02_concepts/code/10_plugging_in_async.py index 331f0b62..dad83733 100644 --- a/docs/02_concepts/code/10_plugging_in_async.py +++ b/docs/02_concepts/code/10_plugging_in_async.py @@ -1,8 +1,7 @@ -from typing import Any +from typing_extensions import override from apify_client import ApifyClientAsync from apify_client.http_clients import HttpClientAsync, HttpResponse -from apify_client.types import Timeout TOKEN = 'MY-APIFY-TOKEN' @@ -10,18 +9,26 @@ class MyHttpClientAsync(HttpClientAsync): """Custom async HTTP client.""" - async def call( + @override + async def send_request( self, *, method: str, url: str, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, - json: Any = None, - stream: bool | None = None, - timeout: Timeout = 'medium', - ) -> HttpResponse: ... + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> HttpResponse: + """Send one request through the custom transport.""" + raise NotImplementedError + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # List the transport's transient failures here, e.g. its timeout + # and connection errors. Returning False for everything opts out + # of transport retries entirely. + return isinstance(exc, TimeoutError) async def main() -> None: diff --git a/docs/02_concepts/code/10_plugging_in_sync.py b/docs/02_concepts/code/10_plugging_in_sync.py index 386281ae..8f0aa362 100644 --- a/docs/02_concepts/code/10_plugging_in_sync.py +++ b/docs/02_concepts/code/10_plugging_in_sync.py @@ -1,8 +1,7 @@ -from typing import Any +from typing_extensions import override from apify_client import ApifyClient from apify_client.http_clients import HttpClient, HttpResponse -from apify_client.types import Timeout TOKEN = 'MY-APIFY-TOKEN' @@ -10,18 +9,26 @@ class MyHttpClient(HttpClient): """Custom sync HTTP client.""" - def call( + @override + def send_request( self, *, method: str, url: str, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, - json: Any = None, - stream: bool | None = None, - timeout: Timeout = 'medium', - ) -> HttpResponse: ... + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> HttpResponse: + """Send one request through the custom transport.""" + raise NotImplementedError + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # List the transport's transient failures here, e.g. its timeout + # and connection errors. Returning False for everything opts out + # of transport retries entirely. + return isinstance(exc, TimeoutError) def main() -> None: diff --git a/docs/03_guides/05_custom_http_client.mdx b/docs/03_guides/05_custom_http_client.mdx index b7314a1c..77e05a7b 100644 --- a/docs/03_guides/05_custom_http_client.mdx +++ b/docs/03_guides/05_custom_http_client.mdx @@ -1,39 +1,46 @@ --- -id: custom-http-client-httpx -title: Use HTTPX as the HTTP client -description: Replace the default Impit HTTP client with one based on HTTPX. +id: custom-http-client +title: Build a custom HTTP client +description: Implement the HTTP client contract with AIOHTTP and requests. --- import ApiLink from '@theme/ApiLink'; +import CodeBlock from '@theme/CodeBlock'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; import CustomHttpClientAsyncExample from '!!raw-loader!./code/05_custom_http_client_async.py'; import CustomHttpClientSyncExample from '!!raw-loader!./code/05_custom_http_client_sync.py'; -This guide shows how to replace the default `ImpitHttpClient` and `ImpitHttpClientAsync` with one based on [HTTPX](https://www.python-httpx.org/). The same approach works for any HTTP library — see [Custom HTTP clients](/api/client/python/docs/concepts/custom-http-clients) for the underlying architecture. - -## Why HTTPX? +This guide implements a custom `HttpClientAsync` with +[AIOHTTP](https://docs.aiohttp.org/) and a custom `HttpClient` with +[requests](https://requests.readthedocs.io/). Neither library satisfies the +`HttpResponse` protocol, so both examples also show how to adapt a +foreign response API. -You might want to use [HTTPX](https://www.python-httpx.org/) instead of the default [Impit](https://github.com/apify/impit)-based client for reasons like: +For an overview of the architecture and the built-in Impit and HTTPX implementations, see +[HTTP clients](../02_concepts/10_custom_http_clients.mdx). -- You already use HTTPX in your project and want a single HTTP stack. -- You need HTTPX-specific features. -- You want fine-grained control over connection pooling or proxy routing. - -## Implementation +## Installation -The implementation involves two steps: +Install the transport alongside the Apify client. Neither AIOHTTP nor requests is an `apify-client` extra: -1. **Extend `HttpClient` (sync) or `HttpClientAsync` (async)** and implement the `call` method that delegates to HTTPX. -2. **Pass it to `ApifyClient.with_custom_http_client`** to create a client that uses your implementation. +```bash +pip install apify-client aiohttp # for the asynchronous client +pip install apify-client requests # for the synchronous client +``` -The `call` method receives parameters like `method`, `url`, `headers`, `params`, `data`, `json`, `stream`, and `timeout`. Map them to the corresponding HTTPX arguments — most map directly, except `data` which becomes HTTPX's `content` parameter and `timeout` which needs conversion from `timedelta` to seconds. +## Implementation -A convenient property of HTTPX is that its `httpx.Response` object already satisfies the `HttpResponse` protocol, so you can return it directly without wrapping. +Each example has three parts: -One part of the contract isn't visible in the method signature: `call` must raise `ApifyApiError` for error responses instead of returning them. The resource clients rely on that error to work correctly. For example, `get` methods translate a 404 raised this way into a `None` return value, and user code handling `ApifyApiError` keeps working. +1. The response adapter, `AiohttpResponse` or `RequestsResponse`, maps the library's own response onto the + `HttpResponse` protocol that resource clients expect. +2. The client, `AiohttpHttpClient` or `RequestsHttpClient`, implements the transport, error-classification, + timeout-classification, and lifecycle hooks. It inherits request preparation, retry handling, timeout growth, + and API error conversion from its base class. +3. `with_custom_http_client()` connects the implementation to the resource clients and applies the API token. The + context manager closes the session at shutdown. @@ -49,16 +56,8 @@ One part of the contract isn't visible in the method signature: `call` must rais :::warning -When using a custom HTTP client, you are responsible for handling retries, timeouts, and error handling yourself. The built-in retry logic with exponential backoff is part of the default `ImpitHttpClient` and is not applied to custom implementations. +These are compact integration examples, not a replacement for all built-in client behavior. A production custom +client should account for transport-specific details such as proxy configuration, TLS settings, redirects, and +response resource cleanup. The shared base provides retries, logging, statistics, timeout growth, and API error +conversion. ::: - -## Going further - -The example above is minimal on purpose. In a production setup, you might want to extend it with: - -- **Retry logic** - Use [HTTPX's event hooks](https://www.python-httpx.org/advanced/event-hooks/) or utilize library like [tenacity](https://tenacity.readthedocs.io/) to retry failed requests. -- **Custom headers** - You can add headers in the `call` method before delegating to HTTPX. -- **Connection lifecycle** - Close the underlying `httpx.Client` when done by adding a `close()` method to your custom client. -- **Proxy support** - You can pass `proxy=...` when creating the `httpx.Client`. -- **Metrics collection** - Track request latency, error rates, or other metrics by adding instrumentation in the `call` method. -- **Logging** - Log requests and responses for debugging or auditing purposes. diff --git a/docs/03_guides/code/05_custom_http_client_async.py b/docs/03_guides/code/05_custom_http_client_async.py index 44c81080..efd1481c 100644 --- a/docs/03_guides/code/05_custom_http_client_async.py +++ b/docs/03_guides/code/05_custom_http_client_async.py @@ -1,75 +1,137 @@ from __future__ import annotations import asyncio -from http import HTTPStatus +import json as jsonlib from typing import TYPE_CHECKING, Any -import httpx +import aiohttp +from typing_extensions import override from apify_client import ApifyClientAsync -from apify_client.errors import ApifyApiError from apify_client.http_clients import HttpClientAsync, HttpResponse if TYPE_CHECKING: - from apify_client.types import Timeout + from collections.abc import AsyncIterator, Iterator, Mapping TOKEN = 'MY-APIFY-TOKEN' -class HttpxClientAsync(HttpClientAsync): - """Custom async HTTP client using HTTPX library.""" +class AiohttpResponse: + """Adapt an aiohttp response to the Apify client's HttpResponse protocol.""" + + def __init__(self, response: aiohttp.ClientResponse) -> None: + self._response = response + self._body: bytes | None = None + + @property + def status_code(self) -> int: + return self._response.status + + @property + def headers(self) -> Mapping[str, str]: + return self._response.headers + + @property + def content(self) -> bytes: + if self._body is None: + raise RuntimeError( + 'The streamed response has not been read yet; ' + 'use aread() or aiter_bytes()' + ) + return self._body + + @property + def text(self) -> str: + encoding = self._response.charset or 'utf-8' + return self.content.decode(encoding, errors='replace') + + def json(self) -> Any: + return jsonlib.loads(self.text) + + def read(self) -> bytes: + return self.content + + async def aread(self) -> bytes: + if self._body is None: + self._body = await self._response.read() + return self._body + + def close(self) -> None: + self._response.close() + + async def aclose(self) -> None: + self._response.release() + await self._response.wait_for_close() + + def iter_bytes(self) -> Iterator[bytes]: + body = self.content + if body: + yield body + + async def aiter_bytes(self) -> AsyncIterator[bytes]: + if self._body is not None: + if self._body: + yield self._body + return + async for chunk in self._response.content.iter_chunked(64 * 1024): + yield chunk + + +class AiohttpHttpClient(HttpClientAsync): + """Minimal custom asynchronous HTTP client backed by aiohttp.""" def __init__(self) -> None: super().__init__() - self._client = httpx.AsyncClient() + self._session = aiohttp.ClientSession() + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance( + exc, aiohttp.ServerTimeoutError + ) + + @override + async def aclose(self) -> None: + await self._session.close() - async def call( + @override + async def send_request( self, *, method: str, url: str, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, - json: Any = None, - stream: bool | None = None, - timeout: Timeout = 'medium', + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, ) -> HttpResponse: - timeout_secs = self._compute_timeout(timeout, attempt=1) or 0 - - # Merge the client's default headers (including authorization) - # with the per-request ones. - headers = self._merge_headers(self._headers, headers) - - response = await self._client.request( + response = await self._session.request( method=method, url=url, headers=headers, - params=params, - content=data, - json=json, - timeout=timeout_secs, + data=content, + timeout=aiohttp.ClientTimeout(total=timeout), ) + adapted_response = AiohttpResponse(response) - # Raising `ApifyApiError` for error responses is part of the `call` - # contract. The resource clients rely on it, e.g. to translate a 404 - # into a `None` return value of `get` methods. - if response.status_code >= HTTPStatus.BAD_REQUEST: - raise ApifyApiError(response, attempt=1, method=method) + if not stream: + await adapted_response.aread() - # httpx.Response satisfies the HttpResponse protocol, - # so it can be returned directly. - return response + return adapted_response + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + return isinstance(exc, (TimeoutError, aiohttp.ClientError)) -async def main() -> None: - client = ApifyClientAsync.with_custom_http_client( - token=TOKEN, - http_client=HttpxClientAsync(), - ) - actor = await client.actor('apify/hello-world').get() - print(actor) +async def main() -> None: + async with AiohttpHttpClient() as http_client: + client = ApifyClientAsync.with_custom_http_client( + token=TOKEN, + http_client=http_client, + ) + actor = await client.actor('apify/hello-world').get() + print(actor) if __name__ == '__main__': diff --git a/docs/03_guides/code/05_custom_http_client_sync.py b/docs/03_guides/code/05_custom_http_client_sync.py index c6716256..996dcef5 100644 --- a/docs/03_guides/code/05_custom_http_client_sync.py +++ b/docs/03_guides/code/05_custom_http_client_sync.py @@ -1,74 +1,138 @@ from __future__ import annotations -from http import HTTPStatus +import json as jsonlib from typing import TYPE_CHECKING, Any -import httpx +import requests +from typing_extensions import override from apify_client import ApifyClient -from apify_client.errors import ApifyApiError from apify_client.http_clients import HttpClient, HttpResponse if TYPE_CHECKING: - from apify_client.types import Timeout + from collections.abc import AsyncIterator, Iterator, Mapping TOKEN = 'MY-APIFY-TOKEN' -class HttpxClient(HttpClient): - """Custom HTTP client using HTTPX library.""" +class RequestsResponse: + """Adapt a requests response to the Apify client's HttpResponse protocol.""" + + def __init__(self, response: requests.Response) -> None: + self._response = response + self._body: bytes | None = None + + @property + def status_code(self) -> int: + return self._response.status_code + + @property + def headers(self) -> Mapping[str, str]: + return self._response.headers + + @property + def content(self) -> bytes: + if self._body is None: + raise RuntimeError( + 'The streamed response has not been read yet; use read() or iter_bytes()' + ) + return self._body + + @property + def text(self) -> str: + encoding = self._response.encoding or 'utf-8' + return self.content.decode(encoding, errors='replace') + + def json(self) -> Any: + return jsonlib.loads(self.text) + + def read(self) -> bytes: + if self._body is None: + self._body = self._response.content + return self._body + + async def aread(self) -> bytes: + return self.read() + + def close(self) -> None: + self._response.close() + + async def aclose(self) -> None: + self.close() + + def iter_bytes(self) -> Iterator[bytes]: + if self._body is not None: + if self._body: + yield self._body + return + yield from self._response.iter_content(64 * 1024) + + async def aiter_bytes(self) -> AsyncIterator[bytes]: + for chunk in self.iter_bytes(): + yield chunk + + +class RequestsHttpClient(HttpClient): + """Minimal custom synchronous HTTP client backed by requests.""" def __init__(self) -> None: super().__init__() - self._client = httpx.Client() + self._session = requests.Session() + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance(exc, requests.Timeout) - def call( + @override + def close(self) -> None: + self._session.close() + + @override + def send_request( self, *, method: str, url: str, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, - json: Any = None, - stream: bool | None = None, - timeout: Timeout = 'medium', + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, ) -> HttpResponse: - timeout_secs = self._compute_timeout(timeout, attempt=1) or 0 - - # Merge the client's default headers (including authorization) - # with the per-request ones. - headers = self._merge_headers(self._headers, headers) - - response = self._client.request( + response = self._session.request( method=method, url=url, headers=headers, - params=params, - content=data, - json=json, - timeout=timeout_secs, + data=content, + timeout=timeout, + stream=stream, + ) + adapted_response = RequestsResponse(response) + + if not stream: + adapted_response.read() + + return adapted_response + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + return isinstance( + exc, + ( + requests.ConnectionError, + requests.Timeout, + requests.exceptions.ChunkedEncodingError, + ), ) - - # Raising `ApifyApiError` for error responses is part of the `call` - # contract. The resource clients rely on it, e.g. to translate a 404 - # into a `None` return value of `get` methods. - if response.status_code >= HTTPStatus.BAD_REQUEST: - raise ApifyApiError(response, attempt=1, method=method) - - # httpx.Response satisfies the HttpResponse protocol, - # so it can be returned directly. - return response def main() -> None: - client = ApifyClient.with_custom_http_client( - token=TOKEN, - http_client=HttpxClient(), - ) - - actor = client.actor('apify/hello-world').get() - print(actor) + with RequestsHttpClient() as http_client: + client = ApifyClient.with_custom_http_client( + token=TOKEN, + http_client=http_client, + ) + actor = client.actor('apify/hello-world').get() + print(actor) if __name__ == '__main__': diff --git a/pyproject.toml b/pyproject.toml index b8922597..cbdbfe2b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -209,12 +209,7 @@ include = ["docs/**/*.py", "website/**/*.py"] unresolved-import = "ignore" [[tool.ty.overrides]] -include = ["docs/**/10_plugging_in_async.py", "docs/**/10_plugging_in_sync.py"] -[tool.ty.overrides.rules] -empty-body = "ignore" - -[[tool.ty.overrides]] -include = ["docs/**/05_custom_http_client_async.py", "docs/**/05_custom_http_client_sync.py"] +include = ["docs/**/05_custom_http_client_async.py"] [tool.ty.overrides.rules] invalid-argument-type = "ignore" From cdeb474da7d08c871d3eb0d9c20db394736dc280 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 27 Aug 2026 11:08:04 +0200 Subject: [PATCH 3/6] docs: Point the HTTPX references at httpx2 --- docs/01_introduction/index.mdx | 5 +++-- docs/02_concepts/10_custom_http_clients.mdx | 11 ++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/01_introduction/index.mdx b/docs/01_introduction/index.mdx index ac49941b..f445bcff 100644 --- a/docs/01_introduction/index.mdx +++ b/docs/01_introduction/index.mdx @@ -64,7 +64,8 @@ For better request-body compression, opt in to `brotli`, which compresses better For details, see [HTTP compression](../02_concepts/13_http_compression.mdx). The client uses [Impit](https://github.com/apify/impit) as its default HTTP transport. To use the built-in -[HTTPX](https://www.python-httpx.org/) transport, install its optional dependency: +[HTTPX](https://github.com/pydantic/httpx2) transport, install its optional dependency. It pulls in `httpx2`, +Pydantic's maintained continuation of HTTPX: @@ -74,7 +75,7 @@ The client uses [Impit](https://github.com/apify/impit) as its default HTTP tran ```bash - conda install conda-forge::apify-client conda-forge::httpx + conda install conda-forge::apify-client conda-forge::httpx2 ``` diff --git a/docs/02_concepts/10_custom_http_clients.mdx b/docs/02_concepts/10_custom_http_clients.mdx index 1015acf7..ae658d4d 100644 --- a/docs/02_concepts/10_custom_http_clients.mdx +++ b/docs/02_concepts/10_custom_http_clients.mdx @@ -20,7 +20,7 @@ import PluggingInAsyncExample from '!!raw-loader!./code/10_plugging_in_async.py' import PluggingInSyncExample from '!!raw-loader!./code/10_plugging_in_sync.py'; The Apify API client uses a pluggable HTTP layer. It ships with an [Impit](https://github.com/apify/impit)-based default, -offers [HTTPX](https://www.python-httpx.org/) as an optional built-in alternative, and accepts fully custom synchronous +offers [HTTPX](https://github.com/pydantic/httpx2) as an optional built-in alternative, and accepts fully custom synchronous or asynchronous implementations. ## Default HTTP client @@ -52,11 +52,12 @@ You can configure the default client through the `HttpxHttpClient` and `HttpxHttpClientAsync`. They use the same request preparation, compression, retry policy, timeout tiers and growth, error handling, logging, and statistics as the default Impit clients, with -[HTTPX](https://www.python-httpx.org/) as the transport. +[HTTPX](https://github.com/pydantic/httpx2) as the transport. HTTPX is an optional dependency. Install `apify-client[httpx]`, then pass the appropriate client to -`ApifyClient.with_custom_http_client`. Impit remains -the default even when the HTTPX extra is installed. +`ApifyClient.with_custom_http_client`. The extra +installs `httpx2`, Pydantic's maintained continuation of HTTPX. Impit remains the default even when the HTTPX extra +is installed. ```bash pip install "apify-client[httpx]" @@ -156,7 +157,7 @@ checkers to catch misspelled or incompatible overrides. | `aiter_bytes() -> AsyncIterator[bytes]` | Iterate body in chunks (async) | :::note -Many HTTP libraries, including our default [Impit](https://github.com/apify/impit) or for example [HTTPX](https://www.python-httpx.org/) already satisfy this protocol out of the box. +Many HTTP libraries, including our default [Impit](https://github.com/apify/impit) or for example [HTTPX](https://github.com/pydantic/httpx2) already satisfy this protocol out of the box. For a streamed response, consume the body inside its context manager with `iter_bytes()` / `aiter_bytes()`, or call `read()` / `aread()` before accessing `content`. Some transports, including HTTPX, intentionally reject `content` on From 968886dc15b62b2af39d22a3c84cae59cb5cffd0 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 27 Aug 2026 11:17:07 +0200 Subject: [PATCH 4/6] docs: Point the remaining HTTPX links at httpx2 --- docs/04_upgrading/upgrading_to_v2.mdx | 2 +- .../version-2.5/04_upgrading/upgrading_to_v2.mdx | 2 +- .../version-3.1/02_concepts/10_custom_http_clients.mdx | 4 ++-- .../version-3.1/03_guides/05_custom_http_client.mdx | 6 +++--- .../version-3.1/04_upgrading/upgrading_to_v2.mdx | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/04_upgrading/upgrading_to_v2.mdx b/docs/04_upgrading/upgrading_to_v2.mdx index ed02202b..c794503f 100644 --- a/docs/04_upgrading/upgrading_to_v2.mdx +++ b/docs/04_upgrading/upgrading_to_v2.mdx @@ -14,7 +14,7 @@ Support for Python 3.9 has been dropped. The Apify Python API Client v2.x now re ## New underlying HTTP library -In v2.0, the Apify Python API client switched from using [`httpx`](https://www.python-httpx.org/) to [`impit`](https://github.com/apify/impit) as the underlying HTTP library. However, this change shouldn't have much impact on the end user. +In v2.0, the Apify Python API client switched from using [`httpx`](https://github.com/pydantic/httpx2) to [`impit`](https://github.com/apify/impit) as the underlying HTTP library. However, this change shouldn't have much impact on the end user. ## API method changes diff --git a/website/versioned_docs/version-2.5/04_upgrading/upgrading_to_v2.mdx b/website/versioned_docs/version-2.5/04_upgrading/upgrading_to_v2.mdx index 6631c716..dd12051e 100644 --- a/website/versioned_docs/version-2.5/04_upgrading/upgrading_to_v2.mdx +++ b/website/versioned_docs/version-2.5/04_upgrading/upgrading_to_v2.mdx @@ -14,7 +14,7 @@ Support for Python 3.9 has been dropped. The Apify Python API Client v2.x now re ## New underlying HTTP library -In v2.0, the Apify Python API client switched from using [`httpx`](https://www.python-httpx.org/) to [`impit`](https://github.com/apify/impit) as the underlying HTTP library. However, this change shouldn't have much impact on the end user. +In v2.0, the Apify Python API client switched from using [`httpx`](https://github.com/pydantic/httpx2) to [`impit`](https://github.com/apify/impit) as the underlying HTTP library. However, this change shouldn't have much impact on the end user. ## API method changes diff --git a/website/versioned_docs/version-3.1/02_concepts/10_custom_http_clients.mdx b/website/versioned_docs/version-3.1/02_concepts/10_custom_http_clients.mdx index 0d2e41c1..ece32fb1 100644 --- a/website/versioned_docs/version-3.1/02_concepts/10_custom_http_clients.mdx +++ b/website/versioned_docs/version-3.1/02_concepts/10_custom_http_clients.mdx @@ -92,7 +92,7 @@ It must return an object satisfying the `HttpRe | `aiter_bytes() -> AsyncIterator[bytes]` | Iterate body in chunks (async) | :::note -Many HTTP libraries, including our default [Impit](https://github.com/apify/impit) or for example [HTTPX](https://www.python-httpx.org/) already satisfy this protocol out of the box. +Many HTTP libraries, including our default [Impit](https://github.com/apify/impit) or for example [HTTPX](https://github.com/pydantic/httpx2) already satisfy this protocol out of the box. ::: ### Plugging it in @@ -122,7 +122,7 @@ When using a custom HTTP client, you are responsible for constructing the reques Custom HTTP clients might be useful when you need to: -- **Use a different HTTP library** - Swap Impit for [httpx](https://www.python-httpx.org/), [requests](https://requests.readthedocs.io/), or [aiohttp](https://docs.aiohttp.org/). +- **Use a different HTTP library** - Swap Impit for [httpx](https://github.com/pydantic/httpx2), [requests](https://requests.readthedocs.io/), or [aiohttp](https://docs.aiohttp.org/). - **Route through a proxy** - Add proxy support or request routing. - **Implement custom retry logic** - Use different backoff strategies or retry conditions. - **Log requests and responses** - Track API calls for debugging or auditing. diff --git a/website/versioned_docs/version-3.1/03_guides/05_custom_http_client.mdx b/website/versioned_docs/version-3.1/03_guides/05_custom_http_client.mdx index b7314a1c..fa9fbc26 100644 --- a/website/versioned_docs/version-3.1/03_guides/05_custom_http_client.mdx +++ b/website/versioned_docs/version-3.1/03_guides/05_custom_http_client.mdx @@ -12,11 +12,11 @@ import CodeBlock from '@theme/CodeBlock'; import CustomHttpClientAsyncExample from '!!raw-loader!./code/05_custom_http_client_async.py'; import CustomHttpClientSyncExample from '!!raw-loader!./code/05_custom_http_client_sync.py'; -This guide shows how to replace the default `ImpitHttpClient` and `ImpitHttpClientAsync` with one based on [HTTPX](https://www.python-httpx.org/). The same approach works for any HTTP library — see [Custom HTTP clients](/api/client/python/docs/concepts/custom-http-clients) for the underlying architecture. +This guide shows how to replace the default `ImpitHttpClient` and `ImpitHttpClientAsync` with one based on [HTTPX](https://github.com/pydantic/httpx2). The same approach works for any HTTP library — see [Custom HTTP clients](/api/client/python/docs/concepts/custom-http-clients) for the underlying architecture. ## Why HTTPX? -You might want to use [HTTPX](https://www.python-httpx.org/) instead of the default [Impit](https://github.com/apify/impit)-based client for reasons like: +You might want to use [HTTPX](https://github.com/pydantic/httpx2) instead of the default [Impit](https://github.com/apify/impit)-based client for reasons like: - You already use HTTPX in your project and want a single HTTP stack. - You need HTTPX-specific features. @@ -56,7 +56,7 @@ When using a custom HTTP client, you are responsible for handling retries, timeo The example above is minimal on purpose. In a production setup, you might want to extend it with: -- **Retry logic** - Use [HTTPX's event hooks](https://www.python-httpx.org/advanced/event-hooks/) or utilize library like [tenacity](https://tenacity.readthedocs.io/) to retry failed requests. +- **Retry logic** - Use [HTTPX's event hooks](https://github.com/pydantic/httpx2) or utilize library like [tenacity](https://tenacity.readthedocs.io/) to retry failed requests. - **Custom headers** - You can add headers in the `call` method before delegating to HTTPX. - **Connection lifecycle** - Close the underlying `httpx.Client` when done by adding a `close()` method to your custom client. - **Proxy support** - You can pass `proxy=...` when creating the `httpx.Client`. diff --git a/website/versioned_docs/version-3.1/04_upgrading/upgrading_to_v2.mdx b/website/versioned_docs/version-3.1/04_upgrading/upgrading_to_v2.mdx index ed02202b..c794503f 100644 --- a/website/versioned_docs/version-3.1/04_upgrading/upgrading_to_v2.mdx +++ b/website/versioned_docs/version-3.1/04_upgrading/upgrading_to_v2.mdx @@ -14,7 +14,7 @@ Support for Python 3.9 has been dropped. The Apify Python API Client v2.x now re ## New underlying HTTP library -In v2.0, the Apify Python API client switched from using [`httpx`](https://www.python-httpx.org/) to [`impit`](https://github.com/apify/impit) as the underlying HTTP library. However, this change shouldn't have much impact on the end user. +In v2.0, the Apify Python API client switched from using [`httpx`](https://github.com/pydantic/httpx2) to [`impit`](https://github.com/apify/impit) as the underlying HTTP library. However, this change shouldn't have much impact on the end user. ## API method changes From a2a03b66429dd6ea3d0ab1a5c7363d8a2cd5d930 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 27 Aug 2026 11:19:29 +0200 Subject: [PATCH 5/6] docs: Revert the HTTPX link change in versioned docs --- .../version-2.5/04_upgrading/upgrading_to_v2.mdx | 2 +- .../version-3.1/02_concepts/10_custom_http_clients.mdx | 4 ++-- .../version-3.1/03_guides/05_custom_http_client.mdx | 6 +++--- .../version-3.1/04_upgrading/upgrading_to_v2.mdx | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/website/versioned_docs/version-2.5/04_upgrading/upgrading_to_v2.mdx b/website/versioned_docs/version-2.5/04_upgrading/upgrading_to_v2.mdx index dd12051e..6631c716 100644 --- a/website/versioned_docs/version-2.5/04_upgrading/upgrading_to_v2.mdx +++ b/website/versioned_docs/version-2.5/04_upgrading/upgrading_to_v2.mdx @@ -14,7 +14,7 @@ Support for Python 3.9 has been dropped. The Apify Python API Client v2.x now re ## New underlying HTTP library -In v2.0, the Apify Python API client switched from using [`httpx`](https://github.com/pydantic/httpx2) to [`impit`](https://github.com/apify/impit) as the underlying HTTP library. However, this change shouldn't have much impact on the end user. +In v2.0, the Apify Python API client switched from using [`httpx`](https://www.python-httpx.org/) to [`impit`](https://github.com/apify/impit) as the underlying HTTP library. However, this change shouldn't have much impact on the end user. ## API method changes diff --git a/website/versioned_docs/version-3.1/02_concepts/10_custom_http_clients.mdx b/website/versioned_docs/version-3.1/02_concepts/10_custom_http_clients.mdx index ece32fb1..0d2e41c1 100644 --- a/website/versioned_docs/version-3.1/02_concepts/10_custom_http_clients.mdx +++ b/website/versioned_docs/version-3.1/02_concepts/10_custom_http_clients.mdx @@ -92,7 +92,7 @@ It must return an object satisfying the `HttpRe | `aiter_bytes() -> AsyncIterator[bytes]` | Iterate body in chunks (async) | :::note -Many HTTP libraries, including our default [Impit](https://github.com/apify/impit) or for example [HTTPX](https://github.com/pydantic/httpx2) already satisfy this protocol out of the box. +Many HTTP libraries, including our default [Impit](https://github.com/apify/impit) or for example [HTTPX](https://www.python-httpx.org/) already satisfy this protocol out of the box. ::: ### Plugging it in @@ -122,7 +122,7 @@ When using a custom HTTP client, you are responsible for constructing the reques Custom HTTP clients might be useful when you need to: -- **Use a different HTTP library** - Swap Impit for [httpx](https://github.com/pydantic/httpx2), [requests](https://requests.readthedocs.io/), or [aiohttp](https://docs.aiohttp.org/). +- **Use a different HTTP library** - Swap Impit for [httpx](https://www.python-httpx.org/), [requests](https://requests.readthedocs.io/), or [aiohttp](https://docs.aiohttp.org/). - **Route through a proxy** - Add proxy support or request routing. - **Implement custom retry logic** - Use different backoff strategies or retry conditions. - **Log requests and responses** - Track API calls for debugging or auditing. diff --git a/website/versioned_docs/version-3.1/03_guides/05_custom_http_client.mdx b/website/versioned_docs/version-3.1/03_guides/05_custom_http_client.mdx index fa9fbc26..b7314a1c 100644 --- a/website/versioned_docs/version-3.1/03_guides/05_custom_http_client.mdx +++ b/website/versioned_docs/version-3.1/03_guides/05_custom_http_client.mdx @@ -12,11 +12,11 @@ import CodeBlock from '@theme/CodeBlock'; import CustomHttpClientAsyncExample from '!!raw-loader!./code/05_custom_http_client_async.py'; import CustomHttpClientSyncExample from '!!raw-loader!./code/05_custom_http_client_sync.py'; -This guide shows how to replace the default `ImpitHttpClient` and `ImpitHttpClientAsync` with one based on [HTTPX](https://github.com/pydantic/httpx2). The same approach works for any HTTP library — see [Custom HTTP clients](/api/client/python/docs/concepts/custom-http-clients) for the underlying architecture. +This guide shows how to replace the default `ImpitHttpClient` and `ImpitHttpClientAsync` with one based on [HTTPX](https://www.python-httpx.org/). The same approach works for any HTTP library — see [Custom HTTP clients](/api/client/python/docs/concepts/custom-http-clients) for the underlying architecture. ## Why HTTPX? -You might want to use [HTTPX](https://github.com/pydantic/httpx2) instead of the default [Impit](https://github.com/apify/impit)-based client for reasons like: +You might want to use [HTTPX](https://www.python-httpx.org/) instead of the default [Impit](https://github.com/apify/impit)-based client for reasons like: - You already use HTTPX in your project and want a single HTTP stack. - You need HTTPX-specific features. @@ -56,7 +56,7 @@ When using a custom HTTP client, you are responsible for handling retries, timeo The example above is minimal on purpose. In a production setup, you might want to extend it with: -- **Retry logic** - Use [HTTPX's event hooks](https://github.com/pydantic/httpx2) or utilize library like [tenacity](https://tenacity.readthedocs.io/) to retry failed requests. +- **Retry logic** - Use [HTTPX's event hooks](https://www.python-httpx.org/advanced/event-hooks/) or utilize library like [tenacity](https://tenacity.readthedocs.io/) to retry failed requests. - **Custom headers** - You can add headers in the `call` method before delegating to HTTPX. - **Connection lifecycle** - Close the underlying `httpx.Client` when done by adding a `close()` method to your custom client. - **Proxy support** - You can pass `proxy=...` when creating the `httpx.Client`. diff --git a/website/versioned_docs/version-3.1/04_upgrading/upgrading_to_v2.mdx b/website/versioned_docs/version-3.1/04_upgrading/upgrading_to_v2.mdx index c794503f..ed02202b 100644 --- a/website/versioned_docs/version-3.1/04_upgrading/upgrading_to_v2.mdx +++ b/website/versioned_docs/version-3.1/04_upgrading/upgrading_to_v2.mdx @@ -14,7 +14,7 @@ Support for Python 3.9 has been dropped. The Apify Python API Client v2.x now re ## New underlying HTTP library -In v2.0, the Apify Python API client switched from using [`httpx`](https://github.com/pydantic/httpx2) to [`impit`](https://github.com/apify/impit) as the underlying HTTP library. However, this change shouldn't have much impact on the end user. +In v2.0, the Apify Python API client switched from using [`httpx`](https://www.python-httpx.org/) to [`impit`](https://github.com/apify/impit) as the underlying HTTP library. However, this change shouldn't have much impact on the end user. ## API method changes From 98365c642f6c4371e9f3360f72e4ed7421d597c0 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Thu, 27 Aug 2026 12:40:54 +0200 Subject: [PATCH 6/6] docs: Refine the HTTP client guide, concepts page, and examples --- README.md | 2 +- docs/01_introduction/index.mdx | 7 +- docs/02_concepts/10_custom_http_clients.mdx | 82 +++++-------------- .../02_concepts/code/10_httpx_client_async.py | 4 +- docs/02_concepts/code/10_httpx_client_sync.py | 8 +- docs/03_guides/05_custom_http_client.mdx | 28 ++----- .../code/05_custom_http_client_async.py | 14 +--- .../code/05_custom_http_client_sync.py | 24 +++--- docs/04_upgrading/upgrading_to_v3.mdx | 2 +- pyproject.toml | 8 -- 10 files changed, 58 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index 444f0a60..98f5c6f6 100644 --- a/README.md +++ b/README.md @@ -202,7 +202,7 @@ The full documentation lives at **[docs.apify.com/api/client/python](https://doc | --- | --- | | [Introduction](https://docs.apify.com/api/client/python/docs) | Overview, prerequisites, and a tour of the client. | | [Quick start](https://docs.apify.com/api/client/python/docs/quick-start) | Authenticate, run an Actor, and fetch its results step by step. | -| [Concepts](https://docs.apify.com/api/client/python/docs/concepts/asyncio-support) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts. | +| [Concepts](https://docs.apify.com/api/client/python/docs/concepts/asyncio-support) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, HTTP clients, timeouts. | | [Guides](https://docs.apify.com/api/client/python/docs/guides/passing-input-to-actor) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), build a custom HTTP client. | | [Upgrading](https://docs.apify.com/api/client/python/docs/upgrading/upgrading-to-v3) | Migrating between major versions. | | [API reference](https://docs.apify.com/api/client/python/reference) | Generated reference for every class, method, and model. | diff --git a/docs/01_introduction/index.mdx b/docs/01_introduction/index.mdx index f445bcff..4f67ba11 100644 --- a/docs/01_introduction/index.mdx +++ b/docs/01_introduction/index.mdx @@ -63,9 +63,7 @@ For better request-body compression, opt in to `brotli`, which compresses better For details, see [HTTP compression](../02_concepts/13_http_compression.mdx). -The client uses [Impit](https://github.com/apify/impit) as its default HTTP transport. To use the built-in -[HTTPX](https://github.com/pydantic/httpx2) transport, install its optional dependency. It pulls in `httpx2`, -Pydantic's maintained continuation of HTTPX: +The client uses [Impit](https://github.com/apify/impit) as its default HTTP transport. To use the built-in [HTTPX](https://github.com/pydantic/httpx2) transport, install the optional `httpx` extra. The extra installs `httpx2`, Pydantic's maintained continuation of HTTPX: @@ -80,8 +78,7 @@ Pydantic's maintained continuation of HTTPX: -See [HTTP clients](../02_concepts/10_custom_http_clients.mdx) for synchronous and asynchronous examples and details -about the shared architecture. +For synchronous and asynchronous examples and details about the shared architecture, see [HTTP clients](../02_concepts/10_custom_http_clients.mdx). ## Quick example diff --git a/docs/02_concepts/10_custom_http_clients.mdx b/docs/02_concepts/10_custom_http_clients.mdx index ae658d4d..625ae01f 100644 --- a/docs/02_concepts/10_custom_http_clients.mdx +++ b/docs/02_concepts/10_custom_http_clients.mdx @@ -19,9 +19,7 @@ import ArchitectureImportsExample from '!!raw-loader!./code/10_architecture_impo import PluggingInAsyncExample from '!!raw-loader!./code/10_plugging_in_async.py'; import PluggingInSyncExample from '!!raw-loader!./code/10_plugging_in_sync.py'; -The Apify API client uses a pluggable HTTP layer. It ships with an [Impit](https://github.com/apify/impit)-based default, -offers [HTTPX](https://github.com/pydantic/httpx2) as an optional built-in alternative, and accepts fully custom synchronous -or asynchronous implementations. +The Apify API client uses a pluggable HTTP layer. It ships with an [Impit](https://github.com/apify/impit)-based default, offers [HTTPX](https://github.com/pydantic/httpx2) as an optional built-in alternative, and accepts custom synchronous or asynchronous implementations. ## Default HTTP client @@ -49,15 +47,9 @@ You can configure the default client through the `HttpxHttpClient` and -`HttpxHttpClientAsync`. They use the same request preparation, -compression, retry policy, timeout tiers and growth, error handling, logging, and statistics as the default Impit clients, with -[HTTPX](https://github.com/pydantic/httpx2) as the transport. +The package also provides `HttpxHttpClient` and `HttpxHttpClientAsync`. They use the same request preparation, compression, retry policy, timeout tiers and growth, error handling, logging, and statistics as the default Impit clients, with [HTTPX](https://github.com/pydantic/httpx2) as the transport. -HTTPX is an optional dependency. Install `apify-client[httpx]`, then pass the appropriate client to -`ApifyClient.with_custom_http_client`. The extra -installs `httpx2`, Pydantic's maintained continuation of HTTPX. Impit remains the default even when the HTTPX extra -is installed. +HTTPX is an optional dependency. Install `apify-client[httpx]`, then pass the appropriate client to `ApifyClient.with_custom_http_client`. The extra installs `httpx2`, Pydantic's maintained continuation of HTTPX. Impit remains the default even when the HTTPX extra is installed. ```bash pip install "apify-client[httpx]" @@ -78,44 +70,25 @@ uv add "apify-client[httpx]" -Configure retries, timeout tiers, default headers, and compression on the HTTPX client instance. The token passed to -`with_custom_http_client` is applied automatically unless the HTTP client already has an `Authorization` header. -The examples use the clients as context managers so their connection pools are closed deterministically. If a context -manager does not fit your application's lifecycle, call `close()` on `HttpxHttpClient` or `await aclose()` on -`HttpxHttpClientAsync` during shutdown. +Configure retries, timeout tiers, default headers, and compression on the HTTPX client instance. The token passed to `with_custom_http_client` is applied automatically unless the HTTP client already has an `Authorization` header. The examples use the clients as context managers so their connection pools are closed deterministically. If a context manager doesn't fit your application's lifecycle, call `close()` on `HttpxHttpClient` or `await aclose()` on `HttpxHttpClientAsync` during shutdown. -Timeout values are passed to the selected transport. Impit treats them as whole-request timeouts, while HTTPX applies -its connect, read, write, and pool timeout semantics. In particular, an HTTPX read timeout limits inactivity between -chunks rather than the total duration of a streamed response. The `no_timeout` option disables HTTPX's timeouts. +Timeout values are passed to the selected transport. Impit enforces them as a deadline for the whole request, body included. HTTPX applies them to each socket operation instead, so a response whose body arrives slowly keeps resetting the timeout and can outlast both the requested timeout and `timeout_max`. The `no_timeout` option disables HTTPX's timeouts. ## Architecture Internally, the HTTP client hierarchy has three layers: -- A common internal base contains configuration and utilities shared by synchronous and asynchronous clients, including - headers, request-body preparation, parameters, compression, and timeout tiers. It is not a public extension point. -- `HttpClient` and `HttpClientAsync` - add the synchronous or asynchronous request pipeline, retry loop, transport hooks, and lifecycle interface. -- The built-in Impit and HTTPX classes inherit directly from the corresponding sync or async class and adapt the - underlying transport. +- A common internal base contains configuration and utilities shared by synchronous and asynchronous clients, including headers, request-body preparation, parameters, compression, and timeout tiers. It isn't a public extension point. +- `HttpClient` and `HttpClientAsync` add the synchronous or asynchronous request pipeline, retry loop, transport hooks, and lifecycle interface. +- The built-in Impit and HTTPX classes inherit directly from the corresponding sync or async class and adapt the underlying transport. -`HttpClient.is_timeout_error(exc)` and `HttpClientAsync.is_timeout_error(exc)` provide the public, transport-neutral way -to determine whether an exception is a timeout. Their shared implementation recognizes Python's `TimeoutError`; -transport adapters override it when their HTTP library defines additional timeout exception types. This lets -higher-level features such as streamed logs classify timeouts without depending on Impit, HTTPX, or private -implementation details. +`HttpClient.is_timeout_error(exc)` and `HttpClientAsync.is_timeout_error(exc)` are the public, transport-neutral way to tell whether an exception is a timeout, so code built on the client, such as streamed logs, doesn't need to know which transport raised it. -Responses use one separate abstraction: +Responses have their own abstraction. `HttpResponse` is a [runtime-checkable protocol](https://docs.python.org/3/library/typing.html#typing.runtime_checkable) that defines the expected response shape. Any object with the required attributes and methods satisfies the protocol, so no inheritance is needed. -- `HttpResponse` - A [runtime-checkable protocol](https://docs.python.org/3/library/typing.html#typing.runtime_checkable) that defines the expected response shape. Any object with the required attributes and methods satisfies the protocol — no inheritance needed. +Custom transport adapters implement the request, error-classification, and lifecycle hooks. They inherit request preparation, retries, timeout growth, API error conversion, logging, and statistics from the base. -To plug in your custom implementation, use the `ApifyClient.with_custom_http_client` class method. - -The built-in Impit and HTTPX classes are thin transport adapters over the request implementation in `HttpClient` and -`HttpClientAsync`. Custom transport adapters implement the request, error-classification, and lifecycle hooks. They -inherit request construction, retries, timeout growth, API error conversion, logging, and statistics from the base. - -All of these are available from the `apify_client.http_clients` module: +The base classes and the response protocol are available from the `apify_client.http_clients` module: {ArchitectureImportsExample} @@ -125,18 +98,12 @@ All of these are available from the `apify_client.http_clients` module: The public `call` method provides the shared request pipeline. A concrete transport implements these hooks: -- `send_request(...)` sends one prepared request and returns an `HttpResponse`. The inherited `call` needs it, so - every transport adapter has to implement it. -- `is_retryable_transport_error(exc)` classifies transport failures for the shared retry loop. The default classifies - nothing as retryable, so a transport that skips it gives up on the first connection failure. -- `is_timeout_error(exc)` identifies transport-specific timeout exceptions for higher-level client features. The - default recognizes Python's `TimeoutError`. Timeout classification is independent of retryability, so a timeout - the retry loop should retry has to be listed in `is_retryable_transport_error` too. -- `close()` or `aclose()` closes resources owned by the transport. The default does nothing, which is correct for a - transport that owns no pool or session. +- `send_request(...)` sends one prepared request and returns an `HttpResponse`, error statuses included. It receives the URL with the query parameters already encoded into it, the headers with the client's default headers already merged in, the body already serialized and compressed, and the timeout for this attempt in seconds. The inherited `call` needs it, so every transport adapter has to implement it. Let the HTTP library's exceptions propagate unwrapped, and leave status handling and `ApifyApiError` to `call`. +- `is_retryable_transport_error(exc)` classifies transport failures for the shared retry loop. The default classifies nothing as retryable, so a transport that doesn't override it gives up on the first connection failure. +- `is_timeout_error(exc)` identifies transport-specific timeout exceptions for higher-level client features. The default recognizes Python's `TimeoutError`. Timeout classification is independent of retryability, so a timeout the retry loop should retry has to be listed in `is_retryable_transport_error` too. +- `close()` or `aclose()` closes resources owned by the transport. The default does nothing, which is correct for a transport that owns no pool or session. -The `@override` decorators in the built-in Impit and HTTPX adapters make these implementations explicit and allow type -checkers to catch misspelled or incompatible overrides. +Decorate your implementations with `@override`, as the built-in Impit and HTTPX adapters do, so a type checker catches a misspelled or incompatible override. ### The HTTP response protocol @@ -159,9 +126,7 @@ checkers to catch misspelled or incompatible overrides. :::note Many HTTP libraries, including our default [Impit](https://github.com/apify/impit) or for example [HTTPX](https://github.com/pydantic/httpx2) already satisfy this protocol out of the box. -For a streamed response, consume the body inside its context manager with `iter_bytes()` / `aiter_bytes()`, or call -`read()` / `aread()` before accessing `content`. Some transports, including HTTPX, intentionally reject `content` on -an unread streamed response. +For a streamed response, consume the body inside the streaming context manager with `iter_bytes()` / `aiter_bytes()`, or call `read()` / `aread()` before accessing `content`. Some transports, including HTTPX, intentionally reject `content` on an unread streamed response. ::: ### Plugging it in @@ -184,15 +149,12 @@ Use the `ApifyClient.wit After that, all API calls made through the client will go through your custom HTTP client. :::warning -If you override `call` itself, your implementation becomes responsible for request preparation, retries, timeouts, API -error conversion, logging, and statistics. Implementing the transport hooks and inheriting `call` keeps the shared -behavior. +If you override `call` itself, your implementation becomes responsible for request preparation, retries, timeouts, API error conversion, logging, and statistics. Implementing the transport hooks and inheriting `call` keeps the shared behavior. ::: ## Use cases -Custom HTTP clients might be useful when the built-in Impit and HTTPX clients do not cover your requirements, for -example when you need to: +Custom HTTP clients might be useful when the built-in Impit and HTTPX clients don't cover your requirements, for example when you need to: - **Use a different HTTP library** - Integrate [requests](https://requests.readthedocs.io/), [aiohttp](https://docs.aiohttp.org/), or another transport. - **Route through a proxy** - Add proxy support or request routing. @@ -201,6 +163,4 @@ example when you need to: - **Modify requests** - Add custom fields, modify the body, or change headers. - **Collect custom metrics** - Measure request latency, track error rates, or count API calls. -For complete synchronous and asynchronous implementations over a transport with a different response API, see -[Build a custom HTTP client](../03_guides/05_custom_http_client.mdx). You can also refer to the -`HttpClient` API reference for the synchronous contract. +For complete synchronous and asynchronous implementations over a transport with a different response API, see [Build a custom HTTP client](../03_guides/05_custom_http_client.mdx). The `HttpClient` and `HttpClientAsync` API references document the full contract. diff --git a/docs/02_concepts/code/10_httpx_client_async.py b/docs/02_concepts/code/10_httpx_client_async.py index 8a7062ed..2c2cbe99 100644 --- a/docs/02_concepts/code/10_httpx_client_async.py +++ b/docs/02_concepts/code/10_httpx_client_async.py @@ -3,11 +3,13 @@ from apify_client import ApifyClientAsync from apify_client.http_clients import HttpxHttpClientAsync +TOKEN = 'MY-APIFY-TOKEN' + async def main() -> None: async with HttpxHttpClientAsync() as http_client: client = ApifyClientAsync.with_custom_http_client( - token='MY-APIFY-TOKEN', + token=TOKEN, http_client=http_client, ) print(await client.actor('apify/hello-world').get()) diff --git a/docs/02_concepts/code/10_httpx_client_sync.py b/docs/02_concepts/code/10_httpx_client_sync.py index f5de3cb3..7a00cd84 100644 --- a/docs/02_concepts/code/10_httpx_client_sync.py +++ b/docs/02_concepts/code/10_httpx_client_sync.py @@ -1,11 +1,17 @@ from apify_client import ApifyClient from apify_client.http_clients import HttpxHttpClient +TOKEN = 'MY-APIFY-TOKEN' + def main() -> None: with HttpxHttpClient() as http_client: client = ApifyClient.with_custom_http_client( - token='MY-APIFY-TOKEN', + token=TOKEN, http_client=http_client, ) print(client.actor('apify/hello-world').get()) + + +if __name__ == '__main__': + main() diff --git a/docs/03_guides/05_custom_http_client.mdx b/docs/03_guides/05_custom_http_client.mdx index 77e05a7b..52a02848 100644 --- a/docs/03_guides/05_custom_http_client.mdx +++ b/docs/03_guides/05_custom_http_client.mdx @@ -1,7 +1,7 @@ --- id: custom-http-client title: Build a custom HTTP client -description: Implement the HTTP client contract with AIOHTTP and requests. +description: Implement the HTTP client contract with aiohttp and requests. --- import ApiLink from '@theme/ApiLink'; @@ -12,18 +12,13 @@ import TabItem from '@theme/TabItem'; import CustomHttpClientAsyncExample from '!!raw-loader!./code/05_custom_http_client_async.py'; import CustomHttpClientSyncExample from '!!raw-loader!./code/05_custom_http_client_sync.py'; -This guide implements a custom `HttpClientAsync` with -[AIOHTTP](https://docs.aiohttp.org/) and a custom `HttpClient` with -[requests](https://requests.readthedocs.io/). Neither library satisfies the -`HttpResponse` protocol, so both examples also show how to adapt a -foreign response API. +This guide implements a custom `HttpClientAsync` with [aiohttp](https://docs.aiohttp.org/) and a custom `HttpClient` with [requests](https://requests.readthedocs.io/). Neither library satisfies the `HttpResponse` protocol, so both examples also show how to adapt a foreign response API. -For an overview of the architecture and the built-in Impit and HTTPX implementations, see -[HTTP clients](../02_concepts/10_custom_http_clients.mdx). +For an overview of the architecture and the built-in Impit and HTTPX implementations, see [HTTP clients](../02_concepts/10_custom_http_clients.mdx). ## Installation -Install the transport alongside the Apify client. Neither AIOHTTP nor requests is an `apify-client` extra: +Install the transport alongside the Apify client. Neither aiohttp nor requests is an `apify-client` extra: ```bash pip install apify-client aiohttp # for the asynchronous client @@ -34,13 +29,9 @@ pip install apify-client requests # for the synchronous client Each example has three parts: -1. The response adapter, `AiohttpResponse` or `RequestsResponse`, maps the library's own response onto the - `HttpResponse` protocol that resource clients expect. -2. The client, `AiohttpHttpClient` or `RequestsHttpClient`, implements the transport, error-classification, - timeout-classification, and lifecycle hooks. It inherits request preparation, retry handling, timeout growth, - and API error conversion from its base class. -3. `with_custom_http_client()` connects the implementation to the resource clients and applies the API token. The - context manager closes the session at shutdown. +1. The response adapter, `AiohttpResponse` or `RequestsResponse`, maps the library's own response onto the `HttpResponse` protocol that resource clients expect. +2. The client, `AiohttpHttpClient` or `RequestsHttpClient`, implements the transport, error-classification, and lifecycle hooks. It inherits request preparation, retry handling, timeout growth, and API error conversion from its base class. Only the requests client also overrides timeout classification. `requests.Timeout` doesn't derive from Python's `TimeoutError`, while aiohttp's timeout errors do, so the inherited default already recognizes them. +3. `with_custom_http_client()` connects the implementation to the resource clients and applies the API token. The context manager closes the session at shutdown. @@ -56,8 +47,5 @@ Each example has three parts: :::warning -These are compact integration examples, not a replacement for all built-in client behavior. A production custom -client should account for transport-specific details such as proxy configuration, TLS settings, redirects, and -response resource cleanup. The shared base provides retries, logging, statistics, timeout growth, and API error -conversion. +These examples are compact integrations, not a replacement for all built-in client behavior. A production custom client should account for transport-specific details such as proxy configuration, TLS settings, redirects, and response resource cleanup. Timeout semantics differ per transport too: the aiohttp example passes the value as a budget for the whole request, while `requests` applies it to each socket read. Both example sessions also keep a shared cookie jar, which replays server cookies on later API requests. The built-in HTTPX client clears it instead. ::: diff --git a/docs/03_guides/code/05_custom_http_client_async.py b/docs/03_guides/code/05_custom_http_client_async.py index efd1481c..c3aeb030 100644 --- a/docs/03_guides/code/05_custom_http_client_async.py +++ b/docs/03_guides/code/05_custom_http_client_async.py @@ -35,8 +35,7 @@ def headers(self) -> Mapping[str, str]: def content(self) -> bytes: if self._body is None: raise RuntimeError( - 'The streamed response has not been read yet; ' - 'use aread() or aiter_bytes()' + 'The streamed response has not been read yet; call aread() first' ) return self._body @@ -60,7 +59,6 @@ def close(self) -> None: self._response.close() async def aclose(self) -> None: - self._response.release() await self._response.wait_for_close() def iter_bytes(self) -> Iterator[bytes]: @@ -85,10 +83,8 @@ def __init__(self) -> None: self._session = aiohttp.ClientSession() @override - def is_timeout_error(self, exc: Exception) -> bool: - return super().is_timeout_error(exc) or isinstance( - exc, aiohttp.ServerTimeoutError - ) + def is_retryable_transport_error(self, exc: Exception) -> bool: + return isinstance(exc, (TimeoutError, aiohttp.ClientError)) @override async def aclose(self) -> None: @@ -119,10 +115,6 @@ async def send_request( return adapted_response - @override - def is_retryable_transport_error(self, exc: Exception) -> bool: - return isinstance(exc, (TimeoutError, aiohttp.ClientError)) - async def main() -> None: async with AiohttpHttpClient() as http_client: diff --git a/docs/03_guides/code/05_custom_http_client_sync.py b/docs/03_guides/code/05_custom_http_client_sync.py index 996dcef5..75386b63 100644 --- a/docs/03_guides/code/05_custom_http_client_sync.py +++ b/docs/03_guides/code/05_custom_http_client_sync.py @@ -34,7 +34,7 @@ def headers(self) -> Mapping[str, str]: def content(self) -> bytes: if self._body is None: raise RuntimeError( - 'The streamed response has not been read yet; use read() or iter_bytes()' + 'The streamed response has not been read yet; call read() first' ) return self._body @@ -83,6 +83,17 @@ def __init__(self) -> None: def is_timeout_error(self, exc: Exception) -> bool: return super().is_timeout_error(exc) or isinstance(exc, requests.Timeout) + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + return isinstance( + exc, + ( + requests.ConnectionError, + requests.Timeout, + requests.exceptions.ChunkedEncodingError, + ), + ) + @override def close(self) -> None: self._session.close() @@ -113,17 +124,6 @@ def send_request( return adapted_response - @override - def is_retryable_transport_error(self, exc: Exception) -> bool: - return isinstance( - exc, - ( - requests.ConnectionError, - requests.Timeout, - requests.exceptions.ChunkedEncodingError, - ), - ) - def main() -> None: with RequestsHttpClient() as http_client: diff --git a/docs/04_upgrading/upgrading_to_v3.mdx b/docs/04_upgrading/upgrading_to_v3.mdx index 1483021e..61a3c0ef 100644 --- a/docs/04_upgrading/upgrading_to_v3.mdx +++ b/docs/04_upgrading/upgrading_to_v3.mdx @@ -140,4 +140,4 @@ Affected types: `ActorJobStatus`, `ActorPermissionLevel`, `ErrorType`, `GeneralA ## Pluggable HTTP client -Additive change, non-breaking. The HTTP layer is now abstracted behind the `HttpClient` and `HttpClientAsync` base classes, so you can swap the underlying transport for your own implementation. The default client built on [Impit](https://github.com/apify/impit) is unchanged and existing code keeps working out of the box. To plug in a custom client, pass an instance to `ApifyClient.with_custom_http_client()` — see [Custom HTTP clients](/api/client/python/docs/concepts/custom-http-clients) for the full walkthrough. +Additive change, non-breaking. The HTTP layer is now abstracted behind the `HttpClient` and `HttpClientAsync` base classes, so you can swap the underlying transport for your own implementation. The default client built on [Impit](https://github.com/apify/impit) is unchanged and existing code keeps working out of the box. To plug in a custom client, pass an instance to `ApifyClient.with_custom_http_client()` — see [HTTP clients](/api/client/python/docs/concepts/custom-http-clients) for the full walkthrough. diff --git a/pyproject.toml b/pyproject.toml index fa27e60a..a2f74f85 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -160,9 +160,6 @@ indent-style = "space" "**/docs/02_concepts/code/10_architecture_imports.py" = [ "F401", # Imported but unused ] -"**/docs/03_guides/code/05_custom_http_client_{async,sync}.py" = [ - "ARG002", # Unused method argument -] "src/apify_client/_{models,typeddicts}.py" = [ "D", # Everything from the pydocstyle "E501", # Line too long @@ -208,11 +205,6 @@ include = ["docs/**/*.py", "website/**/*.py"] [tool.ty.overrides.rules] unresolved-import = "ignore" -[[tool.ty.overrides]] -include = ["docs/**/05_custom_http_client_async.py"] -[tool.ty.overrides.rules] -invalid-argument-type = "ignore" - [tool.coverage.report] exclude_lines = ["pragma: no cover", "if TYPE_CHECKING:", "assert_never()"]