diff --git a/.rules.md b/.rules.md index cced5614..3563212e 100644 --- a/.rules.md +++ b/.rules.md @@ -63,6 +63,7 @@ Docstrings are written on sync clients and **automatically copied** to async cli - `HttpClient`/`HttpClientAsync` — base classes in `http_clients/_base.py` holding the shared request pipeline (retries, timeouts, API errors); transports implement the `send_request`, error-classification, and lifecycle hooks - `ImpitHttpClient`/`ImpitHttpClientAsync` — default implementation (Rust-based Impit) +- `HttpxHttpClient`/`HttpxHttpClientAsync` — built-in alternative behind the `httpx2` optional extra - `HttpResponse` — Protocol (not a concrete class) for response objects - Users can plug in custom HTTP clients via `ApifyClient.with_custom_http_client()` diff --git a/README.md b/README.md index ba5b9084..d9224865 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,17 @@ uv add "apify-client[brotli]" ``` + [Impit](https://github.com/apify/impit) is the default HTTP client and is installed automatically. To use the + built-in [HTTPX](https://github.com/pydantic/httpx2) client instead, install its optional extra and pass + `http_client=HttpxHttpClient()` to `ApifyClient.with_custom_http_client()`. The extra installs `httpx2`, + Pydantic's maintained continuation of HTTPX: + + ```bash + pip install "apify-client[httpx2]" + # or + uv add "apify-client[httpx2]" + ``` + - From [conda-forge](https://anaconda.org/conda-forge/apify-client), it can be installed with [conda](https://docs.conda.io/en/latest/): ```bash @@ -124,7 +135,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)). - **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)). - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)). -- **Pluggable HTTP layer** — swap the default [Impit](https://github.com/apify/impit)-based HTTP client for `httpx`, `requests`, `aiohttp`, or any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). +- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX](https://github.com/pydantic/httpx2) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)). - **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more ([Logging](https://docs.apify.com/api/client/python/docs/concepts/logging)). diff --git a/pyproject.toml b/pyproject.toml index 01e15bc8..dc7409c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ dependencies = [ [project.optional-dependencies] brotli = ["brotli>=1.0.9"] +httpx2 = ["httpx2>=2.0.0"] [project.urls] "Apify Homepage" = "https://apify.com" diff --git a/src/apify_client/http_clients/__init__.py b/src/apify_client/http_clients/__init__.py index 417a0705..b5f3a11a 100644 --- a/src/apify_client/http_clients/__init__.py +++ b/src/apify_client/http_clients/__init__.py @@ -1,10 +1,35 @@ +from apify_client._utils.try_import import install_import_hook as _install_import_hook +from apify_client._utils.try_import import try_import as _try_import from apify_client.http_clients._base import HttpClient, HttpClientAsync, HttpResponse from apify_client.http_clients._impit import ImpitHttpClient, ImpitHttpClientAsync -__all__ = [ - 'HttpClient', - 'HttpClientAsync', - 'HttpResponse', - 'ImpitHttpClient', - 'ImpitHttpClientAsync', -] +_install_import_hook(__name__) + +# `httpx2` is an optional extra, so it's wrapped in try_import. Accessing the HTTPX clients +# without the extra installed raises a clear ImportError instead of failing at package import time. +with _try_import( + __name__, + 'HttpxHttpClient', + 'HttpxHttpClientAsync', + dependency_name='httpx2', +) as _httpx_import: + from apify_client.http_clients._httpx import HttpxHttpClient, HttpxHttpClientAsync + +if _httpx_import.available: + __all__ = [ + 'HttpClient', + 'HttpClientAsync', + 'HttpResponse', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', + 'ImpitHttpClient', + 'ImpitHttpClientAsync', + ] +else: + __all__ = [ + 'HttpClient', + 'HttpClientAsync', + 'HttpResponse', + 'ImpitHttpClient', + 'ImpitHttpClientAsync', + ] diff --git a/src/apify_client/http_clients/_httpx.py b/src/apify_client/http_clients/_httpx.py new file mode 100644 index 00000000..fc39a34f --- /dev/null +++ b/src/apify_client/http_clients/_httpx.py @@ -0,0 +1,258 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +import httpx2 as httpx +from typing_extensions import override + +from apify_client._consts import ( + DEFAULT_MAX_RETRIES, + DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + DEFAULT_TIMEOUT_LONG, + DEFAULT_TIMEOUT_MAX, + DEFAULT_TIMEOUT_MEDIUM, + DEFAULT_TIMEOUT_SHORT, +) +from apify_client._docs import docs_group +from apify_client.http_clients._base import HttpClient, HttpClientAsync + +if TYPE_CHECKING: + from datetime import timedelta + + from apify_client._statistics import ClientStatistics + from apify_client.http_compressors._base import HttpCompressor + + +_PERMANENT_ERRORS = ( + # A request HTTPX rejects before sending it, e.g. one carrying an invalid header value. + httpx.LocalProtocolError, + # A URL scheme HTTPX refuses to speak, which repeating the request cannot change. + httpx.UnsupportedProtocol, + # An over-long redirect chain is a routing loop, which repeating the request cannot break. + httpx.TooManyRedirects, + # Only `Response.raise_for_status()` raises this, and the client never calls it - the shared pipeline decides on + # status codes from the response itself. + httpx.HTTPStatusError, +) +"""HTTPX errors that a retry cannot fix. Everything else in the `httpx.HTTPError` tree counts as transient.""" + + +@docs_group('HTTP clients') +class HttpxHttpClient(HttpClient): + """Synchronous HTTP client for the Apify API built on top of [HTTPX](https://github.com/pydantic/httpx2). + + This client wraps `httpx.Client` and adds automatic retries with exponential backoff for rate-limited + (HTTP 429) and server error (HTTP 5xx) responses. + + HTTPX applies a request timeout to each socket operation rather than to the request as a whole, so a response + whose body arrives slowly keeps resetting it and can outlast both the requested timeout and `timeout_max`. The + default Impit client enforces the same value as a deadline for the whole request, body included. + + Requires the `httpx2` extra: `pip install "apify-client[httpx2]"`. That extra installs `httpx2`, Pydantic's + maintained continuation of HTTPX, which this module imports under the `httpx` name. + """ + + def __init__( + self, + *, + token: str | None = None, + timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT, + timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM, + timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, + timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, + max_retries: int = DEFAULT_MAX_RETRIES, + min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + statistics: ClientStatistics | None = None, + headers: dict[str, str] | None = None, + http_compressor: HttpCompressor | None = None, + ) -> None: + """Initialize the HTTPX-based synchronous HTTP client. + + Args: + token: Apify API token for authentication. + timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). + timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). + timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). + timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. + max_retries: Maximum number of retry attempts for failed requests. + min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). + statistics: Statistics tracker for API calls. Created automatically if not provided. + headers: Additional HTTP headers to include in all requests. + http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. + """ + super().__init__( + token=token, + timeout_short=timeout_short, + timeout_medium=timeout_medium, + timeout_long=timeout_long, + timeout_max=timeout_max, + max_retries=max_retries, + min_delay_between_retries=min_delay_between_retries, + statistics=statistics, + headers=headers, + http_compressor=http_compressor, + ) + + self._httpx_client = httpx.Client( + follow_redirects=True, + event_hooks={'response': [self._clear_response_cookies]}, + ) + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX adds later is retried rather than + # silently treated as fatal. HTTP status code errors are handled by the shared pipeline based on the + # response status code, not here. + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + + @override + def close(self) -> None: + """Close the underlying HTTPX connection pool.""" + self._httpx_client.close() + + @override + def send_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> httpx.Response: + request = self._httpx_client.build_request( + method=method, + url=url, + headers=headers, + content=content, + timeout=timeout, + ) + _restore_explicit_cookie_header(request, headers) + return self._httpx_client.send(request, stream=stream) + + def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_client.cookies.clear() + + +@docs_group('HTTP clients') +class HttpxHttpClientAsync(HttpClientAsync): + """Asynchronous HTTP client for the Apify API built on top of [HTTPX](https://github.com/pydantic/httpx2). + + This client wraps `httpx.AsyncClient` and adds automatic retries with exponential backoff for rate-limited + (HTTP 429) and server error (HTTP 5xx) responses. + + HTTPX applies a request timeout to each socket operation rather than to the request as a whole, so a response + whose body arrives slowly keeps resetting it and can outlast both the requested timeout and `timeout_max`. The + default Impit client enforces the same value as a deadline for the whole request, body included. + + Requires the `httpx2` extra: `pip install "apify-client[httpx2]"`. That extra installs `httpx2`, Pydantic's + maintained continuation of HTTPX, which this module imports under the `httpx` name. + """ + + def __init__( + self, + *, + token: str | None = None, + timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT, + timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM, + timeout_long: timedelta = DEFAULT_TIMEOUT_LONG, + timeout_max: timedelta = DEFAULT_TIMEOUT_MAX, + max_retries: int = DEFAULT_MAX_RETRIES, + min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES, + statistics: ClientStatistics | None = None, + headers: dict[str, str] | None = None, + http_compressor: HttpCompressor | None = None, + ) -> None: + """Initialize the HTTPX-based asynchronous HTTP client. + + Args: + token: Apify API token for authentication. + timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...). + timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...). + timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...). + timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts. + max_retries: Maximum number of retry attempts for failed requests. + min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt). + statistics: Statistics tracker for API calls. Created automatically if not provided. + headers: Additional HTTP headers to include in all requests. + http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`. + """ + super().__init__( + token=token, + timeout_short=timeout_short, + timeout_medium=timeout_medium, + timeout_long=timeout_long, + timeout_max=timeout_max, + max_retries=max_retries, + min_delay_between_retries=min_delay_between_retries, + statistics=statistics, + headers=headers, + http_compressor=http_compressor, + ) + + self._httpx_async_client = httpx.AsyncClient( + follow_redirects=True, + event_hooks={'response': [self._clear_response_cookies]}, + ) + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException) + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in + # `_PERMANENT_ERRORS`. Retrying is the default so a subclass HTTPX adds later is retried rather than + # silently treated as fatal. HTTP status code errors are handled by the shared pipeline based on the + # response status code, not here. + return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS) + + @override + async def aclose(self) -> None: + """Close the underlying asynchronous HTTPX connection pool.""" + await self._httpx_async_client.aclose() + + @override + async def send_request( + self, + *, + method: str, + url: str, + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> httpx.Response: + request = self._httpx_async_client.build_request( + method=method, + url=url, + headers=headers, + content=content, + timeout=timeout, + ) + _restore_explicit_cookie_header(request, headers) + return await self._httpx_async_client.send(request, stream=stream) + + async def _clear_response_cookies(self, _response: httpx.Response) -> None: + """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests.""" + self._httpx_async_client.cookies.clear() + + +def _restore_explicit_cookie_header(request: httpx.Request, headers: dict[str, str]) -> None: + """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar. + + HTTPX drops the `Cookie` header when it builds a redirect request and rebuilds it from the jar, so an explicit + cookie only reaches the first hop of a redirected request. + """ + explicit_cookie = next((value for key, value in headers.items() if key.lower() == 'cookie'), None) + if explicit_cookie is None: + request.headers.pop('cookie', None) + else: + request.headers['cookie'] = explicit_cookie diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1db53b71..e0c9bed6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -2,6 +2,7 @@ import json import os +from dataclasses import dataclass from typing import TYPE_CHECKING import pytest @@ -17,9 +18,35 @@ from apify_client import ApifyClient, ApifyClientAsync from apify_client._consts import DEFAULT_API_URL from apify_client._utils.crypto import create_hmac_signature, create_storage_content_signature +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) if TYPE_CHECKING: - from collections.abc import Generator + from collections.abc import AsyncGenerator, Generator + + +@dataclass(frozen=True) +class HttpClientClasses: + """Synchronous and asynchronous variants of a built-in HTTP client.""" + + sync: type[HttpClient] + async_: type[HttpClientAsync] + + +DEFAULT_HTTP_CLIENT_CLASSES = HttpClientClasses(sync=ImpitHttpClient, async_=ImpitHttpClientAsync) +"""HTTP clients the live-API suite runs with unless a test asks for another transport.""" + +ALL_HTTP_CLIENT_CLASSES = [ + pytest.param(DEFAULT_HTTP_CLIENT_CLASSES, id='impit'), + pytest.param(HttpClientClasses(sync=HttpxHttpClient, async_=HttpxHttpClientAsync), id='httpx'), +] +"""Every built-in HTTP client, for tests that exercise transport behavior rather than an API resource.""" # ============================================================================ @@ -110,17 +137,17 @@ def test_kvs_of_another_user(api_token_2: str) -> Generator[KvsFixture]: @pytest.fixture -def apify_client(api_token: str) -> ApifyClient: - """Sync Apify client instance.""" - api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL - return ApifyClient(api_token, api_url=api_url) +def http_client_classes(request: pytest.FixtureRequest) -> HttpClientClasses: + """Return the sync and async classes of the HTTP client the test runs with. + Defaults to Impit so the live-API suite isn't multiplied by every transport. A transport-level test opts into + the full matrix with `@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True)`. + """ + if not hasattr(request, 'param'): + return DEFAULT_HTTP_CLIENT_CLASSES -@pytest.fixture -def apify_client_async(api_token: str) -> ApifyClientAsync: - """Async Apify client instance.""" - api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL - return ApifyClientAsync(api_token, api_url=api_url) + assert isinstance(request.param, HttpClientClasses) + return request.param @pytest.fixture(params=['sync', 'async']) @@ -130,13 +157,30 @@ def client_type(request: pytest.FixtureRequest) -> str: @pytest.fixture -def client( +async def client( client_type: str, - apify_client: ApifyClient, - apify_client_async: ApifyClientAsync, -) -> ApifyClient | ApifyClientAsync: - """Return sync or async client based on parametrization.""" - return apify_client if client_type == 'sync' else apify_client_async + api_token: str, + http_client_classes: HttpClientClasses, +) -> AsyncGenerator[ApifyClient | ApifyClientAsync]: + """Return each sync/async and HTTP client implementation combination.""" + api_url = os.getenv(API_URL_ENV_VAR) or DEFAULT_API_URL + if client_type == 'sync': + http_client = http_client_classes.sync() + yield ApifyClient.with_custom_http_client( + api_token, + api_url=api_url, + http_client=http_client, + ) + http_client.close() + return + + http_client_async = http_client_classes.async_() + yield ApifyClientAsync.with_custom_http_client( + api_token, + api_url=api_url, + http_client=http_client_async, + ) + await http_client_async.aclose() @pytest.fixture diff --git a/tests/integration/test_apify_client.py b/tests/integration/test_apify_client.py index 126f40b3..4c15eab8 100644 --- a/tests/integration/test_apify_client.py +++ b/tests/integration/test_apify_client.py @@ -4,13 +4,17 @@ from typing import TYPE_CHECKING +import pytest + from .._utils import maybe_await +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import UserPrivateInfo, UserPublicInfo if TYPE_CHECKING: from apify_client import ApifyClient, ApifyClientAsync +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_apify_client(client: ApifyClient | ApifyClientAsync) -> None: """Test basic apify client functionality.""" user_client = client.user('me') diff --git a/tests/integration/test_dataset.py b/tests/integration/test_dataset.py index 333c7229..b7acab4c 100644 --- a/tests/integration/test_dataset.py +++ b/tests/integration/test_dataset.py @@ -18,6 +18,7 @@ maybe_await, poll_until_condition, ) +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import Dataset, DatasetListItem, DatasetStatistics, ListOfDatasets from apify_client._resource_clients.dataset import DatasetItemsPage from apify_client.errors import ApifyApiError @@ -698,6 +699,7 @@ async def get_items() -> DatasetItemsPage: await maybe_await(dataset_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_dataset_stream_items(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming dataset items.""" dataset_name = get_random_resource_name('dataset') diff --git a/tests/integration/test_key_value_store.py b/tests/integration/test_key_value_store.py index 5d1d9238..fee82954 100644 --- a/tests/integration/test_key_value_store.py +++ b/tests/integration/test_key_value_store.py @@ -19,6 +19,7 @@ maybe_sleep, poll_until_condition, ) +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import KeyValueStore, KeyValueStoreKey, ListOfKeys, ListOfKeyValueStores from apify_client.errors import ApifyApiError from apify_client.http_clients import HttpResponse @@ -706,6 +707,7 @@ async def get_keys() -> ListOfKeys: await maybe_await(store_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_key_value_store_stream_record_own(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming a record from one's own key-value store (no signature).""" store_name = get_random_resource_name('kvs') diff --git a/tests/integration/test_log.py b/tests/integration/test_log.py index df687e2f..13db91f8 100644 --- a/tests/integration/test_log.py +++ b/tests/integration/test_log.py @@ -5,7 +5,10 @@ from contextlib import AbstractAsyncContextManager, AbstractContextManager from typing import TYPE_CHECKING +import pytest + from .._utils import maybe_await +from .conftest import ALL_HTTP_CLIENT_CLASSES from apify_client._models import ListOfBuilds, Run from apify_client.http_clients import HttpResponse @@ -72,6 +75,7 @@ async def test_log_get_as_bytes(client: ApifyClient | ApifyClientAsync) -> None: await maybe_await(run_client.delete()) +@pytest.mark.parametrize('http_client_classes', ALL_HTTP_CLIENT_CLASSES, indirect=True) async def test_log_stream_from_run(client: ApifyClient | ApifyClientAsync, *, is_async: bool) -> None: """Test streaming a run's log via the stream() context manager.""" actor = client.actor(HELLO_WORLD_ACTOR) diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 124c12cd..98d48d70 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -7,7 +7,14 @@ from pytest_httpserver import HTTPServer from apify_client import ApifyClient, ApifyClientAsync -from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) if TYPE_CHECKING: from collections.abc import Iterable @@ -43,13 +50,23 @@ def async_client(httpserver: HTTPServer) -> ApifyClientAsync: return ApifyClientAsync(token='test', api_url=httpserver.url_for('/').removesuffix('/')) -@pytest.fixture(params=[pytest.param(ImpitHttpClient, id='impit')]) +@pytest.fixture( + params=[ + pytest.param(ImpitHttpClient, id='impit'), + pytest.param(HttpxHttpClient, id='httpx'), + ] +) def http_client_class(request: pytest.FixtureRequest) -> type[HttpClient]: """Return each built-in synchronous HTTP client class.""" return request.param -@pytest.fixture(params=[pytest.param(ImpitHttpClientAsync, id='impit')]) +@pytest.fixture( + params=[ + pytest.param(ImpitHttpClientAsync, id='impit'), + pytest.param(HttpxHttpClientAsync, id='httpx'), + ] +) def http_client_async_class(request: pytest.FixtureRequest) -> type[HttpClientAsync]: """Return each built-in asynchronous HTTP client class.""" return request.param diff --git a/tests/unit/test_client_headers.py b/tests/unit/test_client_headers.py index 981d6857..3ecda41b 100644 --- a/tests/unit/test_client_headers.py +++ b/tests/unit/test_client_headers.py @@ -6,8 +6,11 @@ from importlib import metadata from typing import TYPE_CHECKING +import httpx2 as httpx from werkzeug import Request, Response +from apify_client.http_clients import HttpxHttpClient, HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync + if TYPE_CHECKING: from pytest_httpserver import HTTPServer @@ -19,6 +22,18 @@ def _parse_accept_encoding(header: str) -> set[str]: return {enc.strip() for enc in header.split(',')} +def _transport_wire_headers( + client_class: type[HttpClient | HttpClientAsync], +) -> tuple[dict[str, str], set[str]]: + """Return the headers the transport adds on its own and the content encodings it advertises.""" + if issubclass(client_class, (ImpitHttpClient, ImpitHttpClientAsync)): + return {}, {'zstd', 'gzip', 'deflate', 'br'} + # HTTPX advertises whichever decoders happen to be installed alongside it, so read the set off the client + # itself rather than hard-coding it and breaking whenever the environment gains or loses a codec. + with httpx.Client() as probe: + return {'Connection': 'keep-alive'}, _parse_accept_encoding(probe.headers['accept-encoding']) + + def _header_handler(request: Request) -> Response: return Response( status=200, @@ -43,15 +58,17 @@ async def test_default_headers_async(httpserver: HTTPServer, http_client_async_c response = await client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_async_class) expected_headers = { 'User-Agent': _get_user_agent(), 'Accept': 'application/json, */*', 'Authorization': 'Bearer placeholder_token', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings def test_default_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient]) -> None: @@ -63,15 +80,17 @@ def test_default_headers_sync(httpserver: HTTPServer, http_client_class: type[Ht response = client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_class) expected_headers = { 'User-Agent': _get_user_agent(), 'Accept': 'application/json, */*', 'Authorization': 'Bearer placeholder_token', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings async def test_headers_async(httpserver: HTTPServer, http_client_async_class: type[HttpClientAsync]) -> None: @@ -86,6 +105,7 @@ async def test_headers_async(httpserver: HTTPServer, http_client_async_class: ty response = await client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_async_class) expected_headers = { 'Test-Header': 'blah', @@ -93,9 +113,10 @@ async def test_headers_async(httpserver: HTTPServer, http_client_async_class: ty 'Accept': 'application/json, */*', 'Authorization': 'strange_value', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient]) -> None: @@ -114,6 +135,7 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient response = client.call(method='GET', url=f'{api_url}/') request_headers = json.loads(response.text)['received_headers'] + transport_headers, expected_encodings = _transport_wire_headers(http_client_class) expected_headers = { 'Test-Header': 'blah', @@ -121,9 +143,10 @@ def test_headers_sync(httpserver: HTTPServer, http_client_class: type[HttpClient 'Accept': 'application/json, */*', 'Authorization': 'strange_value', 'Host': f'{httpserver.host}:{httpserver.port}', + **transport_headers, } assert {k: v for k, v in request_headers.items() if k != 'Accept-Encoding'} == expected_headers - assert _parse_accept_encoding(request_headers['Accept-Encoding']) == {'gzip', 'br', 'zstd', 'deflate'} + assert _parse_accept_encoding(request_headers['Accept-Encoding']) == expected_encodings async def test_per_request_headers_override_defaults_async( @@ -158,3 +181,113 @@ def test_per_request_headers_override_defaults_sync( # WSGI joins duplicate headers into one comma-separated value, so exact equality # also proves the authorization header was sent only once. assert request_headers['Authorization'] == 'Bearer per-request' + + +def _echo_cookie_handler(request: Request) -> Response: + return Response(json.dumps({'cookie': request.headers.get('Cookie')}), content_type='application/json') + + +def test_httpx_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """A Set-Cookie response must not enter HTTPX's shared cookie jar, nor leak into a later API request.""" + httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + client.call(method='GET', url=httpserver.url_for('/set-cookie')) + assert len(client._httpx_client.cookies) == 0 + response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +async def test_httpx_async_does_not_reuse_server_cookies(httpserver: HTTPServer) -> None: + """The asynchronous HTTPX pool also remains stateless between API calls.""" + httpserver.expect_request('/set-cookie').respond_with_data('ok', headers={'Set-Cookie': 'session=secret'}) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + async with HttpxHttpClientAsync() as client: + await client.call(method='GET', url=httpserver.url_for('/set-cookie')) + assert len(client._httpx_async_client.cookies) == 0 + response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +def test_httpx_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: + """A cookie another in-flight request left in the shared jar must not ride along on the next request.""" + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + client._httpx_client.cookies.set('session', 'secret', domain=httpserver.host) + response = client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +async def test_httpx_async_drops_cookies_left_in_the_shared_jar(httpserver: HTTPServer) -> None: + """The asynchronous pool, where concurrent requests really do share one jar, drops leftover cookies too.""" + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + async with HttpxHttpClientAsync() as client: + client._httpx_async_client.cookies.set('session', 'secret', domain=httpserver.host) + response = await client.call(method='GET', url=httpserver.url_for('/echo-cookie')) + + assert response.json() == {'cookie': None} + + +def test_httpx_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: + """A cookie set by a redirecting response must not ride along on the next hop, which HTTPX builds from its jar.""" + httpserver.expect_request('/redirect').respond_with_data( + '', + status=302, + headers={'Set-Cookie': 'session=secret', 'Location': '/echo-cookie'}, + ) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + response = client.call(method='GET', url=httpserver.url_for('/redirect')) + + assert response.json() == {'cookie': None} + + +async def test_httpx_async_does_not_carry_server_cookies_across_a_redirect(httpserver: HTTPServer) -> None: + """The asynchronous pool keeps a redirecting response's cookie off the next hop too.""" + httpserver.expect_request('/redirect').respond_with_data( + '', + status=302, + headers={'Set-Cookie': 'session=secret', 'Location': '/echo-cookie'}, + ) + httpserver.expect_request('/echo-cookie').respond_with_handler(_echo_cookie_handler) + + async with HttpxHttpClientAsync() as client: + response = await client.call(method='GET', url=httpserver.url_for('/redirect')) + + assert response.json() == {'cookie': None} + + +def test_httpx_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: + """Disabling the shared cookie jar must not remove a Cookie header explicitly supplied by the caller.""" + httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) + + with HttpxHttpClient() as client: + response = client.call( + method='GET', + url=httpserver.url_for('/echo-explicit-cookie'), + headers={'Cookie': 'explicit=value'}, + ) + + assert response.json() == {'cookie': 'explicit=value'} + + +async def test_httpx_async_keeps_explicit_cookie_header(httpserver: HTTPServer) -> None: + """The asynchronous pool forwards an explicitly supplied Cookie header as well.""" + httpserver.expect_request('/echo-explicit-cookie').respond_with_handler(_echo_cookie_handler) + + async with HttpxHttpClientAsync() as client: + response = await client.call( + method='GET', + url=httpserver.url_for('/echo-explicit-cookie'), + headers={'Cookie': 'explicit=value'}, + ) + + assert response.json() == {'cookie': 'explicit=value'} diff --git a/tests/unit/test_client_streaming.py b/tests/unit/test_client_streaming.py index d369455e..9e5782e7 100644 --- a/tests/unit/test_client_streaming.py +++ b/tests/unit/test_client_streaming.py @@ -105,7 +105,7 @@ def test_protocol_check_leaves_stream_unread_sync( with client.dataset(DATASET_ID).stream_items(item_format='json') as response: assert isinstance(response, HttpResponse) - # `is_stream_consumed` is transport state, not part of the protocol, but the built-in client exposes it. + # `is_stream_consumed` is transport state, not part of the protocol, but both built-in clients expose it. raw: Any = response assert raw.is_stream_consumed is False @@ -124,6 +124,6 @@ async def test_protocol_check_leaves_stream_unread_async( async with client.dataset(DATASET_ID).stream_items(item_format='json') as response: assert isinstance(response, HttpResponse) - # `is_stream_consumed` is transport state, not part of the protocol, but the built-in client exposes it. + # `is_stream_consumed` is transport state, not part of the protocol, but both built-in clients expose it. raw: Any = response assert raw.is_stream_consumed is False diff --git a/tests/unit/test_client_timeouts.py b/tests/unit/test_client_timeouts.py index b4410acd..a7adaa64 100644 --- a/tests/unit/test_client_timeouts.py +++ b/tests/unit/test_client_timeouts.py @@ -5,16 +5,27 @@ from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock +import httpx2 as httpx import impit import pytest from apify_client._logging import LoggerOnce, logger_name -from apify_client.http_clients import HttpClient, HttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync +from apify_client.http_clients import ( + HttpClient, + HttpClientAsync, + HttpxHttpClient, + HttpxHttpClientAsync, + ImpitHttpClient, + ImpitHttpClientAsync, +) from apify_client.http_clients import _base as http_client_base if TYPE_CHECKING: from _pytest.logging import LogCaptureFixture +UNSET_HTTPX_TIMEOUT = {'connect': None, 'read': None, 'write': None, 'pool': None} +"""What HTTPX stores on a request built with `timeout=None`: every sub-timeout unset, not the client default.""" + @pytest.fixture def fresh_logger_once(monkeypatch: pytest.MonkeyPatch) -> None: @@ -26,6 +37,12 @@ def successful_response() -> Mock: return Mock(status_code=200) +def retryable_error(client: HttpClient | HttpClientAsync) -> Exception: + if isinstance(client, (ImpitHttpClient, ImpitHttpClientAsync)): + return impit.TimeoutException('timeout') + return httpx.ReadTimeout('timeout', request=httpx.Request('GET', 'https://example.com')) + + @pytest.mark.parametrize( ('timeout', 'expected'), [ @@ -160,7 +177,7 @@ def test_dynamic_timeout_sync_client(http_client_class: type[HttpClient], monkey def send_request(*_args: Any, **kwargs: Any) -> Mock: timeouts.append(kwargs['timeout']) if len(timeouts) < 4: - raise impit.TimeoutException('timeout') + raise retryable_error(client) return successful_response() monkeypatch.setattr(client, 'send_request', send_request) @@ -185,7 +202,7 @@ async def test_dynamic_timeout_async_client( async def send_request(*_args: Any, **kwargs: Any) -> Mock: timeouts.append(kwargs['timeout']) if len(timeouts) < 4: - raise impit.TimeoutException('timeout') + raise retryable_error(client) return successful_response() monkeypatch.setattr(client, 'send_request', send_request) @@ -196,8 +213,8 @@ async def send_request(*_args: Any, **kwargs: Any) -> Mock: assert response.status_code == 200 -def test_no_timeout_mapping_for_sync_adapter() -> None: - """The synchronous adapter maps no-timeout to Impit's effectively unbounded value.""" +def test_no_timeout_mapping_for_sync_impit_adapter() -> None: + """The synchronous Impit adapter maps no-timeout to Impit's effectively unbounded value.""" client = ImpitHttpClient() client._impit_client = Mock(request=Mock(return_value=successful_response())) @@ -206,8 +223,8 @@ def test_no_timeout_mapping_for_sync_adapter() -> None: assert client._impit_client.request.call_args.kwargs['timeout'] == 86_400 -async def test_no_timeout_mapping_for_async_adapter() -> None: - """The asynchronous adapter maps no-timeout to Impit's effectively unbounded value.""" +async def test_no_timeout_mapping_for_async_impit_adapter() -> None: + """The asynchronous Impit adapter maps no-timeout to Impit's effectively unbounded value.""" client = ImpitHttpClientAsync() client._impit_async_client = Mock(request=AsyncMock(return_value=successful_response())) @@ -216,3 +233,31 @@ async def test_no_timeout_mapping_for_async_adapter() -> None: ) assert client._impit_async_client.request.call_args.kwargs['timeout'] == 86_400 + + +def test_no_timeout_mapping_for_sync_httpx_adapter(monkeypatch: pytest.MonkeyPatch) -> None: + """The synchronous HTTPX adapter maps no-timeout to every HTTPX sub-timeout being unset.""" + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. + with HttpxHttpClient() as client: + send = Mock(return_value=successful_response()) + monkeypatch.setattr(client._httpx_client, 'send', send) + + client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT + + +async def test_no_timeout_mapping_for_async_httpx_adapter(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX adapter maps no-timeout to every HTTPX sub-timeout being unset.""" + # Only the transport call is stubbed, so the real `build_request` decides what `None` means to HTTPX. + async with HttpxHttpClientAsync() as client: + send = AsyncMock(return_value=successful_response()) + monkeypatch.setattr(client._httpx_async_client, 'send', send) + + await client.send_request( + method='GET', url='https://example.com', headers={}, content=None, timeout=None, stream=False + ) + + assert send.call_args.args[0].extensions['timeout'] == UNSET_HTTPX_TIMEOUT diff --git a/tests/unit/test_http_clients.py b/tests/unit/test_http_clients.py index adbf9ce7..913a5277 100644 --- a/tests/unit/test_http_clients.py +++ b/tests/unit/test_http_clients.py @@ -11,6 +11,7 @@ from unittest.mock import AsyncMock, Mock import brotli +import httpx2 as httpx import impit import pytest @@ -21,6 +22,8 @@ HttpClient, HttpClientAsync, HttpResponse, + HttpxHttpClient, + HttpxHttpClientAsync, ImpitHttpClient, ImpitHttpClientAsync, ) @@ -264,6 +267,24 @@ async def test_http_client_async_creates_async_impit_client() -> None: await client.aclose() +def test_http_client_creates_sync_httpx_client() -> None: + """The synchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" + client = HttpxHttpClient(token='test_token_123') + + assert isinstance(client._httpx_client, httpx.Client) + client.close() + assert client._httpx_client.is_closed + + +async def test_http_client_async_creates_async_httpx_client() -> None: + """The asynchronous HTTPX adapter creates the underlying HTTPX client, and the close hook closes its pool.""" + client = HttpxHttpClientAsync(token='test_token_123') + + assert isinstance(client._httpx_async_client, httpx.AsyncClient) + await client.aclose() + assert client._httpx_async_client.is_closed + + def test_parse_params_none() -> None: """Test _parse_params with None input.""" assert HttpClient._parse_params(None) is None @@ -392,6 +413,70 @@ async def test_async_http_client_classifies_timeout_errors() -> None: assert not client.is_timeout_error(ValueError('test')) +@pytest.mark.parametrize( + 'exc', + [ + # Even the generic base class is transient: HTTPX subclasses it for every failure mode, so an + # unclassified failure is safer to retry. + pytest.param(httpx.HTTPError('unclassified failure'), id='bare HTTPError'), + pytest.param(httpx.TimeoutException('timeout'), id='TimeoutException'), + pytest.param(httpx.NetworkError('network error'), id='NetworkError'), + pytest.param(httpx.RemoteProtocolError('remote protocol error'), id='RemoteProtocolError'), + pytest.param(httpx.DecodingError('decoding error'), id='DecodingError'), + # One `ProxyError` covers both a proxy rejecting the CONNECT tunnel and a 407, so a transient case cannot + # be told from a permanent one - retrying is the safer default. + pytest.param(httpx.ProxyError('proxy error'), id='ProxyError'), + ], +) +def test_httpx_is_retryable_transport_error(exc: Exception) -> None: + """A transient HTTPX transport failure is classified as retryable.""" + with HttpxHttpClient() as client: + assert client.is_retryable_transport_error(exc) + + +@pytest.mark.parametrize( + 'exc', + [ + pytest.param(httpx.LocalProtocolError('invalid header value'), id='LocalProtocolError'), + pytest.param(httpx.UnsupportedProtocol('unsupported scheme'), id='UnsupportedProtocol'), + pytest.param(httpx.TooManyRedirects('too many redirects'), id='TooManyRedirects'), + pytest.param( + httpx.HTTPStatusError( + 'status error', + request=httpx.Request('GET', 'https://example.com'), + response=httpx.Response(500), + ), + id='HTTPStatusError', + ), + # HTTPX reports a bad URL outside the `httpx.HTTPError` tree entirely. + pytest.param(httpx.InvalidURL('unsupported scheme'), id='InvalidURL'), + pytest.param(ValueError('value error'), id='ValueError'), + pytest.param(RuntimeError('runtime error'), id='RuntimeError'), + pytest.param(Exception('generic exception'), id='Exception'), + ], +) +def test_httpx_is_not_retryable_transport_error(exc: Exception) -> None: + """A transport failure a retry cannot fix, and anything outside HTTPX's hierarchy, is not retried.""" + with HttpxHttpClient() as client: + assert not client.is_retryable_transport_error(exc) + + +def test_sync_httpx_client_classifies_timeout_errors() -> None: + """The built-in synchronous HTTPX client exposes transport-neutral timeout classification.""" + with HttpxHttpClient() as client: + assert client.is_timeout_error(TimeoutError('test')) + assert client.is_timeout_error(httpx.TimeoutException('test')) + assert not client.is_timeout_error(ValueError('test')) + + +async def test_async_httpx_client_classifies_timeout_errors() -> None: + """The built-in asynchronous HTTPX client exposes transport-neutral timeout classification.""" + async with HttpxHttpClientAsync() as client: + assert client.is_timeout_error(TimeoutError('test')) + assert client.is_timeout_error(httpx.TimeoutException('test')) + assert not client.is_timeout_error(ValueError('test')) + + def test_permanent_transport_error_is_not_retried() -> None: """A transport error a retry cannot fix fails on the first attempt instead of burning the whole backoff.""" client = ImpitHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) @@ -429,6 +514,58 @@ def test_transient_transport_error_is_retried() -> None: assert request.call_count == 3 +def test_httpx_permanent_transport_error_is_not_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX adapter feeds the same fail-fast classification into the shared pipeline.""" + with HttpxHttpClient(token='test_token', min_delay_between_retries=timedelta(0)) as client: + send = Mock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client._httpx_client, 'send', send) + + with pytest.raises(httpx.UnsupportedProtocol): + client.call(method='GET', url='https://api.test.com/endpoint') + + send.assert_called_once() + + +async def test_httpx_permanent_transport_error_is_not_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX adapter applies the same policy, failing on the first attempt.""" + async with HttpxHttpClientAsync(token='test_token', min_delay_between_retries=timedelta(0)) as client: + send = AsyncMock(side_effect=httpx.UnsupportedProtocol('unsupported scheme')) + monkeypatch.setattr(client._httpx_async_client, 'send', send) + + with pytest.raises(httpx.UnsupportedProtocol): + await client.call(method='GET', url='https://api.test.com/endpoint') + + send.assert_awaited_once() + + +def test_httpx_transient_transport_error_is_retried(monkeypatch: pytest.MonkeyPatch) -> None: + """The HTTPX adapter keeps a transient transport failure inside the shared retry loop.""" + with HttpxHttpClient(token='test_token', max_retries=2, min_delay_between_retries=timedelta(0)) as client: + send = Mock(side_effect=httpx.TimeoutException('timeout')) + monkeypatch.setattr(client._httpx_client, 'send', send) + + with pytest.raises(httpx.TimeoutException): + client.call(method='GET', url='https://api.test.com/endpoint') + + # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. + assert send.call_count == 3 + + +async def test_httpx_transient_transport_error_is_retried_async(monkeypatch: pytest.MonkeyPatch) -> None: + """The asynchronous HTTPX adapter keeps a transient transport failure inside the shared retry loop too.""" + async with HttpxHttpClientAsync( + token='test_token', max_retries=2, min_delay_between_retries=timedelta(0) + ) as client: + send = AsyncMock(side_effect=httpx.TimeoutException('timeout')) + monkeypatch.setattr(client._httpx_async_client, 'send', send) + + with pytest.raises(httpx.TimeoutException): + await client.call(method='GET', url='https://api.test.com/endpoint') + + # `max_retries` attempts inside the backoff loop, plus the final one it makes after the last delay. + assert send.await_count == 3 + + def test_error_response_read_failure_is_retried_and_closed() -> None: """A failure while buffering a streamed error body is retried like a failed send, and the response is closed.""" client = ImpitHttpClient(token='test_token', max_retries=1, min_delay_between_retries=timedelta(0)) diff --git a/tests/unit/test_logging.py b/tests/unit/test_logging.py index 35b10e2e..341fb664 100644 --- a/tests/unit/test_logging.py +++ b/tests/unit/test_logging.py @@ -877,7 +877,7 @@ def test_streamed_log_sync_does_not_leak_exception_on_stream_timeout( http_client_class: type[HttpClient], monkeypatch: pytest.MonkeyPatch, ) -> None: - """The streaming thread ends quietly when the transport times out while reading the log stream.""" + """The streaming thread ends quietly when either transport times out while reading the log stream.""" monkeypatch.setattr(StreamedLog, '_stream_timeout', timedelta(seconds=1)) release_server = threading.Event() @@ -885,7 +885,7 @@ def test_streamed_log_sync_does_not_leak_exception_on_stream_timeout( def _slow_handler(_request: Request) -> Response: def generate_logs() -> Iterator[bytes]: # Emit one complete line, then keep the connection open (as a running Actor would) past the - # client-side total timeout without sending anything more. + # client-side stream timeout without sending anything more. yield b'2025-05-13T07:24:12.588Z ACTOR: still running\n' release_server.wait(timeout=30) @@ -910,7 +910,7 @@ def generate_logs() -> Iterator[bytes]: try: with caplog.at_level(logging.DEBUG, logger=logger_name): thread = streamed_log.start() - # Wait past the 1s total timeout so the streaming request fails inside the thread. + # Wait past the 1s stream timeout so the streaming request fails inside the thread. thread.join(timeout=5) assert not thread.is_alive(), 'streaming thread did not end after the stream timed out' finally: @@ -970,14 +970,14 @@ async def test_streamed_log_async_does_not_error_on_stream_timeout( http_client_async_class: type[HttpClientAsync], monkeypatch: pytest.MonkeyPatch, ) -> None: - """The async streaming task treats a transport stream timeout as an expected terminal condition.""" + """The async streaming task treats either transport's stream timeout as an expected terminal condition.""" monkeypatch.setattr(StreamedLogAsync, '_stream_timeout', timedelta(seconds=1)) release_server = threading.Event() def _slow_handler(_request: Request) -> Response: def generate_logs() -> Iterator[bytes]: - # Emit one complete line, then keep the connection open past the client-side total timeout. + # Emit one complete line, then keep the connection open past the client-side stream timeout. yield b'2025-05-13T07:24:12.588Z ACTOR: still running\n' release_server.wait(timeout=30) @@ -999,7 +999,7 @@ def generate_logs() -> Iterator[bytes]: try: with caplog.at_level(logging.DEBUG, logger=logger_name): task = streamed_log.start() - # The 1s total timeout fails the request inside the task; it must end on its own without our help. + # The 1s stream timeout fails the request inside the task; it must end on its own without our help. done, _pending = await asyncio.wait({task}, timeout=5) assert task in done, 'async streaming task did not end after the stream timed out' finally: diff --git a/tests/unit/test_pluggable_http_client.py b/tests/unit/test_pluggable_http_client.py index 2fd20899..bda71bc7 100644 --- a/tests/unit/test_pluggable_http_client.py +++ b/tests/unit/test_pluggable_http_client.py @@ -2,9 +2,12 @@ import asyncio import json as jsonlib +import subprocess +import sys from dataclasses import dataclass, field from datetime import timedelta from http.client import HTTPConnection +from textwrap import dedent from typing import TYPE_CHECKING, Any from unittest.mock import AsyncMock, Mock from urllib.parse import urlsplit @@ -353,6 +356,8 @@ def test_public_exports() -> None: 'HttpClient', 'HttpClientAsync', 'HttpResponse', + 'HttpxHttpClient', + 'HttpxHttpClientAsync', 'ImpitHttpClient', 'ImpitHttpClientAsync', ): @@ -362,6 +367,47 @@ def test_public_exports() -> None: assert not hasattr(http_clients_module, 'HttpClientBase') +def test_httpx_clients_raise_clear_error_when_extra_missing() -> None: + """Missing HTTPX keeps normal and star imports usable while explicit HTTPX access raises a clear error.""" + script = dedent( + """ + import sys + + class BlockHttpx: + def find_spec(self, name, *_args): + if name == 'httpx2' or name.startswith('httpx2.'): + raise ModuleNotFoundError(f"No module named '{name}'", name='httpx2') + return None + + sys.meta_path.insert(0, BlockHttpx()) + + import apify_client.http_clients as module + assert module.HttpClient is not None + assert module.ImpitHttpClient is not None + + namespace = {} + exec('from apify_client.http_clients import *', namespace) + assert namespace['HttpClient'] is module.HttpClient + assert 'HttpxHttpClient' not in namespace + + for name in ('HttpxHttpClient', 'HttpxHttpClientAsync'): + try: + getattr(module, name) + except ImportError as exc: + assert "No module named 'httpx2'" in str(exc) + else: + raise AssertionError(f'{name} did not raise ImportError') + """ + ) + result = subprocess.run( # noqa: S603 + [sys.executable, '-c', script], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + def test_apify_client_http_client_property_returns_correct_type() -> None: """Test that http_client property returns the correct type.""" # With default diff --git a/uv.lock b/uv.lock index 0f531050..09dbabd2 100644 --- a/uv.lock +++ b/uv.lock @@ -3,7 +3,8 @@ revision = 3 requires-python = ">=3.11" resolution-markers = [ "python_full_version >= '3.14'", - "python_full_version < '3.14'", + "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform == 'emscripten'", + "(python_full_version < '3.14' and sys_platform != 'emscripten') or (python_full_version < '3.12' and sys_platform == 'emscripten')", ] [options] @@ -54,6 +55,9 @@ dependencies = [ brotli = [ { name = "brotli" }, ] +httpx2 = [ + { name = "httpx2" }, +] [package.dev-dependencies] dev = [ @@ -80,12 +84,13 @@ dev = [ requires-dist = [ { name = "brotli", marker = "extra == 'brotli'", specifier = ">=1.0.9" }, { name = "colorama", specifier = ">=0.4.0" }, + { name = "httpx2", marker = "extra == 'httpx2'", specifier = ">=2.0.0" }, { name = "impit", specifier = "~=0.13.0" }, { name = "more-itertools", specifier = ">=10.0.0" }, { name = "pydantic", extras = ["email"], specifier = ">=2.11.0" }, { name = "typing-extensions", specifier = ">=4.6.0" }, ] -provides-extras = ["brotli"] +provides-extras = ["brotli", "httpx2"] [package.metadata.requires-dev] dev = [ @@ -735,6 +740,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/be/ad/f4f0e57345f1870f3e8cb624e058d7eca6e5a27d33bcc3311d9b618734cd/httpcore2-2.12.0.tar.gz", hash = "sha256:9293522bba0aa7c4c8e9e3f040c16575bd8868e155a77fa30c7a9085a5eae648", size = 67548, upload-time = "2026-08-18T13:22:08.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/74/d370e55600d9bcfa0d9794b0166126d49291a3d2b20c268fc98c453a4948/httpcore2-2.12.0-py3-none-any.whl", hash = "sha256:7e04258ce01013d7d615e5b910a3b27fac937d7a95038227e79652b4ba3b4ceb", size = 83074, upload-time = "2026-08-18T13:22:05.854Z" }, +] + [[package]] name = "httpx" version = "0.28.1" @@ -750,6 +768,32 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.12.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "python_full_version >= '3.12' and sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7f/f8/579a8b51e42e38ee32647df9f08aa25643ae788e275cc625b199829c4671/httpx2-2.12.0.tar.gz", hash = "sha256:7631fe9887a8a2275f4a2540e053aa670fcc50742864a9ae7c66e609fdcf12cf", size = 100040, upload-time = "2026-08-18T13:22:09.086Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/95/411ba65569158e862368917aaf56597f3e5fa3b91b0502919638465a08f3/httpx2-2.12.0-py3-none-any.whl", hash = "sha256:cc8b6eecb8661c146b8f89a60e97456ee086e91a784ed31ac450c3a9e613dd36", size = 95427, upload-time = "2026-08-18T13:22:06.834Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "identify" version = "2.6.19" @@ -1508,6 +1552,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/18/c86eb8e0202e32dd3df50d43d7ff9854f8e0603945ff398974c1d91ac1ef/tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90", size = 6675, upload-time = "2025-01-15T12:07:22.074Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "ty" version = "0.0.72"