feat: Add HTTPX-based HTTP client

Author: vdusek

README.md

@@ -57,6 +57,15 @@
     uv add "apify-client[brotli]"
     ```
 
+    [Impit](https://github.com/apify/impit) is the default HTTP client and is installed automatically. To use the
+    built-in [HTTPX](https://www.python-httpx.org/) client instead, install its optional extra:
+
+    ```bash
+    pip install "apify-client[httpx]"
+    # or
+    uv add "apify-client[httpx]"
+    ```
+
 - From [conda-forge](https://anaconda.org/conda-forge/apify-client), it can be installed with [conda](https://docs.conda.io/en/latest/):
 
     ```bash
@@ -124,7 +133,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r
 - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)).
 - **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)).
 - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)).
-- **Pluggable HTTP layer** — swap the default [Impit](https://github.com/apify/impit)-based HTTP client for `httpx`, `requests`, `aiohttp`, or any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)).
+- **Pluggable HTTP layer** — use the default [Impit](https://github.com/apify/impit)-based client, opt in to the built-in [HTTPX](https://www.python-httpx.org/) client, or plug in any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)).
 - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)).
 - **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more ([Logging](https://docs.apify.com/api/client/python/docs/concepts/logging)).
 

pyproject.toml

@@ -34,6 +34,7 @@ dependencies = [
 
 [project.optional-dependencies]
 brotli = ["brotli>=1.0.9"]
+httpx = ["httpx>=0.27.0,<1.0.0"]
 
 [project.urls]
 "Apify Homepage" = "https://apify.com"

src/apify_client/http_clients/__init__.py

@@ -1,10 +1,35 @@
+from apify_client._utils.try_import import install_import_hook as _install_import_hook
+from apify_client._utils.try_import import try_import as _try_import
 from apify_client.http_clients._base import HttpClient, HttpClientAsync, HttpResponse
 from apify_client.http_clients._impit import ImpitHttpClient, ImpitHttpClientAsync
 
-__all__ = [
-    'HttpClient',
-    'HttpClientAsync',
-    'HttpResponse',
-    'ImpitHttpClient',
-    'ImpitHttpClientAsync',
-]
+_install_import_hook(__name__)
+
+# `httpx` is an optional extra, so it's wrapped in try_import. Accessing the HTTPX clients
+# without the extra installed raises a clear ImportError instead of failing at package import time.
+with _try_import(
+    __name__,
+    'HttpxHttpClient',
+    'HttpxHttpClientAsync',
+    dependency_name='httpx',
+) as _httpx_import:
+    from apify_client.http_clients._httpx import HttpxHttpClient, HttpxHttpClientAsync
+
+if _httpx_import.available:
+    __all__ = [
+        'HttpClient',
+        'HttpClientAsync',
+        'HttpResponse',
+        'HttpxHttpClient',
+        'HttpxHttpClientAsync',
+        'ImpitHttpClient',
+        'ImpitHttpClientAsync',
+    ]
+else:
+    __all__ = [
+        'HttpClient',
+        'HttpClientAsync',
+        'HttpResponse',
+        'ImpitHttpClient',
+        'ImpitHttpClientAsync',
+    ]

src/apify_client/http_clients/_httpx.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import httpx
+from typing_extensions import override
+
+from apify_client._consts import (
+    DEFAULT_MAX_RETRIES,
+    DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
+    DEFAULT_TIMEOUT_LONG,
+    DEFAULT_TIMEOUT_MAX,
+    DEFAULT_TIMEOUT_MEDIUM,
+    DEFAULT_TIMEOUT_SHORT,
+)
+from apify_client._docs import docs_group
+from apify_client.http_clients._base import HttpClient, HttpClientAsync
+
+if TYPE_CHECKING:
+    from datetime import timedelta
+
+    from apify_client._statistics import ClientStatistics
+    from apify_client.http_compressors._base import HttpCompressor
+
+
+_PERMANENT_ERRORS = (
+    # A request HTTPX rejects before sending it, e.g. one carrying an invalid header value.
+    httpx.LocalProtocolError,
+    # A URL scheme HTTPX refuses to speak, which repeating the request cannot change.
+    httpx.UnsupportedProtocol,
+    # An over-long redirect chain is a routing loop, which repeating the request cannot break.
+    httpx.TooManyRedirects,
+    # Only `Response.raise_for_status()` raises this, and the client never calls it - the shared pipeline decides on
+    # status codes from the response itself.
+    httpx.HTTPStatusError,
+)
+"""HTTPX errors that a retry cannot fix. Everything else in the `httpx.HTTPError` tree counts as transient."""
+
+
+@docs_group('HTTP clients')
+class HttpxHttpClient(HttpClient):
+    """Synchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/).
+
+    This client wraps `httpx.Client` and adds automatic retries with exponential backoff for rate-limited
+    (HTTP 429) and server error (HTTP 5xx) responses.
+
+    Requires the `httpx` extra: `pip install "apify-client[httpx]"`.
+    """
+
+    def __init__(
+        self,
+        *,
+        token: str | None = None,
+        timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT,
+        timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM,
+        timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
+        timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+        min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
+        statistics: ClientStatistics | None = None,
+        headers: dict[str, str] | None = None,
+        http_compressor: HttpCompressor | None = None,
+    ) -> None:
+        """Initialize the HTTPX-based synchronous HTTP client.
+
+        Args:
+            token: Apify API token for authentication.
+            timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
+            timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
+            timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
+            timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
+            max_retries: Maximum number of retry attempts for failed requests.
+            min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
+            statistics: Statistics tracker for API calls. Created automatically if not provided.
+            headers: Additional HTTP headers to include in all requests.
+            http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
+        """
+        super().__init__(
+            token=token,
+            timeout_short=timeout_short,
+            timeout_medium=timeout_medium,
+            timeout_long=timeout_long,
+            timeout_max=timeout_max,
+            max_retries=max_retries,
+            min_delay_between_retries=min_delay_between_retries,
+            statistics=statistics,
+            headers=headers,
+            http_compressor=http_compressor,
+        )
+
+        self._httpx_client = httpx.Client(
+            follow_redirects=True,
+            event_hooks={'response': [self._clear_response_cookies]},
+        )
+
+    @override
+    def is_timeout_error(self, exc: Exception) -> bool:
+        return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException)
+
+    @override
+    def is_retryable_transport_error(self, exc: Exception) -> bool:
+        # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in
+        # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures
+        # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the
+        # response status code, not here.
+        return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS)
+
+    @override
+    def close(self) -> None:
+        """Close the underlying HTTPX connection pool."""
+        self._httpx_client.close()
+
+    def _clear_response_cookies(self, _response: httpx.Response) -> None:
+        """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests."""
+        self._httpx_client.cookies.clear()
+
+    @override
+    def send_request(
+        self,
+        *,
+        method: str,
+        url: str,
+        headers: dict[str, str],
+        content: bytes | None,
+        timeout: float | None,
+        stream: bool,
+    ) -> httpx.Response:
+        request = self._httpx_client.build_request(
+            method=method,
+            url=url,
+            headers=headers,
+            content=content,
+            timeout=timeout,
+        )
+        _restore_explicit_cookie_header(request, headers)
+        return self._httpx_client.send(request, stream=stream)
+
+
+@docs_group('HTTP clients')
+class HttpxHttpClientAsync(HttpClientAsync):
+    """Asynchronous HTTP client for the Apify API built on top of [HTTPX](https://www.python-httpx.org/).
+
+    This client wraps `httpx.AsyncClient` and adds automatic retries with exponential backoff for rate-limited
+    (HTTP 429) and server error (HTTP 5xx) responses.
+
+    Requires the `httpx` extra: `pip install "apify-client[httpx]"`.
+    """
+
+    def __init__(
+        self,
+        *,
+        token: str | None = None,
+        timeout_short: timedelta = DEFAULT_TIMEOUT_SHORT,
+        timeout_medium: timedelta = DEFAULT_TIMEOUT_MEDIUM,
+        timeout_long: timedelta = DEFAULT_TIMEOUT_LONG,
+        timeout_max: timedelta = DEFAULT_TIMEOUT_MAX,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+        min_delay_between_retries: timedelta = DEFAULT_MIN_DELAY_BETWEEN_RETRIES,
+        statistics: ClientStatistics | None = None,
+        headers: dict[str, str] | None = None,
+        http_compressor: HttpCompressor | None = None,
+    ) -> None:
+        """Initialize the HTTPX-based asynchronous HTTP client.
+
+        Args:
+            token: Apify API token for authentication.
+            timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
+            timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
+            timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
+            timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
+            max_retries: Maximum number of retry attempts for failed requests.
+            min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
+            statistics: Statistics tracker for API calls. Created automatically if not provided.
+            headers: Additional HTTP headers to include in all requests.
+            http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.
+        """
+        super().__init__(
+            token=token,
+            timeout_short=timeout_short,
+            timeout_medium=timeout_medium,
+            timeout_long=timeout_long,
+            timeout_max=timeout_max,
+            max_retries=max_retries,
+            min_delay_between_retries=min_delay_between_retries,
+            statistics=statistics,
+            headers=headers,
+            http_compressor=http_compressor,
+        )
+
+        self._httpx_async_client = httpx.AsyncClient(
+            follow_redirects=True,
+            event_hooks={'response': [self._clear_response_cookies]},
+        )
+
+    @override
+    def is_timeout_error(self, exc: Exception) -> bool:
+        return super().is_timeout_error(exc) or isinstance(exc, httpx.TimeoutException)
+
+    @override
+    def is_retryable_transport_error(self, exc: Exception) -> bool:
+        # Every error from HTTPX's own hierarchy counts as transient except the permanently-failing types listed in
+        # `_PERMANENT_ERRORS`. Retrying is the default because HTTPX also reports genuinely transient failures
+        # through its generic base class. HTTP status code errors are handled by the shared pipeline based on the
+        # response status code, not here.
+        return isinstance(exc, httpx.HTTPError) and not isinstance(exc, _PERMANENT_ERRORS)
+
+    @override
+    async def aclose(self) -> None:
+        """Close the underlying asynchronous HTTPX connection pool."""
+        await self._httpx_async_client.aclose()
+
+    async def _clear_response_cookies(self, _response: httpx.Response) -> None:
+        """Prevent HTTPX's shared cookie jar from leaking server cookies into later API requests."""
+        self._httpx_async_client.cookies.clear()
+
+    @override
+    async def send_request(
+        self,
+        *,
+        method: str,
+        url: str,
+        headers: dict[str, str],
+        content: bytes | None,
+        timeout: float | None,
+        stream: bool,
+    ) -> httpx.Response:
+        request = self._httpx_async_client.build_request(
+            method=method,
+            url=url,
+            headers=headers,
+            content=content,
+            timeout=timeout,
+        )
+        _restore_explicit_cookie_header(request, headers)
+        return await self._httpx_async_client.send(request, stream=stream)
+
+
+def _restore_explicit_cookie_header(request: httpx.Request, headers: dict[str, str]) -> None:
+    """Keep only cookies explicitly supplied for this request, never cookies from HTTPX's shared jar."""
+    explicit_cookie = next((value for key, value in headers.items() if key.lower() == 'cookie'), None)
+    if explicit_cookie is None:
+        request.headers.pop('cookie', None)
+    else:
+        request.headers['cookie'] = explicit_cookie