Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/02_concepts/05_retries.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ By default, the client retries a failed request up to 4 times. The retry interva

You can customize this behavior using the following options in the <ApiLink to="class/ApifyClient">`ApifyClient`</ApiLink> constructor:

- `max_retries`: Defines the maximum number of retry attempts.
- `max_retries`: Defines the maximum number of retry attempts. Set it to `0` to disable retries.
- `min_delay_between_retries`: Sets the minimum delay between retries as a `timedelta`.

Retries with exponential backoff help reduce the load on the server and increase the chances of a successful request.
Expand Down
16 changes: 14 additions & 2 deletions src/apify_client/_apify_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ def __init__(
as well.
api_public_url: The globally accessible URL of the Apify API server. Should be set only if `api_url`
is an internal URL that is not globally accessible. Defaults to https://api.apify.com.
max_retries: How many times to retry a failed request at most.
max_retries: How many times to retry a failed request at most. Set to 0 to disable retries.
min_delay_between_retries: How long will the client wait between retrying requests
(increases exponentially from this value).
timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
Expand All @@ -150,7 +150,13 @@ def __init__(
headers: Additional HTTP headers to include in all API requests.
compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm,
or an `HttpCompressor` instance for finer-grained control.

Raises:
ValueError: If `max_retries` is negative.
"""
if max_retries < 0:
raise ValueError(f'max_retries must be a non-negative integer, got {max_retries}')

# We need to do this because of mocking in tests and default mutable arguments.
api_url = DEFAULT_API_URL if api_url is None else api_url
api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
Expand Down Expand Up @@ -504,7 +510,7 @@ def __init__(
as well.
api_public_url: The globally accessible URL of the Apify API server. Should be set only if `api_url`
is an internal URL that is not globally accessible. Defaults to https://api.apify.com.
max_retries: How many times to retry a failed request at most.
max_retries: How many times to retry a failed request at most. Set to 0 to disable retries.
min_delay_between_retries: How long will the client wait between retrying requests
(increases exponentially from this value).
timeout_short: Default timeout for short-duration API operations (simple CRUD operations, ...).
Expand All @@ -514,7 +520,13 @@ def __init__(
headers: Additional HTTP headers to include in all API requests.
compression: Compression algorithm for request bodies. Pass a string literal to select an algorithm,
or an `HttpCompressor` instance for finer-grained control.

Raises:
ValueError: If `max_retries` is negative.
"""
if max_retries < 0:
raise ValueError(f'max_retries must be a non-negative integer, got {max_retries}')

# We need to do this because of mocking in tests and default mutable arguments.
api_url = DEFAULT_API_URL if api_url is None else api_url
api_public_url = DEFAULT_API_URL if api_public_url is None else api_public_url
Expand Down
14 changes: 7 additions & 7 deletions src/apify_client/http_clients/_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,12 +136,18 @@ def __init__(
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
max_retries: Maximum number of retries for failed requests.
max_retries: Maximum number of retries for failed requests. Set to 0 to disable retries.
min_delay_between_retries: Minimum delay between retries.
statistics: Statistics tracker for API calls. Created automatically if not provided.
headers: Additional HTTP headers to include in all requests.
http_compressor: Compressor used to compress request bodies. Defaults to `GzipHttpCompressor`.

Raises:
ValueError: If `max_retries` is negative.
"""
if max_retries < 0:
raise ValueError(f'max_retries must be a non-negative integer, got {max_retries}')

self._http_compressor = http_compressor if http_compressor is not None else GzipHttpCompressor()
self._timeout_short = timeout_short
self._timeout_medium = timeout_medium
Expand Down Expand Up @@ -598,9 +604,6 @@ def _retry_with_exp_backoff(
random_factor: float = 1,
) -> T:
"""Retry a function with exponential backoff and jitter."""
if max_retries < 1:
raise ValueError(f'max_retries must be at least 1, got {max_retries}')

random_factor = min(max(0, random_factor), 1)
backoff_factor = min(max(1, backoff_factor), 10)
swallow = True
Expand Down Expand Up @@ -825,9 +828,6 @@ async def _retry_with_exp_backoff(
random_factor: float = 1,
) -> T:
"""Retry an async function with exponential backoff and jitter."""
if max_retries < 1:
raise ValueError(f'max_retries must be at least 1, got {max_retries}')

random_factor = min(max(0, random_factor), 1)
backoff_factor = min(max(1, backoff_factor), 10)
swallow = True
Expand Down
4 changes: 2 additions & 2 deletions src/apify_client/http_clients/_httpx2.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def __init__(
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
max_retries: Maximum number of retry attempts for failed requests.
max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries.
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
statistics: Statistics tracker for API calls. Created automatically if not provided.
headers: Additional HTTP headers to include in all requests.
Expand Down Expand Up @@ -179,7 +179,7 @@ def __init__(
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
max_retries: Maximum number of retry attempts for failed requests.
max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries.
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
statistics: Statistics tracker for API calls. Created automatically if not provided.
headers: Additional HTTP headers to include in all requests.
Expand Down
4 changes: 2 additions & 2 deletions src/apify_client/http_clients/_impit.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def __init__(
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
max_retries: Maximum number of retry attempts for failed requests.
max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries.
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
statistics: Statistics tracker for API calls. Created automatically if not provided.
headers: Additional HTTP headers to include in all requests.
Expand Down Expand Up @@ -171,7 +171,7 @@ def __init__(
timeout_medium: Default timeout for medium-duration API operations (batch operations, listing, ...).
timeout_long: Default timeout for long-duration API operations (long-polling, streaming, ...).
timeout_max: Maximum timeout cap for any single request attempt, including tier and per-call timeouts.
max_retries: Maximum number of retry attempts for failed requests.
max_retries: Maximum number of retry attempts for failed requests. Set to 0 to disable retries.
min_delay_between_retries: Minimum delay between retries (increases exponentially with each attempt).
statistics: Statistics tracker for API calls. Created automatically if not provided.
headers: Additional HTTP headers to include in all requests.
Expand Down
36 changes: 31 additions & 5 deletions tests/unit/test_client_errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import pytest
from werkzeug import Response

from apify_client import ApifyClient, ApifyClientAsync
from apify_client.errors import (
ApifyApiError,
ConflictError,
Expand All @@ -25,7 +26,6 @@
from pytest_httpserver import HTTPServer
from werkzeug import Request

from apify_client import ApifyClient, ApifyClientAsync
from apify_client.http_clients import HttpClient, HttpClientAsync

pytestmark = pytest.mark.usefixtures('http_client_classes')
Expand Down Expand Up @@ -195,7 +195,7 @@ def test_apify_api_error_dispatches_5xx_to_server_error(
httpserver.expect_request('/server_error').respond_with_json(
{'error': {'type': 'internal-error', 'message': 'boom'}}, status=503
)
client = http_client_class(max_retries=1)
client = http_client_class(max_retries=0)

with pytest.raises(ServerError) as exc:
client.call(method='GET', url=str(httpserver.url_for('/server_error')))
Expand All @@ -204,6 +204,32 @@ def test_apify_api_error_dispatches_5xx_to_server_error(
assert exc.value.status_code == 503


def test_max_retries_zero_makes_single_attempt(httpserver: HTTPServer) -> None:
"""With `max_retries=0`, a retryable server error is requested once and raised as `ServerError`."""
httpserver.expect_request('/v2/datasets/ds-1').respond_with_json(
{'error': {'type': 'internal-error', 'message': 'boom'}}, status=500
)
client = ApifyClient(token='test', api_url=httpserver.url_for('/').removesuffix('/'), max_retries=0)

with pytest.raises(ServerError):
client.dataset('ds-1').get()

assert len(httpserver.log) == 1


async def test_max_retries_zero_makes_single_attempt_async(httpserver: HTTPServer) -> None:
"""With `max_retries=0`, a retryable server error is requested once and raised as `ServerError`."""
httpserver.expect_request('/v2/datasets/ds-1').respond_with_json(
{'error': {'type': 'internal-error', 'message': 'boom'}}, status=500
)
client = ApifyClientAsync(token='test', api_url=httpserver.url_for('/').removesuffix('/'), max_retries=0)

with pytest.raises(ServerError):
await client.dataset('ds-1').get()

assert len(httpserver.log) == 1


def test_apify_api_error_falls_back_for_unmapped_status(
httpserver: HTTPServer, http_client_class: type[HttpClient]
) -> None:
Expand Down Expand Up @@ -242,8 +268,8 @@ def test_apify_api_error_dispatches_all_mapped_statuses(
httpserver.expect_request('/dispatch_all').respond_with_json(
{'error': {'type': 'some-type', 'message': 'msg'}}, status=status_code
)
# Use max_retries=1 so retryable statuses (429) don't loop during the test.
client = http_client_class(max_retries=1)
# `max_retries=0` keeps retryable statuses (429) from being retried.
client = http_client_class(max_retries=0)

with pytest.raises(expected_cls) as exc:
client.call(method='GET', url=str(httpserver.url_for('/dispatch_all')))
Expand Down Expand Up @@ -272,7 +298,7 @@ def test_apify_api_error_falls_back_for_unparsable_body(
) -> None:
"""When the body can't be parsed, status-based dispatch still applies and `.type` is None."""
httpserver.expect_request('/unparsable').respond_with_data('<not json>', status=418, content_type='text/html')
client = http_client_class(max_retries=1)
client = http_client_class(max_retries=0)

with pytest.raises(ApifyApiError) as exc:
client.call(method='GET', url=str(httpserver.url_for('/unparsable')))
Expand Down
19 changes: 19 additions & 0 deletions tests/unit/test_http_clients.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import impit
import pytest

from apify_client import ApifyClient, ApifyClientAsync
from apify_client._consts import MIN_COMPRESSION_SIZE
from apify_client._logging import logger_name
from apify_client._statistics import ClientStatistics
Expand Down Expand Up @@ -210,6 +211,24 @@ def test_base_http_client_initialization() -> None:
assert isinstance(client2._statistics, ClientStatistics)


@pytest.mark.parametrize(
'client_class',
[
pytest.param(ApifyClient, id='ApifyClient'),
pytest.param(ApifyClientAsync, id='ApifyClientAsync'),
pytest.param(ConcreteHttpClient, id='custom HttpClient'),
pytest.param(ImpitHttpClient, id='ImpitHttpClient'),
pytest.param(ImpitHttpClientAsync, id='ImpitHttpClientAsync'),
pytest.param(Httpx2HttpClient, id='Httpx2HttpClient'),
pytest.param(Httpx2HttpClientAsync, id='Httpx2HttpClientAsync'),
],
)
def test_negative_max_retries_is_rejected_at_construction(client_class: Callable[..., object]) -> None:
"""A negative `max_retries` raises `ValueError` from the constructor, before any request is made."""
with pytest.raises(ValueError, match='max_retries must be a non-negative integer, got -1'):
client_class(max_retries=-1)


def test_http_client_init_headers_override_defaults_case_insensitively() -> None:
"""Constructor headers replace same-named default headers even when their casings differ."""
client = ConcreteHttpClient(token='default_token', headers={'authorization': 'Bearer custom'})
Expand Down
Loading