From 8a976093f5b7e0317bec7af8be403feeb5890d75 Mon Sep 17 00:00:00 2001 From: David Tapiador Date: Tue, 1 Sep 2026 09:59:58 +0200 Subject: [PATCH 1/2] Add retry jitter regression test --- api/datadog/client_retry_test.go | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 api/datadog/client_retry_test.go diff --git a/api/datadog/client_retry_test.go b/api/datadog/client_retry_test.go new file mode 100644 index 00000000000..e0b4761c1c6 --- /dev/null +++ b/api/datadog/client_retry_test.go @@ -0,0 +1,53 @@ +package datadog + +import ( + "net/http" + "testing" + "time" +) + +func TestRetryJitter(t *testing.T) { + tests := []struct { + name string + response *http.Response + minimum time.Duration + }{ + { + name: "server error backoff", + response: &http.Response{StatusCode: http.StatusInternalServerError, Header: http.Header{}}, + minimum: 2 * time.Second, + }, + { + name: "rate limit reset", + response: &http.Response{ + StatusCode: http.StatusTooManyRequests, + Header: http.Header{rateLimitResetHeader: []string{"1"}}, + }, + minimum: time.Second, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := NewConfiguration() + cfg.RetryConfiguration.EnableRetry = true + cfg.RetryConfiguration.RetryJitter = time.Second + client := NewAPIClient(cfg) + + var observedJitter bool + for i := 0; i < 10; i++ { + delay, retry := client.shouldRetryRequest(tc.response, 0) + if !retry { + t.Fatal("expected request to be retried") + } + if *delay < tc.minimum || *delay >= tc.minimum+time.Second { + t.Fatalf("retry delay %s outside [%s, %s)", *delay, tc.minimum, tc.minimum+time.Second) + } + observedJitter = observedJitter || *delay > tc.minimum + } + if !observedJitter { + t.Fatal("expected a jittered retry delay") + } + }) + } +} From ebd348b0173944696fdfa1680007b47311d4a32d Mon Sep 17 00:00:00 2001 From: "ci.datadog-api-spec" Date: Tue, 1 Sep 2026 09:07:17 +0000 Subject: [PATCH 2/2] Regenerate client from commit cb9c4d0 of spec repo --- api/datadog/client.go | 13 +++++- api/datadog/client_retry_test.go | 53 ---------------------- api/datadog/configuration.go | 1 + tests/scenarios/generated-test/test-server | 7 +-- 4 files changed, 16 insertions(+), 58 deletions(-) delete mode 100644 api/datadog/client_retry_test.go diff --git a/api/datadog/client.go b/api/datadog/client.go index 7b570089392..e963974d136 100644 --- a/api/datadog/client.go +++ b/api/datadog/client.go @@ -15,6 +15,7 @@ import ( "io" "log" "math" + "math/rand" "mime/multipart" "net/http" "net/http/httputil" @@ -237,7 +238,7 @@ func (c *APIClient) shouldRetryRequest(response *http.Response, retryCount int) if v := response.Header.Get(rateLimitResetHeader); response.StatusCode == 429 && v != "" { vInt, err := strconv.ParseInt(v, 10, 64) if err == nil { - retryDuration := time.Duration(vInt) * time.Second + retryDuration := time.Duration(vInt)*time.Second + c.retryJitter() return &retryDuration, true } } @@ -251,12 +252,20 @@ func (c *APIClient) shouldRetryRequest(response *http.Response, retryCount int) if c.Cfg.HTTPClient.Timeout > 0 { retryVal = math.Min(float64(c.Cfg.HTTPClient.Timeout/time.Second), retryVal) } - retryDuration := time.Duration(retryVal) * time.Second + retryDuration := time.Duration(retryVal)*time.Second + c.retryJitter() return &retryDuration, true } return nil, false } +func (c *APIClient) retryJitter() time.Duration { + max := c.Cfg.RetryConfiguration.RetryJitter + if max <= 0 { + return 0 + } + return time.Duration(rand.Int63n(int64(max))) +} + // GetConfig allows modification of underlying config for alternate implementations and testing. // Caution: modifying the configuration while live can cause data races and potentially unwanted behavior. func (c *APIClient) GetConfig() *Configuration { diff --git a/api/datadog/client_retry_test.go b/api/datadog/client_retry_test.go deleted file mode 100644 index e0b4761c1c6..00000000000 --- a/api/datadog/client_retry_test.go +++ /dev/null @@ -1,53 +0,0 @@ -package datadog - -import ( - "net/http" - "testing" - "time" -) - -func TestRetryJitter(t *testing.T) { - tests := []struct { - name string - response *http.Response - minimum time.Duration - }{ - { - name: "server error backoff", - response: &http.Response{StatusCode: http.StatusInternalServerError, Header: http.Header{}}, - minimum: 2 * time.Second, - }, - { - name: "rate limit reset", - response: &http.Response{ - StatusCode: http.StatusTooManyRequests, - Header: http.Header{rateLimitResetHeader: []string{"1"}}, - }, - minimum: time.Second, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cfg := NewConfiguration() - cfg.RetryConfiguration.EnableRetry = true - cfg.RetryConfiguration.RetryJitter = time.Second - client := NewAPIClient(cfg) - - var observedJitter bool - for i := 0; i < 10; i++ { - delay, retry := client.shouldRetryRequest(tc.response, 0) - if !retry { - t.Fatal("expected request to be retried") - } - if *delay < tc.minimum || *delay >= tc.minimum+time.Second { - t.Fatalf("retry delay %s outside [%s, %s)", *delay, tc.minimum, tc.minimum+time.Second) - } - observedJitter = observedJitter || *delay > tc.minimum - } - if !observedJitter { - t.Fatal("expected a jittered retry delay") - } - }) - } -} diff --git a/api/datadog/configuration.go b/api/datadog/configuration.go index 5d6e694f662..eab3285dced 100644 --- a/api/datadog/configuration.go +++ b/api/datadog/configuration.go @@ -136,6 +136,7 @@ type RetryConfiguration struct { BackOffBase float64 HTTPRetryTimeout time.Duration MaxRetries int + RetryJitter time.Duration } // NewConfiguration returns a new Configuration object. diff --git a/tests/scenarios/generated-test/test-server b/tests/scenarios/generated-test/test-server index 899230cbccd..a7224b2c64c 100755 --- a/tests/scenarios/generated-test/test-server +++ b/tests/scenarios/generated-test/test-server @@ -16,7 +16,7 @@ import re import tempfile import threading import uuid -from datetime import datetime, timezone +from datetime import UTC, datetime from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -435,7 +435,8 @@ def _json_contains(actual: Any, expected: Any) -> bool: return all(key in actual and _json_contains(actual[key], value) for key, value in expected.items()) if isinstance(expected, list) and isinstance(actual, list): return len(expected) == len(actual) and all( - _json_contains(actual_item, expected_item) for actual_item, expected_item in zip(actual, expected) + _json_contains(actual_item, expected_item) + for actual_item, expected_item in zip(actual, expected, strict=False) ) return actual == expected @@ -460,7 +461,7 @@ def _slug(value: str) -> str: def _now_iso() -> str: - return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + return datetime.now(UTC).isoformat().replace("+00:00", "Z") def _read_json(path: Path) -> dict[str, Any]: