Add configurable retry jitter to Go client

api/datadog/client.go

@@ -15,6 +15,7 @@ import (
 	"io"
 	"log"
 	"math"
+	"math/rand"
 	"mime/multipart"
 	"net/http"
 	"net/http/httputil"
@@ -237,7 +238,7 @@ func (c *APIClient) shouldRetryRequest(response *http.Response, retryCount int)
 	if v := response.Header.Get(rateLimitResetHeader); response.StatusCode == 429 && v != "" {
 		vInt, err := strconv.ParseInt(v, 10, 64)
 		if err == nil {
-			retryDuration := time.Duration(vInt) * time.Second
+			retryDuration := time.Duration(vInt)*time.Second + c.retryJitter()
 			return &retryDuration, true
 		}
 	}
@@ -251,12 +252,20 @@ func (c *APIClient) shouldRetryRequest(response *http.Response, retryCount int)
 		if c.Cfg.HTTPClient.Timeout > 0 {
 			retryVal = math.Min(float64(c.Cfg.HTTPClient.Timeout/time.Second), retryVal)
 		}
-		retryDuration := time.Duration(retryVal) * time.Second
+		retryDuration := time.Duration(retryVal)*time.Second + c.retryJitter()
 		return &retryDuration, true
 	}
 	return nil, false
 }
 
+func (c *APIClient) retryJitter() time.Duration {
+	max := c.Cfg.RetryConfiguration.RetryJitter
+	if max <= 0 {
+		return 0
+	}
+	return time.Duration(rand.Int63n(int64(max)))
+}
+
 // GetConfig allows modification of underlying config for alternate implementations and testing.
 // Caution: modifying the configuration while live can cause data races and potentially unwanted behavior.
 func (c *APIClient) GetConfig() *Configuration {

api/datadog/configuration.go

@@ -136,6 +136,7 @@ type RetryConfiguration struct {
 	BackOffBase       float64
 	HTTPRetryTimeout  time.Duration
 	MaxRetries        int
+	RetryJitter       time.Duration
 }
 
 // NewConfiguration returns a new Configuration object.

tests/scenarios/generated-test/test-server

@@ -16,7 +16,7 @@ import re
 import tempfile
 import threading
 import uuid
-from datetime import datetime, timezone
+from datetime import UTC, datetime
 from http import HTTPStatus
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
@@ -435,7 +435,8 @@ def _json_contains(actual: Any, expected: Any) -> bool:
         return all(key in actual and _json_contains(actual[key], value) for key, value in expected.items())
     if isinstance(expected, list) and isinstance(actual, list):
         return len(expected) == len(actual) and all(
-            _json_contains(actual_item, expected_item) for actual_item, expected_item in zip(actual, expected)
+            _json_contains(actual_item, expected_item)
+            for actual_item, expected_item in zip(actual, expected, strict=False)
         )
     return actual == expected
 
@@ -460,7 +461,7 @@ def _slug(value: str) -> str:
 
 
 def _now_iso() -> str:
-    return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
+    return datetime.now(UTC).isoformat().replace("+00:00", "Z")
 
 
 def _read_json(path: Path) -> dict[str, Any]: