fix: retry transient GitHub API failures and surface real error causes

apps/backend/tests/status.up.test.js

@@ -209,7 +209,16 @@ describe("Test /api/status/up", () => {
     mock.onPost("https://api.github.com/graphql").networkError();
 
     const { req, res } = faker({}, {});
-    await up(req, res);
+    // the retryer sleeps through its transient-backoff schedule
+    // (2 PATs x [1s, 2s, 4s] + jitter) before giving up
+    vi.useFakeTimers();
+    try {
+      const pending = up(req, res);
+      await vi.advanceTimersByTimeAsync(20_000);
+      await pending;
+    } finally {
+      vi.useRealTimers();
+    }
 
     expect(res.setHeader).toHaveBeenCalledWith(
       "Content-Type",
@@ -234,7 +243,16 @@ describe("Test /api/status/up", () => {
     mock.onPost("https://api.github.com/graphql").networkError();
 
     const { req, res } = faker({}, {});
-    await up(req, res);
+    // the retryer sleeps through its transient-backoff schedule
+    // (2 PATs x [1s, 2s, 4s] + jitter) before giving up
+    vi.useFakeTimers();
+    try {
+      const pending = up(req, res);
+      await vi.advanceTimersByTimeAsync(20_000);
+      await pending;
+    } finally {
+      vi.useRealTimers();
+    }
 
     expect(res.setHeader.mock.calls).toEqual([
       ["Content-Type", "application/json"],

packages/core/src/api/gist.js

@@ -2,6 +2,7 @@ import { renderGistCard } from "../cards/gist.js";
 import { findInvalidColor } from "../common/color.js";
 import {
   MissingParamError,
+  describeError,
   retrieveSecondaryMessage,
 } from "../common/error.js";
 import { parseBoolean } from "../common/ops.js";
@@ -102,6 +103,7 @@ export default async (
     if (err instanceof Error) {
       return {
         status: "error - temporary",
+        error: describeError(err),
         content: renderError({
           message: err.message,
           secondaryMessage: retrieveSecondaryMessage(err),

packages/core/src/api/index.js

@@ -2,6 +2,7 @@ import { renderStatsCard } from "../cards/stats.js";
 import { findInvalidColor } from "../common/color.js";
 import {
   MissingParamError,
+  describeError,
   retrieveSecondaryMessage,
 } from "../common/error.js";
 import { parseArray, parseBoolean } from "../common/ops.js";
@@ -171,6 +172,7 @@ export default async (
     if (err instanceof Error) {
       return {
         status: "error - temporary",
+        error: describeError(err),
         content: renderError({
           message: err.message,
           secondaryMessage: retrieveSecondaryMessage(err),

packages/core/src/api/pin.js

@@ -2,6 +2,7 @@ import { renderRepoCard } from "../cards/repo.js";
 import { findInvalidColor } from "../common/color.js";
 import {
   MissingParamError,
+  describeError,
   retrieveSecondaryMessage,
 } from "../common/error.js";
 import { parseArray, parseBoolean } from "../common/ops.js";
@@ -131,6 +132,7 @@ export default async (
     if (err instanceof Error) {
       return {
         status: "error - temporary",
+        error: describeError(err),
         content: renderError({
           message: err.message,
           secondaryMessage: retrieveSecondaryMessage(err),

packages/core/src/api/top-langs.js

@@ -2,6 +2,7 @@ import { renderTopLanguages } from "../cards/top-languages.js";
 import { findInvalidColor } from "../common/color.js";
 import {
   MissingParamError,
+  describeError,
   retrieveSecondaryMessage,
 } from "../common/error.js";
 import { parseArray, parseBoolean } from "../common/ops.js";
@@ -169,6 +170,7 @@ export default async (
     if (err instanceof Error) {
       return {
         status: "error - temporary",
+        error: describeError(err),
         content: renderError({
           message: err.message,
           secondaryMessage: retrieveSecondaryMessage(err),

packages/core/src/api/wakatime.js

@@ -2,6 +2,7 @@ import { renderWakatimeCard } from "../cards/wakatime.js";
 import { findInvalidColor } from "../common/color.js";
 import {
   MissingParamError,
+  describeError,
   retrieveSecondaryMessage,
 } from "../common/error.js";
 import { parseArray, parseBoolean } from "../common/ops.js";
@@ -116,6 +117,7 @@ export default async ({
     if (err instanceof Error) {
       return {
         status: "error - temporary",
+        error: describeError(err),
         content: renderError({
           message: err.message,
           secondaryMessage: retrieveSecondaryMessage(err),

packages/core/src/common/error.ts

@@ -87,10 +87,26 @@ const retrieveSecondaryMessage = (err: Error): string | undefined => {
     : undefined;
 };
 
+/**
+ * Extract structured details from a caught error.
+ *
+ * Callers attach the result to API results as an optional `error` field.
+ * The `status` value itself stays stable, so exact comparisons in
+ * `apps/backend/router.js` and external callers keep working.
+ *
+ * @param err The caught error.
+ * @returns The error type and message.
+ */
+const describeError = (err: Error): { type?: string; message: string } => {
+  const type = "type" in err && typeof err.type === "string" ? err.type : "";
+  return type ? { type, message: err.message } : { message: err.message };
+};
+
 export {
   CustomError,
   MissingParamError,
   SECONDARY_ERROR_MESSAGES,
   TRY_AGAIN_LATER,
+  describeError,
   retrieveSecondaryMessage,
 };

packages/core/src/common/retryer.ts

@@ -28,6 +28,50 @@ function getRandomInt(max: number): number {
   return Math.floor(Math.random() * max);
 }
 
+/**
+ * Delay before each transient retry of the same PAT.
+ *
+ * Transient failures are network-level errors (ECONNRESET, ETIMEDOUT,
+ * socket hang up) and retryable HTTP statuses. Token rotation stays
+ * separate from this backoff.
+ */
+const TRANSIENT_RETRY_DELAYS_MS = [1000, 2000, 4000];
+
+/** Random extra wait added to each transient retry delay. */
+const TRANSIENT_RETRY_JITTER_MS = 250;
+
+/**
+ * HTTP statuses worth a same-token retry. Server-side blips only.
+ * 429 and rate-limit 403 answers skip quick retries entirely: hammering a
+ * limited token violates GitHub's Retry-After / reset guidance and can get
+ * the token blocked. Permanent statuses such as 401/404/422 stay outside
+ * this set.
+ */
+const RETRYABLE_HTTP_STATUS_CODES = new Set([502, 503, 504]);
+
+/**
+ * Wait for `ms` milliseconds.
+ */
+const sleep = (ms: number): Promise<void> => {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+};
+
+/**
+ * Optional overrides for {@link retryer}, mainly for tests.
+ */
+interface RetryerOptions {
+  /**
+   * Delays between transient retries of one PAT.
+   * An empty array disables transient retries.
+   */
+  transientRetryDelaysMs?: Array<number>;
+}
+
+/**
+ * Kind of the failure that caused the latest PAT rotation.
+ */
+type FailureKind = "transient" | "rate-limit" | "credential";
+
 /**
  * A fetcher's Axios response. `TData` is the shape of `response.data`,
  * which is intersected with {@link ResponseErrors} so the retryer can inspect
@@ -55,6 +99,7 @@ const retryer = async <TData = unknown>(
   fetcher: FetcherFunction<TData>,
   variables: Record<string, unknown>,
   pat: string | null = null,
+  { transientRetryDelaysMs = TRANSIENT_RETRY_DELAYS_MS }: RetryerOptions = {},
 ): Promise<FetcherResponse<TData>> => {
   const PATs = pat
     ? [{ name: "user PAT from database", value: pat }]
@@ -65,61 +110,124 @@ const retryer = async <TData = unknown>(
   }
   const startPAT = getRandomInt(PATs.length);
 
+  let lastTransientError: unknown = null;
+  // Kind of the most recent rotation. The final message reflects the last
+  // observed failure, not the first one.
+  let lastFailureKind: FailureKind | null = null;
+
   for (let retries = 0; retries < PATs.length; retries++) {
     const currentPAT = PATs[(startPAT + retries) % PATs.length];
     if (!currentPAT) {
       continue;
     }
 
-    try {
-      const response = await fetcher(
-        variables,
-        currentPAT.value,
-        // used in tests for faking rate limit
-        retries,
-      );
-
-      // react on both type and message-based rate-limit signals.
-      // https://github.com/anuraghazra/github-readme-stats/issues/4425
-      const errors = response.data.errors;
-      const errorType = errors?.[0]?.type;
-      const errorMsg = errors?.[0]?.message ?? "";
-      const isRateLimited =
-        (!!errors && errorType === "RATE_LIMITED") ||
-        /rate limit/i.test(errorMsg);
-
-      if (isRateLimited) {
-        logger.log(`${currentPAT.name} Failed due to rate limiting`);
-      } else {
+    // One transient retry per delay entry. The last pass has no delay left
+    // and rotates to the next PAT instead.
+    for (let attempt = 0; attempt <= transientRetryDelaysMs.length; attempt++) {
+      try {
+        const response = await fetcher(
+          variables,
+          currentPAT.value,
+          // used in tests for faking rate limit
+          retries,
+        );
+
+        // react on both type and message-based rate-limit signals.
+        // https://github.com/anuraghazra/github-readme-stats/issues/4425
+        const errors = response.data.errors;
+        const errorType = errors?.[0]?.type;
+        const errorMsg = errors?.[0]?.message ?? "";
+        const isRateLimited =
+          (!!errors && errorType === "RATE_LIMITED") ||
+          /rate limit/i.test(errorMsg);
+
+        if (isRateLimited) {
+          logger.log(`${currentPAT.name} Failed due to rate limiting`);
+          lastFailureKind = "rate-limit";
+          break; // rotate to next PAT
+        }
         return response;
-      }
-    } catch (err) {
-      const e = err as { response?: FetcherResponse<TData> };
-
-      // network/unexpected error → let caller treat as failure
-      if (!e.response) {
-        throw err;
-      }
+      } catch (err) {
+        const e = err as {
+          response?: FetcherResponse<TData>;
+          isAxiosError?: boolean;
+          message?: unknown;
+        };
+
+        // Rate-limit responses never get quick retries. Quick retries would
+        // violate GitHub's Retry-After / reset guidance and can get the
+        // token blocked, so rotate to the next PAT instead. HTTP 429 and
+        // rate-limit 403 answers both carry this meaning.
+        const status = e.response?.status;
+        const carriesRateLimitMessage = /rate limit/i.test(
+          e.response?.data.message ?? "",
+        );
+        const isRateLimitResponse =
+          status === 429 || (status === 403 && carriesRateLimitMessage);
+
+        if (isRateLimitResponse && e.response) {
+          logger.log(
+            `${currentPAT.name} hit a rate limit (HTTP ${status}), rotating`,
+          );
+          lastFailureKind = "rate-limit";
+          break; // rotate to next PAT
+        }
+
+        // Transient failure: network-level error without a response, or a
+        // retryable HTTP status. Retry the same PAT with backoff before
+        // rotating to the next token.
+        const isTransient =
+          (!e.response && e.isAxiosError === true) ||
+          (!!e.response && RETRYABLE_HTTP_STATUS_CODES.has(e.response.status));
+
+        if (isTransient) {
+          lastTransientError = err;
+          lastFailureKind = "transient";
+          const delayMs = transientRetryDelaysMs[attempt];
+          if (delayMs !== undefined) {
+            logger.log(
+              `${currentPAT.name} transient failure (${String(e.message)}), retrying`,
+            );
+            await sleep(delayMs + getRandomInt(TRANSIENT_RETRY_JITTER_MS));
+            continue;
+          }
+          break; // retries exhausted → rotate to next PAT
+        }
+
+        // non-axios errors are bugs, not transient failures
+        if (!e.response) {
+          throw err;
+        }
+
+        // also checking for bad credentials if any tokens gets invalidated
+        const message = e.response.data.message;
+        const isBadCredential = message === "Bad credentials";
+        const isAccountSuspended =
+          message === "Sorry. Your account was suspended.";
+
+        if (isBadCredential || isAccountSuspended) {
+          logger.log(`${currentPAT.name} Failed due to bad credentials`);
+          lastFailureKind = "credential";
+          break; // rotate to next PAT
+        }
 
-      // also checking for bad credentials if any tokens gets invalidated
-      const message = e.response.data.message;
-      const isBadCredential = message === "Bad credentials";
-      const isAccountSuspended =
-        message === "Sorry. Your account was suspended.";
-
-      if (isBadCredential || isAccountSuspended) {
-        logger.log(`${currentPAT.name} Failed due to bad credentials`);
-      } else {
         // HTTP error with a response → return it for caller-side handling
         return e.response;
       }
     }
   }
 
-  throw new CustomError(
-    "Downtime due to GitHub API rate limiting",
-    CustomError.MAX_RETRY,
-  );
+  // The final message reflects the last failure kind. Claiming "rate
+  // limiting" without a rate limit would mislead the reader. A missing kind
+  // means no PAT slot was usable; keep the historical message there.
+  let reason = "Downtime due to GitHub API rate limiting";
+  if (lastFailureKind === "transient" && lastTransientError instanceof Error) {
+    reason = `GitHub API request failed after transient retries: ${lastTransientError.message}`;
+  } else if (lastFailureKind === "credential") {
+    reason = "GitHub API request failed due to invalid credentials";
+  }
+
+  throw new CustomError(reason, CustomError.MAX_RETRY);
 };
 
 export { retryer };