diff --git a/src/codex/auth-api.ts b/src/codex/auth-api.ts index 8389a15b4..3bb657729 100644 --- a/src/codex/auth-api.ts +++ b/src/codex/auth-api.ts @@ -16,11 +16,13 @@ import { import { deleteCodexAccount, reconcileMainCodexAccountRuntimeState } from "./account-lifecycle"; import { isCodexAccountPaused, setCodexAccountPaused } from "./account-pause"; import { + claimDueCodexQuotaRecoveryProbes, clearCodexAccountCooldown, clearThreadAccountMapForAccount, getEffectiveActiveCodexAccountId, reconcileCodexActiveAfterExclusion, resetCodexRoutingForManualSelection, + settleCodexQuotaRecoveryProbe, } from "./routing"; import { normalizeAccountPoolStickyLimit, @@ -35,6 +37,7 @@ import { clearAccountNeedsReauth, isAccountNeedsReauth, markAccountNeedsReauth } import { clearAccountQuota, getAccountQuota, + isCompleteCodexQuotaRecoverySnapshot, isCodexQuotaExhausted, listAccountQuotas, parseUsageQuota, @@ -53,7 +56,7 @@ export { } from "./quota"; import { extractAccountId, decodeJwtPayload } from "../oauth/chatgpt"; import { getMainAccountPlan, MAIN_CODEX_ACCOUNT_ID, setMainAccountPlan } from "./main-account"; -import { captureConfigGeneration } from "../lib/state-store-sweeper"; +import { captureConfigGeneration, registerStateSweepAfterTick } from "../lib/state-store-sweeper"; import { reconcileLiveStateStores } from "../lib/state-store-registrations"; import { captureMainAccountIdentityGeneration, @@ -816,6 +819,49 @@ async function fetchPoolAccountQuota(accountId: string, forceRefresh = false, co } let primeInFlight: Promise | null = null; +let cooldownRecoveryInFlight: Promise | null = null; + +export async function runCodexCooldownRecoveryProbes(config: OcxConfig, now = Date.now()): Promise { + const openai = config.providers[OPENAI_CODEX_PROVIDER_ID]; + if (!openai + || openai.disabled === true + || !isCanonicalOpenAiForwardProvider(openai) + || providerCodexAccountMode(OPENAI_CODEX_PROVIDER_ID, openai) !== "pool") return; + if (cooldownRecoveryInFlight) return cooldownRecoveryInFlight; + cooldownRecoveryInFlight = (async () => { + const claims = claimDueCodexQuotaRecoveryProbes(config, POOL_QUOTA_REFRESH_CONCURRENCY, now); + await mapWithConcurrency(claims, POOL_QUOTA_REFRESH_CONCURRENCY, async claim => { + const account = configuredPoolAccount(config, claim.accountId); + if (!account) { + settleCodexQuotaRecoveryProbe(claim, false, {}, now); + return; + } + try { + const result = await fetchPoolAccountQuota(claim.accountId, true, account.plan); + // Defence in depth: `spark` is already excluded at the claim site, since generic WHAM + // cannot prove a spark recovery. Keep the settle-side guard so a future claim change + // cannot silently start clearing spark on generic evidence. + const recovered = claim.scope !== "spark" + && isCompleteCodexQuotaRecoverySnapshot(result.freshQuota ?? null, result.freshPlan ?? account.plan); + settleCodexQuotaRecoveryProbe(claim, recovered, { + credentialGeneration: result.freshCredentialGeneration, + }, now); + } catch { + settleCodexQuotaRecoveryProbe(claim, false, {}, now); + } + }); + })().catch(() => { + // Background recovery is best-effort; routing keeps the cooldown on failure. + }).finally(() => { cooldownRecoveryInFlight = null; }); + return cooldownRecoveryInFlight; +} + +export function registerCodexCooldownRecoveryProbeWorker(config: OcxConfig): void { + registerStateSweepAfterTick({ + name: "codex-cooldown-recovery", + afterTick: () => { void runCodexCooldownRecoveryProbes(config); }, + }); +} export interface PrimeCodexPoolQuotasOptions { /** Test seams for proving fenced/recovery priming performs no native-main work. */ @@ -908,6 +954,11 @@ export function clearCodexQuotaPrimeState(): void { primeInFlight = null; } +/** Test-only reset for the worker-level single-flight. */ +export function clearCodexCooldownRecoveryProbeState(): void { + cooldownRecoveryInFlight = null; +} + export function effectiveCodexAuthAccountId(config: OcxConfig): string { return getEffectiveActiveCodexAccountId(config) ?? MAIN_CODEX_ACCOUNT_ID; } diff --git a/src/codex/quota.ts b/src/codex/quota.ts index 7d4f60563..739baf986 100644 --- a/src/codex/quota.ts +++ b/src/codex/quota.ts @@ -67,8 +67,7 @@ export function isCodexQuotaExhausted( plan?: string | null, ): boolean { if (!quota) return false; - const normalizedPlan = plan?.trim().toLowerCase(); - const values = normalizedPlan === "go" || normalizedPlan === "free" + const values = codexQuotaWindowForPlan(plan) === "monthly" ? [quota.monthlyPercent] : [quota.weeklyPercent, quota.monthlyPercent]; return values.some(value => typeof value === "number" @@ -76,6 +75,40 @@ export function isCodexQuotaExhausted( && value >= CODEX_EXHAUSTED_USAGE_PERCENT); } +/** + * Which usage window a plan reports in. This is the SINGLE rule shared by quota + * parsing, exhaustion, and recovery — they must not diverge. + * + * An allowlist of "known" plans was tried here and was wrong: the upstream model + * snapshot alone carries 21 distinct plan strings (`edu_plus`, `finserv`, `k12`, + * `quorum`, `self_serve_business_usage_based`, ...), and `CodexAccount.plan` is an + * unrestricted string, so any list is a list of the plans someone remembered. + * Twelve real plans would have been refused recovery and stayed cooled forever — + * the very defect this unit exists to fix, reintroduced as a typo-shaped hole. + * + * The honest rule is the parser's own: Go and Free report a 30-day window, + * everything else (including an absent plan) reports weekly. Recovery reads the + * window the parser actually wrote rather than second-guessing it. + */ +export function codexQuotaWindowForPlan(plan?: string | null): "monthly" | "weekly" { + const normalized = plan?.trim().toLowerCase(); + return normalized === "go" || normalized === "free" ? "monthly" : "weekly"; +} + +export function isCompleteCodexQuotaRecoverySnapshot( + quota: Pick | null, + plan?: string | null, +): boolean { + if (!quota || isCodexQuotaExhausted(quota, plan)) return false; + // Recovery still fails closed on MISSING EVIDENCE — a credits-only or windowless payload + // carries no usage reading at all and must never clear a cooldown. What it does not do is + // fail closed on an unfamiliar plan NAME, which only ever meant "cooled forever". + const required = codexQuotaWindowForPlan(plan) === "monthly" + ? quota.monthlyPercent + : quota.weeklyPercent; + return typeof required === "number" && Number.isFinite(required); +} + export function normalizeUsagePercent(value: unknown): number | undefined { const numeric = typeof value === "number" ? value @@ -367,7 +400,7 @@ export function parseUsageQuota(data: WhamUsageResponse): Omit = {}; - const thirtyDayOnly = data.plan_type?.trim().toLowerCase() === "go" || data.plan_type?.trim().toLowerCase() === "free"; + const thirtyDayOnly = codexQuotaWindowForPlan(data.plan_type) === "monthly"; const primaryWindow = data.rate_limit.primary_window; const secondaryWindow = data.rate_limit.secondary_window; const tertiaryWindow = data.rate_limit.tertiary_window; diff --git a/src/codex/routing.ts b/src/codex/routing.ts index 63030c5c5..7a47921b0 100644 --- a/src/codex/routing.ts +++ b/src/codex/routing.ts @@ -133,6 +133,18 @@ export type CodexCooldownSource = "retry-after" | "reset-derived" | "default"; */ export type CodexQuotaScope = "shared" | "spark"; +export type CodexQuotaRecoveryProbeClaim = { + accountId: string; + scope?: CodexQuotaScope; + leaseId: string; + cooldownGeneration: number; + credentialGeneration: number; +}; + +export type CodexQuotaRecoveryProbeProof = { + credentialGeneration?: number; +}; + /** * Requests without a resolved native model retain the historic one-account-per- * thread behavior. Requests with a known quota scope get an independent @@ -417,6 +429,119 @@ function canAcquireQuotaProbeLease(health: CodexUpstreamHealth | undefined, now: return now - origin >= CODEX_QUOTA_PROBE_INTERVAL_MS; } +/** + * Claim due reset-derived cooldown probes without consulting account selection. + * Pool credentials only: the main account has no quota-refresh single-flight. + */ +export function claimDueCodexQuotaRecoveryProbes( + config: OcxConfig, + limit: number, + now = Date.now(), +): CodexQuotaRecoveryProbeClaim[] { + const boundedLimit = Math.max(0, Math.floor(limit)); + if (boundedLimit === 0) return []; + const candidates: Array<{ + accountId: string; + scope?: CodexQuotaScope; + health: CodexUpstreamHealth; + credentialGeneration: number; + order: number; + }> = []; + for (const [order, account] of (config.codexAccounts ?? []).entries()) { + if (!isSelectableCodexPoolAccount(account) + || isCodexAccountPaused(config, account.id) + || isAccountNeedsReauth(account.id)) continue; + const record = readCodexAccountRecord(account.id); + if (!record?.credential || record.deletedAt != null) continue; + const due = [ + { scope: undefined, health: upstreamHealth.get(account.id) }, + ...[...(quotaScopedHealth.get(account.id) ?? [])].map(([scope, health]) => ({ scope, health })), + ].filter((entry): entry is { scope?: CodexQuotaScope; health: CodexUpstreamHealth } => + // `spark` is deliberately never claimed. `GET /backend-api/wham/usage` takes no scope + // parameter and returns generic weekly/monthly windows, so its result can never prove a + // spark recovery — a claim here would spend an upstream call to settle `false` every + // time, and (with one claim per account per pass) delay the shared scope that CAN recover. + entry.scope !== "spark" + && entry.health?.cooldownSource === "reset-derived" + && canAcquireQuotaProbeLease(entry.health, now)) + .sort((a, b) => + (a.health.lastProbeAt ?? a.health.cooldownSince ?? 0) + - (b.health.lastProbeAt ?? b.health.cooldownSince ?? 0)); + const candidate = due[0]; + if (candidate) candidates.push({ + accountId: account.id, + ...(candidate.scope ? { scope: candidate.scope } : {}), + health: candidate.health, + credentialGeneration: record.generation, + order, + }); + } + candidates.sort((a, b) => { + const age = (a.health.lastProbeAt ?? a.health.cooldownSince ?? 0) + - (b.health.lastProbeAt ?? b.health.cooldownSince ?? 0); + return age || a.order - b.order; + }); + return candidates.slice(0, boundedLimit).map(candidate => { + const leaseId = randomUUID(); + const next = { + ...candidate.health, + probeLeaseId: leaseId, + probeLeaseGeneration: candidate.health.cooldownGeneration ?? 0, + lastProbeAt: now, + }; + if (candidate.scope) setScopedHealth(candidate.accountId, candidate.scope, next); + else upstreamHealth.set(candidate.accountId, next); + return { + accountId: candidate.accountId, + ...(candidate.scope ? { scope: candidate.scope } : {}), + leaseId, + cooldownGeneration: candidate.health.cooldownGeneration ?? 0, + credentialGeneration: candidate.credentialGeneration, + }; + }); +} + +/** Settle one background recovery claim without mutating account-wide outcome state. */ +export function settleCodexQuotaRecoveryProbe( + claim: CodexQuotaRecoveryProbeClaim, + recovered: boolean, + proof: CodexQuotaRecoveryProbeProof, + now = Date.now(), +): boolean { + const health = claim.scope + ? scopedHealthFor(claim.accountId, claim.scope) + : upstreamHealth.get(claim.accountId); + if (!health || health.probeLeaseId !== claim.leaseId) return false; + const fenced = (health.cooldownGeneration ?? 0) === claim.cooldownGeneration + && (health.probeLeaseGeneration ?? 0) === claim.cooldownGeneration + && claim.credentialGeneration === proof.credentialGeneration + && isCodexAccountGenerationLive(claim.accountId, claim.credentialGeneration); + if (!recovered || !fenced) { + const released = withProbeLeaseReleased(health, now); + if (claim.scope) setScopedHealth(claim.accountId, claim.scope, released); + else upstreamHealth.set(claim.accountId, released); + return false; + } + if (claim.scope) { + deleteScopedHealth(claim.accountId, claim.scope); + } else { + const { + cooldownUntil: _until, + cooldownSince: _since, + cooldownSource: _source, + probeLeaseId: _leaseId, + probeLeaseGeneration: _leaseGeneration, + ...rest + } = health; + upstreamHealth.set(claim.accountId, { + ...rest, + cooldownGeneration: claim.cooldownGeneration + 1, + lastProbeAt: now, + }); + } + return true; +} + /** Acquire the recovery probe for one confirmed model-specific quota group. */ export function tryAcquireCodexQuotaScopeProbeLease( accountId: string, diff --git a/src/server/index.ts b/src/server/index.ts index 1222155f7..57e6f6c9f 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -20,6 +20,7 @@ import { } from "../config"; import { reconcileOAuthProviders } from "../oauth"; import { invalidateCodexModelsCache } from "../codex/catalog"; +import { registerCodexCooldownRecoveryProbeWorker } from "../codex/auth-api"; import { startMemoryWatchdog } from "./memory-watchdog"; import { reconcileLiveStateStores, @@ -416,6 +417,7 @@ export function startServer(port?: number, deps: StartServerDeps = {}) { registerAppOwnedMemorySweepFallback(); configureAppOwnedMemoryBudget(resolveAppOwnedMemoryBudgetBytes(config.appOwnedMemoryBudgetMb)); enforceAppOwnedMemoryBudget(); + registerCodexCooldownRecoveryProbeWorker(config); startStateStoreSweeper(); // Issue #42 Phase 3: opt-in archived auto-cleanup (default OFF). Unref'd hourly // tick for daily/weekly; startup evaluation is fire-and-forget after listen. diff --git a/tests/codex-cooldown-recovery.test.ts b/tests/codex-cooldown-recovery.test.ts new file mode 100644 index 000000000..7ce26fd19 --- /dev/null +++ b/tests/codex-cooldown-recovery.test.ts @@ -0,0 +1,364 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, rmSync } from "node:fs"; +import { join } from "node:path"; +import { + clearCodexCooldownRecoveryProbeState, + clearAccountQuota, + runCodexCooldownRecoveryProbes, + seedCodexAuthAdmissionForTests, +} from "../src/codex/auth-api"; +import { saveCodexAccountCredential } from "../src/codex/account-store"; +import { codexQuotaWindowForPlan, isCompleteCodexQuotaRecoverySnapshot } from "../src/codex/quota"; +import upstreamModels from "../src/codex/data/upstream-models.json"; +import { + CODEX_QUOTA_PROBE_INTERVAL_MS, + clearCodexUpstreamHealth, + getCodexQuotaHealthSnapshot, + recordCodexUpstreamOutcome, + resolveCodexAccountForThread, +} from "../src/codex/routing"; +import type { OcxConfig } from "../src/types"; + +const TEST_DIR = join(import.meta.dir, ".tmp-codex-cooldown-recovery-test"); +const TEST_CODEX_HOME = join(TEST_DIR, "codex"); +const START = 1_800_000_000_000; +let previousOpencodexHome: string | undefined; +let previousCodexHome: string | undefined; +let previousFetch: typeof fetch; + +function makeConfig(ids = ["a", "b"]): OcxConfig { + return { + providers: { + openai: { + adapter: "openai-responses", + baseUrl: "https://chatgpt.com/backend-api/codex", + authMode: "forward", + codexAccountMode: "pool", + }, + }, + defaultProvider: "openai", + activeCodexAccountId: ids[0], + accountPoolStrategy: "fill-first", + codexAccounts: ids.map(id => ({ id, email: `${id}@example.test`, plan: "team", isMain: false })), + } as OcxConfig; +} + +function saveCredential(id: string, suffix = ""): void { + saveCodexAccountCredential(id, { + accessToken: `access-${id}${suffix}`, + refreshToken: `refresh-${id}${suffix}`, + expiresAt: Date.now() + 60 * 60_000, + chatgptAccountId: `acct-${id}${suffix}`, + }); +} + +function cool(config: OcxConfig, id: string, scope: "shared" | "spark" = "shared", now = START): void { + recordCodexUpstreamOutcome(config, id, 429, { + now, + resetAt: now + 60 * 60_000, + modelId: scope === "spark" ? "gpt-5.3-codex-spark" : "gpt-5.6-sol", + }); +} + +function usageResponse(percent = 10, body?: unknown): Response { + return new Response(JSON.stringify(body ?? { + plan_type: "team", + rate_limit: { secondary_window: { used_percent: percent, reset_at: 1_900_000_000 } }, + }), { status: 200, headers: { "Content-Type": "application/json" } }); +} + +function due(at = START): number { + return at + CODEX_QUOTA_PROBE_INTERVAL_MS + 1; +} + +describe("Codex cooldown recovery worker", () => { + beforeEach(() => { + previousOpencodexHome = process.env.OPENCODEX_HOME; + previousCodexHome = process.env.CODEX_HOME; + previousFetch = globalThis.fetch; + if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true }); + mkdirSync(TEST_CODEX_HOME, { recursive: true }); + process.env.OPENCODEX_HOME = TEST_DIR; + process.env.CODEX_HOME = TEST_CODEX_HOME; + clearAccountQuota(); + clearCodexUpstreamHealth(); + clearCodexCooldownRecoveryProbeState(); + }); + + afterEach(() => { + globalThis.fetch = previousFetch; + clearAccountQuota(); + clearCodexUpstreamHealth(); + clearCodexCooldownRecoveryProbeState(); + if (previousOpencodexHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousOpencodexHome; + if (previousCodexHome === undefined) delete process.env.CODEX_HOME; + else process.env.CODEX_HOME = previousCodexHome; + if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true }); + }); + + test("recovers cooled A independently while ordinary routing only selects B", async () => { + const config = makeConfig(); + saveCredential("a"); + saveCredential("b"); + cool(config, "a"); + const routed = [resolveCodexAccountForThread("before", config, due())]; + const authorizations: string[] = []; + globalThis.fetch = async (_input, init) => { + authorizations.push(new Headers(init?.headers).get("Authorization") ?? ""); + return usageResponse(12); + }; + + await runCodexCooldownRecoveryProbes(config, due()); + routed.push(resolveCodexAccountForThread("after", config, due() + 1)); + + expect(authorizations).toEqual(["Bearer access-a"]); + expect(getCodexQuotaHealthSnapshot("a", "shared", due() + 1)).toBeNull(); + expect(routed).toEqual(["b", "b"]); + }); + + test.each([ + ["still exhausted", () => usageResponse(100)], + ["credits only", () => usageResponse(0, { plan_type: "team", rate_limit_reset_credits: { available_count: 1 } })], + ["non-2xx", () => new Response("busy", { status: 503 })], + ["parse failure", () => new Response("not-json", { status: 200 })], + ])("retains the cooldown for %s", async (_name, response) => { + const config = makeConfig(["a"]); + saveCredential("a"); + cool(config, "a"); + globalThis.fetch = async () => response(); + await runCodexCooldownRecoveryProbes(config, due()); + expect(getCodexQuotaHealthSnapshot("a", "shared", due() + 1)).not.toBeNull(); + }); + + test("retains the cooldown after transport timeout", async () => { + const config = makeConfig(["a"]); + saveCredential("a"); + cool(config, "a"); + globalThis.fetch = async () => { throw new DOMException("timed out", "TimeoutError"); }; + await runCodexCooldownRecoveryProbes(config, due()); + expect(getCodexQuotaHealthSnapshot("a", "shared", due() + 1)).not.toBeNull(); + // Same reasoning as the admission case: prove the timed-out claim released its lease by + // requiring a later pass to succeed. + let calls = 0; + globalThis.fetch = async () => { calls += 1; return usageResponse(); }; + const later = due() + CODEX_QUOTA_PROBE_INTERVAL_MS + 1; + await runCodexCooldownRecoveryProbes(config, later); + expect(calls).toBe(1); + expect(getCodexQuotaHealthSnapshot("a", "shared", later + 1)).toBeNull(); + }); + + test("retains and releases a claim when quota admission is busy", async () => { + const config = makeConfig(["a"]); + saveCredential("a"); + cool(config, "a"); + const cleanup = seedCodexAuthAdmissionForTests({ quotaFlights: 16 }); + let calls = 0; + globalThis.fetch = async () => { calls += 1; return usageResponse(); }; + try { + await runCodexCooldownRecoveryProbes(config, due()); + } finally { + cleanup(); + } + expect(calls).toBe(0); + expect(getCodexQuotaHealthSnapshot("a", "shared", due() + 1)).not.toBeNull(); + + // "Releases" has to be proven, not asserted by the test name. A stranded lease is worse + // than the bug being fixed: that account would never be probed again. So lift the admission + // pressure, advance past the probe interval, and require the NEXT pass to reach WHAM and + // actually clear the cooldown — which is only possible if the failed claim released. + const later = due() + CODEX_QUOTA_PROBE_INTERVAL_MS + 1; + await runCodexCooldownRecoveryProbes(config, later); + expect(calls).toBe(1); + expect(getCodexQuotaHealthSnapshot("a", "shared", later + 1)).toBeNull(); + }); + + test("credential replacement during WHAM cannot clear the old cooldown", async () => { + const config = makeConfig(["a"]); + saveCredential("a"); + cool(config, "a"); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + globalThis.fetch = async () => { await gate; return usageResponse(); }; + const run = runCodexCooldownRecoveryProbes(config, due()); + await Promise.resolve(); + saveCredential("a", "-new"); + release(); + await run; + expect(getCodexQuotaHealthSnapshot("a", "shared", due() + 1)).not.toBeNull(); + }); + + test("a newer 429 during WHAM cannot be erased by the older probe", async () => { + const config = makeConfig(["a"]); + saveCredential("a"); + cool(config, "a"); + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + globalThis.fetch = async () => { await gate; return usageResponse(); }; + const run = runCodexCooldownRecoveryProbes(config, due()); + await Promise.resolve(); + recordCodexUpstreamOutcome(config, "a", 429, { + now: due() + 1, + resetAt: due() + 60 * 60_000, + modelId: "gpt-5.6-sol", + }); + release(); + await run; + expect(getCodexQuotaHealthSnapshot("a", "shared", due() + 2)).not.toBeNull(); + }); + + test("concurrent worker passes coalesce into one WHAM request", async () => { + const config = makeConfig(["a"]); + saveCredential("a"); + cool(config, "a"); + let calls = 0; + globalThis.fetch = async () => { calls += 1; await Promise.resolve(); return usageResponse(); }; + await Promise.all([ + runCodexCooldownRecoveryProbes(config, due()), + runCodexCooldownRecoveryProbes(config, due()), + ]); + expect(calls).toBe(1); + }); + + test("shared recovery leaves Spark cooled", async () => { + const config = makeConfig(["a"]); + saveCredential("a"); + cool(config, "a", "shared", START); + cool(config, "a", "spark", START + 1); + globalThis.fetch = async () => usageResponse(); + await runCodexCooldownRecoveryProbes(config, due(START + 1)); + expect(getCodexQuotaHealthSnapshot("a", "shared", due(START + 1) + 1)).toBeNull(); + expect(getCodexQuotaHealthSnapshot("a", "spark", due(START + 1) + 1)).not.toBeNull(); + }); + + test("an older Spark cooldown never starves the shared scope that can recover", async () => { + // Spark is skipped at the claim site: generic WHAM carries no scope and can never prove a + // spark recovery. Claiming it would spend the account's one claim per pass to settle false, + // leaving the shared scope — which this evidence CAN clear — cooled behind it. + const config = makeConfig(["a"]); + saveCredential("a"); + cool(config, "a", "spark", START); + cool(config, "a", "shared", START + 1); + globalThis.fetch = async () => usageResponse(); + await runCodexCooldownRecoveryProbes(config, due(START + 1)); + expect(getCodexQuotaHealthSnapshot("a", "spark", due(START + 1) + 1)).not.toBeNull(); + expect(getCodexQuotaHealthSnapshot("a", "shared", due(START + 1) + 1)).toBeNull(); + }); + + test("a Spark-only cooldown makes no upstream call at all", async () => { + const config = makeConfig(["a"]); + saveCredential("a"); + cool(config, "a", "spark", START); + let calls = 0; + globalThis.fetch = async () => { calls += 1; return usageResponse(); }; + await runCodexCooldownRecoveryProbes(config, due(START)); + expect(calls).toBe(0); + expect(getCodexQuotaHealthSnapshot("a", "spark", due(START) + 1)).not.toBeNull(); + }); + + test("recovery reads the same window the parser wrote, for EVERY plan", () => { + // Derived from the upstream snapshot, not hand-listed. An allowlist was tried here and was + // wrong: 21 distinct plan strings appear in this file alone, and `CodexAccount.plan` is an + // unrestricted string, so any list is a list of the plans someone remembered — and every + // omission means an account cooled forever, which is the defect this unit exists to fix. + const snapshotPlans = new Set(); + const walk = (node: unknown): void => { + if (Array.isArray(node)) { node.forEach(walk); return; } + if (!node || typeof node !== "object") return; + for (const [key, value] of Object.entries(node as Record)) { + if (key === "available_in_plans" && Array.isArray(value)) { + for (const plan of value) if (typeof plan === "string") snapshotPlans.add(plan); + } else walk(value); + } + }; + walk(upstreamModels); + expect(snapshotPlans.size).toBeGreaterThan(9); + + // Pin the rule INDEPENDENTLY first. Deriving the expectation from + // codexQuotaWindowForPlan() below would only prove the loop agrees with itself, so state + // the contract in literals: Go and Free bill on the 30-day window, everything else weekly. + expect(codexQuotaWindowForPlan("go")).toBe("monthly"); + expect(codexQuotaWindowForPlan("free")).toBe("monthly"); + expect(codexQuotaWindowForPlan(" GO ")).toBe("monthly"); + for (const plan of ["plus", "pro", "prolite", "team", "business", "enterprise", "edu", "finserv", "k12"]) { + expect(codexQuotaWindowForPlan(plan)).toBe("weekly"); + } + expect(codexQuotaWindowForPlan(undefined)).toBe("weekly"); + expect(codexQuotaWindowForPlan("")).toBe("weekly"); + // "free_workspace" is not "free": only the exact names take the monthly window. + expect(codexQuotaWindowForPlan("free_workspace")).toBe("weekly"); + + for (const plan of snapshotPlans) { + const monthly = codexQuotaWindowForPlan(plan) === "monthly"; + const filled = monthly ? { monthlyPercent: 12 } : { weeklyPercent: 12 }; + const empty = monthly ? { weeklyPercent: 12 } : { monthlyPercent: 12 }; + expect(isCompleteCodexQuotaRecoverySnapshot(filled, plan)).toBe(true); + // The other window is not evidence for this plan, in either direction. + expect(isCompleteCodexQuotaRecoverySnapshot(empty, plan)).toBe(false); + } + + // Absent plan follows the parser's weekly default. + expect(isCompleteCodexQuotaRecoverySnapshot({ weeklyPercent: 12 }, undefined)).toBe(true); + // Missing EVIDENCE still fails closed — that is the guard that matters. + expect(isCompleteCodexQuotaRecoverySnapshot({}, "plus")).toBe(false); + expect(isCompleteCodexQuotaRecoverySnapshot(null, "plus")).toBe(false); + // An exhausted snapshot is not a recovery no matter how complete it is. + expect(isCompleteCodexQuotaRecoverySnapshot({ weeklyPercent: 100 }, "plus")).toBe(false); + }); + + test.each([ + ["retry-after", { retryAfter: "900" }], + ["default", {}], + ])("never claims %s cooldowns", async (_name, meta) => { + const config = makeConfig(["a"]); + saveCredential("a"); + recordCodexUpstreamOutcome(config, "a", 429, { ...meta, now: START }); + let calls = 0; + globalThis.fetch = async () => { calls += 1; return usageResponse(); }; + await runCodexCooldownRecoveryProbes(config, due()); + expect(calls).toBe(0); + }); + + test("non-pool OpenAI configurations never run recovery probes", async () => { + const config = makeConfig(["a"]); + saveCredential("a"); + cool(config, "a"); + config.providers.openai!.codexAccountMode = "direct"; + let calls = 0; + globalThis.fetch = async () => { calls += 1; return usageResponse(); }; + await runCodexCooldownRecoveryProbes(config, due()); + expect(calls).toBe(0); + expect(getCodexQuotaHealthSnapshot("a", "shared", due() + 1)).not.toBeNull(); + }); + + test("oldest-first fairness: an already-probed account never jumps the queue", async () => { + const ids = ["a", "b", "c", "d", "e", "f"]; + const config = makeConfig(ids); + for (const id of ids) { + saveCredential(id); + cool(config, id); + } + const seen: string[] = []; + globalThis.fetch = async (_input, init) => { + seen.push(new Headers(init?.headers).get("Authorization")?.replace("Bearer access-", "") ?? ""); + return new Response("busy", { status: 503 }); + }; + // Six accounts, four claims per pass. Pass one takes four; the second pass is spaced PAST + // the probe interval so those four are eligible again and genuinely compete with the two + // that were never reached. That competition is the whole test: under stable config-order + // claims the same four would win again and the tail would starve. Tighter spacing would + // pass on any ordering, because a just-probed account is ineligible for five minutes and + // drops out without the sort doing any work. + await runCodexCooldownRecoveryProbes(config, due()); + const firstPass = [...seen]; + expect(firstPass).toHaveLength(4); + + await runCodexCooldownRecoveryProbes(config, due() + CODEX_QUOTA_PROBE_INTERVAL_MS + 1); + const secondPass = seen.slice(4); + const starved = ids.filter(id => !firstPass.includes(id)); + expect(starved).toHaveLength(2); + // The two that waited must be served before any account gets a second turn. + expect(secondPass.slice(0, 2).sort()).toEqual(starved.sort()); + expect(new Set(seen)).toEqual(new Set(ids)); + }); +});