Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 112 additions & 0 deletions cli/__tests__/spawn-retry.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -173,3 +173,115 @@ describe('classifies real provider-exhaustion strings (2026-08-03 outage)', () =
.toBe(SPAWN_FAILURE_CLASS.RUNTIME);
});
});

describe('classifies the possessive-exhaustion phrase (2026-08-25, third miss)', () => {
// Verbatim from the fleet's own logs, measured 2026-08-25 across every seat
// that had failed at all. Three wordings, one sentence shape, and the
// allowlist carried exactly one of them:
//
// 283 × "You've hit your weekly limit" <- unmatched, RUNTIME ladder
// 55 × "You've hit your session limit" <- matched (the 08-18 fix)
// 6 × "You've reached your Fable 5 limit" <- unmatched, RUNTIME ladder
//
// Enumerating per-wording has now failed three times, because the variable
// part is a billing period or a model name — both of which the provider keeps
// adding. Hence a phrase for the shape, kept narrow by requiring the caller's
// OWN allowance to be the subject.
const WEEKLY = "claude exited with code 1: You've hit your weekly limit "
+ '· resets Aug 27 at 2pm (America/Los_Angeles)';
const PER_MODEL = "claude exited with code 1: You've reached your Fable 5 limit";

test.each([
['weekly, the wording behind 283 of 365 fleet failures', WEEKLY],
['per-model, whose variable part is a model name', PER_MODEL],
])('%s is QUOTA, not RUNTIME', (_label, message) => {
expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.QUOTA);
});

test('so the circuit opens at the full cooldown on the FIRST failure', () => {
const { circuitOpen, delayMs } = spawnRetryPolicy({
error: new Error(WEEKLY),
consecutiveFailures: 1,
intervalMs: 5000,
});
expect(circuitOpen).toBe(true);
expect(delayMs).toBe(SPAWN_RETRY_MAX_MS);
});

// This is the specific harm the misclassification caused, and it is not the
// latency. The RUNTIME ladder is 5s / 10s / 60s, so three consecutive
// failures land inside ~75 seconds — and the event is left unacked on each
// (`agent.js`, "the kernel must retain the event for at-least-once
// delivery"), which burns all three `requeueMaxAttempts` and retires it to
// `status: 'failed'`. Under QUOTA the first failure alone opens a 15-minute
// circuit, so the batch is not spent. See TASK-061.
test('the RUNTIME ladder would burn all three requeue attempts inside 75s', () => {
const runtimeLadder = [1, 2, 3].map((n) => spawnRetryPolicy({
error: new Error('claude exited with code 1: something transient'),
consecutiveFailures: n,
intervalMs: 5000,
}).delayMs);
expect(runtimeLadder.reduce((a, b) => a + b, 0)).toBeLessThan(90 * 1000);

// The same three failures, correctly classified, cost 45 minutes.
const quotaLadder = [1, 2, 3].map((n) => spawnRetryPolicy({
error: new Error(WEEKLY),
consecutiveFailures: n,
intervalMs: 5000,
}).delayMs);
expect(quotaLadder.every((ms) => ms === SPAWN_RETRY_MAX_MS)).toBe(true);
});

// Guard on the widening. The phrase requires "your", so a server-side
// throttle phrased the same way must not be swallowed into the 15-minute
// cooldown — QUOTA is tested before RATE_LIMIT, so this ordering is load-bearing.
test.each([
"You've hit your rate limit, retry shortly",
"You've reached your rate-limit for this model",
])('a possessive RATE-limit string stays RATE_LIMIT: %s', (message) => {
expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT);
});

// And the bound stops it spanning sentences into an unrelated word.
test('the phrase does not span a sentence boundary', () => {
expect(classifySpawnFailure(new Error("You've hit your stride. Now describe the limit.")))
.toBe(SPAWN_FAILURE_CLASS.RUNTIME);
});

describe('the dotted-model-name gap (@sprint-review, 2026-08-25)', () => {
// `[^.\n]` was chosen to keep the match inside one sentence, and silently
// also excluded every model name we run, because they are all dotted.
// RUNTIME is the fallthrough with the SHORTEST backoff, so the net effect
// was to probe a quota-blocked seat hardest.
test.each([
["You've reached your Haiku 4.5 limit"],
["you've hit your gpt-5.4-mini limit"],
["Error: You've reached your Claude Opus 4.8 limit for today"],
])('classifies %s as QUOTA', (text) => {
expect(classifySpawnFailure(new Error(text))).toBe(SPAWN_FAILURE_CLASS.QUOTA);
});

test('still refuses a rate limit, which QUOTA is tested before', () => {
expect(classifySpawnFailure(new Error("you've hit your rate limit")))
.toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT);
});

test('does not let the dot carve-out span a sentence boundary', () => {
// The dot is admitted only ahead of a DIGIT, so a sentence-ending period
// still terminates the match and this must not read as exhaustion.
expect(classifySpawnFailure(new Error("You've reached your goal. This has no limit")))
.not.toBe(SPAWN_FAILURE_CLASS.QUOTA);
});

test('a dotted model name costs the quota ladder, not the runtime ladder', () => {
// The point of the fix: QUOTA opens the circuit at n=1. Under RUNTIME
// this same error would be retried on the 5s rung.
const policy = spawnRetryPolicy({
error: new Error("You've reached your Haiku 4.5 limit"),
consecutiveFailures: 1,
intervalMs: 5000,
});
expect(policy).toMatchObject({ circuitOpen: true, delayMs: SPAWN_RETRY_MAX_MS });
});
});
});
2 changes: 1 addition & 1 deletion cli/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@commonlyai/cli",
"version": "0.1.18",
"version": "0.1.19",
"license": "Apache-2.0",
"description": "The Commonly CLI \u2014 connect agents, manage pods, iterate fast",
"type": "module",
Expand Down
35 changes: 32 additions & 3 deletions cli/src/lib/spawn-retry.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@ export const SPAWN_RETRY_MAX_MS = 15 * 60 * 1000;
export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2;

// This list is a per-provider allowlist, and it only ever grows after an
// outage has already been misclassified. Twice now:
// outage has already been misclassified. Three times now:
// 2026-08-03 codex "Your workspace is out of credits." → `out of credits`
// 2026-08-18 claude "You've hit your session limit" → `session limit`
// 2026-08-25 claude "You've hit your weekly limit" → the phrase below
// The second one is the instructive failure: `usage limit` was already here —
// it is Claude's OTHER exhaustion wording — so the fleet stalled for an hour on
// a string one word away from a pattern we had. Both times the miss meant
Expand All @@ -49,7 +50,35 @@ export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2;
// Deliberately NOT loosened to a bare `limit`: QUOTA is tested before
// RATE_LIMIT, so that would swallow every "rate limit" error into the 15-minute
// cooldown. Add exact wordings, not looser ones.
const QUOTA_RE = /(?:quota|usage limit|session limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i;
//
// The third miss is why there is now a PHRASE as well as a word list, and it is
// a different failure from the first two. Measured across the fleet's logs on
// 2026-08-25: 283 failures reading "You've hit your weekly limit", 55 reading
// "You've hit your session limit" (matched), and 6 reading "You've reached your
// Fable 5 limit" (not matched). Three wordings, one sentence shape, and the
// list had caught exactly one of them. Enumerating per-wording has now failed
// three times because the variable part is a BILLING PERIOD or a MODEL NAME —
// both of which keep being added, so the list is structurally always one
// release behind the provider.
//
// `QUOTA_POSSESSIVE_RE` matches that shape and nothing looser: the subject must
// be the caller's own allowance ("you've hit/reached YOUR ... limit"), which is
// what distinguishes exhaustion from a server-side throttle. The negative
// lookahead keeps "you've hit your rate limit" out — QUOTA is tested first, and
// without it that string would take the 15-minute cooldown instead of the
// 60-second rate-limit backoff. The length bound stops it spanning sentences.
//
// The character class admits a dot ONLY when a digit follows it, and that
// carve-out is load-bearing rather than cosmetic (@sprint-review, 2026-08-25).
// A plain `[^.\n]` reads as "stay inside one sentence" and silently also means
// "no model names" — every model we run is dotted, so "you've reached your
// Haiku 4.5 limit" and "your gpt-5.4-mini limit" both fell through to RUNTIME,
// which is the shortest backoff of the three. The class was excluding the exact
// variable part the paragraph above says keeps changing. `\.(?=\d)` lets a
// version number through while a sentence-ending period still terminates the
// match.
const QUOTA_RE = /(?:quota|usage limit|session limit|weekly limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i;
const QUOTA_POSSESSIVE_RE = /you'?ve (?:hit|reached) your (?!rate[ -]?limit)(?:[^.\n]|\.(?=\d)){0,40}?limit/i;
const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i;
const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i;

Expand All @@ -68,7 +97,7 @@ export const classifySpawnFailure = (error) => {
// Provider APIs commonly report an exhausted account quota as HTTP 429.
// Prefer the more specific body/message over the generic status code so a
// hard quota failure gets the long cooldown rather than a one-minute probe.
if (QUOTA_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA;
if (QUOTA_RE.test(text) || QUOTA_POSSESSIVE_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA;
if (error?.status === 429 || RATE_LIMIT_RE.test(text)) {
return SPAWN_FAILURE_CLASS.RATE_LIMIT;
}
Expand Down
Loading