From 8d8ea4f49295ddb2691bcd2b7940166bbfba717d Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:37:18 -0700 Subject: [PATCH 1/2] fix(cli): "You've hit your weekly limit" is QUOTA, not RUNTIME MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third time the exhaustion allowlist has missed a provider wording, and this one accounts for most of the fleet's spawn failures. Measured across every seat's log on 2026-08-25: 283 x "You've hit your weekly limit" <- unmatched -> RUNTIME 55 x "You've hit your session limit" <- matched (the 08-18 fix) 6 x "You've reached your Fable 5 limit" <- unmatched -> RUNTIME Three wordings, one sentence shape, and the list carried exactly one of them. Enumerating per-wording has now failed three times because the variable part is a billing period or a model name — both of which the provider keeps adding, so an exact-wording list is structurally always one release behind. Adds `weekly limit` to the list AND a phrase for the shape, `QUOTA_POSSESSIVE_RE`, requiring the caller's own allowance to be the subject ("you've hit/reached YOUR ... limit"). A negative lookahead keeps "you've hit your rate limit" in RATE_LIMIT — QUOTA is tested first, so without it that string would take the 15-minute cooldown instead of the 60s backoff — and a length bound stops it spanning sentences. WHY THIS IS NOT JUST LATENCY. The RUNTIME ladder is 5s/10s/60s, so three consecutive failures land inside ~75 seconds. Each leaves the event unacked by design (`agent.js`: "the kernel must retain the event for at-least-once delivery"), which burns all three `requeueMaxAttempts` and retires the row to `status: 'failed'` with "requeue cap exhausted after 3 delivery attempts without an ack" — false, since no model turn ever saw it. Measured on this seat today: 22 failures over 8 event ids, seven appearing exactly 3 times then never again. Fleet-wide the same 3-then-silence pattern covers 92 events across 6 seats, and no event anywhere appears more than 3 times. Under QUOTA the first failure alone opens a 15-minute circuit, so the attempt budget is not spent. The cap's own misuse is a separate defect and is filed as TASK-061; this fixes the trigger that fires it most often. Negative control: reverting the classifier reddens exactly the four tests about the new wordings and leaves the two widening-guards green. Full CLI suite 342 passed / 24 suites. Co-Authored-By: Claude Opus 5 --- cli/__tests__/spawn-retry.test.mjs | 75 ++++++++++++++++++++++++++++++ cli/src/lib/spawn-retry.js | 25 ++++++++-- 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/cli/__tests__/spawn-retry.test.mjs b/cli/__tests__/spawn-retry.test.mjs index 599bd097d..8abee5a80 100644 --- a/cli/__tests__/spawn-retry.test.mjs +++ b/cli/__tests__/spawn-retry.test.mjs @@ -173,3 +173,78 @@ describe('classifies real provider-exhaustion strings (2026-08-03 outage)', () = .toBe(SPAWN_FAILURE_CLASS.RUNTIME); }); }); + +describe('classifies the possessive-exhaustion phrase (2026-08-25, third miss)', () => { + // Verbatim from the fleet's own logs, measured 2026-08-25 across every seat + // that had failed at all. Three wordings, one sentence shape, and the + // allowlist carried exactly one of them: + // + // 283 × "You've hit your weekly limit" <- unmatched, RUNTIME ladder + // 55 × "You've hit your session limit" <- matched (the 08-18 fix) + // 6 × "You've reached your Fable 5 limit" <- unmatched, RUNTIME ladder + // + // Enumerating per-wording has now failed three times, because the variable + // part is a billing period or a model name — both of which the provider keeps + // adding. Hence a phrase for the shape, kept narrow by requiring the caller's + // OWN allowance to be the subject. + const WEEKLY = "claude exited with code 1: You've hit your weekly limit " + + '· resets Aug 27 at 2pm (America/Los_Angeles)'; + const PER_MODEL = "claude exited with code 1: You've reached your Fable 5 limit"; + + test.each([ + ['weekly, the wording behind 283 of 365 fleet failures', WEEKLY], + ['per-model, whose variable part is a model name', PER_MODEL], + ])('%s is QUOTA, not RUNTIME', (_label, message) => { + expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + test('so the circuit opens at the full cooldown on the FIRST failure', () => { + const { circuitOpen, delayMs } = spawnRetryPolicy({ + error: new Error(WEEKLY), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(circuitOpen).toBe(true); + expect(delayMs).toBe(SPAWN_RETRY_MAX_MS); + }); + + // This is the specific harm the misclassification caused, and it is not the + // latency. The RUNTIME ladder is 5s / 10s / 60s, so three consecutive + // failures land inside ~75 seconds — and the event is left unacked on each + // (`agent.js`, "the kernel must retain the event for at-least-once + // delivery"), which burns all three `requeueMaxAttempts` and retires it to + // `status: 'failed'`. Under QUOTA the first failure alone opens a 15-minute + // circuit, so the batch is not spent. See TASK-061. + test('the RUNTIME ladder would burn all three requeue attempts inside 75s', () => { + const runtimeLadder = [1, 2, 3].map((n) => spawnRetryPolicy({ + error: new Error('claude exited with code 1: something transient'), + consecutiveFailures: n, + intervalMs: 5000, + }).delayMs); + expect(runtimeLadder.reduce((a, b) => a + b, 0)).toBeLessThan(90 * 1000); + + // The same three failures, correctly classified, cost 45 minutes. + const quotaLadder = [1, 2, 3].map((n) => spawnRetryPolicy({ + error: new Error(WEEKLY), + consecutiveFailures: n, + intervalMs: 5000, + }).delayMs); + expect(quotaLadder.every((ms) => ms === SPAWN_RETRY_MAX_MS)).toBe(true); + }); + + // Guard on the widening. The phrase requires "your", so a server-side + // throttle phrased the same way must not be swallowed into the 15-minute + // cooldown — QUOTA is tested before RATE_LIMIT, so this ordering is load-bearing. + test.each([ + "You've hit your rate limit, retry shortly", + "You've reached your rate-limit for this model", + ])('a possessive RATE-limit string stays RATE_LIMIT: %s', (message) => { + expect(classifySpawnFailure(new Error(message))).toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT); + }); + + // And the bound stops it spanning sentences into an unrelated word. + test('the phrase does not span a sentence boundary', () => { + expect(classifySpawnFailure(new Error("You've hit your stride. Now describe the limit."))) + .toBe(SPAWN_FAILURE_CLASS.RUNTIME); + }); +}); diff --git a/cli/src/lib/spawn-retry.js b/cli/src/lib/spawn-retry.js index 10eab0234..52640daa0 100644 --- a/cli/src/lib/spawn-retry.js +++ b/cli/src/lib/spawn-retry.js @@ -20,9 +20,10 @@ export const SPAWN_RETRY_MAX_MS = 15 * 60 * 1000; export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2; // This list is a per-provider allowlist, and it only ever grows after an -// outage has already been misclassified. Twice now: +// outage has already been misclassified. Three times now: // 2026-08-03 codex "Your workspace is out of credits." → `out of credits` // 2026-08-18 claude "You've hit your session limit" → `session limit` +// 2026-08-25 claude "You've hit your weekly limit" → the phrase below // The second one is the instructive failure: `usage limit` was already here — // it is Claude's OTHER exhaustion wording — so the fleet stalled for an hour on // a string one word away from a pattern we had. Both times the miss meant @@ -49,7 +50,25 @@ export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2; // Deliberately NOT loosened to a bare `limit`: QUOTA is tested before // RATE_LIMIT, so that would swallow every "rate limit" error into the 15-minute // cooldown. Add exact wordings, not looser ones. -const QUOTA_RE = /(?:quota|usage limit|session limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; +// +// The third miss is why there is now a PHRASE as well as a word list, and it is +// a different failure from the first two. Measured across the fleet's logs on +// 2026-08-25: 283 failures reading "You've hit your weekly limit", 55 reading +// "You've hit your session limit" (matched), and 6 reading "You've reached your +// Fable 5 limit" (not matched). Three wordings, one sentence shape, and the +// list had caught exactly one of them. Enumerating per-wording has now failed +// three times because the variable part is a BILLING PERIOD or a MODEL NAME — +// both of which keep being added, so the list is structurally always one +// release behind the provider. +// +// `QUOTA_POSSESSIVE_RE` matches that shape and nothing looser: the subject must +// be the caller's own allowance ("you've hit/reached YOUR ... limit"), which is +// what distinguishes exhaustion from a server-side throttle. The negative +// lookahead keeps "you've hit your rate limit" out — QUOTA is tested first, and +// without it that string would take the 15-minute cooldown instead of the +// 60-second rate-limit backoff. The length bound stops it spanning sentences. +const QUOTA_RE = /(?:quota|usage limit|session limit|weekly limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; +const QUOTA_POSSESSIVE_RE = /you'?ve (?:hit|reached) your (?!rate[ -]?limit)[^.\n]{0,40}?limit/i; const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i; const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i; @@ -68,7 +87,7 @@ export const classifySpawnFailure = (error) => { // Provider APIs commonly report an exhausted account quota as HTTP 429. // Prefer the more specific body/message over the generic status code so a // hard quota failure gets the long cooldown rather than a one-minute probe. - if (QUOTA_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA; + if (QUOTA_RE.test(text) || QUOTA_POSSESSIVE_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA; if (error?.status === 429 || RATE_LIMIT_RE.test(text)) { return SPAWN_FAILURE_CLASS.RATE_LIMIT; } From c67fb6cd2989a674f42b9990badd7eed57d469ce Mon Sep 17 00:00:00 2001 From: Lily Shen <115414357+lilyshen0722@users.noreply.github.com> Date: Tue, 25 Aug 2026 00:57:13 -0700 Subject: [PATCH 2/2] fix(cli): the shape matcher excluded the model names it was written for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit @sprint-review gating #1215: `[^.\n]{0,40}?` reads as "stay inside one sentence" and silently also means "no model names", because every model we run is dotted. So the strings the possessive matcher exists to catch — You've reached your Haiku 4.5 limit you've hit your gpt-5.4-mini limit fell through to RUNTIME, which is the shortest backoff of the three rungs. A quota-blocked seat therefore got probed hardest, which is the exact failure this PR's own header narrates: `session limit` was one word away from `usage limit`, and this was one model name away. The paragraph above the regex already said the variable part is "a BILLING PERIOD or a MODEL NAME". The character class was excluding one of the two things the comment names. Enumeration failing three times is what motivated the shape match; the shape match then reintroduced the enumeration bug in its terminator. Fix is sprint-review's: `(?:[^.\n]|\.(?=\d))` admits a dot only ahead of a digit, so a version number passes and a sentence-ending period still stops the match. Rate-limit exclusion is untouched — QUOTA is tested first, so the negative lookahead stays load-bearing. Six tests, including the sentence-boundary case and a ladder-cost assertion (QUOTA opens the circuit at n=1; RUNTIME would retry in 5s). Probe: reverting the carve-out reddens exactly the four new cases, 27 others still green. Full spawn-retry suite 31 passed. Version 0.1.18 -> 0.1.19: main moved to 0.1.18 after #1166, so the branch's existing bump had gone stale and the `Source changed => version bumped` guard was red on identity rather than on omission. Co-Authored-By: Claude Opus 5 --- cli/__tests__/spawn-retry.test.mjs | 37 ++++++++++++++++++++++++++++++ cli/package.json | 2 +- cli/src/lib/spawn-retry.js | 12 +++++++++- 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/cli/__tests__/spawn-retry.test.mjs b/cli/__tests__/spawn-retry.test.mjs index 8abee5a80..64eee7137 100644 --- a/cli/__tests__/spawn-retry.test.mjs +++ b/cli/__tests__/spawn-retry.test.mjs @@ -247,4 +247,41 @@ describe('classifies the possessive-exhaustion phrase (2026-08-25, third miss)', expect(classifySpawnFailure(new Error("You've hit your stride. Now describe the limit."))) .toBe(SPAWN_FAILURE_CLASS.RUNTIME); }); + + describe('the dotted-model-name gap (@sprint-review, 2026-08-25)', () => { + // `[^.\n]` was chosen to keep the match inside one sentence, and silently + // also excluded every model name we run, because they are all dotted. + // RUNTIME is the fallthrough with the SHORTEST backoff, so the net effect + // was to probe a quota-blocked seat hardest. + test.each([ + ["You've reached your Haiku 4.5 limit"], + ["you've hit your gpt-5.4-mini limit"], + ["Error: You've reached your Claude Opus 4.8 limit for today"], + ])('classifies %s as QUOTA', (text) => { + expect(classifySpawnFailure(new Error(text))).toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + test('still refuses a rate limit, which QUOTA is tested before', () => { + expect(classifySpawnFailure(new Error("you've hit your rate limit"))) + .toBe(SPAWN_FAILURE_CLASS.RATE_LIMIT); + }); + + test('does not let the dot carve-out span a sentence boundary', () => { + // The dot is admitted only ahead of a DIGIT, so a sentence-ending period + // still terminates the match and this must not read as exhaustion. + expect(classifySpawnFailure(new Error("You've reached your goal. This has no limit"))) + .not.toBe(SPAWN_FAILURE_CLASS.QUOTA); + }); + + test('a dotted model name costs the quota ladder, not the runtime ladder', () => { + // The point of the fix: QUOTA opens the circuit at n=1. Under RUNTIME + // this same error would be retried on the 5s rung. + const policy = spawnRetryPolicy({ + error: new Error("You've reached your Haiku 4.5 limit"), + consecutiveFailures: 1, + intervalMs: 5000, + }); + expect(policy).toMatchObject({ circuitOpen: true, delayMs: SPAWN_RETRY_MAX_MS }); + }); + }); }); diff --git a/cli/package.json b/cli/package.json index fe18ee1a5..bf3549222 100644 --- a/cli/package.json +++ b/cli/package.json @@ -1,6 +1,6 @@ { "name": "@commonlyai/cli", - "version": "0.1.18", + "version": "0.1.19", "license": "Apache-2.0", "description": "The Commonly CLI \u2014 connect agents, manage pods, iterate fast", "type": "module", diff --git a/cli/src/lib/spawn-retry.js b/cli/src/lib/spawn-retry.js index 52640daa0..75b9be153 100644 --- a/cli/src/lib/spawn-retry.js +++ b/cli/src/lib/spawn-retry.js @@ -67,8 +67,18 @@ export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2; // lookahead keeps "you've hit your rate limit" out — QUOTA is tested first, and // without it that string would take the 15-minute cooldown instead of the // 60-second rate-limit backoff. The length bound stops it spanning sentences. +// +// The character class admits a dot ONLY when a digit follows it, and that +// carve-out is load-bearing rather than cosmetic (@sprint-review, 2026-08-25). +// A plain `[^.\n]` reads as "stay inside one sentence" and silently also means +// "no model names" — every model we run is dotted, so "you've reached your +// Haiku 4.5 limit" and "your gpt-5.4-mini limit" both fell through to RUNTIME, +// which is the shortest backoff of the three. The class was excluding the exact +// variable part the paragraph above says keeps changing. `\.(?=\d)` lets a +// version number through while a sentence-ending period still terminates the +// match. const QUOTA_RE = /(?:quota|usage limit|session limit|weekly limit|credit balance|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; -const QUOTA_POSSESSIVE_RE = /you'?ve (?:hit|reached) your (?!rate[ -]?limit)[^.\n]{0,40}?limit/i; +const QUOTA_POSSESSIVE_RE = /you'?ve (?:hit|reached) your (?!rate[ -]?limit)(?:[^.\n]|\.(?=\d)){0,40}?limit/i; const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i; const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i;