diff --git a/CHANGELOG.md b/CHANGELOG.md index d1881ae..9e2ab86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ This project follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) fo ## [Unreleased] +### Added + +- **`agy` dispatches now report real token usage instead of always `null`.** All twelve `agy-*` `edit_exists`/`read_only` transports now pass `--output-format json` and declare `usage_from` (`lib/cli-usage.js`'s existing structured-output interpreter already documented and tested this exact envelope shape — `{"status","response","usage":{"input_tokens","output_tokens","cache_read_tokens"}}` — but no registry entry had ever wired it up). Verified live against `agy` 1.2.0 that this composes correctly with the print-timeout `failure_markers` fix below: the JSON envelope goes to stdout and the print-timeout notice to stderr as two separate streams, so the marker still forces a failure on a non-blank stdout, and the parsed usage (zero, on that path) still comes through rather than being lost to a raw-JSON fallback. + ### Fixed - **`agy`'s own print-timeout watchdog exiting 0 was scored as a success.** `agy --print-timeout` firing mid-turn writes `[agy] print timeout after with turn in progress; returning partial output` to stderr and exits 0 regardless of whether the turn produced anything — confirmed live against `agy` 1.2.0. When the model had streamed some visible prose before the cutoff, that non-blank stdout was enough for the existing empty-run guard to wave the run through: `agy-claude-opus-4-6-thinking` and `agy-gpt-oss-120b-medium` were each logged `outcome:"success"` after running ~486s (within seconds of the registry's `--print-timeout 8m`) with no file changed. The `failure_markers` mechanism already exists for exactly this class (see `kiro`'s "Monthly request limit reached"); the agy family's six registry entries just never declared one. All twelve `agy-*` `edit_exists`/`read_only` transports now declare `failure_markers: ["with turn in progress; returning partial output"]`, so a print-timeout with nothing to show for it is a failure — a genuine partial edit made before the cutoff is untouched, since `hasSubstantiveOutput` still wins when real files changed. diff --git a/agents.yaml b/agents.yaml index b498696..1693d31 100644 --- a/agents.yaml +++ b/agents.yaml @@ -1133,14 +1133,31 @@ agents: card_required: false token_limits: note: "Antigravity account CLI -- capped by an account-wide quota window ('resets in ~Xh'), not published numeric tpm/rpm/rpd." + # How to read token usage out of agy's envelope (lib/cli-usage.js). Verified + # live 2026-09-10: a single JSON object (not an event array), the answer + # under `.response`, no cost field. Before this, every agy dispatch reported + # no tokens at all — the same blind spot `claude`'s usage_from closed above. + # A mismatch costs nothing: parseCliUsage returns null on any discrepancy + # and the caller keeps raw stdout as the answer. + # + # The envelope also carries `usage.thinking_tokens`, deliberately left + # unmapped: parseCliUsage's spec vocabulary has no slot for it today (only + # tokens_in/tokens_out/cost_usd/cache_read/cache_write), and adding one is a + # cli-usage.js code change, not a registry edit — out of scope here. + usage_from: + kind: json + text: response + tokens_in: usage.input_tokens + tokens_out: usage.output_tokens + cache_read: usage.cache_read_tokens transports: edit_exists: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model gemini-3.6-flash-high" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --model gemini-3.6-flash-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] read_only: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model gemini-3.6-flash-high" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --mode plan --model gemini-3.6-flash-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] @@ -1157,14 +1174,21 @@ agents: card_required: false token_limits: note: "Antigravity account CLI -- capped by an account-wide quota window ('resets in ~Xh'), not published numeric tpm/rpm/rpd." + # See agy-gemini-3.6-flash-high above for why this shape is declared. + usage_from: + kind: json + text: response + tokens_in: usage.input_tokens + tokens_out: usage.output_tokens + cache_read: usage.cache_read_tokens transports: edit_exists: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model gemini-3.1-pro-high" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --model gemini-3.1-pro-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] read_only: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model gemini-3.1-pro-high" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --mode plan --model gemini-3.1-pro-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] @@ -1187,14 +1211,21 @@ agents: card_required: false token_limits: note: "Antigravity account CLI -- capped by an account-wide quota window ('resets in ~Xh'), not published numeric tpm/rpm/rpd." + # See agy-gemini-3.6-flash-high above for why this shape is declared. + usage_from: + kind: json + text: response + tokens_in: usage.input_tokens + tokens_out: usage.output_tokens + cache_read: usage.cache_read_tokens transports: edit_exists: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model gemini-3.7-flash-high" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --model gemini-3.7-flash-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] read_only: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model gemini-3.7-flash-high" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --mode plan --model gemini-3.7-flash-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] @@ -1211,14 +1242,21 @@ agents: card_required: false token_limits: note: "Antigravity account CLI -- capped by an account-wide quota window ('resets in ~Xh'), not published numeric tpm/rpm/rpd." + # See agy-gemini-3.6-flash-high above for why this shape is declared. + usage_from: + kind: json + text: response + tokens_in: usage.input_tokens + tokens_out: usage.output_tokens + cache_read: usage.cache_read_tokens transports: edit_exists: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model claude-sonnet-4-6" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --model claude-sonnet-4-6" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] read_only: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model claude-sonnet-4-6" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --mode plan --model claude-sonnet-4-6" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] @@ -1235,14 +1273,21 @@ agents: card_required: false token_limits: note: "Antigravity account CLI -- capped by an account-wide quota window ('resets in ~Xh'), not published numeric tpm/rpm/rpd." + # See agy-gemini-3.6-flash-high above for why this shape is declared. + usage_from: + kind: json + text: response + tokens_in: usage.input_tokens + tokens_out: usage.output_tokens + cache_read: usage.cache_read_tokens transports: edit_exists: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model claude-opus-4-6-thinking" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --model claude-opus-4-6-thinking" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] read_only: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model claude-opus-4-6-thinking" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --mode plan --model claude-opus-4-6-thinking" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] @@ -1259,14 +1304,21 @@ agents: card_required: false token_limits: note: "Antigravity account CLI -- capped by an account-wide quota window ('resets in ~Xh'), not published numeric tpm/rpm/rpd." + # See agy-gemini-3.6-flash-high above for why this shape is declared. + usage_from: + kind: json + text: response + tokens_in: usage.input_tokens + tokens_out: usage.output_tokens + cache_read: usage.cache_read_tokens transports: edit_exists: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model gpt-oss-120b-medium" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --model gpt-oss-120b-medium" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] read_only: - cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model gpt-oss-120b-medium" + cmd: "agy --dangerously-skip-permissions --print-timeout 8m --output-format json --mode plan --model gpt-oss-120b-medium" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" failure_markers: ["with turn in progress; returning partial output"] diff --git a/lib/cli-usage.test.js b/lib/cli-usage.test.js index f88b084..b0e14a2 100644 --- a/lib/cli-usage.test.js +++ b/lib/cli-usage.test.js @@ -63,6 +63,31 @@ test('reads the agy envelope: a single object, answer under .response', () => { assert.equal(u.cost_usd, null, 'agy reports no cost and none is invented'); }); +test('reads agy\'s print-timeout envelope: zero usage and empty response, not a mismatch', () => { + // Captured live 2026-09-10: when agy's own --print-timeout fires mid-turn, + // --output-format json still returns a well-formed envelope — status: + // "ERROR", response: "", usage all zero. num() treats 0 as a real number + // (not "absent"), so this must NOT parse as a mismatch (which would fall + // back to raw stdout as the answer) — it must extract text:"" and + // tokens 0/0, so the caller's answerText is genuinely empty and the + // existing emptyRunReason/failure_markers guards see the true shape of the + // run instead of a JSON blob that merely LOOKS non-blank. + const AGY_PRINT_TIMEOUT_OUT = JSON.stringify({ + conversation_id: 'c1408cb3-a6fa-4508-a136-afa41470a192', + status: 'ERROR', + response: '', + error: 'The stream was interrupted. Please continue the task you were working on.', + duration_seconds: 12.823739, + num_turns: 1, + usage: { input_tokens: 0, output_tokens: 0, thinking_tokens: 0, cache_read_tokens: 0, total_tokens: 0 }, + }); + const u = parseCliUsage(AGY_PRINT_TIMEOUT_OUT, AGY_SPEC); + assert.ok(u, 'a well-formed envelope with all-zero usage must still parse, not fall back to raw stdout'); + assert.equal(u.text, ''); + assert.equal(u.tokens_in, 0); + assert.equal(u.tokens_out, 0); +}); + test('takes the LAST matching event, because a multi-turn run emits several', () => { const out = JSON.stringify([ { type: 'result', result: 'first', usage: { input_tokens: 1, output_tokens: 1 } }, diff --git a/lib/false-success.test.js b/lib/false-success.test.js index c37e75e..2e72fd6 100644 --- a/lib/false-success.test.js +++ b/lib/false-success.test.js @@ -247,6 +247,97 @@ test("end to end: without the declared marker, agy's print-timeout reads as a fa assert.equal(result.exitCode, 0, "documents the pre-fix false success this registry change closes"); }); +test("end to end: usage_from and failure_markers compose — real agy's --output-format json shape", async () => { + // Real agy entries now declare BOTH usage_from (to unwrap --output-format + // json) and failure_markers (this fix). This is the actual production + // shape: the JSON envelope goes to stdout, the print-timeout notice goes to + // stderr separately — confirmed live against agy 1.2.0. Neither mechanism + // may undermine the other: the marker must still force a failure even + // though stdout is a non-blank JSON blob, AND the parsed usage (zero, but a + // real zero) must still come through rather than being lost. + const { runDispatch } = await import("./dispatch.js"); + const dir = tmpdir(); + const fakeCli = path.join(dir, "fake-agy-json.sh"); + const envelope = JSON.stringify({ + conversation_id: "c1408cb3-a6fa-4508-a136-afa41470a192", + status: "ERROR", + response: "", + error: "The stream was interrupted. Please continue the task you were working on.", + num_turns: 1, + usage: { input_tokens: 0, output_tokens: 0, thinking_tokens: 0, cache_read_tokens: 0, total_tokens: 0 }, + }); + fs.writeFileSync( + fakeCli, + `#!/bin/sh\necho '${envelope}'\necho '${AGY_PRINT_TIMEOUT_STDERR}' 1>&2\nexit 0\n`, + { mode: 0o755 }, + ); + const entry = { + id: "fake-agy-json-and-markers", + usage_from: { kind: "json", text: "response", tokens_in: "usage.input_tokens", tokens_out: "usage.output_tokens" }, + transports: { + edit_exists: { + cmd: `sh ${fakeCli}`, + failure_markers: ["with turn in progress; returning partial output"], + }, + }, + }; + const result = await runDispatch(entry, "do the thing", { timeoutMs: 30000 }, "edit_exists"); + assert.equal(result.exitCode, 1, "the marker must still force a failure even though stdout is a non-blank JSON envelope"); + assert.equal(result.tokens_in, 0, "usage_from must still unwrap real (zero) usage, not lose it to the marker check"); + assert.equal(result.tokens_out, 0); + assert.equal(result.output, "", "the unwrapped answer is the empty response field, not the raw JSON envelope"); +}); + +test("end to end: a successful agy dispatch under --output-format json reports real usage", async () => { + // The common path, not just the failure one: a real file edit under + // --output-format json (the actual production shape now that all agy + // entries declare it) must stay a success AND report the real numbers + // instead of null. Envelope captured live against agy 1.2.0. + const { runDispatch } = await import("./dispatch.js"); + const dir = tmpdir(); + const fakeCli = path.join(dir, "fake-agy-success.sh"); + const envelope = JSON.stringify({ + conversation_id: "e13be7ad-7a7c-442a-a5a6-cbd32842bbee", + status: "SUCCESS", + response: "I have created the file hello.txt with the exact content `hello world`.\n", + num_turns: 1, + usage: { input_tokens: 15406, output_tokens: 272, thinking_tokens: 0, cache_read_tokens: 44698, total_tokens: 15678 }, + }); + fs.writeFileSync( + fakeCli, + // printf, not echo: this shell's `echo` interpolates the `\n` INSIDE the + // JSON string's `response` text as a real newline byte, which breaks + // JSON.parse ("Bad control character") — an artifact of faking the CLI + // through a shell script, not something real agy (which writes its own + // process output) is subject to. + `#!/bin/sh\ntouch "$PWD/hello.txt"\nprintf '%s' '${envelope}'\nexit 0\n`, + { mode: 0o755 }, + ); + const entry = { + id: "fake-agy-json-success", + usage_from: { + kind: "json", + text: "response", + tokens_in: "usage.input_tokens", + tokens_out: "usage.output_tokens", + cache_read: "usage.cache_read_tokens", + }, + transports: { + edit_exists: { + cmd: `sh ${fakeCli}`, + failure_markers: ["with turn in progress; returning partial output"], + }, + }, + }; + const result = await runDispatch(entry, "create hello.txt", { timeoutMs: 30000 }, "edit_exists"); + assert.equal(result.exitCode, 0, "a genuine success must stay a success under JSON output"); + assert.equal(result.tokens_in, 15406); + assert.equal(result.tokens_out, 272); + assert.equal(result.cache_read, 44698); + assert.equal(result.output, "I have created the file hello.txt with the exact content `hello world`.\n"); + assert.ok(result.files.some((f) => f.path === "hello.txt"), "the real file edit is still reported"); +}); + // ── The same class on the HTTP path ──────────────────────────────────────────── // Deferred once with a recorded reason, then closed by separating the two halves // it had been treating as one: a provider error in a 200 ENVELOPE is structured diff --git a/lib/registry.transports.test.js b/lib/registry.transports.test.js index 438b319..39bef80 100644 --- a/lib/registry.transports.test.js +++ b/lib/registry.transports.test.js @@ -220,6 +220,27 @@ const RECORDED_ENVELOPES = { ]), expect: { text: "Ok", tokens_in: 10, tokens_out: 315, cache_read: 21675, cache_write: 16104, cost_usd: 0.0377 }, }, + // Captured live 2026-09-10 against agy 1.2.0 (`agy --output-format json + // --model gemini-3.6-flash-low --print "Create a file named hello.txt + // containing exactly the text: hello world"`) while verifying the fix for + // the print-timeout false-success class: a single JSON object, not an event + // array, the answer under `.response`, no cost field at all. + agy: { + stdout: JSON.stringify({ + conversation_id: "e13be7ad-7a7c-442a-a5a6-cbd32842bbee", + status: "SUCCESS", + response: "I have created the file [hello.txt](file:///tmp/agy-json-test/hello.txt) with the exact content `hello world`.\n", + duration_seconds: 9.486174, + num_turns: 1, + usage: { input_tokens: 15406, output_tokens: 272, thinking_tokens: 0, cache_read_tokens: 44698, total_tokens: 15678 }, + }), + expect: { + text: "I have created the file [hello.txt](file:///tmp/agy-json-test/hello.txt) with the exact content `hello world`.\n", + tokens_in: 15406, + tokens_out: 272, + cache_read: 44698, + }, + }, }; test("every bundled usage_from parses the CLI output that was actually recorded", () => {