From fa3418f7e0df45ba56217aa80a0f939cb3485722 Mon Sep 17 00:00:00 2001 From: Vlad Feninets Date: Thu, 10 Sep 2026 10:21:04 +0300 Subject: [PATCH] fix(agy): declare failure_markers for the print-timeout watchdog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agy's own `--print-timeout` fires mid-turn, writes "... with turn in progress; returning partial output" to stderr, and exits 0 regardless of whether anything was accomplished. When the model had streamed some visible prose before the cutoff, that non-blank stdout was enough for the existing empty-run guard to wave the run through: dispatch-log shows agy-claude-opus-4-6-thinking and agy-gpt-oss-120b-medium each logged outcome:"success" after running ~486s (within seconds of the registry's --print-timeout 8m) with no file changed. Confirmed live against the real agy 1.2.0 binary. The failure_markers mechanism already exists for exactly this class (see kiro's "Monthly request limit reached"); the agy family just never declared one. All twelve agy-* edit_exists/read_only entries now do. hasSubstantiveOutput still wins when a real edit landed before the cutoff, so genuine partial completions are unaffected. Consensus: 3/4 reviewers responded (groq-gpt-oss-120b, qwencloud-qwen-flash-character, qwencloud-qwen-plus-character), all APPROVE, no critical issues — only optional doc suggestions, one of which (a explanatory comment) is included here. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 4 ++ agents.yaml | 23 ++++++++++++ lib/false-success.test.js | 78 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c57101..6287bf1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ This project follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/) fo ## [Unreleased] +### Fixed + +- **`agy`'s own print-timeout watchdog exiting 0 was scored as a success.** `agy --print-timeout` firing mid-turn writes `[agy] print timeout after with turn in progress; returning partial output` to stderr and exits 0 regardless of whether the turn produced anything — confirmed live against `agy` 1.2.0. When the model had streamed some visible prose before the cutoff, that non-blank stdout was enough for the existing empty-run guard to wave the run through: `agy-claude-opus-4-6-thinking` and `agy-gpt-oss-120b-medium` were each logged `outcome:"success"` after running ~486s (within seconds of the registry's `--print-timeout 8m`) with no file changed. The `failure_markers` mechanism already exists for exactly this class (see `kiro`'s "Monthly request limit reached"); the agy family's six registry entries just never declared one. All twelve `agy-*` `edit_exists`/`read_only` transports now declare `failure_markers: ["with turn in progress; returning partial output"]`, so a print-timeout with nothing to show for it is a failure — a genuine partial edit made before the cutoff is untouched, since `hasSubstantiveOutput` still wins when real files changed. + ## [0.61.0] - 2026-09-09 ### Fixed diff --git a/agents.yaml b/agents.yaml index 2a22260..b498696 100644 --- a/agents.yaml +++ b/agents.yaml @@ -1109,6 +1109,17 @@ agents: # process still exits 0, so the dispatch trailer reports success for a # request the model never received. `prompt_flag: "--print"` below tells # runDispatch to place `--print` immediately before the real prompt instead. + # + # `failure_markers` on every entry below: agy's OWN `--print-timeout` + # watchdog exits 0 and writes "... with turn in progress; returning + # partial output" to stderr when it fires mid-turn. If the model had + # already streamed some visible prose, that alone reads as a real answer to + # the generic empty-run guard even though nothing was actually finished — + # confirmed live against two agy models running ~486s (the registry's + # `--print-timeout 8m`, plus overhead) and logged as a plain success with no + # file changed. The marker forces that specific shape back to a failure + # UNLESS real files changed first (see hasSubstantiveOutput in dispatch.js), + # so a genuine partial edit made before the cutoff is untouched. - id: agy-gemini-3.6-flash-high provider: antigravity model: gemini-3.6-flash-high @@ -1127,10 +1138,12 @@ agents: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model gemini-3.6-flash-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] read_only: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model gemini-3.6-flash-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] - id: agy-gemini-3.1-pro-high provider: antigravity model: gemini-3.1-pro-high @@ -1149,10 +1162,12 @@ agents: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model gemini-3.1-pro-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] read_only: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model gemini-3.1-pro-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] # gemini-3.7-flash supersedes 3.6-flash as the quick/free variant in this # family (added 2026-08-31, discovered live via `agy models` on a fresh # Antigravity login). Kept both -3.6 and -3.7 -high for now rather than @@ -1177,10 +1192,12 @@ agents: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model gemini-3.7-flash-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] read_only: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model gemini-3.7-flash-high" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] - id: agy-claude-sonnet-4-6 provider: antigravity model: claude-sonnet-4-6 @@ -1199,10 +1216,12 @@ agents: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model claude-sonnet-4-6" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] read_only: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model claude-sonnet-4-6" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] - id: agy-claude-opus-4-6-thinking provider: antigravity model: claude-opus-4-6-thinking @@ -1221,10 +1240,12 @@ agents: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model claude-opus-4-6-thinking" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] read_only: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model claude-opus-4-6-thinking" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] - id: agy-gpt-oss-120b-medium provider: antigravity model: gpt-oss-120b-medium @@ -1243,10 +1264,12 @@ agents: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --model gpt-oss-120b-medium" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] read_only: cmd: "agy --dangerously-skip-permissions --print-timeout 8m --mode plan --model gpt-oss-120b-medium" cwd_flag: "--add-dir {cwd}" prompt_flag: "--print" + failure_markers: ["with turn in progress; returning partial output"] # ============ Primary orchestrator (subscription CLI) ============ # These are strong-tier agentic CLIs backed by an operator subscription — not diff --git a/lib/false-success.test.js b/lib/false-success.test.js index 41a7a2d..c37e75e 100644 --- a/lib/false-success.test.js +++ b/lib/false-success.test.js @@ -169,6 +169,84 @@ test("end to end: the same CLI without declared markers keeps its old behaviour" assert.equal(result.exitCode, 0, "an entry that declares nothing must be unaffected"); }); +// ── agy's own print-timeout watchdog ─────────────────────────────────────── +// Recorded live (2026-09-10): agy-claude-opus-4-6-thinking and +// agy-gpt-oss-120b-medium each ran ~486s — within seconds of the registry's +// `--print-timeout 8m` — exited 0, and were logged outcome:"success" with no +// file changed. Reproduced locally against the real `agy` 1.2.0 binary: a +// short `--print-timeout` against a deliberately slow prompt writes exactly +// "[agy] print timeout after with turn in progress; returning +// partial output" to stderr and exits 0. +// +// That reproduction alone is NOT the production shape, and building the test +// on it first gave a false failure: with stdout still empty, the EXISTING +// emptyRunExitCode guard (empty stdout + no files -> exit 1) already catches +// it, marker or no marker. The class this marker exists for needs the model +// to have streamed some visible prose before the watchdog cut it off — the +// realistic case for an 8-minute "thinking" turn — which makes stdout +// non-blank and is exactly what lets emptyRunExitCode wave the run through. +const AGY_PRINT_TIMEOUT_STDERR = "[agy] print timeout after 6s with turn in progress; returning partial output"; +const AGY_PARTIAL_PROSE = "Here's my plan so far: I'll start by looking at"; + +test("agy's print-timeout marker with no files is a failure, not a success", () => { + assert.equal( + declaredMarkerExitCode(0, ["with turn in progress; returning partial output"], AGY_PARTIAL_PROSE, AGY_PRINT_TIMEOUT_STDERR, []), + 1, + ); +}); + +test("agy's print-timeout marker is ignored when the turn left real work behind", () => { + // The watchdog can fire AFTER a real edit already landed (e.g. the model + // wrote the file, then kept reasoning about further changes until cut off). + // That is genuine partial completion, not the empty-turn class this marker + // targets, and hasSubstantiveOutput is what tells the two apart. + const files = [{ path: "changed.md", bytes: 42 }]; + assert.equal( + declaredMarkerExitCode(0, ["with turn in progress; returning partial output"], AGY_PARTIAL_PROSE, AGY_PRINT_TIMEOUT_STDERR, files), + 0, + ); +}); + +test("end to end: agy's print-timeout marker with partial prose and no files is a failure", async () => { + const { runDispatch } = await import("./dispatch.js"); + const dir = tmpdir(); + const fakeCli = path.join(dir, "fake-agy.sh"); + fs.writeFileSync( + fakeCli, + `#!/bin/sh\necho '${AGY_PARTIAL_PROSE}'\necho '${AGY_PRINT_TIMEOUT_STDERR}' 1>&2\nexit 0\n`, + { mode: 0o755 }, + ); + const entry = { + id: "fake-agy-print-timeout", + transports: { + edit_exists: { + cmd: `sh ${fakeCli}`, + failure_markers: ["with turn in progress; returning partial output"], + }, + }, + }; + const result = await runDispatch(entry, "do the thing", { timeoutMs: 30000 }, "edit_exists"); + assert.equal(result.exitCode, 1, "partial prose plus a print-timeout marker and no files must be a failure"); +}); + +test("end to end: without the declared marker, agy's print-timeout reads as a false success", async () => { + // This is the bug exactly as observed in production: the same CLI output, + // on an entry that declares no failure_markers, keeps the old (wrong) + // behaviour — non-blank stdout is enough for emptyRunExitCode to wave it + // through even though the turn was cut off having done nothing. + const { runDispatch } = await import("./dispatch.js"); + const dir = tmpdir(); + const fakeCli = path.join(dir, "fake-agy.sh"); + fs.writeFileSync( + fakeCli, + `#!/bin/sh\necho '${AGY_PARTIAL_PROSE}'\necho '${AGY_PRINT_TIMEOUT_STDERR}' 1>&2\nexit 0\n`, + { mode: 0o755 }, + ); + const entry = { id: "fake-agy-no-markers", transports: { edit_exists: { cmd: `sh ${fakeCli}` } } }; + const result = await runDispatch(entry, "do the thing", { timeoutMs: 30000 }, "edit_exists"); + assert.equal(result.exitCode, 0, "documents the pre-fix false success this registry change closes"); +}); + // ── The same class on the HTTP path ──────────────────────────────────────────── // Deferred once with a recorded reason, then closed by separating the two halves // it had been treating as one: a provider error in a 200 ENVELOPE is structured