From 9fdf6ab1b8867c2a9728cd95e5f52e5951aa4fee Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Mon, 27 Jul 2026 08:38:46 +0100 Subject: [PATCH] fix(prose-tests): keep the walk's closing turn, and name the one live difference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both spec-entry cases failed on a claim about the handoff block their walks emitted correctly. The block was missing from the record, so the asserter had nothing to judge and rightly refused to pass an unevidenced step. Four walks, two cases, one cause. The cause is not the parser. Running the hook's own extraction over the settled transcript returns every turn including the one it missed, so the file simply did not hold that turn yet when the hook read it. This hook runs inside the stop sequence rather than after it: the agent has stopped thinking, and the runtime is still writing down the last thing it said. The turn is not lost, only not yet on disk — the stop payload carries it as `last_assistant_message`, which is the runtime handing it over at exactly the moment it is unreadable anywhere else. It is appended as the closing turn when it is not already there. Reading the transcript later instead, from the assert command, would also have worked, but it splits the record across two components and leaves the world holding a pointer rather than the walk; the world stays self-contained this way. Appending only when absent is not deduplication. Two identical turns that are genuinely in the transcript both survive, because a step that ran twice is something the asserter has to see. Truncation is reconsidered with it. The caps were never about disk — the logs are throwaway — but about the asserter's prompt, which every recorded action is read into. That justifies trimming a file body no claim ever rests on, and does not justify trimming evidence: the walk log now caps nothing at all, and a command's output gets a ceiling that bounds a runaway without ever reaching real output. Separately, the one genuine difference between a walk and a live session is written down rather than rediscovered each run. An inline directive is substituted when a skill loads live; a walk reads the prose as a file, so the walker takes the fallback the prose supplies for that case. Correct, expected, and not a marker. The asserter is also told what it costs — the primary arm is the one real sessions take and it is never exercised here, so a claim resting on the substituted value is unprovable rather than passed. It is named as the only such exemption: anything else resembling an environment quirk is a finding. Co-Authored-By: Claude Opus 5 --- .claude/agents/prose-asserter.md | 16 ++++++ .claude/agents/prose-walker.md | 5 ++ tests/prose/lib/record-action.cjs | 39 ++++++++++--- tests/scripts/test-prose-record-action.cjs | 64 ++++++++++++++++++++++ 4 files changed, 115 insertions(+), 9 deletions(-) diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index ca0785349..c6e7ff162 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -89,6 +89,22 @@ record exactly like one the prose asked for — only the walk says which it was. So a claim that something was not done is answered by both together: the record for whether it happened, the walk for whose doing it was. +## One known difference between here and a live session + +An inline `` !`command` `` directive is substituted when a skill loads +live. A walk reads the prose as a file, so the substitution never happens +and the walker takes the fallback the prose supplies for that case. That +is correct behaviour and neither a failure nor a marker — do not report +it, and do not fail a step for the fallback arm having run. + +It is worth knowing what it costs, though: the primary arm is the one a +real session almost always takes, and here it is never exercised. A claim +that depends on the substituted value cannot be answered by a walk, and +should be judged unprovable rather than passed or failed. + +This is the only such difference. Anything else that looks like an +environment quirk is a finding, not an exemption. + ## An invalid walk is not a failing walk Check the **recorded actions**, not the walk, for where the walk diff --git a/.claude/agents/prose-walker.md b/.claude/agents/prose-walker.md index 758d17e69..63c768459 100644 --- a/.claude/agents/prose-walker.md +++ b/.claude/agents/prose-walker.md @@ -71,6 +71,11 @@ any harness substitutions. Follow it exactly. errored. Use only the tools the walk itself requires. - Do not read the case directory (`tests/prose/cases/…`). It holds the expected result, and seeing it invalidates the run. +- **An inline `` !`command` `` directive will not have run.** That + substitution happens when a skill is loaded live; here the prose is read + as a file, so the literal backtick line is what you see. The prose gives + a fallback for exactly this — take it. It is expected, it is not a + `DEVIATION`, and it needs no marker. ## Markers diff --git a/tests/prose/lib/record-action.cjs b/tests/prose/lib/record-action.cjs index c58e78ac7..89684a383 100644 --- a/tests/prose/lib/record-action.cjs +++ b/tests/prose/lib/record-action.cjs @@ -42,12 +42,16 @@ const LOG = '.walk-actions.log'; const WALK = '.walk-transcript.log'; const VIOLATIONS = 'tests/prose/.agent-tool-use.log'; const WORLD = /(^|[\s"'`])(\/[^\s"'`]*\/prose-world-[A-Za-z0-9]+)/; +// These caps exist to protect the asserter's prompt, never to save disk — +// every recorded action is read into it, and a walk makes twenty-odd file +// reads whose bodies are whole skill files. They apply only where the +// content is incidental. A file a walker read back is incidental: no claim +// ever rests on those bytes. A command's output is not — it settles what +// the prose was shown, whether a gate rendered empty, whether a menu had +// entries — so it gets a ceiling generous enough that real evidence is +// never trimmed, bounding only a runaway that would swamp the prompt. const MAX_OUTPUT = 400; -// A command's output is the evidence that settles what the prose was -// shown — whether a gate rendered empty, whether a menu had entries. It -// gets the room a whole rendered section needs; a file read does not, -// since no claim turns on the bytes a walker read back. -const MAX_COMMAND_OUTPUT = 2000; +const MAX_COMMAND_OUTPUT = 10000; function read() { try { @@ -136,13 +140,30 @@ function fromTranscript(transcriptPath) { * already; this lifts them into the world beside the action log, where * the judging is done. Nothing is asked of the walker, which is what * makes it dependable. + * + * The final turn comes from the payload rather than the transcript. This + * hook runs inside the stop sequence, not after it, and the last thing + * the agent said is still being appended to the transcript as we read it + * — verified: the same extraction over the settled file returns the turn + * this misses. That turn is where a flow's closing emission lives, a + * handoff block among them, so losing it costs exactly the evidence a + * final claim rests on. `last_assistant_message` is the runtime handing + * it over directly, at the one moment it is not yet on disk. + * + * It is appended only when it is not already the closing turn — the race + * does not always fire. That is not deduplication: two identical turns + * genuinely in the transcript still both appear, because a step that ran + * twice is something the asserter must see. */ -function writeWalk(world, turns) { - if (!turns || !turns.length) return; +function writeWalk(world, turns, closing) { + const all = turns ? turns.slice() : []; + const tail = (closing || '').trim(); + if (tail && all[all.length - 1] !== tail) all.push(tail); + if (!all.length) return; try { fs.writeFileSync( path.join(world, WALK), - `${turns.join('\n\n---\n\n').split(world).join('.')}\n`, + `${all.join('\n\n---\n\n').split(world).join('.')}\n`, ); } catch { /* a hook must never break what it observes */ } } @@ -203,7 +224,7 @@ function main() { ); } else if (stop) { parts.push(flatten(payload.last_assistant_message, MAX_OUTPUT).split(world).join('.')); - writeWalk(world, traced.turns); + writeWalk(world, traced.turns, payload.last_assistant_message); } try { diff --git a/tests/scripts/test-prose-record-action.cjs b/tests/scripts/test-prose-record-action.cjs index c0b3b1129..c3823ccdb 100644 --- a/tests/scripts/test-prose-record-action.cjs +++ b/tests/scripts/test-prose-record-action.cjs @@ -233,6 +233,70 @@ describe('prose recorder — the stop event', () => { assert.ok(walk.indexOf('ENTERED') < walk.indexOf('Which feature?'), 'in the order they happened'); }); + it('appends the closing turn the stop race leaves out of the transcript', () => { + // The hook runs inside the stop sequence: the agent's last message is + // still being appended as it reads. That turn is where a closing + // emission lives, so the payload's copy stands in for it. + const transcript = writeTranscript([ + { message: { model: 'claude-sonnet-5', content: [{ type: 'text', text: 'ENTERED: § Step 4' }] }, cwd: world }, + ]); + fire({ + hook_event_name: 'SubagentStop', + agent_type: 'prose-walker', + agent_transcript_path: transcript, + last_assistant_message: 'EMITTED (handoff block):\n Source: .workflows/pay/discussion/pay.md', + }); + const walk = fs.readFileSync(path.join(world, '.walk-transcript.log'), 'utf8'); + assert.ok(walk.includes('ENTERED: § Step 4'), 'the transcript turns survive'); + assert.ok(walk.includes('Source: .workflows/pay/discussion/pay.md'), 'and the closing turn arrives'); + assert.ok(walk.indexOf('Step 4') < walk.indexOf('Source:'), 'in that order'); + }); + + it('does not double-record a closing turn the transcript already caught', () => { + const closing = 'STOPPED: end of flow'; + const transcript = writeTranscript([ + { message: { model: 'x', content: [{ type: 'text', text: 'ENTERED: § Step 4' }] }, cwd: world }, + { message: { content: [{ type: 'text', text: closing }] } }, + ]); + fire({ + hook_event_name: 'SubagentStop', + agent_type: 'prose-walker', + agent_transcript_path: transcript, + last_assistant_message: closing, + }); + const walk = fs.readFileSync(path.join(world, '.walk-transcript.log'), 'utf8'); + assert.equal(walk.split(closing).length - 1, 1, 'recorded once, not twice'); + }); + + it('keeps a genuine repeat — a step that ran twice must stay visible', () => { + const twice = 'ENTERED: § Step 4'; + const transcript = writeTranscript([ + { message: { model: 'x', content: [{ type: 'text', text: twice }] }, cwd: world }, + { message: { content: [{ type: 'text', text: twice }] } }, + ]); + fire({ + hook_event_name: 'SubagentStop', + agent_type: 'prose-walker', + agent_transcript_path: transcript, + last_assistant_message: 'STOPPED: done', + }); + const walk = fs.readFileSync(path.join(world, '.walk-transcript.log'), 'utf8'); + assert.equal(walk.split(twice).length - 1, 2, 'both occurrences survive'); + }); + + it('records the closing turn whole — a final emission is evidence', () => { + const long = `EMITTED:\n${'x'.repeat(3000)}`; + fire({ + hook_event_name: 'SubagentStop', + agent_type: 'prose-walker', + agent_transcript_path: writeTranscript([{ message: { model: 'x' }, cwd: world }]), + last_assistant_message: long, + }); + const walk = fs.readFileSync(path.join(world, '.walk-transcript.log'), 'utf8'); + assert.ok(!walk.includes('[truncated]'), 'the walk log caps nothing'); + assert.ok(walk.includes('x'.repeat(3000))); + }); + it('writes no walk file when the transcript holds no turns', () => { fire({ hook_event_name: 'SubagentStop',