diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index ca0785349..c6e7ff162 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -89,6 +89,22 @@ record exactly like one the prose asked for — only the walk says which it was. So a claim that something was not done is answered by both together: the record for whether it happened, the walk for whose doing it was. +## One known difference between here and a live session + +An inline `` !`command` `` directive is substituted when a skill loads +live. A walk reads the prose as a file, so the substitution never happens +and the walker takes the fallback the prose supplies for that case. That +is correct behaviour and neither a failure nor a marker — do not report +it, and do not fail a step for the fallback arm having run. + +It is worth knowing what it costs, though: the primary arm is the one a +real session almost always takes, and here it is never exercised. A claim +that depends on the substituted value cannot be answered by a walk, and +should be judged unprovable rather than passed or failed. + +This is the only such difference. Anything else that looks like an +environment quirk is a finding, not an exemption. + ## An invalid walk is not a failing walk Check the **recorded actions**, not the walk, for where the walk diff --git a/.claude/agents/prose-walker.md b/.claude/agents/prose-walker.md index 758d17e69..63c768459 100644 --- a/.claude/agents/prose-walker.md +++ b/.claude/agents/prose-walker.md @@ -71,6 +71,11 @@ any harness substitutions. Follow it exactly. errored. Use only the tools the walk itself requires. - Do not read the case directory (`tests/prose/cases/…`). It holds the expected result, and seeing it invalidates the run. +- **An inline `` !`command` `` directive will not have run.** That + substitution happens when a skill is loaded live; here the prose is read + as a file, so the literal backtick line is what you see. The prose gives + a fallback for exactly this — take it. It is expected, it is not a + `DEVIATION`, and it needs no marker. ## Markers diff --git a/tests/prose/lib/record-action.cjs b/tests/prose/lib/record-action.cjs index c58e78ac7..89684a383 100644 --- a/tests/prose/lib/record-action.cjs +++ b/tests/prose/lib/record-action.cjs @@ -42,12 +42,16 @@ const LOG = '.walk-actions.log'; const WALK = '.walk-transcript.log'; const VIOLATIONS = 'tests/prose/.agent-tool-use.log'; const WORLD = /(^|[\s"'`])(\/[^\s"'`]*\/prose-world-[A-Za-z0-9]+)/; +// These caps exist to protect the asserter's prompt, never to save disk — +// every recorded action is read into it, and a walk makes twenty-odd file +// reads whose bodies are whole skill files. They apply only where the +// content is incidental. A file a walker read back is incidental: no claim +// ever rests on those bytes. A command's output is not — it settles what +// the prose was shown, whether a gate rendered empty, whether a menu had +// entries — so it gets a ceiling generous enough that real evidence is +// never trimmed, bounding only a runaway that would swamp the prompt. const MAX_OUTPUT = 400; -// A command's output is the evidence that settles what the prose was -// shown — whether a gate rendered empty, whether a menu had entries. It -// gets the room a whole rendered section needs; a file read does not, -// since no claim turns on the bytes a walker read back. -const MAX_COMMAND_OUTPUT = 2000; +const MAX_COMMAND_OUTPUT = 10000; function read() { try { @@ -136,13 +140,30 @@ function fromTranscript(transcriptPath) { * already; this lifts them into the world beside the action log, where * the judging is done. Nothing is asked of the walker, which is what * makes it dependable. + * + * The final turn comes from the payload rather than the transcript. This + * hook runs inside the stop sequence, not after it, and the last thing + * the agent said is still being appended to the transcript as we read it + * — verified: the same extraction over the settled file returns the turn + * this misses. That turn is where a flow's closing emission lives, a + * handoff block among them, so losing it costs exactly the evidence a + * final claim rests on. `last_assistant_message` is the runtime handing + * it over directly, at the one moment it is not yet on disk. + * + * It is appended only when it is not already the closing turn — the race + * does not always fire. That is not deduplication: two identical turns + * genuinely in the transcript still both appear, because a step that ran + * twice is something the asserter must see. */ -function writeWalk(world, turns) { - if (!turns || !turns.length) return; +function writeWalk(world, turns, closing) { + const all = turns ? turns.slice() : []; + const tail = (closing || '').trim(); + if (tail && all[all.length - 1] !== tail) all.push(tail); + if (!all.length) return; try { fs.writeFileSync( path.join(world, WALK), - `${turns.join('\n\n---\n\n').split(world).join('.')}\n`, + `${all.join('\n\n---\n\n').split(world).join('.')}\n`, ); } catch { /* a hook must never break what it observes */ } } @@ -203,7 +224,7 @@ function main() { ); } else if (stop) { parts.push(flatten(payload.last_assistant_message, MAX_OUTPUT).split(world).join('.')); - writeWalk(world, traced.turns); + writeWalk(world, traced.turns, payload.last_assistant_message); } try { diff --git a/tests/scripts/test-prose-record-action.cjs b/tests/scripts/test-prose-record-action.cjs index c0b3b1129..c3823ccdb 100644 --- a/tests/scripts/test-prose-record-action.cjs +++ b/tests/scripts/test-prose-record-action.cjs @@ -233,6 +233,70 @@ describe('prose recorder — the stop event', () => { assert.ok(walk.indexOf('ENTERED') < walk.indexOf('Which feature?'), 'in the order they happened'); }); + it('appends the closing turn the stop race leaves out of the transcript', () => { + // The hook runs inside the stop sequence: the agent's last message is + // still being appended as it reads. That turn is where a closing + // emission lives, so the payload's copy stands in for it. + const transcript = writeTranscript([ + { message: { model: 'claude-sonnet-5', content: [{ type: 'text', text: 'ENTERED: § Step 4' }] }, cwd: world }, + ]); + fire({ + hook_event_name: 'SubagentStop', + agent_type: 'prose-walker', + agent_transcript_path: transcript, + last_assistant_message: 'EMITTED (handoff block):\n Source: .workflows/pay/discussion/pay.md', + }); + const walk = fs.readFileSync(path.join(world, '.walk-transcript.log'), 'utf8'); + assert.ok(walk.includes('ENTERED: § Step 4'), 'the transcript turns survive'); + assert.ok(walk.includes('Source: .workflows/pay/discussion/pay.md'), 'and the closing turn arrives'); + assert.ok(walk.indexOf('Step 4') < walk.indexOf('Source:'), 'in that order'); + }); + + it('does not double-record a closing turn the transcript already caught', () => { + const closing = 'STOPPED: end of flow'; + const transcript = writeTranscript([ + { message: { model: 'x', content: [{ type: 'text', text: 'ENTERED: § Step 4' }] }, cwd: world }, + { message: { content: [{ type: 'text', text: closing }] } }, + ]); + fire({ + hook_event_name: 'SubagentStop', + agent_type: 'prose-walker', + agent_transcript_path: transcript, + last_assistant_message: closing, + }); + const walk = fs.readFileSync(path.join(world, '.walk-transcript.log'), 'utf8'); + assert.equal(walk.split(closing).length - 1, 1, 'recorded once, not twice'); + }); + + it('keeps a genuine repeat — a step that ran twice must stay visible', () => { + const twice = 'ENTERED: § Step 4'; + const transcript = writeTranscript([ + { message: { model: 'x', content: [{ type: 'text', text: twice }] }, cwd: world }, + { message: { content: [{ type: 'text', text: twice }] } }, + ]); + fire({ + hook_event_name: 'SubagentStop', + agent_type: 'prose-walker', + agent_transcript_path: transcript, + last_assistant_message: 'STOPPED: done', + }); + const walk = fs.readFileSync(path.join(world, '.walk-transcript.log'), 'utf8'); + assert.equal(walk.split(twice).length - 1, 2, 'both occurrences survive'); + }); + + it('records the closing turn whole — a final emission is evidence', () => { + const long = `EMITTED:\n${'x'.repeat(3000)}`; + fire({ + hook_event_name: 'SubagentStop', + agent_type: 'prose-walker', + agent_transcript_path: writeTranscript([{ message: { model: 'x' }, cwd: world }]), + last_assistant_message: long, + }); + const walk = fs.readFileSync(path.join(world, '.walk-transcript.log'), 'utf8'); + assert.ok(!walk.includes('[truncated]'), 'the walk log caps nothing'); + assert.ok(walk.includes('x'.repeat(3000))); + }); + it('writes no walk file when the transcript holds no turns', () => { fire({ hook_event_name: 'SubagentStop',