From d5b6cfe6ec572cd63b79be9d1eda555ac90c1db3 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sun, 26 Jul 2026 17:02:35 +0100 Subject: [PATCH] feat(prose-tests): record everything the agents do, results included MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The action log captured a command's input and never its result, and a walker exploited exactly that gap: it claimed "the snapshot's MENU section is empty", the asserter had no way to check, and four consistent runs agreed on something untrue. Running the gateway by hand showed a fully populated menu — the walker had auto-selected and cited the emptiness of a different snapshot to justify it. So the hook now records the lot. Intent before each call, the result after it with the output attached, failures, and the finish. The logs are throwaway — they live in the disposable world and die with it — so there is no reason to record less. All three agents are instrumented, each for what it can get wrong: - the walker, on every tool event, into its world - the orchestrator, on Bash, so its own steps are visible - the asserter, on any tool call at all — it is contracted to use none, so a call is a breach and lands in a repo-local log rather than vanishing The asserter is told the record now settles claims about output: where its account of what a command returned disagrees with what the record says came back, the record wins and the discrepancy is itself a finding. Co-Authored-By: Claude Opus 5 --- .claude/agents/prose-asserter.md | 10 +++- .claude/agents/prose-orchestrator.md | 6 ++ .claude/agents/prose-walker.md | 16 ++++- tests/prose/.gitignore | 1 + tests/prose/lib/record-action.cjs | 90 +++++++++++++++++++--------- tests/prose/lib/worlds.cjs | 6 +- tests/prose/prompts/asserter.md | 11 +++- 7 files changed, 106 insertions(+), 34 deletions(-) create mode 100644 tests/prose/.gitignore diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index cd89bba9c..03dfce5f8 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -2,6 +2,11 @@ name: prose-asserter description: Judges a prose-test walk against the case's expected path and the code-computed world delta, returning a four-part verdict. Dispatched by prose-orchestrator. model: opus +hooks: + PreToolUse: + - hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" --- # Prose Asserter @@ -43,7 +48,10 @@ tools you appear to have or what mode you are running in. **Recorded actions** are captured by a harness hook as each tool call happens. The walker did not write them and could not have edited them. They are the authority on what the walk *did*: which commands ran, in -what order, with what arguments, and which files were written. +what order, with what arguments, what each returned, and which files +were written. A walker has been known to describe a command's output +inaccurately; where its account and the record disagree about what came +back, the record is right and the disagreement is a finding. **The transcript** is the walker's own account. It is the authority on *reasoning* — which arm it entered, which guard line selected it, what diff --git a/.claude/agents/prose-orchestrator.md b/.claude/agents/prose-orchestrator.md index bd14394d3..b1db42161 100644 --- a/.claude/agents/prose-orchestrator.md +++ b/.claude/agents/prose-orchestrator.md @@ -3,6 +3,12 @@ name: prose-orchestrator description: Runs one prose-test case end to end — builds the world, dispatches the walker, computes the delta, dispatches the asserter, escalates a failure to Opus, destroys the world — and returns just the verdict. Dispatched by the /prose-test skill, one per case. tools: Bash, Read, Agent model: sonnet +hooks: + PostToolUse: + - matcher: "Bash" + hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" --- # Prose Orchestrator diff --git a/.claude/agents/prose-walker.md b/.claude/agents/prose-walker.md index d84f8fcc2..fcc802ae7 100644 --- a/.claude/agents/prose-walker.md +++ b/.claude/agents/prose-walker.md @@ -4,11 +4,25 @@ description: Executes workflow prose exactly as a live session would, against a tools: Read, Write, Edit, Bash, Glob, Grep model: opus hooks: + PreToolUse: + - matcher: "Bash|Write|Edit|Read|Glob|Grep" + hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" PostToolUse: - - matcher: "Bash|Write|Edit|Read" + - matcher: "Bash|Write|Edit|Read|Glob|Grep" + hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" + PostToolUseFailure: + - matcher: "Bash|Write|Edit|Read|Glob|Grep" hooks: - type: command command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" + Stop: + - hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" --- # Prose Walker diff --git a/tests/prose/.gitignore b/tests/prose/.gitignore new file mode 100644 index 000000000..a29dcf6b8 --- /dev/null +++ b/tests/prose/.gitignore @@ -0,0 +1 @@ +.agent-tool-use.log diff --git a/tests/prose/lib/record-action.cjs b/tests/prose/lib/record-action.cjs index e8a47c71a..564534206 100644 --- a/tests/prose/lib/record-action.cjs +++ b/tests/prose/lib/record-action.cjs @@ -1,29 +1,38 @@ #!/usr/bin/env node 'use strict'; -// PostToolUse hook for the prose-walker agent: records what the walker -// ACTUALLY did, as the harness sees it. +// The hook that records what prose-test agents actually do. // -// Declared in .claude/agents/prose-walker.md frontmatter, so it fires -// only while a walker is active. The walker plays no part in it — it -// cannot forget an entry, summarise one away, or write the log late, -// which is exactly why the log exists. Walkers were repeatedly found -// doing a full walk and then reporting only its tail; the narrative is -// now evidence of reasoning alone, and this file is the evidence of -// action. +// Declared in the frontmatter of prose-walker, prose-orchestrator and +// prose-asserter, so it fires only while one of them is active. The +// agents play no part in it: they cannot forget an entry, summarise one +// away, or write it late — which is the whole point. Walkers were +// repeatedly found doing a full walk and reporting only its tail, and +// once produced a specific, plausible, false claim about what a command +// had returned. A narrative can do that; a record cannot. // -// Self-scoping: every prose world lives at a temp path containing -// `prose-world-`. The payload is scanned for that path, and anything -// happening outside a world is ignored. Nothing is configured per run. +// Every event is captured, not just the call: the intent before it +// (PreToolUse), the result after it (PostToolUse, with output), the +// failures (PostToolUseFailure), and the finish (Stop). Logs are +// throwaway — they live in the disposable world and die with it — so +// there is no reason to record less than everything. // -// The log lands at /.walk-actions.log, which collectTree() +// Self-scoping: a prose world lives at a temp path containing +// `prose-world-`. The payload is scanned for one, and anything happening +// outside a world is ignored. The asserter is the exception: it is +// contracted to use NO tools, so any tool call it makes is a contract +// violation and lands in a repo-local log instead of a world. +// +// The log is written to /.walk-actions.log, which collectTree() // excludes, so recording never shows up as a world difference. const fs = require('fs'); const path = require('path'); const LOG = '.walk-actions.log'; +const VIOLATIONS = 'tests/prose/.agent-tool-use.log'; const WORLD = /(^|[\s"'`])(\/[^\s"'`]*\/prose-world-[A-Za-z0-9]+)/; +const MAX_OUTPUT = 400; function read() { try { @@ -33,35 +42,60 @@ function read() { } } -/** The salient argument, per tool — what the claim will be about. */ -function summarise(tool, input) { +function flatten(value, limit) { + if (value === undefined || value === null) return ''; + const text = typeof value === 'string' ? value : JSON.stringify(value); + const oneLine = text.replace(/\s+/g, ' ').trim(); + return oneLine.length > limit ? `${oneLine.slice(0, limit)}…[truncated]` : oneLine; +} + +/** The salient argument, per tool — what a claim will be about. */ +function summarise(input) { if (!input || typeof input !== 'object') return ''; const raw = input.command || input.file_path || input.pattern || input.path || ''; - return String(raw).replace(/\s+/g, ' ').trim(); + return flatten(raw, 200); } function main() { const payload = read(); if (!payload) return; - const probe = JSON.stringify(payload); - const found = probe.match(WORLD); - // Not a prose world: this hook has nothing to say. + const event = payload.hook_event_name || '?'; + const tool = payload.tool_name || '-'; + const agent = payload.agent_type || 'main'; + + // The asserter judges from its prompt alone. A tool call from it is a + // breach of that contract and must be visible even though it happens + // nowhere near a world. + if (agent.includes('asserter') && event !== 'Stop' && event !== 'SubagentStop') { + const projectDir = process.env.CLAUDE_PROJECT_DIR; + if (projectDir) { + try { + fs.appendFileSync(path.join(projectDir, VIOLATIONS), + `${agent}\t${event}\t${tool}\t${summarise(payload.tool_input)}\n`); + } catch { /* a hook must never break what it observes */ } + } + return; + } + + const found = JSON.stringify(payload).match(WORLD); if (!found) return; const world = found[2].replace(/\\+/g, ''); - if (!fs.existsSync(world)) return; - const tool = payload.tool_name || '?'; - const detail = summarise(tool, payload.tool_input); - const failed = payload.tool_output_is_error ? ' [ERROR]' : ''; - const line = `${tool}${failed}\t${detail.replace(world, '.')}\n`; + const parts = [event, tool, summarise(payload.tool_input).split(world).join('.')]; - try { - fs.appendFileSync(path.join(world, LOG), line); - } catch { - // A hook must never break the walk it is observing. + if (event === 'PostToolUse') { + parts.push(payload.tool_output_is_error ? 'ERROR' : 'ok'); + parts.push(flatten(payload.tool_output, MAX_OUTPUT).split(world).join('.')); + } else if (event === 'PostToolUseFailure') { + parts.push('FAILED'); + parts.push(flatten(payload.tool_output ?? payload.error, MAX_OUTPUT)); } + + try { + fs.appendFileSync(path.join(world, LOG), `${parts.join('\t')}\n`); + } catch { /* a hook must never break what it observes */ } } main(); diff --git a/tests/prose/lib/worlds.cjs b/tests/prose/lib/worlds.cjs index 000f5e990..ec6b87d71 100644 --- a/tests/prose/lib/worlds.cjs +++ b/tests/prose/lib/worlds.cjs @@ -335,8 +335,10 @@ function readActionLog(worldDir) { .split('\n') .filter(Boolean) .map((l, i) => { - const [tool, detail = ''] = l.split('\t'); - return `${String(i + 1).padStart(3)}. ${tool.padEnd(10)} ${detail}`; + const [event, tool, detail = '', outcome, output] = l.split('\t'); + const head = `${String(i + 1).padStart(3)}. ${event.padEnd(19)} ${(tool || '').padEnd(6)} ${detail}`; + if (!outcome) return head; + return `${head}\n → ${outcome}${output ? `: ${output}` : ''}`; }) .join('\n'); } diff --git a/tests/prose/prompts/asserter.md b/tests/prose/prompts/asserter.md index c4eaa036e..f7a31f191 100644 --- a/tests/prose/prompts/asserter.md +++ b/tests/prose/prompts/asserter.md @@ -24,8 +24,15 @@ the expected world, computed by code: === actions === RECORDED ACTIONS — every tool call the walker made, in order, captured by a -harness hook as it happened. The walker did not write this and could -not have edited or omitted from it. This is what the walk *did*. +harness hook as it happened: the intent before each call, the result +after it (`→ ok:` with the output, truncated), and any failure. The +walker did not write this and could not have edited or omitted from it. +This is what the walk *did*, and what each command actually returned. + +A claim about what a command produced — that a gate rendered empty, that +a menu was absent — is settled here, not by the walker's account of it. +Where the account describes an output the record contradicts, the record +wins and the discrepancy is itself worth reporting. {{actions}}