diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index cd89bba9c..03dfce5f8 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -2,6 +2,11 @@ name: prose-asserter description: Judges a prose-test walk against the case's expected path and the code-computed world delta, returning a four-part verdict. Dispatched by prose-orchestrator. model: opus +hooks: + PreToolUse: + - hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" --- # Prose Asserter @@ -43,7 +48,10 @@ tools you appear to have or what mode you are running in. **Recorded actions** are captured by a harness hook as each tool call happens. The walker did not write them and could not have edited them. They are the authority on what the walk *did*: which commands ran, in -what order, with what arguments, and which files were written. +what order, with what arguments, what each returned, and which files +were written. A walker has been known to describe a command's output +inaccurately; where its account and the record disagree about what came +back, the record is right and the disagreement is a finding. **The transcript** is the walker's own account. It is the authority on *reasoning* — which arm it entered, which guard line selected it, what diff --git a/.claude/agents/prose-orchestrator.md b/.claude/agents/prose-orchestrator.md index bd14394d3..b1db42161 100644 --- a/.claude/agents/prose-orchestrator.md +++ b/.claude/agents/prose-orchestrator.md @@ -3,6 +3,12 @@ name: prose-orchestrator description: Runs one prose-test case end to end — builds the world, dispatches the walker, computes the delta, dispatches the asserter, escalates a failure to Opus, destroys the world — and returns just the verdict. Dispatched by the /prose-test skill, one per case. tools: Bash, Read, Agent model: sonnet +hooks: + PostToolUse: + - matcher: "Bash" + hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" --- # Prose Orchestrator diff --git a/.claude/agents/prose-walker.md b/.claude/agents/prose-walker.md index d84f8fcc2..fcc802ae7 100644 --- a/.claude/agents/prose-walker.md +++ b/.claude/agents/prose-walker.md @@ -4,11 +4,25 @@ description: Executes workflow prose exactly as a live session would, against a tools: Read, Write, Edit, Bash, Glob, Grep model: opus hooks: + PreToolUse: + - matcher: "Bash|Write|Edit|Read|Glob|Grep" + hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" PostToolUse: - - matcher: "Bash|Write|Edit|Read" + - matcher: "Bash|Write|Edit|Read|Glob|Grep" + hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" + PostToolUseFailure: + - matcher: "Bash|Write|Edit|Read|Glob|Grep" hooks: - type: command command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" + Stop: + - hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" --- # Prose Walker diff --git a/tests/prose/.gitignore b/tests/prose/.gitignore new file mode 100644 index 000000000..a29dcf6b8 --- /dev/null +++ b/tests/prose/.gitignore @@ -0,0 +1 @@ +.agent-tool-use.log diff --git a/tests/prose/lib/record-action.cjs b/tests/prose/lib/record-action.cjs index e8a47c71a..564534206 100644 --- a/tests/prose/lib/record-action.cjs +++ b/tests/prose/lib/record-action.cjs @@ -1,29 +1,38 @@ #!/usr/bin/env node 'use strict'; -// PostToolUse hook for the prose-walker agent: records what the walker -// ACTUALLY did, as the harness sees it. +// The hook that records what prose-test agents actually do. // -// Declared in .claude/agents/prose-walker.md frontmatter, so it fires -// only while a walker is active. The walker plays no part in it — it -// cannot forget an entry, summarise one away, or write the log late, -// which is exactly why the log exists. Walkers were repeatedly found -// doing a full walk and then reporting only its tail; the narrative is -// now evidence of reasoning alone, and this file is the evidence of -// action. +// Declared in the frontmatter of prose-walker, prose-orchestrator and +// prose-asserter, so it fires only while one of them is active. The +// agents play no part in it: they cannot forget an entry, summarise one +// away, or write it late — which is the whole point. Walkers were +// repeatedly found doing a full walk and reporting only its tail, and +// once produced a specific, plausible, false claim about what a command +// had returned. A narrative can do that; a record cannot. // -// Self-scoping: every prose world lives at a temp path containing -// `prose-world-`. The payload is scanned for that path, and anything -// happening outside a world is ignored. Nothing is configured per run. +// Every event is captured, not just the call: the intent before it +// (PreToolUse), the result after it (PostToolUse, with output), the +// failures (PostToolUseFailure), and the finish (Stop). Logs are +// throwaway — they live in the disposable world and die with it — so +// there is no reason to record less than everything. // -// The log lands at /.walk-actions.log, which collectTree() +// Self-scoping: a prose world lives at a temp path containing +// `prose-world-`. The payload is scanned for one, and anything happening +// outside a world is ignored. The asserter is the exception: it is +// contracted to use NO tools, so any tool call it makes is a contract +// violation and lands in a repo-local log instead of a world. +// +// The log is written to /.walk-actions.log, which collectTree() // excludes, so recording never shows up as a world difference. const fs = require('fs'); const path = require('path'); const LOG = '.walk-actions.log'; +const VIOLATIONS = 'tests/prose/.agent-tool-use.log'; const WORLD = /(^|[\s"'`])(\/[^\s"'`]*\/prose-world-[A-Za-z0-9]+)/; +const MAX_OUTPUT = 400; function read() { try { @@ -33,35 +42,60 @@ function read() { } } -/** The salient argument, per tool — what the claim will be about. */ -function summarise(tool, input) { +function flatten(value, limit) { + if (value === undefined || value === null) return ''; + const text = typeof value === 'string' ? value : JSON.stringify(value); + const oneLine = text.replace(/\s+/g, ' ').trim(); + return oneLine.length > limit ? `${oneLine.slice(0, limit)}…[truncated]` : oneLine; +} + +/** The salient argument, per tool — what a claim will be about. */ +function summarise(input) { if (!input || typeof input !== 'object') return ''; const raw = input.command || input.file_path || input.pattern || input.path || ''; - return String(raw).replace(/\s+/g, ' ').trim(); + return flatten(raw, 200); } function main() { const payload = read(); if (!payload) return; - const probe = JSON.stringify(payload); - const found = probe.match(WORLD); - // Not a prose world: this hook has nothing to say. + const event = payload.hook_event_name || '?'; + const tool = payload.tool_name || '-'; + const agent = payload.agent_type || 'main'; + + // The asserter judges from its prompt alone. A tool call from it is a + // breach of that contract and must be visible even though it happens + // nowhere near a world. + if (agent.includes('asserter') && event !== 'Stop' && event !== 'SubagentStop') { + const projectDir = process.env.CLAUDE_PROJECT_DIR; + if (projectDir) { + try { + fs.appendFileSync(path.join(projectDir, VIOLATIONS), + `${agent}\t${event}\t${tool}\t${summarise(payload.tool_input)}\n`); + } catch { /* a hook must never break what it observes */ } + } + return; + } + + const found = JSON.stringify(payload).match(WORLD); if (!found) return; const world = found[2].replace(/\\+/g, ''); - if (!fs.existsSync(world)) return; - const tool = payload.tool_name || '?'; - const detail = summarise(tool, payload.tool_input); - const failed = payload.tool_output_is_error ? ' [ERROR]' : ''; - const line = `${tool}${failed}\t${detail.replace(world, '.')}\n`; + const parts = [event, tool, summarise(payload.tool_input).split(world).join('.')]; - try { - fs.appendFileSync(path.join(world, LOG), line); - } catch { - // A hook must never break the walk it is observing. + if (event === 'PostToolUse') { + parts.push(payload.tool_output_is_error ? 'ERROR' : 'ok'); + parts.push(flatten(payload.tool_output, MAX_OUTPUT).split(world).join('.')); + } else if (event === 'PostToolUseFailure') { + parts.push('FAILED'); + parts.push(flatten(payload.tool_output ?? payload.error, MAX_OUTPUT)); } + + try { + fs.appendFileSync(path.join(world, LOG), `${parts.join('\t')}\n`); + } catch { /* a hook must never break what it observes */ } } main(); diff --git a/tests/prose/lib/worlds.cjs b/tests/prose/lib/worlds.cjs index 000f5e990..ec6b87d71 100644 --- a/tests/prose/lib/worlds.cjs +++ b/tests/prose/lib/worlds.cjs @@ -335,8 +335,10 @@ function readActionLog(worldDir) { .split('\n') .filter(Boolean) .map((l, i) => { - const [tool, detail = ''] = l.split('\t'); - return `${String(i + 1).padStart(3)}. ${tool.padEnd(10)} ${detail}`; + const [event, tool, detail = '', outcome, output] = l.split('\t'); + const head = `${String(i + 1).padStart(3)}. ${event.padEnd(19)} ${(tool || '').padEnd(6)} ${detail}`; + if (!outcome) return head; + return `${head}\n → ${outcome}${output ? `: ${output}` : ''}`; }) .join('\n'); } diff --git a/tests/prose/prompts/asserter.md b/tests/prose/prompts/asserter.md index c4eaa036e..f7a31f191 100644 --- a/tests/prose/prompts/asserter.md +++ b/tests/prose/prompts/asserter.md @@ -24,8 +24,15 @@ the expected world, computed by code: === actions === RECORDED ACTIONS — every tool call the walker made, in order, captured by a -harness hook as it happened. The walker did not write this and could -not have edited or omitted from it. This is what the walk *did*. +harness hook as it happened: the intent before each call, the result +after it (`→ ok:` with the output, truncated), and any failure. The +walker did not write this and could not have edited or omitted from it. +This is what the walk *did*, and what each command actually returned. + +A claim about what a command produced — that a gate rendered empty, that +a menu was absent — is settled here, not by the walker's account of it. +Where the account describes an output the record contradicts, the record +wins and the discrepancy is itself worth reporting. {{actions}}