From 586474383d9b941dd146ff508fab396176918666 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Mon, 27 Jul 2026 15:05:20 +0100 Subject: [PATCH] fix(prose-tests): the world log is the walker's record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first two rows of the quick-fix walk's action log were not the walk's. They were the orchestrator's own commands — the world it had just built, the prompt it had just emitted — landing in the walker's log because their payloads carry the world path and the recorder scopes by finding one. The record of the walk opened with actions no walk ran. Noise today, a hazard tomorrow: the checks match substrings against those rows, so an orchestrator command could satisfy a calls_include or trip a calls_exclude on a walk that did neither. The recorder now gates world-scoped writes on the walker: nothing else writes into a world's action log or walk transcript. The asserter's violation path is untouched — its tool calls still land in the repo-local log, since a breach of the no-tools contract must be visible wherever it happens. The orchestrator's own recording hook goes with it; it existed to record everything, and everything it recorded in a world was someone else's record. Co-Authored-By: Claude Opus 5 --- .claude/agents/prose-orchestrator.md | 6 ------ tests/prose/lib/record-action.cjs | 11 +++++++++-- tests/scripts/test-prose-record-action.cjs | 15 +++++++++++++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.claude/agents/prose-orchestrator.md b/.claude/agents/prose-orchestrator.md index 95511d0c9..04735107f 100644 --- a/.claude/agents/prose-orchestrator.md +++ b/.claude/agents/prose-orchestrator.md @@ -3,12 +3,6 @@ name: prose-orchestrator description: Runs one prose-test case end to end — builds the world, dispatches the walker, computes the delta, dispatches the asserter, reruns a failure from a fresh world to confirm it, destroys the world — and returns just the verdict. Dispatched by the /prose-test skill, one per case. tools: Bash, Read, Agent model: sonnet -hooks: - PostToolUse: - - matcher: "Bash" - hooks: - - type: command - command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" --- # Prose Orchestrator diff --git a/tests/prose/lib/record-action.cjs b/tests/prose/lib/record-action.cjs index ceb311a59..6681a6a42 100644 --- a/tests/prose/lib/record-action.cjs +++ b/tests/prose/lib/record-action.cjs @@ -3,8 +3,8 @@ // The hook that records what prose-test agents actually do. // -// Declared in the frontmatter of prose-walker, prose-orchestrator and -// prose-asserter, so it fires only while one of them is active. The +// Declared in the frontmatter of prose-walker and prose-asserter, so it +// fires only while one of them is active. The // agents play no part in it: they cannot forget an entry, summarise one // away, or write it late — which is the whole point. Walkers were // repeatedly found doing a full walk and reporting only its tail, and @@ -201,6 +201,13 @@ function main() { // every other event is absent from it — resolve it from the transcript. const stop = event === 'Stop' || event === 'SubagentStop'; const traced = stop ? fromTranscript(payload.agent_transcript_path) : null; + // The world log is the walker's record. The orchestrator shares this + // hook, and its own commands carry world paths in their payloads — the + // world it just built, the prompt it just emitted — so without this + // gate its rows land in the walker's log ahead of the walk itself, + // padding the record and handing the checks substrings no walk ran. + if (!agent.includes('walker')) return; + const found = JSON.stringify(payload).match(WORLD); const world = found ? found[2].replace(/\\+/g, '') : traced && traced.world; if (!world || !fs.existsSync(world)) return; diff --git a/tests/scripts/test-prose-record-action.cjs b/tests/scripts/test-prose-record-action.cjs index 172a8cd2a..d865158af 100644 --- a/tests/scripts/test-prose-record-action.cjs +++ b/tests/scripts/test-prose-record-action.cjs @@ -164,6 +164,21 @@ describe('prose recorder — tool events', () => { assert.ok(row.includes('Usage: engine '), 'and says what came back'); }); + it('keeps the orchestrator out of the walker\'s record', () => { + // The orchestrator's own commands carry world paths — the world it + // just built, the prompt it just emitted — and its rows were landing + // in the walker's log ahead of the walk itself, padding the record + // and handing the checks substrings no walk ran. + fire({ + hook_event_name: 'PostToolUse', + tool_name: 'Bash', + agent_type: 'prose-orchestrator', + tool_input: { command: 'node tests/prose/run.cjs world some-case' }, + tool_response: { stdout: `{"world":"${world}","case":"some-case"}` }, + }); + assert.deepEqual(logLines(), [], 'no orchestrator row reaches the world log'); + }); + it('ignores anything happening outside a world', () => { fire({ hook_event_name: 'PreToolUse',