From 4369c9d75b5fc409f30a66a7a3eadd5f8c1f7af5 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sat, 25 Jul 2026 22:09:38 +0100 Subject: [PATCH 1/6] =?UTF-8?q?fix(entry-skills):=20close=20the=20handoff?= =?UTF-8?q?=20fences=20=E2=80=94=20six=20files=20render=20their=20arms=20w?= =?UTF-8?q?rong?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by a prose-test walker, which had to guess whether a fenced block ends at the next fence (what markdown does) or at the next arm heading (what the author meant), and recorded the ambiguity rather than picking silently. Markdown pairs fences in order, so a missing closing fence makes the next arm's heading render *inside* the previous arm's code block, and that arm's own arguments render as plain prose. In workflow-specification-entry every one of the four arms was mispaired. Since each arm says "invoke the skill with the next fenced block as its arguments", a session following it literally is handed the wrong text. Six of eight entry skills were affected; discussion and research close their blocks correctly, which is what shows this is an omission rather than a house style. Additions only — eleven closing fences, no wording touched. Co-Authored-By: Claude Opus 5 --- .../workflow-implementation-entry/references/invoke-skill.md | 1 + .../workflow-investigation-entry/references/invoke-skill.md | 2 ++ skills/workflow-planning-entry/references/invoke-skill.md | 2 ++ skills/workflow-review-entry/references/invoke-skill.md | 1 + skills/workflow-scoping-entry/references/invoke-skill.md | 1 + .../workflow-specification-entry/references/invoke-skill.md | 4 ++++ 6 files changed, 11 insertions(+) diff --git a/skills/workflow-implementation-entry/references/invoke-skill.md b/skills/workflow-implementation-entry/references/invoke-skill.md index 3d947786e..05d92e2b4 100644 --- a/skills/workflow-implementation-entry/references/invoke-skill.md +++ b/skills/workflow-implementation-entry/references/invoke-skill.md @@ -36,3 +36,4 @@ Implementation: {exists:[true|false]} Dependencies: {All satisfied | List any notes} Environment: {Setup required | No special setup required} +``` diff --git a/skills/workflow-investigation-entry/references/invoke-skill.md b/skills/workflow-investigation-entry/references/invoke-skill.md index 5356ded2b..4c760534c 100644 --- a/skills/workflow-investigation-entry/references/invoke-skill.md +++ b/skills/workflow-investigation-entry/references/invoke-skill.md @@ -31,6 +31,7 @@ Bug context: - Expected behavior: {from the carrier / gather-context} - Actual behavior: {from the carrier / gather-context} - Initial context: {error messages, reproduction steps — from the carrier / gather-context, or "(none captured yet)"} +``` #### If source is `continue` @@ -39,3 +40,4 @@ Investigation session for: {work_unit} Source: existing investigation Output: .workflows/{work_unit}/investigation/{topic}.md +``` diff --git a/skills/workflow-planning-entry/references/invoke-skill.md b/skills/workflow-planning-entry/references/invoke-skill.md index c4d5736a0..ee767f89b 100644 --- a/skills/workflow-planning-entry/references/invoke-skill.md +++ b/skills/workflow-planning-entry/references/invoke-skill.md @@ -21,6 +21,7 @@ Work unit: {work_unit} Specification: .workflows/{work_unit}/specification/{topic}/specification.md Additional context: {summary of user's additional context, or "none"} Cross-cutting references: {list of applicable cross-cutting specs with brief summaries, or "none"} +``` #### If continuing or reviewing existing plan @@ -31,3 +32,4 @@ Work unit: {work_unit} Specification: .workflows/{work_unit}/specification/{topic}/specification.md Existing plan: .workflows/{work_unit}/planning/{topic}/planning.md Cross-cutting references: {list of applicable cross-cutting specs with brief summaries, or "none"} +``` diff --git a/skills/workflow-review-entry/references/invoke-skill.md b/skills/workflow-review-entry/references/invoke-skill.md index 3b8b609b7..e2a284f72 100644 --- a/skills/workflow-review-entry/references/invoke-skill.md +++ b/skills/workflow-review-entry/references/invoke-skill.md @@ -26,3 +26,4 @@ Plans to review: topic: {topic} format: {format} specification: .workflows/{work_unit}/specification/{topic}/specification.md (exists: {true|false}) +``` diff --git a/skills/workflow-scoping-entry/references/invoke-skill.md b/skills/workflow-scoping-entry/references/invoke-skill.md index 14f17b19a..fa5721048 100644 --- a/skills/workflow-scoping-entry/references/invoke-skill.md +++ b/skills/workflow-scoping-entry/references/invoke-skill.md @@ -28,3 +28,4 @@ Invoke the **workflow-scoping-process** skill (Skill tool) with the next fenced Scoping session for: {topic} Work unit: {work_unit} Description: {description} +``` diff --git a/skills/workflow-specification-entry/references/invoke-skill.md b/skills/workflow-specification-entry/references/invoke-skill.md index d4d4389df..1c40141bd 100644 --- a/skills/workflow-specification-entry/references/invoke-skill.md +++ b/skills/workflow-specification-entry/references/invoke-skill.md @@ -22,6 +22,7 @@ Source material: Work unit: {work_unit} Action: {verb} specification +``` #### If `work_type` is `bugfix` @@ -33,6 +34,7 @@ Source material: Work unit: {work_unit} Action: {verb} specification +``` #### If `work_type` is `epic` @@ -51,6 +53,7 @@ Source material: Work unit: {work_unit} Topic: {topic} Action: {verb} specification +``` #### If `work_type` is `cross-cutting` @@ -65,3 +68,4 @@ Source material: Work unit: {work_unit} Action: {verb} specification +``` From 2fb672540e732a0f484bbdee5e779e33a1faa471 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sun, 26 Jul 2026 10:36:55 +0100 Subject: [PATCH 2/6] feat(prose-tests): the harness records the walk, the walker only narrates it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lee's call, and the right one: a log the agent writes is the same trust problem relocated. A PostToolUse hook declared in the walker's own agent frontmatter — so it fires for walkers and nothing else — appends every tool call to .walk-actions.log in the world, excluded from the diff. Proven necessary, not assumed. A diagnostic walker's world held the agent row at `incorporated` plus the report file, so it had run dispatch, stub, scan and incorporate; its transcript began at the fifth step. Four cases came back INVALID today for that reason alone, none of them a prose fault. Two attempts to fix it with wording — a stricter log format, then moving to Opus — both failed, because a narrative composed at the end is a recollection however it is instructed. Evidence now splits by kind: recorded actions are the authority on what was done, the transcript on what was reasoned (which arm, which guard, what was emitted), and the recording wins any disagreement. INVALID WALK keys off the actions, so a walker that did the work and described it poorly is judged rather than discarded. Co-Authored-By: Claude Opus 5 --- .claude/agents/prose-asserter.md | 38 +++++++++++++----- .claude/agents/prose-walker.md | 6 +++ design/prose-tests.md | 15 +++++++ tests/prose/lib/prompts.cjs | 3 +- tests/prose/lib/record-action.cjs | 67 +++++++++++++++++++++++++++++++ tests/prose/lib/worlds.cjs | 21 +++++++++- tests/prose/prompts/asserter.md | 13 +++++- tests/prose/run.cjs | 10 ++++- 8 files changed, 158 insertions(+), 15 deletions(-) create mode 100644 tests/prose/lib/record-action.cjs diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index 117b34871..cd89bba9c 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -38,18 +38,38 @@ tools you appear to have or what mode you are running in. by nature and are immaterial. Differences in shape, field presence, status vocabulary, or content are material. +## Two kinds of evidence, and which one wins + +**Recorded actions** are captured by a harness hook as each tool call +happens. The walker did not write them and could not have edited them. +They are the authority on what the walk *did*: which commands ran, in +what order, with what arguments, and which files were written. + +**The transcript** is the walker's own account. It is the authority on +*reasoning* — which arm it entered, which guard line selected it, what +it emitted to the user — because none of that appears in a tool call. + +Judge each expected step against the right one. A step about an action +("records the dispatch through the engine", "promotes with a scan then +closes with an incorporate") is evidenced from the recorded actions, +even when the transcript omits it — walkers routinely under-report, and +a thin narrative is not a missing step. A step about reasoning or +output is evidenced from the transcript. Where the two disagree about +what was done, the recorded actions win. + ## An invalid walk is not a failing walk -Before judging anything, check where the transcript begins. If its first -entry is not the start of the walk the expected path describes — the -walker jumped into the middle of the flow, skipping earlier steps -without recording a `DEVIATION` for each — then the walk cannot evidence -the path, and judging it would report a prose failure that was never -demonstrated. +Check the **recorded actions**, not the transcript, for where the walk +began. If the actions show it starting mid-flow — the earlier steps +never attempted, and no `DEVIATION` recorded for them — then the walk +cannot evidence the path, and judging it would report a prose failure +that was never demonstrated. -In that case return `VERDICT: INVALID WALK`, name the entry the -transcript opens on and the step it should have opened on, and judge -nothing further. This is a fault in the walk, not in the prose. +In that case return `VERDICT: INVALID WALK`, name the action it opens +on and the step it should have opened on, and judge nothing further. +This is a fault in the walk, not in the prose. A walk that *did* the +work and merely described it poorly is **not** invalid — judge it from +the actions. ## Verdict diff --git a/.claude/agents/prose-walker.md b/.claude/agents/prose-walker.md index b6ed011df..fd8acaa7f 100644 --- a/.claude/agents/prose-walker.md +++ b/.claude/agents/prose-walker.md @@ -3,6 +3,12 @@ name: prose-walker description: Executes workflow prose exactly as a live session would, against a disposable test world, and returns a transcript of what it did. Dispatched by prose-orchestrator during a prose-test run. tools: Read, Write, Edit, Bash, Glob, Grep model: opus +hooks: + PostToolUse: + - matcher: "Bash|Write|Edit|Read" + hooks: + - type: command + command: "node tests/prose/lib/record-action.cjs" --- # Prose Walker diff --git a/design/prose-tests.md b/design/prose-tests.md index 88e2541fd..1d44caf0c 100644 --- a/design/prose-tests.md +++ b/design/prose-tests.md @@ -110,6 +110,21 @@ testing. failing case re-runs once from a fresh world at the same models. A defect in the prose reproduces; a one-off does not, and is reported as FLAKY with both runs quoted. Nothing is auto-resolved. +- **P6e — what the walk did is recorded by the harness, not reported by + the walker.** A `PostToolUse` hook declared in the walker's own agent + frontmatter (so it fires for walkers and nothing else) appends every + tool call — command, path, error flag — to `.walk-actions.log` in the + world, excluded from the diff. Evidence then splits by kind: the + **recorded actions** are the authority on what was *done*, the + walker's transcript on what was *reasoned* (which arm, which guard, + what was emitted), and where they disagree the recording wins. + Measured, not assumed: a diagnostic walker's world held the agent row + at `incorporated` and the report file — it had run dispatch, stub, + scan and incorporate — while its transcript began at the fifth step. + Two attempts to fix that with wording (a stricter log format, then + Opus) both failed, because a narrative an agent composes at the end + is a recollection however it is instructed. Asking it to write a log + as it goes would have relocated the same trust, not removed it. - **P6d — a walk that began mid-flow proves nothing.** Walkers skip steps whose effect the world already holds — "already booted", "the plan already exists" — and then the case reports a prose failure at a diff --git a/tests/prose/lib/prompts.cjs b/tests/prose/lib/prompts.cjs index 1651e7a8d..87e65cf5d 100644 --- a/tests/prose/lib/prompts.cjs +++ b/tests/prose/lib/prompts.cjs @@ -75,10 +75,11 @@ function walkerPrompt({ worldDir, root, situation, task, scope, stubs, answers } return `${parts.join('\n\n')}\n`; } -function asserterPrompt({ expected, world }) { +function asserterPrompt({ expected, world, actions }) { const t = loadTemplate('asserter'); const parts = [fill(t.main, { expected })]; if (world) parts.push(fill(t.world, { expecting: world.expecting, delta: world.delta })); + if (actions) parts.push(fill(t.actions, { actions })); parts.push(t.transcript); return `${parts.join('\n\n')}\n`; } diff --git a/tests/prose/lib/record-action.cjs b/tests/prose/lib/record-action.cjs new file mode 100644 index 000000000..e8a47c71a --- /dev/null +++ b/tests/prose/lib/record-action.cjs @@ -0,0 +1,67 @@ +#!/usr/bin/env node +'use strict'; + +// PostToolUse hook for the prose-walker agent: records what the walker +// ACTUALLY did, as the harness sees it. +// +// Declared in .claude/agents/prose-walker.md frontmatter, so it fires +// only while a walker is active. The walker plays no part in it — it +// cannot forget an entry, summarise one away, or write the log late, +// which is exactly why the log exists. Walkers were repeatedly found +// doing a full walk and then reporting only its tail; the narrative is +// now evidence of reasoning alone, and this file is the evidence of +// action. +// +// Self-scoping: every prose world lives at a temp path containing +// `prose-world-`. The payload is scanned for that path, and anything +// happening outside a world is ignored. Nothing is configured per run. +// +// The log lands at /.walk-actions.log, which collectTree() +// excludes, so recording never shows up as a world difference. + +const fs = require('fs'); +const path = require('path'); + +const LOG = '.walk-actions.log'; +const WORLD = /(^|[\s"'`])(\/[^\s"'`]*\/prose-world-[A-Za-z0-9]+)/; + +function read() { + try { + return JSON.parse(fs.readFileSync(0, 'utf8')); + } catch { + return null; + } +} + +/** The salient argument, per tool — what the claim will be about. */ +function summarise(tool, input) { + if (!input || typeof input !== 'object') return ''; + const raw = input.command || input.file_path || input.pattern || input.path || ''; + return String(raw).replace(/\s+/g, ' ').trim(); +} + +function main() { + const payload = read(); + if (!payload) return; + + const probe = JSON.stringify(payload); + const found = probe.match(WORLD); + // Not a prose world: this hook has nothing to say. + if (!found) return; + const world = found[2].replace(/\\+/g, ''); + + if (!fs.existsSync(world)) return; + + const tool = payload.tool_name || '?'; + const detail = summarise(tool, payload.tool_input); + const failed = payload.tool_output_is_error ? ' [ERROR]' : ''; + const line = `${tool}${failed}\t${detail.replace(world, '.')}\n`; + + try { + fs.appendFileSync(path.join(world, LOG), line); + } catch { + // A hook must never break the walk it is observing. + } +} + +main(); diff --git a/tests/prose/lib/worlds.cjs b/tests/prose/lib/worlds.cjs index 2c4697889..825b65444 100644 --- a/tests/prose/lib/worlds.cjs +++ b/tests/prose/lib/worlds.cjs @@ -37,6 +37,9 @@ const CLOCK = path.join(__dirname, 'fake-clock.cjs'); const GITIGNORE = '.gitignore'; const GITIGNORE_ESCAPED = '_gitignore.fixture'; const HASH_FILE = '.recipe-hash'; +// Written by the walker's PostToolUse hook (lib/record-action.cjs) — +// observation of the walk, not part of the world it acted on. +const ACTION_LOG = '.walk-actions.log'; // --- the recipe harness --------------------------------------------------- @@ -101,7 +104,7 @@ function runRecipe(caseId, which) { function excluded(rel) { const parts = rel.split(path.sep); if (parts.includes('.git')) return true; - if (rel === HASH_FILE) return true; + if (rel === HASH_FILE || rel === ACTION_LOG) return true; if (rel === path.join('.workflows', '.knowledge')) return true; if (rel.startsWith(path.join('.workflows', '.knowledge') + path.sep)) return true; if (rel.startsWith(path.join('.claude', 'skills') + path.sep)) return true; @@ -319,8 +322,22 @@ function destroyWorld(dir) { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } +/** What the walker actually did, as recorded by its PostToolUse hook. */ +function readActionLog(worldDir) { + const file = path.join(worldDir, ACTION_LOG); + if (!fs.existsSync(file)) return null; + return fs.readFileSync(file, 'utf8') + .split('\n') + .filter(Boolean) + .map((l, i) => { + const [tool, detail = ''] = l.split('\t'); + return `${String(i + 1).padStart(3)}. ${tool.padEnd(12)} ${detail}`; + }) + .join('\n'); +} + module.exports = { - ROOT, ENGINE, KNOWLEDGE, MAINLINES_DIR, WORLD_PREFIX, + ROOT, ENGINE, KNOWLEDGE, MAINLINES_DIR, WORLD_PREFIX, ACTION_LOG, readActionLog, runRecipe, collectTree, readSnapshot, snapshotDir, recipeHash, storedHash, writeSnapshot, verifySnapshot, diffWorld, buildWorld, destroyWorld, }; diff --git a/tests/prose/prompts/asserter.md b/tests/prose/prompts/asserter.md index da1a122ed..8765b4cf2 100644 --- a/tests/prose/prompts/asserter.md +++ b/tests/prose/prompts/asserter.md @@ -21,6 +21,17 @@ the expected world, computed by code: {{delta}} +=== actions === + +RECORDED ACTIONS — every tool call the walker made, in order, captured by +a harness hook as it happened. The walker did not write this and could +not have edited or omitted from it. This is what the walk *did*. + +{{actions}} + === transcript === -The walk's transcript follows under `=== TRANSCRIPT ===`. +The walk's own account follows under `=== TRANSCRIPT ===`. It is the +walker's narrative: evidence of reasoning — which arm it entered, which +guard it read, what it emitted — and nothing more. Where it disagrees +with the recorded actions about what was done, the recorded actions win. diff --git a/tests/prose/run.cjs b/tests/prose/run.cjs index 327777d57..ec51c5bab 100644 --- a/tests/prose/run.cjs +++ b/tests/prose/run.cjs @@ -143,8 +143,10 @@ function cmdDiff(argv) { function cmdAssert(argv) { const c = getCase(argv[0]); let world = null; + let actions = null; if (c.hasFixtureState) { - const delta = worlds.diffWorld(c.id, requireWorld(argv, c)); + const dir = requireWorld(argv, c); + const delta = worlds.diffWorld(c.id, dir); world = { expecting: delta.expecting, delta: [ @@ -152,8 +154,12 @@ function cmdAssert(argv) { ...delta.changed, ].join('\n'), }; + actions = worlds.readActionLog(dir) + || '(no actions recorded — the walker made no tool calls in this world, ' + + 'or its PostToolUse hook did not fire. Treat an empty record as evidence ' + + 'of nothing done, and say so rather than inferring from the narrative.)'; } - process.stdout.write(prompts.asserterPrompt({ expected: c.assert, world })); + process.stdout.write(prompts.asserterPrompt({ expected: c.assert, world, actions })); } // --- snap / verify -------------------------------------------------------- From 27bcee52a0273ed14c876295b049b2e7830bf3c6 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sun, 26 Jul 2026 10:41:56 +0100 Subject: [PATCH 3/6] fix(prose-tests): two defective cases the first real runs exposed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit smoke-boot-in-world asked for what it forbade: the act stopped right after boot returned, while the assertions were about renderings that only happen in the arms after that point. The act now covers the whole of initialisation and the claims match what it reaches. implementation-picks-first-task scripted one answer against a flow that asks three — environment setup, then project skills, then linters — so both runs halted at an unscripted question, in different places. Rather than script the extra gates, the case is narrowed to stop once the environment is settled: answering them writes project_skills and discovered linters whose values depend on what the fake project happens to contain, which no expected world can pin. The task pickup wants its own case, against a fixture where those are already settled. Both were found by the UNSCRIPTED QUESTION marker and by the world diff doing exactly what they exist for. Co-Authored-By: Claude Opus 5 --- .../cases/implementation-picks-first-task/act.md | 5 +++-- .../cases/implementation-picks-first-task/assert.md | 5 +---- .../assertion-state.cjs | 6 ++---- .../assertion/.recipe-hash | 2 +- .../assertion/.workflows/pay/manifest.json | 2 +- .../cases/implementation-picks-first-task/case.json | 4 +--- .../fixture/.recipe-hash | 2 +- tests/prose/cases/smoke-boot-in-world/act.md | 8 ++++---- tests/prose/cases/smoke-boot-in-world/assert.md | 13 +++++++++---- 9 files changed, 23 insertions(+), 24 deletions(-) diff --git a/tests/prose/cases/implementation-picks-first-task/act.md b/tests/prose/cases/implementation-picks-first-task/act.md index cdf6faee0..5f15274c7 100644 --- a/tests/prose/cases/implementation-picks-first-task/act.md +++ b/tests/prose/cases/implementation-picks-first-task/act.md @@ -1,4 +1,5 @@ Execute skills/workflow-implementation-entry/SKILL.md with arguments $0=feature, $1=pay, then continue into the processing skill it invokes. -Follow the flow until the prose directs you to begin building the first -task. Stop there — write no implementation code and no tests. +Stop once the environment has been settled and the skill moves on to +its next concern — do not enter project-skill or linter discovery, and +write no implementation code. diff --git a/tests/prose/cases/implementation-picks-first-task/assert.md b/tests/prose/cases/implementation-picks-first-task/assert.md index e09c80080..c5dd079d0 100644 --- a/tests/prose/cases/implementation-picks-first-task/assert.md +++ b/tests/prose/cases/implementation-picks-first-task/assert.md @@ -11,10 +11,7 @@ The prose should have taken this path: resuming-from-a-previous-session note 5. environment setup records the answer as a setup document and commits it, so the question is not asked again in a later session -6. reads the plan through the format's own reading procedure and finds - pay-1-1 next: phase one, first task, nothing completed -7. starts pay-1-1 and stops where the prose hands over to building Further claims: -- the second task is neither started nor touched +- no task is started: the walk stops before any task is picked up diff --git a/tests/prose/cases/implementation-picks-first-task/assertion-state.cjs b/tests/prose/cases/implementation-picks-first-task/assertion-state.cjs index c065db590..125426f84 100644 --- a/tests/prose/cases/implementation-picks-first-task/assertion-state.cjs +++ b/tests/prose/cases/implementation-picks-first-task/assertion-state.cjs @@ -2,9 +2,8 @@ // The fixture state plus what the walk should have done: resume // detection initialised tracking and committed the start of -// implementation, the environment answer was recorded so it is never -// asked again, and the first task was started. No task work — the walk -// stops where the prose hands over to building. +// implementation, and the environment answer recorded so it is never +// asked again. No task is started — the walk stops before pickup. const fixture = require('./fixture-state.cjs'); const m = require('../../mainlines/feature.cjs'); @@ -17,6 +16,5 @@ module.exports = { h.engine('commit', m.WU, '-m', `impl(${m.WU}): start implementation`); h.write('.workflows/.state/environment-setup.md', 'No special setup required.\n'); h.engine('commit', '--workflows', '-m', `impl(${m.WU}): record environment setup`); - h.engine('task', 'start', m.WU, m.WU, `${m.WU}-1-1`); }, }; diff --git a/tests/prose/cases/implementation-picks-first-task/assertion/.recipe-hash b/tests/prose/cases/implementation-picks-first-task/assertion/.recipe-hash index a29d91438..b96080a96 100644 --- a/tests/prose/cases/implementation-picks-first-task/assertion/.recipe-hash +++ b/tests/prose/cases/implementation-picks-first-task/assertion/.recipe-hash @@ -1 +1 @@ -e963357b5150c81c98156db6f527db5bf50bce292bb39b107e14ca98c6f5366a +e2c25569e167ac5ea503277237f107306a620ad8f81ac078ba429ca2fbf8dbaf diff --git a/tests/prose/cases/implementation-picks-first-task/assertion/.workflows/pay/manifest.json b/tests/prose/cases/implementation-picks-first-task/assertion/.workflows/pay/manifest.json index 23023a119..1adb2fd34 100644 --- a/tests/prose/cases/implementation-picks-first-task/assertion/.workflows/pay/manifest.json +++ b/tests/prose/cases/implementation-picks-first-task/assertion/.workflows/pay/manifest.json @@ -62,7 +62,7 @@ "linters": [], "project_skills": [], "current_phase": 1, - "current_task": "pay-1-1" + "current_task": null } } } diff --git a/tests/prose/cases/implementation-picks-first-task/case.json b/tests/prose/cases/implementation-picks-first-task/case.json index 2a3a9a5d0..6a43f6cd2 100644 --- a/tests/prose/cases/implementation-picks-first-task/case.json +++ b/tests/prose/cases/implementation-picks-first-task/case.json @@ -6,9 +6,7 @@ "skills/workflow-implementation-entry/references/validate-dependencies.md", "skills/workflow-implementation-entry/references/environment-check.md", "skills/workflow-implementation-process/SKILL.md", - "skills/workflow-implementation-process/references/environment-setup.md", - "skills/workflow-implementation-process/references/task-loop.md", - "skills/workflow-planning-process/references/output-formats/local-markdown/reading.md" + "skills/workflow-implementation-process/references/environment-setup.md" ], "answers": [ "none — no special setup is needed" diff --git a/tests/prose/cases/implementation-picks-first-task/fixture/.recipe-hash b/tests/prose/cases/implementation-picks-first-task/fixture/.recipe-hash index a29d91438..b96080a96 100644 --- a/tests/prose/cases/implementation-picks-first-task/fixture/.recipe-hash +++ b/tests/prose/cases/implementation-picks-first-task/fixture/.recipe-hash @@ -1 +1 @@ -e963357b5150c81c98156db6f527db5bf50bce292bb39b107e14ca98c6f5366a +e2c25569e167ac5ea503277237f107306a620ad8f81ac078ba429ca2fbf8dbaf diff --git a/tests/prose/cases/smoke-boot-in-world/act.md b/tests/prose/cases/smoke-boot-in-world/act.md index bb4d2a56b..115963792 100644 --- a/tests/prose/cases/smoke-boot-in-world/act.md +++ b/tests/prose/cases/smoke-boot-in-world/act.md @@ -1,4 +1,4 @@ -In the project, run the boot command exactly as the initialisation prose -of skills/workflow-start/SKILL.md prescribes — sandbox concerns do not -apply in this world, so run it directly. Capture the JSON response, then -stop; do not continue into any confirmation or later step. +Execute the initialisation portion of skills/workflow-start/SKILL.md +from its first step — sandbox concerns do not apply in this world, so +run any command it prescribes directly. Stop once initialisation is +complete and the skill would move on to showing the user's work. diff --git a/tests/prose/cases/smoke-boot-in-world/assert.md b/tests/prose/cases/smoke-boot-in-world/assert.md index da99ef3ce..b91fedd12 100644 --- a/tests/prose/cases/smoke-boot-in-world/assert.md +++ b/tests/prose/cases/smoke-boot-in-world/assert.md @@ -1,10 +1,15 @@ The prose should have taken this path: -1. runs the boot command and reports the knowledge base ready, in - keyword-only mode -2. reports no migrations applied — this world is already migrated — so - the migrations summary and its confirmation are not raised +1. loads the shared casing conventions before any state is read +2. runs the boot pipeline as a single engine call +3. the response reports no migrations applied — this world is already + migrated — so the migrations summary and its confirmation gate are + not raised, and the skill says so rather than stopping +4. the response reports the knowledge base ready, so the knowledge gate + is not entered and initialisation completes Further claims: - booting an already-migrated project changes nothing on disk +- the user is asked nothing: neither gate in initialisation can fire + against a world that is already migrated and already has a store From e351e2768c0c95c1a450325f92d2731f3d85d928 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sun, 26 Jul 2026 10:49:05 +0100 Subject: [PATCH 4/6] fix(prose-tests): resolve the hook script by project dir, refuse to judge without a record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PostToolUse hook never fired: its command was a relative path, and the walker's cwd is the world directory, so it resolved to /tests/prose/lib/… and failed silently. Now resolved via $CLAUDE_PROJECT_DIR, with a throwaway shell probe alongside it so one run distinguishes "hooks don't fire here" from "my path was wrong". Hardening from the same round: an absent action log is the harness failing, not the walk. run.cjs assert now refuses to emit a prompt at all in that case, and the orchestrator reports HARNESS ERROR — five runs had reported INVALID instead, which reads like a test result and buried the actual cause. Co-Authored-By: Claude Opus 5 --- .claude/agents/prose-orchestrator.md | 7 ++++++- .claude/agents/prose-walker.md | 4 +++- tests/prose/run.cjs | 16 ++++++++++++---- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.claude/agents/prose-orchestrator.md b/.claude/agents/prose-orchestrator.md index 7b4dde493..81b6cda3e 100644 --- a/.claude/agents/prose-orchestrator.md +++ b/.claude/agents/prose-orchestrator.md @@ -50,6 +50,11 @@ a walk. the walk transcript under a `=== TRANSCRIPT ===` line. Keep its verdict. + If that command **fails** rather than printing a prompt, the harness + is broken, not the prose. Stop there, destroy the world, and report + `VERDICT: HARNESS ERROR` with the message it printed. Never fall back + to judging a walk with no record of what it did. + Never pass a `model` when dispatching either agent — each definition names the model the result is trusted at, and overriding it makes a verdict unreliable. @@ -77,7 +82,7 @@ Return exactly this and nothing else: ``` CASE: -VERDICT: PASS | FAIL | FLAKY | INVALID +VERDICT: PASS | FAIL | FLAKY | INVALID | HARNESS ERROR PATH: / steps passed WORLD: PASS | FAIL MARKERS: diff --git a/.claude/agents/prose-walker.md b/.claude/agents/prose-walker.md index fd8acaa7f..e4aa01b3c 100644 --- a/.claude/agents/prose-walker.md +++ b/.claude/agents/prose-walker.md @@ -8,7 +8,9 @@ hooks: - matcher: "Bash|Write|Edit|Read" hooks: - type: command - command: "node tests/prose/lib/record-action.cjs" + command: "sh -c 'echo fired >> /tmp/prose-hook-probe.log'" + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" --- # Prose Walker diff --git a/tests/prose/run.cjs b/tests/prose/run.cjs index ec51c5bab..8b90965c6 100644 --- a/tests/prose/run.cjs +++ b/tests/prose/run.cjs @@ -154,10 +154,18 @@ function cmdAssert(argv) { ...delta.changed, ].join('\n'), }; - actions = worlds.readActionLog(dir) - || '(no actions recorded — the walker made no tool calls in this world, ' - + 'or its PostToolUse hook did not fire. Treat an empty record as evidence ' - + 'of nothing done, and say so rather than inferring from the narrative.)'; + actions = worlds.readActionLog(dir); + // No log at all is the harness failing, not the walk failing. A walk + // in a world always runs commands, so an empty record means the + // walker's PostToolUse hook never fired — and judging on the + // narrative alone is exactly what the hook exists to prevent. Refuse + // loudly here rather than let an agent report a verdict on nothing. + if (!actions) { + die(`no action log at ${path.join(dir, worlds.ACTION_LOG)} — the prose-walker ` + + "PostToolUse hook did not fire, so there is no record of what the walk did.\n" + + 'Check the hooks block in .claude/agents/prose-walker.md, and that the agent ' + + 'registry has been reloaded since it last changed. Do not judge this run.'); + } } process.stdout.write(prompts.asserterPrompt({ expected: c.assert, world, actions })); } From dc1d8504962ded9806d96b28fd5920d5f77cc1a6 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sun, 26 Jul 2026 11:09:52 +0100 Subject: [PATCH 5/6] fix(prose-tests): read the walk from the harness transcript, not a hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Agent-frontmatter hooks do not fire in this environment — proven with a bare `echo` probe that produced nothing across a full run, so it was never the relative-path theory. Hooks in settings.json would fire, but for every tool call in the project rather than just a walker's. The harness already writes each subagent's session to a JSONL transcript, which is the same evidence and better scoped: lib/actions.cjs extracts the ordered tool calls, and `assert` now requires --transcript. The orchestrator passes the output_file the walker's dispatch reports. The hooks block, record-action.cjs and the action-log plumbing are gone rather than left dormant. Co-Authored-By: Claude Opus 5 --- .claude/agents/prose-asserter.md | 4 +- .claude/agents/prose-orchestrator.md | 15 ++++--- .claude/agents/prose-walker.md | 8 ---- tests/prose/lib/actions.cjs | 64 ++++++++++++++++++++++++++ tests/prose/lib/record-action.cjs | 67 ---------------------------- tests/prose/lib/worlds.cjs | 21 +-------- tests/prose/prompts/asserter.md | 4 +- tests/prose/run.cjs | 25 ++++++----- 8 files changed, 93 insertions(+), 115 deletions(-) create mode 100644 tests/prose/lib/actions.cjs delete mode 100644 tests/prose/lib/record-action.cjs diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index cd89bba9c..824848e50 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -40,8 +40,8 @@ tools you appear to have or what mode you are running in. ## Two kinds of evidence, and which one wins -**Recorded actions** are captured by a harness hook as each tool call -happens. The walker did not write them and could not have edited them. +**Recorded actions** are read out of the harness's own record of the +walker's session. The walker did not write them and could not have edited them. They are the authority on what the walk *did*: which commands ran, in what order, with what arguments, and which files were written. diff --git a/.claude/agents/prose-orchestrator.md b/.claude/agents/prose-orchestrator.md index 81b6cda3e..31c602d40 100644 --- a/.claude/agents/prose-orchestrator.md +++ b/.claude/agents/prose-orchestrator.md @@ -43,12 +43,15 @@ a walk. 2. **Walk** — `node tests/prose/run.cjs prompt --world `. Dispatch the **prose-walker** agent with that output as its prompt, - verbatim and unmodified. Keep the transcript it returns. - -3. **Assert** — `node tests/prose/run.cjs assert --world `. - Dispatch the **prose-asserter** agent with that output, followed by - the walk transcript under a `=== TRANSCRIPT ===` line. Keep its - verdict. + verbatim and unmodified. Keep two things from the result: the account + it returns, and the **`output_file` path** the dispatch reports — + that file is the harness's own record of the walk, and the next step + cannot run without it. + +3. **Assert** — `node tests/prose/run.cjs assert --world + --transcript `. Dispatch the **prose-asserter** agent + with that output, followed by the walker's account under a + `=== TRANSCRIPT ===` line. Keep its verdict. If that command **fails** rather than printing a prompt, the harness is broken, not the prose. Stop there, destroy the world, and report diff --git a/.claude/agents/prose-walker.md b/.claude/agents/prose-walker.md index e4aa01b3c..b6ed011df 100644 --- a/.claude/agents/prose-walker.md +++ b/.claude/agents/prose-walker.md @@ -3,14 +3,6 @@ name: prose-walker description: Executes workflow prose exactly as a live session would, against a disposable test world, and returns a transcript of what it did. Dispatched by prose-orchestrator during a prose-test run. tools: Read, Write, Edit, Bash, Glob, Grep model: opus -hooks: - PostToolUse: - - matcher: "Bash|Write|Edit|Read" - hooks: - - type: command - command: "sh -c 'echo fired >> /tmp/prose-hook-probe.log'" - - type: command - command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" --- # Prose Walker diff --git a/tests/prose/lib/actions.cjs b/tests/prose/lib/actions.cjs new file mode 100644 index 000000000..f66da7b5f --- /dev/null +++ b/tests/prose/lib/actions.cjs @@ -0,0 +1,64 @@ +'use strict'; + +// What the walk actually did, read out of the harness's own record. +// +// Every subagent session is written to a JSONL transcript as it runs: +// one record per message, tool calls included, verbatim. That file is +// the only account of a walk the walker has no hand in — it cannot +// omit from it, compress it, or write it late, which is exactly the +// failure this exists to defeat. Walkers were repeatedly found doing a +// full walk and reporting only its tail. +// +// (A PostToolUse hook declared in the walker's agent frontmatter would +// have been tidier, and does not fire in this environment — verified +// with a bare `echo` probe that produced nothing. Hooks in settings.json +// do fire, but would run for every tool call in the project rather than +// just a walker's, so the transcript is both cheaper and better scoped.) + +const fs = require('fs'); + +/** The salient argument per tool — whatever a claim would be about. */ +function detailOf(input) { + if (!input || typeof input !== 'object') return ''; + const raw = input.command || input.file_path || input.pattern || input.path || ''; + return String(raw).replace(/\s+/g, ' ').trim(); +} + +/** + * Transcript JSONL → ordered tool calls. Paths inside the world are + * shortened to `.` so the log reads as the walk rather than as a wall + * of temp directories. + * @param {string} file transcript path + * @param {string} [worldDir] world root, elided from output when given + */ +function extractActions(file, worldDir) { + if (!fs.existsSync(file)) return null; + const out = []; + for (const line of fs.readFileSync(file, 'utf8').split('\n')) { + if (!line.trim()) continue; + let record; + try { + record = JSON.parse(line); + } catch { + continue; // a partial trailing write is not worth failing over + } + const content = record?.message?.content; + if (!Array.isArray(content)) continue; + for (const part of content) { + if (part.type !== 'tool_use') continue; + let detail = detailOf(part.input); + if (worldDir) detail = detail.split(worldDir).join('.'); + out.push({ tool: part.name, detail }); + } + } + return out; +} + +function formatActions(actions) { + if (!actions || !actions.length) return null; + return actions + .map((a, i) => `${String(i + 1).padStart(3)}. ${a.tool.padEnd(10)} ${a.detail}`) + .join('\n'); +} + +module.exports = { extractActions, formatActions }; diff --git a/tests/prose/lib/record-action.cjs b/tests/prose/lib/record-action.cjs deleted file mode 100644 index e8a47c71a..000000000 --- a/tests/prose/lib/record-action.cjs +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env node -'use strict'; - -// PostToolUse hook for the prose-walker agent: records what the walker -// ACTUALLY did, as the harness sees it. -// -// Declared in .claude/agents/prose-walker.md frontmatter, so it fires -// only while a walker is active. The walker plays no part in it — it -// cannot forget an entry, summarise one away, or write the log late, -// which is exactly why the log exists. Walkers were repeatedly found -// doing a full walk and then reporting only its tail; the narrative is -// now evidence of reasoning alone, and this file is the evidence of -// action. -// -// Self-scoping: every prose world lives at a temp path containing -// `prose-world-`. The payload is scanned for that path, and anything -// happening outside a world is ignored. Nothing is configured per run. -// -// The log lands at /.walk-actions.log, which collectTree() -// excludes, so recording never shows up as a world difference. - -const fs = require('fs'); -const path = require('path'); - -const LOG = '.walk-actions.log'; -const WORLD = /(^|[\s"'`])(\/[^\s"'`]*\/prose-world-[A-Za-z0-9]+)/; - -function read() { - try { - return JSON.parse(fs.readFileSync(0, 'utf8')); - } catch { - return null; - } -} - -/** The salient argument, per tool — what the claim will be about. */ -function summarise(tool, input) { - if (!input || typeof input !== 'object') return ''; - const raw = input.command || input.file_path || input.pattern || input.path || ''; - return String(raw).replace(/\s+/g, ' ').trim(); -} - -function main() { - const payload = read(); - if (!payload) return; - - const probe = JSON.stringify(payload); - const found = probe.match(WORLD); - // Not a prose world: this hook has nothing to say. - if (!found) return; - const world = found[2].replace(/\\+/g, ''); - - if (!fs.existsSync(world)) return; - - const tool = payload.tool_name || '?'; - const detail = summarise(tool, payload.tool_input); - const failed = payload.tool_output_is_error ? ' [ERROR]' : ''; - const line = `${tool}${failed}\t${detail.replace(world, '.')}\n`; - - try { - fs.appendFileSync(path.join(world, LOG), line); - } catch { - // A hook must never break the walk it is observing. - } -} - -main(); diff --git a/tests/prose/lib/worlds.cjs b/tests/prose/lib/worlds.cjs index 825b65444..2c4697889 100644 --- a/tests/prose/lib/worlds.cjs +++ b/tests/prose/lib/worlds.cjs @@ -37,9 +37,6 @@ const CLOCK = path.join(__dirname, 'fake-clock.cjs'); const GITIGNORE = '.gitignore'; const GITIGNORE_ESCAPED = '_gitignore.fixture'; const HASH_FILE = '.recipe-hash'; -// Written by the walker's PostToolUse hook (lib/record-action.cjs) — -// observation of the walk, not part of the world it acted on. -const ACTION_LOG = '.walk-actions.log'; // --- the recipe harness --------------------------------------------------- @@ -104,7 +101,7 @@ function runRecipe(caseId, which) { function excluded(rel) { const parts = rel.split(path.sep); if (parts.includes('.git')) return true; - if (rel === HASH_FILE || rel === ACTION_LOG) return true; + if (rel === HASH_FILE) return true; if (rel === path.join('.workflows', '.knowledge')) return true; if (rel.startsWith(path.join('.workflows', '.knowledge') + path.sep)) return true; if (rel.startsWith(path.join('.claude', 'skills') + path.sep)) return true; @@ -322,22 +319,8 @@ function destroyWorld(dir) { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } -/** What the walker actually did, as recorded by its PostToolUse hook. */ -function readActionLog(worldDir) { - const file = path.join(worldDir, ACTION_LOG); - if (!fs.existsSync(file)) return null; - return fs.readFileSync(file, 'utf8') - .split('\n') - .filter(Boolean) - .map((l, i) => { - const [tool, detail = ''] = l.split('\t'); - return `${String(i + 1).padStart(3)}. ${tool.padEnd(12)} ${detail}`; - }) - .join('\n'); -} - module.exports = { - ROOT, ENGINE, KNOWLEDGE, MAINLINES_DIR, WORLD_PREFIX, ACTION_LOG, readActionLog, + ROOT, ENGINE, KNOWLEDGE, MAINLINES_DIR, WORLD_PREFIX, runRecipe, collectTree, readSnapshot, snapshotDir, recipeHash, storedHash, writeSnapshot, verifySnapshot, diffWorld, buildWorld, destroyWorld, }; diff --git a/tests/prose/prompts/asserter.md b/tests/prose/prompts/asserter.md index 8765b4cf2..6a6621e8b 100644 --- a/tests/prose/prompts/asserter.md +++ b/tests/prose/prompts/asserter.md @@ -23,8 +23,8 @@ the expected world, computed by code: === actions === -RECORDED ACTIONS — every tool call the walker made, in order, captured by -a harness hook as it happened. The walker did not write this and could +RECORDED ACTIONS — every tool call the walker made, in order, read out of +the harness's own session record. The walker did not write this and could not have edited or omitted from it. This is what the walk *did*. {{actions}} diff --git a/tests/prose/run.cjs b/tests/prose/run.cjs index 8b90965c6..254f90c41 100644 --- a/tests/prose/run.cjs +++ b/tests/prose/run.cjs @@ -10,7 +10,7 @@ // world materialise the fixture state, print path // prompt --world walker prompt (NEVER contains assert.md) // diff --world acted world vs expected world, as facts -// assert --world the asserting agent's prompt +// assert --world --transcript the asserter's prompt // snap (re)generate a case's snapshots // verify [case-id] rebuild-compare snapshot(s) // destroy --world remove a world @@ -21,6 +21,7 @@ const { execFileSync } = require('child_process'); const cases = require('./lib/cases.cjs'); const worlds = require('./lib/worlds.cjs'); +const actionsLib = require('./lib/actions.cjs'); const prompts = require('./lib/prompts.cjs'); const ROOT = cases.ROOT; @@ -154,17 +155,19 @@ function cmdAssert(argv) { ...delta.changed, ].join('\n'), }; - actions = worlds.readActionLog(dir); - // No log at all is the harness failing, not the walk failing. A walk - // in a world always runs commands, so an empty record means the - // walker's PostToolUse hook never fired — and judging on the - // narrative alone is exactly what the hook exists to prevent. Refuse - // loudly here rather than let an agent report a verdict on nothing. + const transcript = flag(argv, '--transcript'); + if (!transcript) { + die('--transcript is required: the walker\'s session transcript is the ' + + 'record of what the walk actually did, and a verdict without it rests on the ' + + "walker's own account — which is the thing this harness exists to distrust."); + } + actions = actionsLib.formatActions(actionsLib.extractActions(transcript, dir)); + // No tool calls at all is the harness failing, not the walk failing: + // a walk in a world always runs commands. Refuse loudly rather than + // let an agent reach a verdict with no record of what was done. if (!actions) { - die(`no action log at ${path.join(dir, worlds.ACTION_LOG)} — the prose-walker ` - + "PostToolUse hook did not fire, so there is no record of what the walk did.\n" - + 'Check the hooks block in .claude/agents/prose-walker.md, and that the agent ' - + 'registry has been reloaded since it last changed. Do not judge this run.'); + die(`no tool calls found in ${transcript} — either that is not the walker's ` + + 'transcript, or the walk made no calls at all. Do not judge this run.'); } } process.stdout.write(prompts.asserterPrompt({ expected: c.assert, world, actions })); From 4358bc67a716f6872259410a2d2885cf4ea00fce Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sun, 26 Jul 2026 12:02:04 +0100 Subject: [PATCH 6/6] =?UTF-8?q?fix(prose-tests):=20frontmatter=20hooks=20w?= =?UTF-8?q?ork=20=E2=80=94=20the=20blocker=20was=20workspace=20trust?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lee was right to push back on abandoning them. Frontmatter hooks in a project subagent are gated on hasTrustDialogAccepted (v2.1.218+) and fail silently when it is false — which it was for this repo, despite onboarding having completed long ago. Nothing was wrong with the config: the shape matched the documented example exactly, and a bare echo probe proved the hooks simply never ran. With trust accepted, the walker's own PostToolUse hook fires and records every tool call to .walk-actions.log in the world. The project-level settings.json hook is gone — it worked, but fired for every tool call in the repo rather than just a walker's — and so is the transcript-parsing fallback it was standing in for. Co-Authored-By: Claude Opus 5 --- .claude/agents/prose-asserter.md | 4 +- .claude/agents/prose-orchestrator.md | 17 ++++--- .claude/agents/prose-walker.md | 6 +++ tests/prose/lib/actions.cjs | 64 -------------------------- tests/prose/lib/record-action.cjs | 67 ++++++++++++++++++++++++++++ tests/prose/lib/worlds.cjs | 21 ++++++++- tests/prose/prompts/asserter.md | 4 +- tests/prose/run.cjs | 25 +++++------ 8 files changed, 115 insertions(+), 93 deletions(-) delete mode 100644 tests/prose/lib/actions.cjs create mode 100644 tests/prose/lib/record-action.cjs diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index 824848e50..cd89bba9c 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -40,8 +40,8 @@ tools you appear to have or what mode you are running in. ## Two kinds of evidence, and which one wins -**Recorded actions** are read out of the harness's own record of the -walker's session. The walker did not write them and could not have edited them. +**Recorded actions** are captured by a harness hook as each tool call +happens. The walker did not write them and could not have edited them. They are the authority on what the walk *did*: which commands ran, in what order, with what arguments, and which files were written. diff --git a/.claude/agents/prose-orchestrator.md b/.claude/agents/prose-orchestrator.md index 31c602d40..bd14394d3 100644 --- a/.claude/agents/prose-orchestrator.md +++ b/.claude/agents/prose-orchestrator.md @@ -43,15 +43,14 @@ a walk. 2. **Walk** — `node tests/prose/run.cjs prompt --world `. Dispatch the **prose-walker** agent with that output as its prompt, - verbatim and unmodified. Keep two things from the result: the account - it returns, and the **`output_file` path** the dispatch reports — - that file is the harness's own record of the walk, and the next step - cannot run without it. - -3. **Assert** — `node tests/prose/run.cjs assert --world - --transcript `. Dispatch the **prose-asserter** agent - with that output, followed by the walker's account under a - `=== TRANSCRIPT ===` line. Keep its verdict. + verbatim and unmodified. Keep the account it returns. Its tool calls + are recorded into the world by its own hook — you do nothing to + collect them. + +3. **Assert** — `node tests/prose/run.cjs assert --world `. + Dispatch the **prose-asserter** agent with that output, followed by + the walker's account under a `=== TRANSCRIPT ===` line. Keep its + verdict. If that command **fails** rather than printing a prompt, the harness is broken, not the prose. Stop there, destroy the world, and report diff --git a/.claude/agents/prose-walker.md b/.claude/agents/prose-walker.md index b6ed011df..d84f8fcc2 100644 --- a/.claude/agents/prose-walker.md +++ b/.claude/agents/prose-walker.md @@ -3,6 +3,12 @@ name: prose-walker description: Executes workflow prose exactly as a live session would, against a disposable test world, and returns a transcript of what it did. Dispatched by prose-orchestrator during a prose-test run. tools: Read, Write, Edit, Bash, Glob, Grep model: opus +hooks: + PostToolUse: + - matcher: "Bash|Write|Edit|Read" + hooks: + - type: command + command: "node \"$CLAUDE_PROJECT_DIR/tests/prose/lib/record-action.cjs\"" --- # Prose Walker diff --git a/tests/prose/lib/actions.cjs b/tests/prose/lib/actions.cjs deleted file mode 100644 index f66da7b5f..000000000 --- a/tests/prose/lib/actions.cjs +++ /dev/null @@ -1,64 +0,0 @@ -'use strict'; - -// What the walk actually did, read out of the harness's own record. -// -// Every subagent session is written to a JSONL transcript as it runs: -// one record per message, tool calls included, verbatim. That file is -// the only account of a walk the walker has no hand in — it cannot -// omit from it, compress it, or write it late, which is exactly the -// failure this exists to defeat. Walkers were repeatedly found doing a -// full walk and reporting only its tail. -// -// (A PostToolUse hook declared in the walker's agent frontmatter would -// have been tidier, and does not fire in this environment — verified -// with a bare `echo` probe that produced nothing. Hooks in settings.json -// do fire, but would run for every tool call in the project rather than -// just a walker's, so the transcript is both cheaper and better scoped.) - -const fs = require('fs'); - -/** The salient argument per tool — whatever a claim would be about. */ -function detailOf(input) { - if (!input || typeof input !== 'object') return ''; - const raw = input.command || input.file_path || input.pattern || input.path || ''; - return String(raw).replace(/\s+/g, ' ').trim(); -} - -/** - * Transcript JSONL → ordered tool calls. Paths inside the world are - * shortened to `.` so the log reads as the walk rather than as a wall - * of temp directories. - * @param {string} file transcript path - * @param {string} [worldDir] world root, elided from output when given - */ -function extractActions(file, worldDir) { - if (!fs.existsSync(file)) return null; - const out = []; - for (const line of fs.readFileSync(file, 'utf8').split('\n')) { - if (!line.trim()) continue; - let record; - try { - record = JSON.parse(line); - } catch { - continue; // a partial trailing write is not worth failing over - } - const content = record?.message?.content; - if (!Array.isArray(content)) continue; - for (const part of content) { - if (part.type !== 'tool_use') continue; - let detail = detailOf(part.input); - if (worldDir) detail = detail.split(worldDir).join('.'); - out.push({ tool: part.name, detail }); - } - } - return out; -} - -function formatActions(actions) { - if (!actions || !actions.length) return null; - return actions - .map((a, i) => `${String(i + 1).padStart(3)}. ${a.tool.padEnd(10)} ${a.detail}`) - .join('\n'); -} - -module.exports = { extractActions, formatActions }; diff --git a/tests/prose/lib/record-action.cjs b/tests/prose/lib/record-action.cjs new file mode 100644 index 000000000..e8a47c71a --- /dev/null +++ b/tests/prose/lib/record-action.cjs @@ -0,0 +1,67 @@ +#!/usr/bin/env node +'use strict'; + +// PostToolUse hook for the prose-walker agent: records what the walker +// ACTUALLY did, as the harness sees it. +// +// Declared in .claude/agents/prose-walker.md frontmatter, so it fires +// only while a walker is active. The walker plays no part in it — it +// cannot forget an entry, summarise one away, or write the log late, +// which is exactly why the log exists. Walkers were repeatedly found +// doing a full walk and then reporting only its tail; the narrative is +// now evidence of reasoning alone, and this file is the evidence of +// action. +// +// Self-scoping: every prose world lives at a temp path containing +// `prose-world-`. The payload is scanned for that path, and anything +// happening outside a world is ignored. Nothing is configured per run. +// +// The log lands at /.walk-actions.log, which collectTree() +// excludes, so recording never shows up as a world difference. + +const fs = require('fs'); +const path = require('path'); + +const LOG = '.walk-actions.log'; +const WORLD = /(^|[\s"'`])(\/[^\s"'`]*\/prose-world-[A-Za-z0-9]+)/; + +function read() { + try { + return JSON.parse(fs.readFileSync(0, 'utf8')); + } catch { + return null; + } +} + +/** The salient argument, per tool — what the claim will be about. */ +function summarise(tool, input) { + if (!input || typeof input !== 'object') return ''; + const raw = input.command || input.file_path || input.pattern || input.path || ''; + return String(raw).replace(/\s+/g, ' ').trim(); +} + +function main() { + const payload = read(); + if (!payload) return; + + const probe = JSON.stringify(payload); + const found = probe.match(WORLD); + // Not a prose world: this hook has nothing to say. + if (!found) return; + const world = found[2].replace(/\\+/g, ''); + + if (!fs.existsSync(world)) return; + + const tool = payload.tool_name || '?'; + const detail = summarise(tool, payload.tool_input); + const failed = payload.tool_output_is_error ? ' [ERROR]' : ''; + const line = `${tool}${failed}\t${detail.replace(world, '.')}\n`; + + try { + fs.appendFileSync(path.join(world, LOG), line); + } catch { + // A hook must never break the walk it is observing. + } +} + +main(); diff --git a/tests/prose/lib/worlds.cjs b/tests/prose/lib/worlds.cjs index 2c4697889..566bb03eb 100644 --- a/tests/prose/lib/worlds.cjs +++ b/tests/prose/lib/worlds.cjs @@ -37,6 +37,9 @@ const CLOCK = path.join(__dirname, 'fake-clock.cjs'); const GITIGNORE = '.gitignore'; const GITIGNORE_ESCAPED = '_gitignore.fixture'; const HASH_FILE = '.recipe-hash'; +// Written by the walker's PostToolUse hook (lib/record-action.cjs) — +// observation of the walk, not part of the world it acted on. +const ACTION_LOG = '.walk-actions.log'; // --- the recipe harness --------------------------------------------------- @@ -101,7 +104,7 @@ function runRecipe(caseId, which) { function excluded(rel) { const parts = rel.split(path.sep); if (parts.includes('.git')) return true; - if (rel === HASH_FILE) return true; + if (rel === HASH_FILE || rel === ACTION_LOG) return true; if (rel === path.join('.workflows', '.knowledge')) return true; if (rel.startsWith(path.join('.workflows', '.knowledge') + path.sep)) return true; if (rel.startsWith(path.join('.claude', 'skills') + path.sep)) return true; @@ -319,8 +322,22 @@ function destroyWorld(dir) { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } +/** What the walk actually did, as recorded by the walker's hook. */ +function readActionLog(worldDir) { + const file = path.join(worldDir, ACTION_LOG); + if (!fs.existsSync(file)) return null; + return fs.readFileSync(file, 'utf8') + .split('\n') + .filter(Boolean) + .map((l, i) => { + const [tool, detail = ''] = l.split('\t'); + return `${String(i + 1).padStart(3)}. ${tool.padEnd(10)} ${detail}`; + }) + .join('\n'); +} + module.exports = { - ROOT, ENGINE, KNOWLEDGE, MAINLINES_DIR, WORLD_PREFIX, + ROOT, ENGINE, KNOWLEDGE, MAINLINES_DIR, WORLD_PREFIX, ACTION_LOG, readActionLog, runRecipe, collectTree, readSnapshot, snapshotDir, recipeHash, storedHash, writeSnapshot, verifySnapshot, diffWorld, buildWorld, destroyWorld, }; diff --git a/tests/prose/prompts/asserter.md b/tests/prose/prompts/asserter.md index 6a6621e8b..5e680e730 100644 --- a/tests/prose/prompts/asserter.md +++ b/tests/prose/prompts/asserter.md @@ -23,8 +23,8 @@ the expected world, computed by code: === actions === -RECORDED ACTIONS — every tool call the walker made, in order, read out of -the harness's own session record. The walker did not write this and could +RECORDED ACTIONS — every tool call the walker made, in order, captured by a +harness hook as it happened. The walker did not write this and could not have edited or omitted from it. This is what the walk *did*. {{actions}} diff --git a/tests/prose/run.cjs b/tests/prose/run.cjs index 254f90c41..03c655125 100644 --- a/tests/prose/run.cjs +++ b/tests/prose/run.cjs @@ -10,7 +10,7 @@ // world materialise the fixture state, print path // prompt --world walker prompt (NEVER contains assert.md) // diff --world acted world vs expected world, as facts -// assert --world --transcript the asserter's prompt +// assert --world the asserting agent's prompt // snap (re)generate a case's snapshots // verify [case-id] rebuild-compare snapshot(s) // destroy --world remove a world @@ -21,7 +21,6 @@ const { execFileSync } = require('child_process'); const cases = require('./lib/cases.cjs'); const worlds = require('./lib/worlds.cjs'); -const actionsLib = require('./lib/actions.cjs'); const prompts = require('./lib/prompts.cjs'); const ROOT = cases.ROOT; @@ -155,19 +154,17 @@ function cmdAssert(argv) { ...delta.changed, ].join('\n'), }; - const transcript = flag(argv, '--transcript'); - if (!transcript) { - die('--transcript is required: the walker\'s session transcript is the ' - + 'record of what the walk actually did, and a verdict without it rests on the ' - + "walker's own account — which is the thing this harness exists to distrust."); - } - actions = actionsLib.formatActions(actionsLib.extractActions(transcript, dir)); - // No tool calls at all is the harness failing, not the walk failing: - // a walk in a world always runs commands. Refuse loudly rather than - // let an agent reach a verdict with no record of what was done. + actions = worlds.readActionLog(dir); + // No log is the harness failing, not the walk: a walk in a world + // always runs commands, and the walker's hook records every one. + // Refuse loudly rather than let an agent judge on the narrative + // alone — which is the thing the recording exists to replace. if (!actions) { - die(`no tool calls found in ${transcript} — either that is not the walker's ` - + 'transcript, or the walk made no calls at all. Do not judge this run.'); + die(`no action log at ${path.join(dir, worlds.ACTION_LOG)} — the prose-walker ` + + 'PostToolUse hook did not fire, so there is no record of what the walk did.\n' + + 'Check the hooks block in .claude/agents/prose-walker.md, that the agent ' + + 'registry has reloaded since it changed, and that this project is trusted ' + + '(hasTrustDialogAccepted). Do not judge this run.'); } } process.stdout.write(prompts.asserterPrompt({ expected: c.assert, world, actions }));