From 2ee5855807f1311b8d7d1e13dad9d945e6f4e30a Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sun, 26 Jul 2026 20:35:43 +0100 Subject: [PATCH] test(prose): only walks that can be observed, and checks that survive the trip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two findings from running all nine cases. Eight passed on Sonnet, with every invariant derived from the prose in the previous change holding up against a live walk. The ninth, smoke-start-boot-structure, came back INVALID twice. It was the one structure-only case — read the prose, trace the logic, execute nothing — so it built no world, and with no world the hook records no actions and there is no delta. The asserter was handed an expected path and no evidence of any kind, and refused to judge it. Correctly: nothing about that walk was observable, so nothing it claimed could be answered. It was scaffolding from before real cases existed, and its one claim — casing conventions load before the boot pipeline — is covered by start-lists-active-work, which walks the same skill against a world. It goes, and a world becomes required, which removes the category rather than leaving a trap for the next author. The branches that served it go with it: the structural walker prompt, the world: null response, the no-world arms in prompt, diff and assert. The second finding is subtler. The orchestrator's verdict block had no line for the deterministic checks, so the asserter reported them and the orchestrator dropped them. Nine PASSes came back and the only reason to believe the checks passed was the rule that a failing one forces FAIL — an inference, about the one part of the result that exists precisely so nothing has to infer anything. They now travel through to the block and into the skill's table, unsummarised. Co-Authored-By: Claude Opus 5 --- .claude/agents/prose-orchestrator.md | 7 +- .claude/skills/prose-test/SKILL.md | 9 +- .../cases/smoke-start-boot-structure/act.md | 5 -- .../smoke-start-boot-structure/assert.md | 7 -- .../smoke-start-boot-structure/case.json | 7 -- tests/prose/lib/cases.cjs | 17 +--- tests/prose/lib/prompts.cjs | 6 +- tests/prose/prompts/walker.md | 6 -- tests/prose/run.cjs | 86 ++++++++----------- 9 files changed, 54 insertions(+), 96 deletions(-) delete mode 100644 tests/prose/cases/smoke-start-boot-structure/act.md delete mode 100644 tests/prose/cases/smoke-start-boot-structure/assert.md delete mode 100644 tests/prose/cases/smoke-start-boot-structure/case.json diff --git a/.claude/agents/prose-orchestrator.md b/.claude/agents/prose-orchestrator.md index fee116010..f38e81fad 100644 --- a/.claude/agents/prose-orchestrator.md +++ b/.claude/agents/prose-orchestrator.md @@ -44,8 +44,7 @@ a walk. ## Steps 1. **Build the world** — `node tests/prose/run.cjs world `. The - response carries the path, or `world: null` for a structure-only case - (skip `--world` everywhere below when so). + response carries the path to the world it built. 2. **Walk** — `node tests/prose/run.cjs prompt --world `. Dispatch the **prose-walker** agent with that output as its prompt, @@ -104,6 +103,10 @@ CASE: MODEL: VERDICT: PASS | FAIL | FLAKY | INVALID | HARNESS ERROR +CHECKS: PATH: / steps passed WORLD: PASS | FAIL MARKERS: diff --git a/.claude/skills/prose-test/SKILL.md b/.claude/skills/prose-test/SKILL.md index 54b9f812b..bbc3178c6 100644 --- a/.claude/skills/prose-test/SKILL.md +++ b/.claude/skills/prose-test/SKILL.md @@ -38,8 +38,13 @@ about an expected result can leak into a later dispatch. ## Step 3: Collate Report a verdict table from what the orchestrators returned: case id, -model, verdict, path steps passed, world, markers. Quote the evidence -line for every failure. +model, verdict, deterministic checks, path steps passed, world, markers. +Quote the evidence line for every failure. + +Carry the checks through as reported. They were decided in code before +any agent saw the case, so they are the one part of a verdict that rests +on nothing's judgement — summarising them, or taking a green overall +verdict as proof they passed, puts an assumption back where a fact was. The model column comes from the harness record, not from any agent's say-so. An edited agent definition does not reach a running session until diff --git a/tests/prose/cases/smoke-start-boot-structure/act.md b/tests/prose/cases/smoke-start-boot-structure/act.md deleted file mode 100644 index cc3f6a902..000000000 --- a/tests/prose/cases/smoke-start-boot-structure/act.md +++ /dev/null @@ -1,5 +0,0 @@ -Structure-only: read the initialisation portion of -skills/workflow-start/SKILL.md — everything the skill does before any -work is shown or routed. Establish what initialisation consists of and -in what order its parts run. Stop there; do not trace the rest of the -skill. diff --git a/tests/prose/cases/smoke-start-boot-structure/assert.md b/tests/prose/cases/smoke-start-boot-structure/assert.md deleted file mode 100644 index 058974b50..000000000 --- a/tests/prose/cases/smoke-start-boot-structure/assert.md +++ /dev/null @@ -1,7 +0,0 @@ -The prose should have taken this path: - -1. casing conventions are loaded before the boot pipeline runs -2. the boot pipeline is declared mandatory — it must complete before the - skill proceeds -3. the knowledge gate is reached only after boot, and branches on what - boot reported rather than on its own checks diff --git a/tests/prose/cases/smoke-start-boot-structure/case.json b/tests/prose/cases/smoke-start-boot-structure/case.json deleted file mode 100644 index c47d0f93f..000000000 --- a/tests/prose/cases/smoke-start-boot-structure/case.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "origin": "framework smoke \u2014 a structure-only walk, no world", - "entry": "workflow-start", - "files": [ - "skills/workflow-start/SKILL.md#Boot" - ] -} diff --git a/tests/prose/lib/cases.cjs b/tests/prose/lib/cases.cjs index 371f8cc37..1293e452c 100644 --- a/tests/prose/lib/cases.cjs +++ b/tests/prose/lib/cases.cjs @@ -216,6 +216,10 @@ function validateCorpus(cases) { } } + if (!c.hasFixtureState) { + errors.push(`${at}: no ${FILES.fixtureState} — a case with no world records no ` + + 'actions and produces no delta, so nothing it claims can be answered'); + } if (!c.act) errors.push(`${at}: no ${FILES.act}`); if (!c.assert) errors.push(`${at}: no ${FILES.assert} — the expected trace is what catches a silent repair`); @@ -226,21 +230,9 @@ function validateCorpus(cases) { errors.push(`${at}: world "claims" and ${FILES.assertionState} are exclusive — ` + 'a world a recipe can build must be built'); } - if (c.worldMode === 'claims' && !c.hasFixtureState) { - errors.push(`${at}: world "claims" needs a world to mutate`); - } for (const e of invariants.declarationErrors(c.invariants)) { errors.push(`${at}: ${FILES.meta} ${e}`); } - if (c.invariants && !c.hasFixtureState) { - errors.push(`${at}: invariants without ${FILES.fixtureState} — no walk to check`); - } - if (c.hasAssertionState && !c.hasFixtureState) { - errors.push(`${at}: ${FILES.assertionState} without ${FILES.fixtureState} — nothing to act on`); - } - if (c.situation && !c.hasFixtureState) { - errors.push(`${at}: ${FILES.situation} describes a world this case does not build`); - } for (const which of ['fixtureState', 'assertionState']) { try { requireState(c.id, which); @@ -254,7 +246,6 @@ function validateCorpus(cases) { if (!s.trigger || !s.trigger.trim()) { errors.push(`${at}: stub "${s.name}" declares no trigger — the case owns the moment`); } - if (!c.hasFixtureState) errors.push(`${at}: stub "${s.name}" needs a world to fire in`); } } return errors; diff --git a/tests/prose/lib/prompts.cjs b/tests/prose/lib/prompts.cjs index c15a26f11..c1132c34c 100644 --- a/tests/prose/lib/prompts.cjs +++ b/tests/prose/lib/prompts.cjs @@ -55,11 +55,9 @@ function indent(text, by = ' ') { return text.split('\n').map((l) => (l.length ? by + l : l)).join('\n'); } -function walkerPrompt({ worldDir, root, situation, task, scope, stubs, answers }) { +function walkerPrompt({ worldDir, situation, task, scope, stubs, answers }) { const t = loadTemplate('walker'); - const parts = [ - worldDir ? fill(t.world, { world_dir: worldDir }) : fill(t.structural, { root }), - ]; + const parts = [fill(t.world, { world_dir: worldDir })]; if (situation) parts.push(fill(t.situation, { situation })); parts.push(fill(t.task, { task, scope })); if (answers) parts.push(fill(t.answers, { answers })); diff --git a/tests/prose/prompts/walker.md b/tests/prose/prompts/walker.md index 693c2d4ac..f1017f38d 100644 --- a/tests/prose/prompts/walker.md +++ b/tests/prose/prompts/walker.md @@ -15,12 +15,6 @@ Project directory — your cwd for EVERY command: {{world_dir}} The workflow skills are installed at .claude/skills/ inside that project. Mutations are expected and safe: the project is a disposable test world. -=== structural === -Structure-only walk: read the named prose and trace the logic. Execute -nothing — no commands, no writes. - -Repository root: {{root}} - === situation === SITUATION — where the project stands as you begin: diff --git a/tests/prose/run.cjs b/tests/prose/run.cjs index 166f0b115..fc3732ac7 100644 --- a/tests/prose/run.cjs +++ b/tests/prose/run.cjs @@ -96,10 +96,6 @@ function cmdSelect(argv) { function cmdWorld(argv) { const c = getCase(argv[0]); - if (!c.hasFixtureState) { - process.stdout.write(`${JSON.stringify({ world: null, note: 'structure-only case — no world' })}\n`); - return; - } process.stdout.write(`${JSON.stringify({ world: worlds.buildWorld(c.id), case: c.id })}\n`); } @@ -113,11 +109,10 @@ function cmdDestroy(argv) { function cmdPrompt(argv) { const c = getCase(argv[0]); - const worldDir = c.hasFixtureState ? requireWorld(argv, c) : null; + const worldDir = requireWorld(argv, c); process.stdout.write(prompts.walkerPrompt({ worldDir, - root: ROOT, situation: c.situation, task: c.act, scope: c.files @@ -132,10 +127,6 @@ function cmdPrompt(argv) { function cmdDiff(argv) { const c = getCase(argv[0]); - if (!c.hasFixtureState) { - process.stdout.write(`${JSON.stringify({ note: 'structure-only case — no world to diff' }, null, 2)}\n`); - return; - } process.stdout.write(`${JSON.stringify(worlds.diffWorld(c.id, requireWorld(argv, c)), null, 2)}\n`); } @@ -143,45 +134,40 @@ function cmdDiff(argv) { function cmdAssert(argv) { const c = getCase(argv[0]); - let world = null; - let actions = null; - let walk = null; - let checks = null; - if (c.hasFixtureState) { - const dir = requireWorld(argv, c); - const delta = worlds.diffWorld(c.id, dir, c.worldMode === 'claims'); - world = { - expecting: delta.expecting, - delta: [ - JSON.stringify({ added: delta.added, removed: delta.removed, identical: delta.identical }, null, 2), - ...delta.changed, - ].join('\n'), - }; - actions = worlds.readActionLog(dir); - // No log is the harness failing, not the walk: a walk in a world - // always runs commands, and the walker's hook records every one. - // Refuse loudly rather than let an agent judge on the narrative - // alone — which is the thing the recording exists to replace. - if (!actions) { - die(`no action log at ${path.join(dir, worlds.ACTION_LOG)} — the prose-walker ` - + 'PostToolUse hook did not fire, so there is no record of what the walk did.\n' - + 'Check the hooks block in .claude/agents/prose-walker.md, that the agent ' - + 'registry has reloaded since it changed, and that this project is trusted ' - + '(hasTrustDialogAccepted). Do not judge this run.'); - } - checks = invariants.format(invariants.check(worlds.readActionRows(dir), c.invariants)); - walk = worlds.readWalkLog(dir); - // Same stance as the action log: the walk is harness-captured, so its - // absence is a broken hook, not a quiet walk. Judging without it would - // fall back to whatever the walker chose to say at the end — the very - // summary this record exists to replace. - if (!walk) { - die(`no walk log at ${path.join(dir, worlds.WALK_LOG)} — the prose-walker ` - + 'SubagentStop hook did not fire, so there is no turn-by-turn record of ' - + 'the walk.\nCheck the hooks block in .claude/agents/prose-walker.md and ' - + 'that the agent registry has reloaded since it changed. Do not judge this run.'); - } + const dir = requireWorld(argv, c); + const delta = worlds.diffWorld(c.id, dir, c.worldMode === 'claims'); + const world = { + expecting: delta.expecting, + delta: [ + JSON.stringify({ added: delta.added, removed: delta.removed, identical: delta.identical }, null, 2), + ...delta.changed, + ].join('\n'), + }; + const actions = worlds.readActionLog(dir); + // No log is the harness failing, not the walk: a walk in a world + // always runs commands, and the walker's hook records every one. + // Refuse loudly rather than let an agent judge on the narrative + // alone — which is the thing the recording exists to replace. + if (!actions) { + die(`no action log at ${path.join(dir, worlds.ACTION_LOG)} — the prose-walker ` + + 'PostToolUse hook did not fire, so there is no record of what the walk did.\n' + + 'Check the hooks block in .claude/agents/prose-walker.md, that the agent ' + + 'registry has reloaded since it changed, and that this project is trusted ' + + '(hasTrustDialogAccepted). Do not judge this run.'); + } + const checks = invariants.format(invariants.check(worlds.readActionRows(dir), c.invariants)); + const walk = worlds.readWalkLog(dir); + // Same stance as the action log: the walk is harness-captured, so its + // absence is a broken hook, not a quiet walk. Judging without it would + // fall back to whatever the walker chose to say at the end — the very + // summary this record exists to replace. + if (!walk) { + die(`no walk log at ${path.join(dir, worlds.WALK_LOG)} — the prose-walker ` + + 'SubagentStop hook did not fire, so there is no turn-by-turn record of ' + + 'the walk.\nCheck the hooks block in .claude/agents/prose-walker.md and ' + + 'that the agent registry has reloaded since it changed. Do not judge this run.'); } + const substitutions = c.stubs.length ? c.stubs.map((s) => { const stub = cases.readStub(s.name); @@ -197,7 +183,7 @@ function cmdAssert(argv) { function statesOf(c) { return [ - c.hasFixtureState ? 'fixture' : null, + 'fixture', c.hasAssertionState ? 'assertion' : null, ].filter(Boolean); } @@ -245,7 +231,7 @@ function cmdList() { for (const c of all) { const stubs = c.stubs.length ? ` stubs=${c.stubs.map((s) => s.name).join(',')}` : ''; const expects = c.hasAssertionState ? 'assertion state' : 'no change'; - process.stdout.write(`${c.id}\n ${c.hasFixtureState ? 'fixture' : 'no world'} → ${expects}${stubs}\n`); + process.stdout.write(`${c.id}\n fixture → ${expects}${stubs}\n`); } process.stdout.write(`\n${all.length} cases, ${cases.listStubs().length} stubs`); process.stdout.write(errors.length ? `, ${errors.length} VALIDATION ERRORS:\n` : ', corpus valid\n');