From 7fbbbd80e88255089bd4d28da34893ce40d62402 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Mon, 27 Jul 2026 10:07:21 +0100 Subject: [PATCH] fix(prose-tests): prescribe the cross-check, drop a check that cannot fire, lint the scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things the full run turned up, none of them failures. The asserter had begun comparing the action record against the walker's narrative unprompted, and reporting where they diverged. It is the right instinct and it was nobody's instruction, so a run that depended on it was depending on a coincidence. It is now required, with both directions named: narrated but not recorded means the walker described something it did not do, and the step resting on that claim fails; recorded but not narrated is usually orientation and fails nothing, but is reported, because a call that touched state and went unmentioned is the shape a silent repair takes. The two streams exist so neither is taken on trust, which only holds if they are actually set against each other. engine_before_write was declared on every case and could only ever fire on the three that write state. Reporting N/A seven times is honest but it is still a check dressed as coverage, so it is declared only where a walk can trip it. The check itself stays: it is the anti-fabrication guard for anything that lands state, and cases that land state are the direction we are heading. The scope lint is new. A case's file list is not documentation — it decides whether editing a file selects that case for a run, so prose the walk goes through but the case omits can be changed without the test that covers it ever running again. Every case has such gaps. The list cannot be built by following links: the reachable set is far wider than the walked one, and a case that declared everything reachable would be selected for branches it deliberately stops before. So it is computed from the walk's own Reads and reported as a finding about the case, correcting nothing. Co-Authored-By: Claude Opus 5 --- .claude/agents/prose-asserter.md | 17 +++++++++ .../case.json | 1 - .../case.json | 1 - .../case.json | 1 - .../prose/cases/review-entry-fresh/case.json | 1 - .../spec-entry-from-discussion/case.json | 1 - .../spec-entry-from-investigation/case.json | 1 - .../cases/start-lists-active-work/case.json | 1 - tests/prose/lib/invariants.cjs | 34 +++++++++++++++++- tests/prose/lib/prompts.cjs | 3 +- tests/prose/prompts/asserter.md | 13 +++++++ tests/prose/run.cjs | 9 +++-- tests/scripts/test-prose-invariants.cjs | 36 +++++++++++++++++++ 13 files changed, 108 insertions(+), 11 deletions(-) diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index c6e7ff162..0809870fe 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -83,6 +83,23 @@ closes with an incorporate") is evidenced from the recorded actions. A step about reasoning or output is evidenced from the walk. Where the two disagree about what was done, the recorded actions win. +**Read them against each other, always, before ruling on anything.** Two +divergences matter and both are reported under markers: + +- **Narrated but not recorded** — the walk claims a command or a file + write the record does not contain. Treat the record as right. A step + resting on that claim fails, and the discrepancy is a finding in its + own right, because it means the walker described something it did not + do. +- **Recorded but not narrated** — the record holds a call the walk never + mentions. Usually harmless orientation, and not a failure by itself. + Report it anyway: a call that touched state and went unmentioned is the + shape a silent repair takes. + +This comparison is required, not a courtesy. The two streams exist so +that neither has to be taken on trust, and that only holds if they are +actually set against each other every time. + The record settles *that* a call was made, never *why*. A call the walker made during a wrong turn of its own, and then abandoned, appears in the record exactly like one the prose asked for — only the walk says which it diff --git a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json index ac5027dd2..b0f907864 100644 --- a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json @@ -6,7 +6,6 @@ "skills/workflow-shared/references/ensure-discovery-item.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "manifest get pay.discussion.pay status" ], diff --git a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json index 64a64307a..d11bcfca0 100644 --- a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json @@ -6,7 +6,6 @@ "skills/workflow-investigation-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "manifest get crash-fix.investigation.crash-fix status", "render phase-note" diff --git a/tests/prose/cases/planning-entry-asks-for-late-context/case.json b/tests/prose/cases/planning-entry-asks-for-late-context/case.json index 0ea0dade9..072803bfd 100644 --- a/tests/prose/cases/planning-entry-asks-for-late-context/case.json +++ b/tests/prose/cases/planning-entry-asks-for-late-context/case.json @@ -11,7 +11,6 @@ "continue \u2014 nothing has changed since the specification" ], "invariants": { - "engine_before_write": true, "calls_include": [ "render entry-gate pay.planning.pay", "manifest list", diff --git a/tests/prose/cases/review-entry-fresh/case.json b/tests/prose/cases/review-entry-fresh/case.json index 6bfcb1428..1bac663c7 100644 --- a/tests/prose/cases/review-entry-fresh/case.json +++ b/tests/prose/cases/review-entry-fresh/case.json @@ -6,7 +6,6 @@ "skills/workflow-review-entry/references/validate-phase.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "render entry-gate pay.review.pay" ], diff --git a/tests/prose/cases/spec-entry-from-discussion/case.json b/tests/prose/cases/spec-entry-from-discussion/case.json index 484e517cc..bbae6fa2a 100644 --- a/tests/prose/cases/spec-entry-from-discussion/case.json +++ b/tests/prose/cases/spec-entry-from-discussion/case.json @@ -8,7 +8,6 @@ "skills/workflow-specification-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "render entry-gate pay.specification.pay" ], diff --git a/tests/prose/cases/spec-entry-from-investigation/case.json b/tests/prose/cases/spec-entry-from-investigation/case.json index 2c38f89d1..a2fd8f96b 100644 --- a/tests/prose/cases/spec-entry-from-investigation/case.json +++ b/tests/prose/cases/spec-entry-from-investigation/case.json @@ -7,7 +7,6 @@ "skills/workflow-specification-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "render entry-gate crash-fix.specification.crash-fix" ], diff --git a/tests/prose/cases/start-lists-active-work/case.json b/tests/prose/cases/start-lists-active-work/case.json index 93fc70255..a588da9ab 100644 --- a/tests/prose/cases/start-lists-active-work/case.json +++ b/tests/prose/cases/start-lists-active-work/case.json @@ -5,7 +5,6 @@ "skills/workflow-start/SKILL.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "engine.cjs boot", "gateway.cjs" diff --git a/tests/prose/lib/invariants.cjs b/tests/prose/lib/invariants.cjs index d9dcbd631..a9e3944f3 100644 --- a/tests/prose/lib/invariants.cjs +++ b/tests/prose/lib/invariants.cjs @@ -48,6 +48,38 @@ function isWrite(row) { const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude', 'calls_in_order']; +/** + * Prose the walk actually opened, as repo-relative paths. + * + * A world holds the skills at `.claude/skills/`; the corpus names them at + * `skills/`. Same file, two addresses, so one is translated to the other. + */ +function proseRead(rows) { + const seen = new Set(); + for (const row of rows) { + if (row.tool !== 'Read') continue; + const m = row.detail.match(/\.claude\/skills\/(.+\.md)$/); + if (m) seen.add(`skills/${m[1]}`); + } + return seen; +} + +/** + * Prose a walk went through but the case never declared. + * + * `files` is not documentation: it decides whether a change to a file + * selects this case for a run. A file the walk traverses and the case + * omits is prose that can be edited without ever re-running the test that + * covers it. Reported, never corrected — which files belong is an + * authoring judgement, and the reachable set is far wider than the walked + * one, so a list assembled by following every link would select this case + * for branches it deliberately stops before. + */ +function undeclaredProse(rows, declared) { + const listed = new Set(declared || []); + return [...proseRead(rows)].filter((f) => !listed.has(f)).sort(); +} + /** The commands the walk ran, in order. */ function commands(rows) { return rows.filter((r) => r.tool === 'Bash' && r.event === 'PreToolUse').map((r) => r.detail); @@ -184,4 +216,4 @@ function declarationErrors(declared) { return errors; } -module.exports = { check, format, declarationErrors, NAMES }; +module.exports = { check, format, declarationErrors, undeclaredProse, NAMES }; diff --git a/tests/prose/lib/prompts.cjs b/tests/prose/lib/prompts.cjs index c1132c34c..bbc70688c 100644 --- a/tests/prose/lib/prompts.cjs +++ b/tests/prose/lib/prompts.cjs @@ -73,12 +73,13 @@ function walkerPrompt({ worldDir, situation, task, scope, stubs, answers }) { return `${parts.join('\n\n')}\n`; } -function asserterPrompt({ expected, world, actions, checks, walk, substitutions }) { +function asserterPrompt({ expected, world, actions, checks, walk, substitutions, scope }) { const t = loadTemplate('asserter'); const parts = [fill(t.main, { expected })]; if (world) parts.push(fill(t.world, { expecting: world.expecting, delta: world.delta })); if (actions) parts.push(fill(t.actions, { actions })); if (checks) parts.push(fill(t.checks, { checks })); + if (scope) parts.push(fill(t.scope, { scope })); if (substitutions) parts.push(fill(t.substitutions, { substitutions })); if (walk) parts.push(fill(t.walk, { walk })); return `${parts.join('\n\n')}\n`; diff --git a/tests/prose/prompts/asserter.md b/tests/prose/prompts/asserter.md index a012670c5..dac165e84 100644 --- a/tests/prose/prompts/asserter.md +++ b/tests/prose/prompts/asserter.md @@ -53,6 +53,19 @@ verdict. {{checks}} +=== scope === + +UNDECLARED PROSE — files this walk opened that the case does not list, +computed from the record. + +The case's file list decides whether editing a file selects this case for +a run, so prose missing from it can be changed without the test that +covers it ever running again. This is a finding about the case, never +about the prose or the walk: report it under markers and let it affect +nothing else. + +{{scope}} + === substitutions === HARNESS SUBSTITUTIONS the case armed for this walk. Where the recorded diff --git a/tests/prose/run.cjs b/tests/prose/run.cjs index fc3732ac7..9af456393 100644 --- a/tests/prose/run.cjs +++ b/tests/prose/run.cjs @@ -155,7 +155,9 @@ function cmdAssert(argv) { + 'registry has reloaded since it changed, and that this project is trusted ' + '(hasTrustDialogAccepted). Do not judge this run.'); } - const checks = invariants.format(invariants.check(worlds.readActionRows(dir), c.invariants)); + const rows = worlds.readActionRows(dir); + const checks = invariants.format(invariants.check(rows, c.invariants)); + const undeclared = invariants.undeclaredProse(rows, c.files.map((f) => f.path)); const walk = worlds.readWalkLog(dir); // Same stance as the action log: the walk is harness-captured, so its // absence is a broken hook, not a quiet walk. Judging without it would @@ -175,7 +177,10 @@ function cmdAssert(argv) { }).join('\n') : null; process.stdout.write( - prompts.asserterPrompt({ expected: c.assert, world, actions, checks, walk, substitutions }), + prompts.asserterPrompt({ + expected: c.assert, world, actions, checks, walk, substitutions, + scope: undeclared.length ? undeclared.map((f) => `- ${f}`).join('\n') : null, + }), ); } diff --git a/tests/scripts/test-prose-invariants.cjs b/tests/scripts/test-prose-invariants.cjs index 867abaaa6..b96a82ff1 100644 --- a/tests/scripts/test-prose-invariants.cjs +++ b/tests/scripts/test-prose-invariants.cjs @@ -289,3 +289,39 @@ describe('entry points — where a walk may begin', () => { assert.deepEqual(cases.validateCorpus(cases.loadAllCases()), []); }); }); + +describe('undeclared prose — the case list against what the walk opened', () => { + const rd = (detail) => ({ event: 'PreToolUse', tool: 'Read', detail }); + + it('names prose the walk opened that the case never declared', () => { + const rows = [ + rd('./.claude/skills/workflow-specification-entry/SKILL.md'), + rd('./.claude/skills/workflow-specification-entry/references/validate-phase.md'), + ]; + assert.deepEqual( + invariants.undeclaredProse(rows, ['skills/workflow-specification-entry/SKILL.md']), + ['skills/workflow-specification-entry/references/validate-phase.md'], + ); + }); + + it('is quiet when the list already covers the walk', () => { + const rows = [rd('./.claude/skills/workflow-review-entry/SKILL.md')]; + assert.deepEqual(invariants.undeclaredProse(rows, ['skills/workflow-review-entry/SKILL.md']), []); + }); + + it('ignores files that are not prose — a walk reads world state too', () => { + const rows = [rd('./.workflows/pay/manifest.json'), rd('./.workflows/pay/discussion/pay.md')]; + assert.deepEqual(invariants.undeclaredProse(rows, []), []); + }); + + it('ignores commands — only what was opened counts', () => { + const rows = [bash('cd . && cat .claude/skills/workflow-start/SKILL.md')]; + assert.deepEqual(invariants.undeclaredProse(rows, []), []); + }); + + it('reports each file once however often it was reopened', () => { + const f = './.claude/skills/workflow-start/references/active-work.md'; + assert.deepEqual(invariants.undeclaredProse([rd(f), rd(f)], []), + ['skills/workflow-start/references/active-work.md']); + }); +});