diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index c6e7ff162..0809870fe 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -83,6 +83,23 @@ closes with an incorporate") is evidenced from the recorded actions. A step about reasoning or output is evidenced from the walk. Where the two disagree about what was done, the recorded actions win. +**Read them against each other, always, before ruling on anything.** Two +divergences matter and both are reported under markers: + +- **Narrated but not recorded** — the walk claims a command or a file + write the record does not contain. Treat the record as right. A step + resting on that claim fails, and the discrepancy is a finding in its + own right, because it means the walker described something it did not + do. +- **Recorded but not narrated** — the record holds a call the walk never + mentions. Usually harmless orientation, and not a failure by itself. + Report it anyway: a call that touched state and went unmentioned is the + shape a silent repair takes. + +This comparison is required, not a courtesy. The two streams exist so +that neither has to be taken on trust, and that only holds if they are +actually set against each other every time. + The record settles *that* a call was made, never *why*. A call the walker made during a wrong turn of its own, and then abandoned, appears in the record exactly like one the prose asked for — only the walk says which it diff --git a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json index ac5027dd2..b0f907864 100644 --- a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json @@ -6,7 +6,6 @@ "skills/workflow-shared/references/ensure-discovery-item.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "manifest get pay.discussion.pay status" ], diff --git a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json index 64a64307a..d11bcfca0 100644 --- a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json @@ -6,7 +6,6 @@ "skills/workflow-investigation-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "manifest get crash-fix.investigation.crash-fix status", "render phase-note" diff --git a/tests/prose/cases/planning-entry-asks-for-late-context/case.json b/tests/prose/cases/planning-entry-asks-for-late-context/case.json index 0ea0dade9..072803bfd 100644 --- a/tests/prose/cases/planning-entry-asks-for-late-context/case.json +++ b/tests/prose/cases/planning-entry-asks-for-late-context/case.json @@ -11,7 +11,6 @@ "continue \u2014 nothing has changed since the specification" ], "invariants": { - "engine_before_write": true, "calls_include": [ "render entry-gate pay.planning.pay", "manifest list", diff --git a/tests/prose/cases/review-entry-fresh/case.json b/tests/prose/cases/review-entry-fresh/case.json index 6bfcb1428..1bac663c7 100644 --- a/tests/prose/cases/review-entry-fresh/case.json +++ b/tests/prose/cases/review-entry-fresh/case.json @@ -6,7 +6,6 @@ "skills/workflow-review-entry/references/validate-phase.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "render entry-gate pay.review.pay" ], diff --git a/tests/prose/cases/spec-entry-from-discussion/case.json b/tests/prose/cases/spec-entry-from-discussion/case.json index 484e517cc..bbae6fa2a 100644 --- a/tests/prose/cases/spec-entry-from-discussion/case.json +++ b/tests/prose/cases/spec-entry-from-discussion/case.json @@ -8,7 +8,6 @@ "skills/workflow-specification-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "render entry-gate pay.specification.pay" ], diff --git a/tests/prose/cases/spec-entry-from-investigation/case.json b/tests/prose/cases/spec-entry-from-investigation/case.json index 2c38f89d1..a2fd8f96b 100644 --- a/tests/prose/cases/spec-entry-from-investigation/case.json +++ b/tests/prose/cases/spec-entry-from-investigation/case.json @@ -7,7 +7,6 @@ "skills/workflow-specification-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "render entry-gate crash-fix.specification.crash-fix" ], diff --git a/tests/prose/cases/start-lists-active-work/case.json b/tests/prose/cases/start-lists-active-work/case.json index 93fc70255..a588da9ab 100644 --- a/tests/prose/cases/start-lists-active-work/case.json +++ b/tests/prose/cases/start-lists-active-work/case.json @@ -5,7 +5,6 @@ "skills/workflow-start/SKILL.md" ], "invariants": { - "engine_before_write": true, "calls_include": [ "engine.cjs boot", "gateway.cjs" diff --git a/tests/prose/lib/invariants.cjs b/tests/prose/lib/invariants.cjs index d9dcbd631..a9e3944f3 100644 --- a/tests/prose/lib/invariants.cjs +++ b/tests/prose/lib/invariants.cjs @@ -48,6 +48,38 @@ function isWrite(row) { const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude', 'calls_in_order']; +/** + * Prose the walk actually opened, as repo-relative paths. + * + * A world holds the skills at `.claude/skills/`; the corpus names them at + * `skills/`. Same file, two addresses, so one is translated to the other. + */ +function proseRead(rows) { + const seen = new Set(); + for (const row of rows) { + if (row.tool !== 'Read') continue; + const m = row.detail.match(/\.claude\/skills\/(.+\.md)$/); + if (m) seen.add(`skills/${m[1]}`); + } + return seen; +} + +/** + * Prose a walk went through but the case never declared. + * + * `files` is not documentation: it decides whether a change to a file + * selects this case for a run. A file the walk traverses and the case + * omits is prose that can be edited without ever re-running the test that + * covers it. Reported, never corrected — which files belong is an + * authoring judgement, and the reachable set is far wider than the walked + * one, so a list assembled by following every link would select this case + * for branches it deliberately stops before. + */ +function undeclaredProse(rows, declared) { + const listed = new Set(declared || []); + return [...proseRead(rows)].filter((f) => !listed.has(f)).sort(); +} + /** The commands the walk ran, in order. */ function commands(rows) { return rows.filter((r) => r.tool === 'Bash' && r.event === 'PreToolUse').map((r) => r.detail); @@ -184,4 +216,4 @@ function declarationErrors(declared) { return errors; } -module.exports = { check, format, declarationErrors, NAMES }; +module.exports = { check, format, declarationErrors, undeclaredProse, NAMES }; diff --git a/tests/prose/lib/prompts.cjs b/tests/prose/lib/prompts.cjs index c1132c34c..bbc70688c 100644 --- a/tests/prose/lib/prompts.cjs +++ b/tests/prose/lib/prompts.cjs @@ -73,12 +73,13 @@ function walkerPrompt({ worldDir, situation, task, scope, stubs, answers }) { return `${parts.join('\n\n')}\n`; } -function asserterPrompt({ expected, world, actions, checks, walk, substitutions }) { +function asserterPrompt({ expected, world, actions, checks, walk, substitutions, scope }) { const t = loadTemplate('asserter'); const parts = [fill(t.main, { expected })]; if (world) parts.push(fill(t.world, { expecting: world.expecting, delta: world.delta })); if (actions) parts.push(fill(t.actions, { actions })); if (checks) parts.push(fill(t.checks, { checks })); + if (scope) parts.push(fill(t.scope, { scope })); if (substitutions) parts.push(fill(t.substitutions, { substitutions })); if (walk) parts.push(fill(t.walk, { walk })); return `${parts.join('\n\n')}\n`; diff --git a/tests/prose/prompts/asserter.md b/tests/prose/prompts/asserter.md index a012670c5..dac165e84 100644 --- a/tests/prose/prompts/asserter.md +++ b/tests/prose/prompts/asserter.md @@ -53,6 +53,19 @@ verdict. {{checks}} +=== scope === + +UNDECLARED PROSE — files this walk opened that the case does not list, +computed from the record. + +The case's file list decides whether editing a file selects this case for +a run, so prose missing from it can be changed without the test that +covers it ever running again. This is a finding about the case, never +about the prose or the walk: report it under markers and let it affect +nothing else. + +{{scope}} + === substitutions === HARNESS SUBSTITUTIONS the case armed for this walk. Where the recorded diff --git a/tests/prose/run.cjs b/tests/prose/run.cjs index fc3732ac7..9af456393 100644 --- a/tests/prose/run.cjs +++ b/tests/prose/run.cjs @@ -155,7 +155,9 @@ function cmdAssert(argv) { + 'registry has reloaded since it changed, and that this project is trusted ' + '(hasTrustDialogAccepted). Do not judge this run.'); } - const checks = invariants.format(invariants.check(worlds.readActionRows(dir), c.invariants)); + const rows = worlds.readActionRows(dir); + const checks = invariants.format(invariants.check(rows, c.invariants)); + const undeclared = invariants.undeclaredProse(rows, c.files.map((f) => f.path)); const walk = worlds.readWalkLog(dir); // Same stance as the action log: the walk is harness-captured, so its // absence is a broken hook, not a quiet walk. Judging without it would @@ -175,7 +177,10 @@ function cmdAssert(argv) { }).join('\n') : null; process.stdout.write( - prompts.asserterPrompt({ expected: c.assert, world, actions, checks, walk, substitutions }), + prompts.asserterPrompt({ + expected: c.assert, world, actions, checks, walk, substitutions, + scope: undeclared.length ? undeclared.map((f) => `- ${f}`).join('\n') : null, + }), ); } diff --git a/tests/scripts/test-prose-invariants.cjs b/tests/scripts/test-prose-invariants.cjs index 867abaaa6..b96a82ff1 100644 --- a/tests/scripts/test-prose-invariants.cjs +++ b/tests/scripts/test-prose-invariants.cjs @@ -289,3 +289,39 @@ describe('entry points — where a walk may begin', () => { assert.deepEqual(cases.validateCorpus(cases.loadAllCases()), []); }); }); + +describe('undeclared prose — the case list against what the walk opened', () => { + const rd = (detail) => ({ event: 'PreToolUse', tool: 'Read', detail }); + + it('names prose the walk opened that the case never declared', () => { + const rows = [ + rd('./.claude/skills/workflow-specification-entry/SKILL.md'), + rd('./.claude/skills/workflow-specification-entry/references/validate-phase.md'), + ]; + assert.deepEqual( + invariants.undeclaredProse(rows, ['skills/workflow-specification-entry/SKILL.md']), + ['skills/workflow-specification-entry/references/validate-phase.md'], + ); + }); + + it('is quiet when the list already covers the walk', () => { + const rows = [rd('./.claude/skills/workflow-review-entry/SKILL.md')]; + assert.deepEqual(invariants.undeclaredProse(rows, ['skills/workflow-review-entry/SKILL.md']), []); + }); + + it('ignores files that are not prose — a walk reads world state too', () => { + const rows = [rd('./.workflows/pay/manifest.json'), rd('./.workflows/pay/discussion/pay.md')]; + assert.deepEqual(invariants.undeclaredProse(rows, []), []); + }); + + it('ignores commands — only what was opened counts', () => { + const rows = [bash('cd . && cat .claude/skills/workflow-start/SKILL.md')]; + assert.deepEqual(invariants.undeclaredProse(rows, []), []); + }); + + it('reports each file once however often it was reopened', () => { + const f = './.claude/skills/workflow-start/references/active-work.md'; + assert.deepEqual(invariants.undeclaredProse([rd(f), rd(f)], []), + ['skills/workflow-start/references/active-work.md']); + }); +});