From c4b258911d1e1830d7e37e2c3643e98d722770dd Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sun, 26 Jul 2026 20:18:08 +0100 Subject: [PATCH] test(prose): give the eight read-only cases something that can fail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every entry case declared exactly one check, `engine_before_write`, and those walks write nothing — so it reported "no workflow state was written" and passed, every time, whatever the prose did. Eight cases with nothing enforceable behind them, all showing green. Two changes so that cannot recur quietly. A check with nothing to examine now reports N/A rather than PASS. A green tick for absence of coverage is how this went unnoticed in the first place; the distinction is now on the page. And the eight get real declarations, drawn from the claims they already make. Their "further claims" turn out to be not-called assertions almost word for word — "the entry skill records nothing", "an entry that only validates writes nothing", "the knowledge base is queried, never written to" — so each becomes a calls_exclude over `topic start`, `topic reopen`, `manifest set`, `knowledge index`. What each case says it reads becomes a calls_include over the gate it must render. calls_in_order is new, and is the one thing presence could never settle: a gate read *after* the arm it was supposed to select proves the arm was chosen some other way. Declared commands must appear as a subsequence, with anything permitted in between. P9 is corrected too. It demanded every case run until state landed, which was wrong twice: it treated file changes as the only hard evidence, when recorded actions are equally deterministic, and it would have pushed short entry walks into needing conversation stubs — trading live prose for hand-written fiction. A handoff is a fine ending. The rule is that every claim must have a deterministic answer; the stop point follows from that rather than being legislated. The declarations are derived from the prose and not yet from a live run. Any that prove wrong get corrected on this branch before it lands. Co-Authored-By: Claude Opus 5 --- design/prose-tests.md | 36 +++++++----- .../case.json | 10 +++- .../case.json | 14 ++++- .../case.json | 16 ++++- .../prose/cases/review-entry-fresh/case.json | 9 ++- .../spec-entry-from-discussion/case.json | 11 +++- .../spec-entry-from-investigation/case.json | 11 +++- .../cases/start-lists-active-work/case.json | 10 +++- tests/prose/lib/invariants.cjs | 54 +++++++++++++++-- tests/scripts/test-prose-invariants.cjs | 58 +++++++++++++++++++ 10 files changed, 203 insertions(+), 26 deletions(-) diff --git a/design/prose-tests.md b/design/prose-tests.md index 1970bcd02..ab7be189c 100644 --- a/design/prose-tests.md +++ b/design/prose-tests.md @@ -174,20 +174,28 @@ testing. A failure is a finished result; diagnosing it is a separate, human-led act. This is stated in prose in each definition rather than relied on from tool restrictions, which an agent may hold regardless. -- **P9 — a case starts where a session starts, and runs until state - lands.** The only real openings are `workflow-start` (the user's), a - `workflow-*-entry` skill (what a bridge plan file invokes after a - context clear), and `workflow-discovery` (epic continuation). Navigation - and processing skills are always reached mid-session; a reference is - never entered directly. A walk carries only the context it accumulates, - so one begun in the middle is judged under conditions the prose was - never written for, and its verdict means nothing in either direction. - Enforced in `lib/cases.cjs` by the required `entry` field, not left to - authoring discipline. The same rule ends a case: stop at a gate and the - delta is empty, the claims turn into descriptions of a display, and the - asserter is left comparing the walker's account of what it showed - against the case's account of what it should have shown — with no - independent ground truth anywhere in the loop. +- **P9 — a case starts where a session starts, and stops where its + claims can be answered.** The only real openings are `workflow-start` + (the user's), a `workflow-*-entry` skill (what a bridge plan file + invokes after a context clear), and `workflow-discovery` (epic + continuation). Navigation and processing skills are always reached + mid-session; a reference is never entered directly. A walk carries only + the context it accumulates, so one begun in the middle is judged under + conditions the prose was never written for, and its verdict means + nothing in either direction. Enforced in `lib/cases.cjs` by the required + `entry` field, not left to authoring discipline. + + The far end is governed by the claims, not by the clock or by state. A + handoff is a perfectly good ending: the recorded actions are as + deterministic as a world delta, so which gates were read, what they + returned, and what was never called are all hard facts at that point. + What is *not* allowed is a claim with no deterministic answer — a claim + about text put on screen rests on the walker's account alone, since no + hook observes an emission. Regulate the claims and the stop point falls + out of them. (An earlier draft demanded every case run until state + landed. That was wrong twice over: it treated file changes as the only + hard evidence, and it would have pushed short entry walks into needing + conversation stubs, trading live prose for hand-written fiction.) - **P10 — what code can decide, code decides.** A case declares invariants over the recorded actions (`lib/invariants.cjs`) that run diff --git a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json index 514a9adfa..ac5027dd2 100644 --- a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json @@ -6,6 +6,14 @@ "skills/workflow-shared/references/ensure-discovery-item.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "manifest get pay.discussion.pay status" + ], + "calls_exclude": [ + "topic start", + "topic reopen", + "manifest set" + ] } } diff --git a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json index 8f64940f7..64a64307a 100644 --- a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json @@ -6,6 +6,18 @@ "skills/workflow-investigation-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "manifest get crash-fix.investigation.crash-fix status", + "render phase-note" + ], + "calls_in_order": [ + "manifest get crash-fix.investigation.crash-fix status", + "render phase-note" + ], + "calls_exclude": [ + "topic start", + "topic reopen" + ] } } diff --git a/tests/prose/cases/planning-entry-asks-for-late-context/case.json b/tests/prose/cases/planning-entry-asks-for-late-context/case.json index 8ff061076..0ea0dade9 100644 --- a/tests/prose/cases/planning-entry-asks-for-late-context/case.json +++ b/tests/prose/cases/planning-entry-asks-for-late-context/case.json @@ -11,6 +11,20 @@ "continue \u2014 nothing has changed since the specification" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "render entry-gate pay.planning.pay", + "manifest list", + "knowledge.cjs query" + ], + "calls_in_order": [ + "render entry-gate pay.planning.pay", + "knowledge.cjs query" + ], + "calls_exclude": [ + "knowledge.cjs index", + "knowledge.cjs remove", + "topic reopen" + ] } } diff --git a/tests/prose/cases/review-entry-fresh/case.json b/tests/prose/cases/review-entry-fresh/case.json index a365e2cd2..6bfcb1428 100644 --- a/tests/prose/cases/review-entry-fresh/case.json +++ b/tests/prose/cases/review-entry-fresh/case.json @@ -6,6 +6,13 @@ "skills/workflow-review-entry/references/validate-phase.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "render entry-gate pay.review.pay" + ], + "calls_exclude": [ + "topic reopen", + "topic start" + ] } } diff --git a/tests/prose/cases/spec-entry-from-discussion/case.json b/tests/prose/cases/spec-entry-from-discussion/case.json index 95045863f..484e517cc 100644 --- a/tests/prose/cases/spec-entry-from-discussion/case.json +++ b/tests/prose/cases/spec-entry-from-discussion/case.json @@ -8,6 +8,15 @@ "skills/workflow-specification-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "render entry-gate pay.specification.pay" + ], + "calls_exclude": [ + "topic reopen", + "topic supersede", + "manifest set", + "manifest apply" + ] } } diff --git a/tests/prose/cases/spec-entry-from-investigation/case.json b/tests/prose/cases/spec-entry-from-investigation/case.json index c48f2332e..2c38f89d1 100644 --- a/tests/prose/cases/spec-entry-from-investigation/case.json +++ b/tests/prose/cases/spec-entry-from-investigation/case.json @@ -7,6 +7,15 @@ "skills/workflow-specification-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "render entry-gate crash-fix.specification.crash-fix" + ], + "calls_exclude": [ + "topic reopen", + "topic supersede", + "manifest set", + "manifest apply" + ] } } diff --git a/tests/prose/cases/start-lists-active-work/case.json b/tests/prose/cases/start-lists-active-work/case.json index 8cd90c53f..93fc70255 100644 --- a/tests/prose/cases/start-lists-active-work/case.json +++ b/tests/prose/cases/start-lists-active-work/case.json @@ -5,6 +5,14 @@ "skills/workflow-start/SKILL.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "engine.cjs boot", + "gateway.cjs" + ], + "calls_in_order": [ + "engine.cjs boot", + "gateway.cjs" + ] } } diff --git a/tests/prose/lib/invariants.cjs b/tests/prose/lib/invariants.cjs index d7243e458..d9dcbd631 100644 --- a/tests/prose/lib/invariants.cjs +++ b/tests/prose/lib/invariants.cjs @@ -18,9 +18,14 @@ // "invariants": { // "engine_before_write": true, // no .workflows write out of nowhere // "calls_include": ["task init"], // these commands must have run -// "calls_exclude": ["task start"] // these must not have +// "calls_exclude": ["task start"], // these must not have +// "calls_in_order": ["a", "b"] // and these in this sequence // } // +// A check that could not have failed reports N/A rather than PASS. A +// green tick for "there was nothing to examine" is how a corpus comes to +// look covered while enforcing nothing. +// // Deliberately not a pinned call sequence. A recorded-and-replayed // sequence would freeze whatever the walker happened to do, which is how // a test starts certifying broken prose instead of catching it. @@ -41,7 +46,7 @@ function isWrite(row) { return row.tool === 'Bash' && SHELL_WRITE.test(row.detail); } -const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude']; +const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude', 'calls_in_order']; /** The commands the walk ran, in order. */ function commands(rows) { @@ -60,7 +65,11 @@ function engineBeforeWrite(rows) { ); const firstWrite = rows.findIndex(isWrite); if (firstWrite === -1) { - return { ok: true, detail: 'no workflow state was written' }; + // Nothing was written, so there was nothing for this check to catch. + // Reporting that as a pass would dress absence of coverage up as + // coverage — which is exactly how a corpus of read-only cases came to + // look green while enforcing nothing at all. + return { ok: true, vacuous: true, detail: 'no workflow state was written' }; } if (firstEngine === -1) { return { ok: false, detail: `wrote ${rows[firstWrite].detail} having never called the engine` }; @@ -90,6 +99,31 @@ function callsExclude(rows, forbidden) { : { ok: true, detail: `ran none of: ${forbidden.join(', ')}` }; } +/** + * Order carries meaning a presence check cannot: a gate read after the arm + * it was supposed to select proves the arm was chosen some other way. The + * declared commands must appear as a subsequence — other calls may fall + * between them, but never out of sequence. + */ +function callsInOrder(rows, sequence) { + const ran = commands(rows); + let at = 0; + for (const wanted of sequence) { + const found = ran.findIndex((c, i) => i >= at && c.includes(wanted)); + if (found === -1) { + const seen = sequence.slice(0, sequence.indexOf(wanted)); + return { + ok: false, + detail: seen.length + ? `"${wanted}" never ran after ${seen.map((s) => `"${s}"`).join(' → ')}` + : `"${wanted}" never ran`, + }; + } + at = found + 1; + } + return { ok: true, detail: `ran in order: ${sequence.join(' → ')}` }; +} + /** * Run a case's declared invariants against its recorded actions. * Returns one result per declared check, in declaration order. @@ -106,6 +140,9 @@ function check(rows, declared) { if (declared.calls_exclude && declared.calls_exclude.length) { results.push({ name: 'calls_exclude', ...callsExclude(rows, declared.calls_exclude) }); } + if (declared.calls_in_order && declared.calls_in_order.length) { + results.push({ name: 'calls_in_order', ...callsInOrder(rows, declared.calls_in_order) }); + } return results; } @@ -113,7 +150,10 @@ function check(rows, declared) { function format(results) { if (!results.length) return null; return results - .map((r) => `${r.ok ? 'PASS' : 'FAIL'} ${r.name} — ${r.detail}`) + .map((r) => { + const verdict = r.ok ? (r.vacuous ? 'N/A ' : 'PASS') : 'FAIL'; + return `${verdict} ${r.name} — ${r.detail}`; + }) .join('\n'); } @@ -130,11 +170,15 @@ function declarationErrors(declared) { if ('engine_before_write' in declared && typeof declared.engine_before_write !== 'boolean') { errors.push('engine_before_write must be true or false'); } - for (const key of ['calls_include', 'calls_exclude']) { + for (const key of ['calls_include', 'calls_exclude', 'calls_in_order']) { if (!(key in declared)) continue; const value = declared[key]; if (!Array.isArray(value) || value.some((v) => typeof v !== 'string' || !v.trim())) { errors.push(`${key} must be an array of non-empty strings`); + continue; + } + if (key === 'calls_in_order' && value.length < 2) { + errors.push('calls_in_order needs at least two commands — one has no order'); } } return errors; diff --git a/tests/scripts/test-prose-invariants.cjs b/tests/scripts/test-prose-invariants.cjs index 8179495f1..867abaaa6 100644 --- a/tests/scripts/test-prose-invariants.cjs +++ b/tests/scripts/test-prose-invariants.cjs @@ -141,6 +141,64 @@ describe('calls_include / calls_exclude', () => { }); }); +describe('calls_in_order — presence is not sequence', () => { + it('passes when the declared calls appear in sequence', () => { + const rows = [ + bash(`${ENGINE} render entry-gate pay.planning.pay`), + bash(`${ENGINE} manifest list`), + bash('cd . && node .claude/skills/workflow-knowledge/scripts/knowledge.cjs query x'), + ]; + const [result] = invariants.check(rows, { + calls_in_order: ['render entry-gate pay.planning.pay', 'knowledge.cjs query'], + }); + assert.equal(result.ok, true); + }); + + it('fails when they ran in the wrong order — the arm was chosen some other way', () => { + const rows = [ + bash('cd . && node .claude/skills/workflow-knowledge/scripts/knowledge.cjs query x'), + bash(`${ENGINE} render entry-gate pay.planning.pay`), + ]; + const [result] = invariants.check(rows, { + calls_in_order: ['render entry-gate pay.planning.pay', 'knowledge.cjs query'], + }); + assert.equal(result.ok, false); + assert.match(result.detail, /"knowledge.cjs query" never ran after "render entry-gate/); + }); + + it('tolerates other calls falling between them', () => { + const rows = [bash(`${ENGINE} a`), bash(`${ENGINE} unrelated`), bash(`${ENGINE} b`)]; + assert.equal(invariants.check(rows, { calls_in_order: ['a', 'b'] })[0].ok, true); + }); + + it('fails when a declared call never ran at all', () => { + const rows = [bash(`${ENGINE} a`)]; + const [result] = invariants.check(rows, { calls_in_order: ['a', 'b'] }); + assert.equal(result.ok, false); + assert.match(result.detail, /"b" never ran after "a"/); + }); + + it('needs two commands to mean anything', () => { + assert.match(invariants.declarationErrors({ calls_in_order: ['a'] })[0], /at least two/); + }); +}); + +describe('a check that could not fail says so', () => { + it('reports N/A, not PASS, when nothing was written to examine', () => { + const out = invariants.format( + invariants.check([bash(`${ENGINE} boot`)], { engine_before_write: true }), + ); + assert.match(out, /^N\/A {3}engine_before_write/); // padded to align with PASS/FAIL + assert.ok(!out.includes('PASS'), 'absence of coverage must not read as coverage'); + }); + + it('still reports PASS when it genuinely had something to check', () => { + const rows = [bash(`${ENGINE} task init pay pay`), wrote('./.workflows/a.md')]; + const out = invariants.format(invariants.check(rows, { engine_before_write: true })); + assert.match(out, /^PASS {2}engine_before_write/); + }); +}); + describe('declaration', () => { it('runs every declared check, and only those', () => { const rows = [bash(`${ENGINE} boot`)];