diff --git a/design/prose-tests.md b/design/prose-tests.md index 1970bcd02..ab7be189c 100644 --- a/design/prose-tests.md +++ b/design/prose-tests.md @@ -174,20 +174,28 @@ testing. A failure is a finished result; diagnosing it is a separate, human-led act. This is stated in prose in each definition rather than relied on from tool restrictions, which an agent may hold regardless. -- **P9 — a case starts where a session starts, and runs until state - lands.** The only real openings are `workflow-start` (the user's), a - `workflow-*-entry` skill (what a bridge plan file invokes after a - context clear), and `workflow-discovery` (epic continuation). Navigation - and processing skills are always reached mid-session; a reference is - never entered directly. A walk carries only the context it accumulates, - so one begun in the middle is judged under conditions the prose was - never written for, and its verdict means nothing in either direction. - Enforced in `lib/cases.cjs` by the required `entry` field, not left to - authoring discipline. The same rule ends a case: stop at a gate and the - delta is empty, the claims turn into descriptions of a display, and the - asserter is left comparing the walker's account of what it showed - against the case's account of what it should have shown — with no - independent ground truth anywhere in the loop. +- **P9 — a case starts where a session starts, and stops where its + claims can be answered.** The only real openings are `workflow-start` + (the user's), a `workflow-*-entry` skill (what a bridge plan file + invokes after a context clear), and `workflow-discovery` (epic + continuation). Navigation and processing skills are always reached + mid-session; a reference is never entered directly. A walk carries only + the context it accumulates, so one begun in the middle is judged under + conditions the prose was never written for, and its verdict means + nothing in either direction. Enforced in `lib/cases.cjs` by the required + `entry` field, not left to authoring discipline. + + The far end is governed by the claims, not by the clock or by state. A + handoff is a perfectly good ending: the recorded actions are as + deterministic as a world delta, so which gates were read, what they + returned, and what was never called are all hard facts at that point. + What is *not* allowed is a claim with no deterministic answer — a claim + about text put on screen rests on the walker's account alone, since no + hook observes an emission. Regulate the claims and the stop point falls + out of them. (An earlier draft demanded every case run until state + landed. That was wrong twice over: it treated file changes as the only + hard evidence, and it would have pushed short entry walks into needing + conversation stubs, trading live prose for hand-written fiction.) - **P10 — what code can decide, code decides.** A case declares invariants over the recorded actions (`lib/invariants.cjs`) that run diff --git a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json index 514a9adfa..ac5027dd2 100644 --- a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json @@ -6,6 +6,14 @@ "skills/workflow-shared/references/ensure-discovery-item.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "manifest get pay.discussion.pay status" + ], + "calls_exclude": [ + "topic start", + "topic reopen", + "manifest set" + ] } } diff --git a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json index 8f64940f7..64a64307a 100644 --- a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json @@ -6,6 +6,18 @@ "skills/workflow-investigation-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "manifest get crash-fix.investigation.crash-fix status", + "render phase-note" + ], + "calls_in_order": [ + "manifest get crash-fix.investigation.crash-fix status", + "render phase-note" + ], + "calls_exclude": [ + "topic start", + "topic reopen" + ] } } diff --git a/tests/prose/cases/planning-entry-asks-for-late-context/case.json b/tests/prose/cases/planning-entry-asks-for-late-context/case.json index 8ff061076..0ea0dade9 100644 --- a/tests/prose/cases/planning-entry-asks-for-late-context/case.json +++ b/tests/prose/cases/planning-entry-asks-for-late-context/case.json @@ -11,6 +11,20 @@ "continue \u2014 nothing has changed since the specification" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "render entry-gate pay.planning.pay", + "manifest list", + "knowledge.cjs query" + ], + "calls_in_order": [ + "render entry-gate pay.planning.pay", + "knowledge.cjs query" + ], + "calls_exclude": [ + "knowledge.cjs index", + "knowledge.cjs remove", + "topic reopen" + ] } } diff --git a/tests/prose/cases/review-entry-fresh/case.json b/tests/prose/cases/review-entry-fresh/case.json index a365e2cd2..6bfcb1428 100644 --- a/tests/prose/cases/review-entry-fresh/case.json +++ b/tests/prose/cases/review-entry-fresh/case.json @@ -6,6 +6,13 @@ "skills/workflow-review-entry/references/validate-phase.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "render entry-gate pay.review.pay" + ], + "calls_exclude": [ + "topic reopen", + "topic start" + ] } } diff --git a/tests/prose/cases/spec-entry-from-discussion/case.json b/tests/prose/cases/spec-entry-from-discussion/case.json index 95045863f..484e517cc 100644 --- a/tests/prose/cases/spec-entry-from-discussion/case.json +++ b/tests/prose/cases/spec-entry-from-discussion/case.json @@ -8,6 +8,15 @@ "skills/workflow-specification-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "render entry-gate pay.specification.pay" + ], + "calls_exclude": [ + "topic reopen", + "topic supersede", + "manifest set", + "manifest apply" + ] } } diff --git a/tests/prose/cases/spec-entry-from-investigation/case.json b/tests/prose/cases/spec-entry-from-investigation/case.json index c48f2332e..2c38f89d1 100644 --- a/tests/prose/cases/spec-entry-from-investigation/case.json +++ b/tests/prose/cases/spec-entry-from-investigation/case.json @@ -7,6 +7,15 @@ "skills/workflow-specification-entry/references/invoke-skill.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "render entry-gate crash-fix.specification.crash-fix" + ], + "calls_exclude": [ + "topic reopen", + "topic supersede", + "manifest set", + "manifest apply" + ] } } diff --git a/tests/prose/cases/start-lists-active-work/case.json b/tests/prose/cases/start-lists-active-work/case.json index 8cd90c53f..93fc70255 100644 --- a/tests/prose/cases/start-lists-active-work/case.json +++ b/tests/prose/cases/start-lists-active-work/case.json @@ -5,6 +5,14 @@ "skills/workflow-start/SKILL.md" ], "invariants": { - "engine_before_write": true + "engine_before_write": true, + "calls_include": [ + "engine.cjs boot", + "gateway.cjs" + ], + "calls_in_order": [ + "engine.cjs boot", + "gateway.cjs" + ] } } diff --git a/tests/prose/lib/invariants.cjs b/tests/prose/lib/invariants.cjs index d7243e458..d9dcbd631 100644 --- a/tests/prose/lib/invariants.cjs +++ b/tests/prose/lib/invariants.cjs @@ -18,9 +18,14 @@ // "invariants": { // "engine_before_write": true, // no .workflows write out of nowhere // "calls_include": ["task init"], // these commands must have run -// "calls_exclude": ["task start"] // these must not have +// "calls_exclude": ["task start"], // these must not have +// "calls_in_order": ["a", "b"] // and these in this sequence // } // +// A check that could not have failed reports N/A rather than PASS. A +// green tick for "there was nothing to examine" is how a corpus comes to +// look covered while enforcing nothing. +// // Deliberately not a pinned call sequence. A recorded-and-replayed // sequence would freeze whatever the walker happened to do, which is how // a test starts certifying broken prose instead of catching it. @@ -41,7 +46,7 @@ function isWrite(row) { return row.tool === 'Bash' && SHELL_WRITE.test(row.detail); } -const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude']; +const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude', 'calls_in_order']; /** The commands the walk ran, in order. */ function commands(rows) { @@ -60,7 +65,11 @@ function engineBeforeWrite(rows) { ); const firstWrite = rows.findIndex(isWrite); if (firstWrite === -1) { - return { ok: true, detail: 'no workflow state was written' }; + // Nothing was written, so there was nothing for this check to catch. + // Reporting that as a pass would dress absence of coverage up as + // coverage — which is exactly how a corpus of read-only cases came to + // look green while enforcing nothing at all. + return { ok: true, vacuous: true, detail: 'no workflow state was written' }; } if (firstEngine === -1) { return { ok: false, detail: `wrote ${rows[firstWrite].detail} having never called the engine` }; @@ -90,6 +99,31 @@ function callsExclude(rows, forbidden) { : { ok: true, detail: `ran none of: ${forbidden.join(', ')}` }; } +/** + * Order carries meaning a presence check cannot: a gate read after the arm + * it was supposed to select proves the arm was chosen some other way. The + * declared commands must appear as a subsequence — other calls may fall + * between them, but never out of sequence. + */ +function callsInOrder(rows, sequence) { + const ran = commands(rows); + let at = 0; + for (const wanted of sequence) { + const found = ran.findIndex((c, i) => i >= at && c.includes(wanted)); + if (found === -1) { + const seen = sequence.slice(0, sequence.indexOf(wanted)); + return { + ok: false, + detail: seen.length + ? `"${wanted}" never ran after ${seen.map((s) => `"${s}"`).join(' → ')}` + : `"${wanted}" never ran`, + }; + } + at = found + 1; + } + return { ok: true, detail: `ran in order: ${sequence.join(' → ')}` }; +} + /** * Run a case's declared invariants against its recorded actions. * Returns one result per declared check, in declaration order. @@ -106,6 +140,9 @@ function check(rows, declared) { if (declared.calls_exclude && declared.calls_exclude.length) { results.push({ name: 'calls_exclude', ...callsExclude(rows, declared.calls_exclude) }); } + if (declared.calls_in_order && declared.calls_in_order.length) { + results.push({ name: 'calls_in_order', ...callsInOrder(rows, declared.calls_in_order) }); + } return results; } @@ -113,7 +150,10 @@ function check(rows, declared) { function format(results) { if (!results.length) return null; return results - .map((r) => `${r.ok ? 'PASS' : 'FAIL'} ${r.name} — ${r.detail}`) + .map((r) => { + const verdict = r.ok ? (r.vacuous ? 'N/A ' : 'PASS') : 'FAIL'; + return `${verdict} ${r.name} — ${r.detail}`; + }) .join('\n'); } @@ -130,11 +170,15 @@ function declarationErrors(declared) { if ('engine_before_write' in declared && typeof declared.engine_before_write !== 'boolean') { errors.push('engine_before_write must be true or false'); } - for (const key of ['calls_include', 'calls_exclude']) { + for (const key of ['calls_include', 'calls_exclude', 'calls_in_order']) { if (!(key in declared)) continue; const value = declared[key]; if (!Array.isArray(value) || value.some((v) => typeof v !== 'string' || !v.trim())) { errors.push(`${key} must be an array of non-empty strings`); + continue; + } + if (key === 'calls_in_order' && value.length < 2) { + errors.push('calls_in_order needs at least two commands — one has no order'); } } return errors; diff --git a/tests/scripts/test-prose-invariants.cjs b/tests/scripts/test-prose-invariants.cjs index 8179495f1..867abaaa6 100644 --- a/tests/scripts/test-prose-invariants.cjs +++ b/tests/scripts/test-prose-invariants.cjs @@ -141,6 +141,64 @@ describe('calls_include / calls_exclude', () => { }); }); +describe('calls_in_order — presence is not sequence', () => { + it('passes when the declared calls appear in sequence', () => { + const rows = [ + bash(`${ENGINE} render entry-gate pay.planning.pay`), + bash(`${ENGINE} manifest list`), + bash('cd . && node .claude/skills/workflow-knowledge/scripts/knowledge.cjs query x'), + ]; + const [result] = invariants.check(rows, { + calls_in_order: ['render entry-gate pay.planning.pay', 'knowledge.cjs query'], + }); + assert.equal(result.ok, true); + }); + + it('fails when they ran in the wrong order — the arm was chosen some other way', () => { + const rows = [ + bash('cd . && node .claude/skills/workflow-knowledge/scripts/knowledge.cjs query x'), + bash(`${ENGINE} render entry-gate pay.planning.pay`), + ]; + const [result] = invariants.check(rows, { + calls_in_order: ['render entry-gate pay.planning.pay', 'knowledge.cjs query'], + }); + assert.equal(result.ok, false); + assert.match(result.detail, /"knowledge.cjs query" never ran after "render entry-gate/); + }); + + it('tolerates other calls falling between them', () => { + const rows = [bash(`${ENGINE} a`), bash(`${ENGINE} unrelated`), bash(`${ENGINE} b`)]; + assert.equal(invariants.check(rows, { calls_in_order: ['a', 'b'] })[0].ok, true); + }); + + it('fails when a declared call never ran at all', () => { + const rows = [bash(`${ENGINE} a`)]; + const [result] = invariants.check(rows, { calls_in_order: ['a', 'b'] }); + assert.equal(result.ok, false); + assert.match(result.detail, /"b" never ran after "a"/); + }); + + it('needs two commands to mean anything', () => { + assert.match(invariants.declarationErrors({ calls_in_order: ['a'] })[0], /at least two/); + }); +}); + +describe('a check that could not fail says so', () => { + it('reports N/A, not PASS, when nothing was written to examine', () => { + const out = invariants.format( + invariants.check([bash(`${ENGINE} boot`)], { engine_before_write: true }), + ); + assert.match(out, /^N\/A {3}engine_before_write/); // padded to align with PASS/FAIL + assert.ok(!out.includes('PASS'), 'absence of coverage must not read as coverage'); + }); + + it('still reports PASS when it genuinely had something to check', () => { + const rows = [bash(`${ENGINE} task init pay pay`), wrote('./.workflows/a.md')]; + const out = invariants.format(invariants.check(rows, { engine_before_write: true })); + assert.match(out, /^PASS {2}engine_before_write/); + }); +}); + describe('declaration', () => { it('runs every declared check, and only those', () => { const rows = [bash(`${ENGINE} boot`)];