From e198646bb775d37e99b6eb71aefad0842b8855d2 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sun, 26 Jul 2026 19:33:02 +0100 Subject: [PATCH 1/2] feat(prose-tests): decide in code what an agent should not be deciding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The world delta proves the outcome. It cannot prove the outcome was earned: a walk that ignored every instruction and wrote the expected files directly lands exactly the same state as one that followed the prose. Only the order of what it did separates them, and that has been recorded all along without anything reading it. Cases can now declare invariants over the recorded actions, checked before the asserter sees anything: engine_before_write no workflow state written without first consulting the engine — a skill learns what it knows through the engine, so a write that precedes every call did not come from following the prose calls_include commands the walk must have run calls_exclude commands it must not have A verdict computed in code cannot drift between runs, and the asserter is told it may not overturn one. Where a check can carry a claim, the claim moves: implementation-picks-first-task no longer asks an agent whether a task was started, it asserts that neither `task start` nor `task next` appears in the record. Deliberately not a pinned call sequence. Recording one and replaying it would freeze whatever the walker happened to do, which is how a test starts certifying broken prose rather than catching it. The declarations are hand-written, like every other part of a case. Every case that builds a world takes engine_before_write — it costs nothing and it is the fabrication detector. The four cases retired in the next change are left untouched. Co-Authored-By: Claude Opus 5 --- .claude/agents/prose-asserter.md | 7 +- package.json | 2 +- .../case.json | 7 +- .../implementation-picks-first-task/assert.md | 1 - .../implementation-picks-first-task/case.json | 16 +- .../case.json | 7 +- .../case.json | 9 +- .../prose/cases/review-entry-fresh/case.json | 7 +- .../spec-entry-from-discussion/case.json | 7 +- .../spec-entry-from-investigation/case.json | 7 +- .../cases/start-lists-active-work/case.json | 7 +- tests/prose/lib/cases.cjs | 11 +- tests/prose/lib/invariants.cjs | 133 +++++++++++++++ tests/prose/lib/prompts.cjs | 3 +- tests/prose/lib/worlds.cjs | 15 +- tests/prose/prompts/asserter.md | 17 ++ tests/prose/run.cjs | 5 +- tests/scripts/test-prose-invariants.cjs | 156 ++++++++++++++++++ 18 files changed, 392 insertions(+), 25 deletions(-) create mode 100644 tests/prose/lib/invariants.cjs create mode 100644 tests/scripts/test-prose-invariants.cjs diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index 1541c6592..ad27af794 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -87,6 +87,10 @@ Return exactly this and nothing else: as recorded. If no such line exists, say `unrecorded`. A verdict is only as trustworthy as the model that produced the walk, so it is stated before anything is judged, never inferred, and never omitted. +0b. **Checks** — every deterministic check and its computed verdict, + repeated as given. Omit the section only when the prompt carried none. + A FAILing check fails the case on its own, whatever the path and world + show; it was decided in code and is not yours to revisit. 1. **Path** — one line per expected step: PASS or FAIL, each PASS quoting the line of the walk that shows it, each FAIL stating what is missing or contradicting. @@ -97,4 +101,5 @@ Return exactly this and nothing else: `DEVIATION` in the walk. Each is a finding in its own right, even when everything else passed. 4. **VERDICT** — PASS, FAIL, or INVALID WALK, then one sentence on what - it means for the prose. + it means for the prose. Any FAILing deterministic check makes this + FAIL. diff --git a/package.json b/package.json index 50661ef01..29d011eb5 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build": "node build/knowledge.build.js", "knowledge:repl": "node scripts/knowledge-repl.js", "typecheck": "tsc -p tsconfig.json", - "test": "node --test tests/scripts/test-render.cjs tests/scripts/test-engine-gateway.cjs tests/scripts/test-engine-epic-projections.cjs tests/scripts/test-engine-start-projections.cjs tests/scripts/test-engine-workunit-projections.cjs tests/scripts/test-engine-discussion-map.cjs tests/scripts/test-engine-discovery-map.cjs tests/scripts/test-engine-tasks.cjs tests/scripts/test-engine-render-surfaces.cjs tests/scripts/test-engine-transactions.cjs tests/scripts/test-engine-workunit-create.cjs tests/scripts/test-engine-workunit-pivot.cjs tests/scripts/test-engine-workunit-absorb.cjs tests/scripts/test-engine-workunit-promote.cjs tests/scripts/test-engine-discovery-session.cjs tests/scripts/test-engine-cache.cjs tests/scripts/test-engine-manifest-io.cjs tests/scripts/test-engine-manifest-fields.cjs tests/scripts/test-engine-boot.cjs tests/scripts/test-migration-orchestrator.cjs tests/scripts/test-migration-047.cjs tests/scripts/test-migration-048.cjs tests/scripts/test-migration-049.cjs tests/scripts/test-migration-050.cjs tests/scripts/test-migration-051.cjs tests/scripts/test-migration-052.cjs tests/scripts/test-engine-specification-projections.cjs tests/scripts/test-engine-discovery-projections.cjs tests/scripts/test-gateway-for-start.cjs tests/scripts/test-gateway-for-specification.cjs tests/scripts/test-gateway-for-workflow-continue-epic.cjs tests/scripts/test-gateway-for-workflow-continue-feature.cjs tests/scripts/test-gateway-for-workflow-continue-bugfix.cjs tests/scripts/test-gateway-for-workflow-continue-quickfix.cjs tests/scripts/test-gateway-for-workflow-continue-cross-cutting.cjs tests/scripts/test-gateway-for-discovery-skill.cjs tests/scripts/test-gateway-for-bridge.cjs tests/scripts/test-reads-derivations.cjs tests/scripts/test-ensure-discovery-item.cjs tests/scripts/test-legacy-research-split.cjs tests/scripts/test-knowledge-chunker.cjs tests/scripts/test-knowledge-config.cjs tests/scripts/test-knowledge-discovery.cjs tests/scripts/test-knowledge-embeddings.cjs tests/scripts/test-knowledge-integration.cjs tests/scripts/test-knowledge-openai.cjs tests/scripts/test-knowledge-openai-compatible.cjs tests/scripts/test-knowledge-openai-integration.cjs tests/scripts/test-knowledge-progress.cjs tests/scripts/test-knowledge-rerank.cjs tests/scripts/test-knowledge-retry.cjs tests/scripts/test-knowledge-setup-forms.cjs tests/scripts/test-knowledge-store.cjs tests/scripts/test-conventions-lint.cjs tests/scripts/test-pipeline-simulation.cjs tests/scripts/test-engine-agent-state.cjs tests/scripts/test-prose-record-action.cjs tests/scripts/test-prose-cases.cjs tests/scripts/test-prose-snapshots.cjs", + "test": "node --test tests/scripts/test-render.cjs tests/scripts/test-engine-gateway.cjs tests/scripts/test-engine-epic-projections.cjs tests/scripts/test-engine-start-projections.cjs tests/scripts/test-engine-workunit-projections.cjs tests/scripts/test-engine-discussion-map.cjs tests/scripts/test-engine-discovery-map.cjs tests/scripts/test-engine-tasks.cjs tests/scripts/test-engine-render-surfaces.cjs tests/scripts/test-engine-transactions.cjs tests/scripts/test-engine-workunit-create.cjs tests/scripts/test-engine-workunit-pivot.cjs tests/scripts/test-engine-workunit-absorb.cjs tests/scripts/test-engine-workunit-promote.cjs tests/scripts/test-engine-discovery-session.cjs tests/scripts/test-engine-cache.cjs tests/scripts/test-engine-manifest-io.cjs tests/scripts/test-engine-manifest-fields.cjs tests/scripts/test-engine-boot.cjs tests/scripts/test-migration-orchestrator.cjs tests/scripts/test-migration-047.cjs tests/scripts/test-migration-048.cjs tests/scripts/test-migration-049.cjs tests/scripts/test-migration-050.cjs tests/scripts/test-migration-051.cjs tests/scripts/test-migration-052.cjs tests/scripts/test-engine-specification-projections.cjs tests/scripts/test-engine-discovery-projections.cjs tests/scripts/test-gateway-for-start.cjs tests/scripts/test-gateway-for-specification.cjs tests/scripts/test-gateway-for-workflow-continue-epic.cjs tests/scripts/test-gateway-for-workflow-continue-feature.cjs tests/scripts/test-gateway-for-workflow-continue-bugfix.cjs tests/scripts/test-gateway-for-workflow-continue-quickfix.cjs tests/scripts/test-gateway-for-workflow-continue-cross-cutting.cjs tests/scripts/test-gateway-for-discovery-skill.cjs tests/scripts/test-gateway-for-bridge.cjs tests/scripts/test-reads-derivations.cjs tests/scripts/test-ensure-discovery-item.cjs tests/scripts/test-legacy-research-split.cjs tests/scripts/test-knowledge-chunker.cjs tests/scripts/test-knowledge-config.cjs tests/scripts/test-knowledge-discovery.cjs tests/scripts/test-knowledge-embeddings.cjs tests/scripts/test-knowledge-integration.cjs tests/scripts/test-knowledge-openai.cjs tests/scripts/test-knowledge-openai-compatible.cjs tests/scripts/test-knowledge-openai-integration.cjs tests/scripts/test-knowledge-progress.cjs tests/scripts/test-knowledge-rerank.cjs tests/scripts/test-knowledge-retry.cjs tests/scripts/test-knowledge-setup-forms.cjs tests/scripts/test-knowledge-store.cjs tests/scripts/test-conventions-lint.cjs tests/scripts/test-pipeline-simulation.cjs tests/scripts/test-engine-agent-state.cjs tests/scripts/test-prose-invariants.cjs tests/scripts/test-prose-record-action.cjs tests/scripts/test-prose-cases.cjs tests/scripts/test-prose-snapshots.cjs", "test:migrations": "for t in tests/scripts/test-migration-*.sh; do bash $t || exit 1; done", "test:cli": "for t in tests/scripts/test-engine-manifest.sh tests/scripts/test-inbox-promotion.sh tests/scripts/test-knowledge-cli.sh tests/scripts/test-knowledge-build.sh; do bash $t || exit 1; done" }, diff --git a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json index f5bc80e4b..85b404f92 100644 --- a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json @@ -1,7 +1,10 @@ { - "origin": "feature mainline — single-phase work seeds from its durable carrier", + "origin": "feature mainline \u2014 single-phase work seeds from its durable carrier", "files": [ "skills/workflow-discussion-entry/SKILL.md", "skills/workflow-shared/references/ensure-discovery-item.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/implementation-picks-first-task/assert.md b/tests/prose/cases/implementation-picks-first-task/assert.md index 97e435949..0445421ef 100644 --- a/tests/prose/cases/implementation-picks-first-task/assert.md +++ b/tests/prose/cases/implementation-picks-first-task/assert.md @@ -16,4 +16,3 @@ The prose should have taken this path: Further claims: - the setup document is written once, by the setup step and nowhere else -- no task is started: the walk stops before any task is picked up diff --git a/tests/prose/cases/implementation-picks-first-task/case.json b/tests/prose/cases/implementation-picks-first-task/case.json index 20268377d..cf3393720 100644 --- a/tests/prose/cases/implementation-picks-first-task/case.json +++ b/tests/prose/cases/implementation-picks-first-task/case.json @@ -1,5 +1,5 @@ { - "origin": "feature mainline — entry through resume detection to the first task", + "origin": "feature mainline \u2014 entry through resume detection to the first task", "files": [ "skills/workflow-implementation-entry/SKILL.md", "skills/workflow-implementation-entry/references/validate-phase.md", @@ -8,6 +8,16 @@ "skills/workflow-implementation-process/references/environment-setup.md" ], "answers": [ - "none — no special setup is needed" - ] + "none \u2014 no special setup is needed" + ], + "invariants": { + "engine_before_write": true, + "calls_include": [ + "task init" + ], + "calls_exclude": [ + "task start", + "task next" + ] + } } diff --git a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json index 3eb224b9c..71758ea1b 100644 --- a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json @@ -1,7 +1,10 @@ { - "origin": "bugfix mainline — a bug shaped in discovery is never re-interrogated", + "origin": "bugfix mainline \u2014 a bug shaped in discovery is never re-interrogated", "files": [ "skills/workflow-investigation-entry/SKILL.md", "skills/workflow-investigation-entry/references/invoke-skill.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/planning-entry-asks-for-late-context/case.json b/tests/prose/cases/planning-entry-asks-for-late-context/case.json index e5e2d1d46..83b632fd8 100644 --- a/tests/prose/cases/planning-entry-asks-for-late-context/case.json +++ b/tests/prose/cases/planning-entry-asks-for-late-context/case.json @@ -1,5 +1,5 @@ { - "origin": "feature mainline — planning offers a last chance to add context", + "origin": "feature mainline \u2014 planning offers a last chance to add context", "files": [ "skills/workflow-planning-entry/SKILL.md", "skills/workflow-planning-entry/references/validate-spec.md", @@ -7,6 +7,9 @@ "skills/workflow-planning-entry/references/cross-cutting-context.md" ], "answers": [ - "continue — nothing has changed since the specification" - ] + "continue \u2014 nothing has changed since the specification" + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/review-entry-fresh/case.json b/tests/prose/cases/review-entry-fresh/case.json index c67cada98..e19d4dfa4 100644 --- a/tests/prose/cases/review-entry-fresh/case.json +++ b/tests/prose/cases/review-entry-fresh/case.json @@ -1,7 +1,10 @@ { - "origin": "feature mainline — review refuses to start until delivery is complete", + "origin": "feature mainline \u2014 review refuses to start until delivery is complete", "files": [ "skills/workflow-review-entry/SKILL.md", "skills/workflow-review-entry/references/validate-phase.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/spec-entry-from-discussion/case.json b/tests/prose/cases/spec-entry-from-discussion/case.json index f380e57ce..f7f4aa938 100644 --- a/tests/prose/cases/spec-entry-from-discussion/case.json +++ b/tests/prose/cases/spec-entry-from-discussion/case.json @@ -1,9 +1,12 @@ { - "origin": "feature mainline — a feature's spec draws on its discussion", + "origin": "feature mainline \u2014 a feature's spec draws on its discussion", "files": [ "skills/workflow-specification-entry/SKILL.md", "skills/workflow-specification-entry/references/validate-source.md", "skills/workflow-specification-entry/references/validate-phase.md", "skills/workflow-specification-entry/references/invoke-skill.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/spec-entry-from-investigation/case.json b/tests/prose/cases/spec-entry-from-investigation/case.json index 3eed6746c..e0849f435 100644 --- a/tests/prose/cases/spec-entry-from-investigation/case.json +++ b/tests/prose/cases/spec-entry-from-investigation/case.json @@ -1,8 +1,11 @@ { - "origin": "bugfix mainline — the same entry skill, sourced from an investigation", + "origin": "bugfix mainline \u2014 the same entry skill, sourced from an investigation", "files": [ "skills/workflow-specification-entry/SKILL.md", "skills/workflow-specification-entry/references/validate-source.md", "skills/workflow-specification-entry/references/invoke-skill.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/start-lists-active-work/case.json b/tests/prose/cases/start-lists-active-work/case.json index 59711da9f..c4b7667c2 100644 --- a/tests/prose/cases/start-lists-active-work/case.json +++ b/tests/prose/cases/start-lists-active-work/case.json @@ -1,6 +1,9 @@ { - "origin": "feature mainline — the entry skill surfaces work without touching it", + "origin": "feature mainline \u2014 the entry skill surfaces work without touching it", "files": [ "skills/workflow-start/SKILL.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/lib/cases.cjs b/tests/prose/lib/cases.cjs index a4b3d79df..e26d82620 100644 --- a/tests/prose/lib/cases.cjs +++ b/tests/prose/lib/cases.cjs @@ -9,7 +9,8 @@ // // case.json the values code branches on: // { origin, files[], answers[], stubs{name: trigger}, -// world: "claims" (optional) } +// world: "claims" (optional), +// invariants{} (optional) — see lib/invariants.cjs } // fixture.md optional. Prose describing the starting world — // what has already happened, where the session // stands. Given to the walker. @@ -45,6 +46,7 @@ const fs = require('fs'); const path = require('path'); +const invariants = require('./invariants.cjs'); const ROOT = path.join(__dirname, '../../..'); const PROSE_DIR = path.join(ROOT, 'tests/prose'); @@ -104,6 +106,7 @@ function loadCase(id) { }), answers: meta.answers || [], worldMode: meta.world || null, + invariants: meta.invariants || null, stubs: Object.entries(meta.stubs || {}).map(([name, trigger]) => ({ name, trigger })), situation: readIf(dir, FILES.situation), act: readIf(dir, FILES.act), @@ -189,6 +192,12 @@ function validateCorpus(cases) { if (c.worldMode === 'claims' && !c.hasFixtureState) { errors.push(`${at}: world "claims" needs a world to mutate`); } + for (const e of invariants.declarationErrors(c.invariants)) { + errors.push(`${at}: ${FILES.meta} ${e}`); + } + if (c.invariants && !c.hasFixtureState) { + errors.push(`${at}: invariants without ${FILES.fixtureState} — no walk to check`); + } if (c.hasAssertionState && !c.hasFixtureState) { errors.push(`${at}: ${FILES.assertionState} without ${FILES.fixtureState} — nothing to act on`); } diff --git a/tests/prose/lib/invariants.cjs b/tests/prose/lib/invariants.cjs new file mode 100644 index 000000000..bb664dc04 --- /dev/null +++ b/tests/prose/lib/invariants.cjs @@ -0,0 +1,133 @@ +'use strict'; + +// Deterministic checks over the recorded actions. +// +// The world delta proves the outcome. It cannot prove the outcome was +// reached by following the prose: a walker that ignored every instruction +// and wrote the expected files directly lands the same delta. Only the +// order of what it actually did separates the two, and that is recorded. +// +// These checks read that record in code and decide before any agent sees +// the case. A judgement an agent never makes is a judgement that cannot +// drift — which is the whole reason they exist, and why they are worth +// having even though they can only ever cover the coarse shape of a walk. +// +// Declared per case in case.json, hand-written like every other part of a +// case: +// +// "invariants": { +// "engine_before_write": true, // no .workflows write out of nowhere +// "calls_include": ["task init"], // these commands must have run +// "calls_exclude": ["task start"] // these must not have +// } +// +// Deliberately not a pinned call sequence. A recorded-and-replayed +// sequence would freeze whatever the walker happened to do, which is how +// a test starts certifying broken prose instead of catching it. + +const ENGINE_CALL = /\/(engine|gateway)\.cjs\b/; +const WRITE_TOOLS = new Set(['Write', 'Edit', 'NotebookEdit']); +const WORKFLOWS = '.workflows/'; + +const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude']; + +/** The commands the walk ran, in order. */ +function commands(rows) { + return rows.filter((r) => r.tool === 'Bash' && r.event === 'PreToolUse').map((r) => r.detail); +} + +/** + * A write the prose could only have reached by consulting state first. + * Reading a file is not enough — the engine is how a workflow skill + * learns anything, so a walk that writes workflow state having never + * called it did not get there by following the prose. + */ +function engineBeforeWrite(rows) { + const firstEngine = rows.findIndex( + (r) => r.tool === 'Bash' && ENGINE_CALL.test(r.detail), + ); + const firstWrite = rows.findIndex( + (r) => WRITE_TOOLS.has(r.tool) && r.detail.includes(WORKFLOWS), + ); + if (firstWrite === -1) { + return { ok: true, detail: 'no workflow state was written' }; + } + if (firstEngine === -1) { + return { ok: false, detail: `wrote ${rows[firstWrite].detail} having never called the engine` }; + } + if (firstEngine > firstWrite) { + return { + ok: false, + detail: `wrote ${rows[firstWrite].detail} before any engine call`, + }; + } + return { ok: true, detail: 'every workflow write followed an engine call' }; +} + +function callsInclude(rows, wanted) { + const ran = commands(rows); + const missing = wanted.filter((w) => !ran.some((c) => c.includes(w))); + return missing.length + ? { ok: false, detail: `never ran: ${missing.join(', ')}` } + : { ok: true, detail: `ran all of: ${wanted.join(', ')}` }; +} + +function callsExclude(rows, forbidden) { + const ran = commands(rows); + const found = forbidden.filter((f) => ran.some((c) => c.includes(f))); + return found.length + ? { ok: false, detail: `ran what it should not have: ${found.join(', ')}` } + : { ok: true, detail: `ran none of: ${forbidden.join(', ')}` }; +} + +/** + * Run a case's declared invariants against its recorded actions. + * Returns one result per declared check, in declaration order. + */ +function check(rows, declared) { + if (!declared) return []; + const results = []; + if (declared.engine_before_write) { + results.push({ name: 'engine_before_write', ...engineBeforeWrite(rows) }); + } + if (declared.calls_include && declared.calls_include.length) { + results.push({ name: 'calls_include', ...callsInclude(rows, declared.calls_include) }); + } + if (declared.calls_exclude && declared.calls_exclude.length) { + results.push({ name: 'calls_exclude', ...callsExclude(rows, declared.calls_exclude) }); + } + return results; +} + +/** The checks as the asserter reads them — verdict first, computed by code. */ +function format(results) { + if (!results.length) return null; + return results + .map((r) => `${r.ok ? 'PASS' : 'FAIL'} ${r.name} — ${r.detail}`) + .join('\n'); +} + +/** Shape errors in a case's declaration, for corpus validation. */ +function declarationErrors(declared) { + if (declared === undefined || declared === null) return []; + const errors = []; + if (typeof declared !== 'object' || Array.isArray(declared)) { + return ['invariants must be an object']; + } + for (const key of Object.keys(declared)) { + if (!NAMES.includes(key)) errors.push(`unknown invariant "${key}" (known: ${NAMES.join(', ')})`); + } + if ('engine_before_write' in declared && typeof declared.engine_before_write !== 'boolean') { + errors.push('engine_before_write must be true or false'); + } + for (const key of ['calls_include', 'calls_exclude']) { + if (!(key in declared)) continue; + const value = declared[key]; + if (!Array.isArray(value) || value.some((v) => typeof v !== 'string' || !v.trim())) { + errors.push(`${key} must be an array of non-empty strings`); + } + } + return errors; +} + +module.exports = { check, format, declarationErrors, NAMES }; diff --git a/tests/prose/lib/prompts.cjs b/tests/prose/lib/prompts.cjs index b6a7cb1a4..c15a26f11 100644 --- a/tests/prose/lib/prompts.cjs +++ b/tests/prose/lib/prompts.cjs @@ -75,11 +75,12 @@ function walkerPrompt({ worldDir, root, situation, task, scope, stubs, answers } return `${parts.join('\n\n')}\n`; } -function asserterPrompt({ expected, world, actions, walk, substitutions }) { +function asserterPrompt({ expected, world, actions, checks, walk, substitutions }) { const t = loadTemplate('asserter'); const parts = [fill(t.main, { expected })]; if (world) parts.push(fill(t.world, { expecting: world.expecting, delta: world.delta })); if (actions) parts.push(fill(t.actions, { actions })); + if (checks) parts.push(fill(t.checks, { checks })); if (substitutions) parts.push(fill(t.substitutions, { substitutions })); if (walk) parts.push(fill(t.walk, { walk })); return `${parts.join('\n\n')}\n`; diff --git a/tests/prose/lib/worlds.cjs b/tests/prose/lib/worlds.cjs index 6cc652a77..14b570076 100644 --- a/tests/prose/lib/worlds.cjs +++ b/tests/prose/lib/worlds.cjs @@ -331,6 +331,19 @@ function destroyWorld(dir) { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } +/** The recorded actions as rows, for checks that run in code. */ +function readActionRows(worldDir) { + const file = path.join(worldDir, ACTION_LOG); + if (!fs.existsSync(file)) return []; + return fs.readFileSync(file, 'utf8') + .split('\n') + .filter(Boolean) + .map((l) => { + const [event, tool, detail = '', outcome = null, output = null] = l.split('\t'); + return { event, tool, detail, outcome, output }; + }); +} + /** What the walk actually did, as recorded by the walker's hook. */ function readActionLog(worldDir) { const file = path.join(worldDir, ACTION_LOG); @@ -356,7 +369,7 @@ function readWalkLog(worldDir) { module.exports = { ROOT, ENGINE, KNOWLEDGE, MAINLINES_DIR, WORLD_PREFIX, - ACTION_LOG, readActionLog, WALK_LOG, readWalkLog, + ACTION_LOG, readActionLog, readActionRows, WALK_LOG, readWalkLog, runRecipe, collectTree, readSnapshot, snapshotDir, recipeHash, storedHash, writeSnapshot, verifySnapshot, diffWorld, buildWorld, destroyWorld, }; diff --git a/tests/prose/prompts/asserter.md b/tests/prose/prompts/asserter.md index ff8f54bc6..a012670c5 100644 --- a/tests/prose/prompts/asserter.md +++ b/tests/prose/prompts/asserter.md @@ -36,6 +36,23 @@ wins and the discrepancy is itself worth reporting. {{actions}} +=== checks === + +DETERMINISTIC CHECKS — run by code over the recorded actions, before you +saw this case. The verdicts below are computed, not judged. + +They exist because a world can be reached without following the prose: a +walk that ignored the instructions and wrote the expected files directly +lands the same state as one that earned it. Only the order of what was +done separates them. + +Take each verdict as given. A FAIL here is a failure of the case, and you +may not overturn it, explain it away, or weigh it against the walk — the +walk is the walker's account, and these are not. Report every one in your +verdict. + +{{checks}} + === substitutions === HARNESS SUBSTITUTIONS the case armed for this walk. Where the recorded diff --git a/tests/prose/run.cjs b/tests/prose/run.cjs index bc6c4b251..166f0b115 100644 --- a/tests/prose/run.cjs +++ b/tests/prose/run.cjs @@ -22,6 +22,7 @@ const { execFileSync } = require('child_process'); const cases = require('./lib/cases.cjs'); const worlds = require('./lib/worlds.cjs'); const prompts = require('./lib/prompts.cjs'); +const invariants = require('./lib/invariants.cjs'); const ROOT = cases.ROOT; @@ -145,6 +146,7 @@ function cmdAssert(argv) { let world = null; let actions = null; let walk = null; + let checks = null; if (c.hasFixtureState) { const dir = requireWorld(argv, c); const delta = worlds.diffWorld(c.id, dir, c.worldMode === 'claims'); @@ -167,6 +169,7 @@ function cmdAssert(argv) { + 'registry has reloaded since it changed, and that this project is trusted ' + '(hasTrustDialogAccepted). Do not judge this run.'); } + checks = invariants.format(invariants.check(worlds.readActionRows(dir), c.invariants)); walk = worlds.readWalkLog(dir); // Same stance as the action log: the walk is harness-captured, so its // absence is a broken hook, not a quiet walk. Judging without it would @@ -186,7 +189,7 @@ function cmdAssert(argv) { }).join('\n') : null; process.stdout.write( - prompts.asserterPrompt({ expected: c.assert, world, actions, walk, substitutions }), + prompts.asserterPrompt({ expected: c.assert, world, actions, checks, walk, substitutions }), ); } diff --git a/tests/scripts/test-prose-invariants.cjs b/tests/scripts/test-prose-invariants.cjs new file mode 100644 index 000000000..1a0577a58 --- /dev/null +++ b/tests/scripts/test-prose-invariants.cjs @@ -0,0 +1,156 @@ +'use strict'; + +// Deterministic checks over a walk's recorded actions. +// +// These are the assertions no agent makes, so they are the ones that +// cannot drift between runs. What they mainly buy is skip-detection: a +// walker that ignored the prose and wrote the expected files directly +// produces the right world, and only the order of what it did gives it +// away. + +const { describe, it } = require('node:test'); +const assert = require('node:assert'); + +const invariants = require('../prose/lib/invariants.cjs'); + +const ENGINE = 'cd . && node .claude/skills/workflow-engine/scripts/engine.cjs'; + +/** Rows in the shape worlds.readActionRows produces. */ +const bash = (detail) => ({ event: 'PreToolUse', tool: 'Bash', detail }); +const wrote = (detail) => ({ event: 'PreToolUse', tool: 'Write', detail }); +const read = (detail) => ({ event: 'PreToolUse', tool: 'Read', detail }); + +const verdicts = (results) => Object.fromEntries(results.map((r) => [r.name, r.ok])); + +describe('engine_before_write — the skip-to-the-end detector', () => { + const declared = { engine_before_write: true }; + + it('passes a walk that consulted the engine before writing state', () => { + const rows = [ + read('./.claude/skills/workflow-implementation-process/SKILL.md'), + bash(`${ENGINE} task init pay pay`), + wrote('./.workflows/.state/environment-setup.md'), + ]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); + + it('fails a walk that wrote workflow state having never called the engine', () => { + const rows = [ + read('./.claude/skills/workflow-implementation-process/SKILL.md'), + wrote('./.workflows/pay/discussion/pay.md'), + ]; + const [result] = invariants.check(rows, declared); + assert.equal(result.ok, false); + assert.match(result.detail, /never called the engine/); + }); + + it('fails a walk that wrote first and called the engine afterwards', () => { + const rows = [ + wrote('./.workflows/pay/discussion/pay.md'), + bash(`${ENGINE} discussion-map set pay pay x decided`), + ]; + const [result] = invariants.check(rows, declared); + assert.equal(result.ok, false); + assert.match(result.detail, /before any engine call/); + }); + + it('passes a read-only walk, which writes no state to justify', () => { + const rows = [read('./.claude/skills/workflow-start/SKILL.md'), bash(`${ENGINE} boot`)]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); + + it('ignores writes outside .workflows — a walk may touch a scratch file', () => { + const rows = [wrote('./notes.txt')]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); + + it('counts a gateway call as consulting state, as the prose does', () => { + const rows = [ + bash('cd . && node .claude/skills/workflow-start/scripts/gateway.cjs view'), + wrote('./.workflows/pay/manifest.json'), + ]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); +}); + +describe('calls_include / calls_exclude', () => { + it('fails when a command the case requires never ran', () => { + const rows = [bash(`${ENGINE} boot`)]; + const [result] = invariants.check(rows, { calls_include: ['task init', 'boot'] }); + assert.equal(result.ok, false); + assert.match(result.detail, /never ran: task init/); + assert.ok(!result.detail.includes('boot,'), 'and names only what is missing'); + }); + + it('passes when every required command ran', () => { + const rows = [bash(`${ENGINE} task init pay pay`), bash(`${ENGINE} boot`)]; + assert.equal(invariants.check(rows, { calls_include: ['task init', 'boot'] })[0].ok, true); + }); + + it('fails when a forbidden command ran — the walk went too far', () => { + const rows = [bash(`${ENGINE} task init pay pay`), bash(`${ENGINE} task start pay pay`)]; + const [result] = invariants.check(rows, { calls_exclude: ['task start'] }); + assert.equal(result.ok, false); + assert.match(result.detail, /task start/); + }); + + it('passes when no forbidden command ran', () => { + const rows = [bash(`${ENGINE} task init pay pay`)]; + assert.equal(invariants.check(rows, { calls_exclude: ['task start'] })[0].ok, true); + }); + + it('reads commands only — a file whose name matches is not a call', () => { + const rows = [read('./.claude/skills/workflow-engine/task-start-notes.md')]; + assert.equal(invariants.check(rows, { calls_exclude: ['task start'] })[0].ok, true); + assert.equal(invariants.check(rows, { calls_include: ['task start'] })[0].ok, false); + }); +}); + +describe('declaration', () => { + it('runs every declared check, and only those', () => { + const rows = [bash(`${ENGINE} boot`)]; + const results = invariants.check(rows, { engine_before_write: true, calls_include: ['boot'] }); + assert.deepEqual(verdicts(results), { engine_before_write: true, calls_include: true }); + }); + + it('runs nothing when a case declares nothing', () => { + assert.deepEqual(invariants.check([bash('x')], null), []); + assert.deepEqual(invariants.check([bash('x')], {}), []); + }); + + it('skips a check switched off rather than treating it as declared', () => { + assert.deepEqual(invariants.check([bash('x')], { engine_before_write: false }), []); + }); + + it('formats verdicts computed-first, so they read as facts', () => { + const out = invariants.format(invariants.check([wrote('./.workflows/a.md')], { engine_before_write: true })); + assert.match(out, /^FAIL {2}engine_before_write — /); + }); + + it('formats nothing when there is nothing to report', () => { + assert.equal(invariants.format([]), null); + }); +}); + +describe('declaration validation', () => { + it('accepts an absent declaration', () => { + assert.deepEqual(invariants.declarationErrors(undefined), []); + assert.deepEqual(invariants.declarationErrors(null), []); + }); + + it('rejects an unknown check, which would otherwise pass silently', () => { + const errors = invariants.declarationErrors({ calls_includes: ['x'] }); + assert.equal(errors.length, 1); + assert.match(errors[0], /unknown invariant "calls_includes"/); + }); + + it('rejects the wrong type for each known check', () => { + assert.match(invariants.declarationErrors({ engine_before_write: 'yes' })[0], /true or false/); + assert.match(invariants.declarationErrors({ calls_include: 'task init' })[0], /array of non-empty strings/); + assert.match(invariants.declarationErrors({ calls_exclude: [''] })[0], /array of non-empty strings/); + }); + + it('rejects a declaration that is not an object', () => { + assert.match(invariants.declarationErrors(['engine_before_write'])[0], /must be an object/); + }); +}); From 73b1b54bc327f66204cbd7e0dca95112ba263929 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Sun, 26 Jul 2026 19:52:11 +0100 Subject: [PATCH 2/2] fix(prose-tests): a shell redirect writes state too MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engine_before_write looked only at the write tools, so a walk that created workflow state with `printf … > .workflows/…` was reported as having written nothing at all — the check passed by vacancy, on the one walk it most needed to examine. Found by running it: an Opus walk of implementation-picks-first-task built the setup document through the shell while a Sonnet walk of the same case used the Write tool. Same world, same delta, and only one of them was being checked. Redirects, tee, cp, mv and install into the workflow directory now count as writes. Segment separators bound the match, so a redirect elsewhere in a compound command is not mistaken for one into `.workflows`, and reading state is still reading. Co-Authored-By: Claude Opus 5 --- tests/prose/lib/invariants.cjs | 16 ++++++++--- tests/scripts/test-prose-invariants.cjs | 35 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/tests/prose/lib/invariants.cjs b/tests/prose/lib/invariants.cjs index bb664dc04..d7243e458 100644 --- a/tests/prose/lib/invariants.cjs +++ b/tests/prose/lib/invariants.cjs @@ -28,6 +28,18 @@ const ENGINE_CALL = /\/(engine|gateway)\.cjs\b/; const WRITE_TOOLS = new Set(['Write', 'Edit', 'NotebookEdit']); const WORKFLOWS = '.workflows/'; +// A file is not only written through the write tools. A walker reaches +// for the shell as readily — `printf … > .workflows/…` creates state +// just as surely, and an Opus walk was observed doing exactly that while +// a tool-only check reported that nothing had been written at all. +// Segment separators bound the match so a redirect elsewhere in a +// compound command is not mistaken for one into the workflow directory. +const SHELL_WRITE = /(>>?|\btee\b|\bcp\b|\bmv\b|\binstall\b)[^|;&]*\.workflows\//; + +function isWrite(row) { + if (WRITE_TOOLS.has(row.tool)) return row.detail.includes(WORKFLOWS); + return row.tool === 'Bash' && SHELL_WRITE.test(row.detail); +} const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude']; @@ -46,9 +58,7 @@ function engineBeforeWrite(rows) { const firstEngine = rows.findIndex( (r) => r.tool === 'Bash' && ENGINE_CALL.test(r.detail), ); - const firstWrite = rows.findIndex( - (r) => WRITE_TOOLS.has(r.tool) && r.detail.includes(WORKFLOWS), - ); + const firstWrite = rows.findIndex(isWrite); if (firstWrite === -1) { return { ok: true, detail: 'no workflow state was written' }; } diff --git a/tests/scripts/test-prose-invariants.cjs b/tests/scripts/test-prose-invariants.cjs index 1a0577a58..4d79cd855 100644 --- a/tests/scripts/test-prose-invariants.cjs +++ b/tests/scripts/test-prose-invariants.cjs @@ -64,6 +64,41 @@ describe('engine_before_write — the skip-to-the-end detector', () => { assert.equal(invariants.check(rows, declared)[0].ok, true); }); + it('counts a shell redirect as a write — a walker reaches for it readily', () => { + // Observed live: an Opus walk created the setup document with printf + // rather than the Write tool, and a tool-only check reported that + // nothing had been written at all. + const rows = [ + bash("cd . && mkdir -p .workflows/.state && printf 'No special setup required.\n' > .workflows/.state/environment-setup.md"), + ]; + const [result] = invariants.check(rows, declared); + assert.equal(result.ok, false); + assert.match(result.detail, /never called the engine/); + }); + + it('counts tee, cp and mv into the workflow directory', () => { + for (const command of [ + 'cd . && echo x | tee .workflows/a.md', + 'cd . && cp /tmp/a.md .workflows/a.md', + 'cd . && mv /tmp/a.md .workflows/a.md', + ]) { + assert.equal(invariants.check([bash(command)], declared)[0].ok, false, command); + } + }); + + it('does not mistake reading workflow state for writing it', () => { + const rows = [ + bash('cd . && cat .workflows/pay/manifest.json 2>&1'), + bash('cd . && grep -r pay .workflows/ | head -5'), + ]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); + + it('does not let a redirect elsewhere in a compound command count', () => { + const rows = [bash('cd . && cat notes.md > /tmp/out.txt && ls .workflows/')]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); + it('counts a gateway call as consulting state, as the prose does', () => { const rows = [ bash('cd . && node .claude/skills/workflow-start/scripts/gateway.cjs view'),