diff --git a/.claude/agents/prose-asserter.md b/.claude/agents/prose-asserter.md index 1541c6592..ad27af794 100644 --- a/.claude/agents/prose-asserter.md +++ b/.claude/agents/prose-asserter.md @@ -87,6 +87,10 @@ Return exactly this and nothing else: as recorded. If no such line exists, say `unrecorded`. A verdict is only as trustworthy as the model that produced the walk, so it is stated before anything is judged, never inferred, and never omitted. +0b. **Checks** — every deterministic check and its computed verdict, + repeated as given. Omit the section only when the prompt carried none. + A FAILing check fails the case on its own, whatever the path and world + show; it was decided in code and is not yours to revisit. 1. **Path** — one line per expected step: PASS or FAIL, each PASS quoting the line of the walk that shows it, each FAIL stating what is missing or contradicting. @@ -97,4 +101,5 @@ Return exactly this and nothing else: `DEVIATION` in the walk. Each is a finding in its own right, even when everything else passed. 4. **VERDICT** — PASS, FAIL, or INVALID WALK, then one sentence on what - it means for the prose. + it means for the prose. Any FAILing deterministic check makes this + FAIL. diff --git a/package.json b/package.json index 50661ef01..29d011eb5 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "build": "node build/knowledge.build.js", "knowledge:repl": "node scripts/knowledge-repl.js", "typecheck": "tsc -p tsconfig.json", - "test": "node --test tests/scripts/test-render.cjs tests/scripts/test-engine-gateway.cjs tests/scripts/test-engine-epic-projections.cjs tests/scripts/test-engine-start-projections.cjs tests/scripts/test-engine-workunit-projections.cjs tests/scripts/test-engine-discussion-map.cjs tests/scripts/test-engine-discovery-map.cjs tests/scripts/test-engine-tasks.cjs tests/scripts/test-engine-render-surfaces.cjs tests/scripts/test-engine-transactions.cjs tests/scripts/test-engine-workunit-create.cjs tests/scripts/test-engine-workunit-pivot.cjs tests/scripts/test-engine-workunit-absorb.cjs tests/scripts/test-engine-workunit-promote.cjs tests/scripts/test-engine-discovery-session.cjs tests/scripts/test-engine-cache.cjs tests/scripts/test-engine-manifest-io.cjs tests/scripts/test-engine-manifest-fields.cjs tests/scripts/test-engine-boot.cjs tests/scripts/test-migration-orchestrator.cjs tests/scripts/test-migration-047.cjs tests/scripts/test-migration-048.cjs tests/scripts/test-migration-049.cjs tests/scripts/test-migration-050.cjs tests/scripts/test-migration-051.cjs tests/scripts/test-migration-052.cjs tests/scripts/test-engine-specification-projections.cjs tests/scripts/test-engine-discovery-projections.cjs tests/scripts/test-gateway-for-start.cjs tests/scripts/test-gateway-for-specification.cjs tests/scripts/test-gateway-for-workflow-continue-epic.cjs tests/scripts/test-gateway-for-workflow-continue-feature.cjs tests/scripts/test-gateway-for-workflow-continue-bugfix.cjs tests/scripts/test-gateway-for-workflow-continue-quickfix.cjs tests/scripts/test-gateway-for-workflow-continue-cross-cutting.cjs tests/scripts/test-gateway-for-discovery-skill.cjs tests/scripts/test-gateway-for-bridge.cjs tests/scripts/test-reads-derivations.cjs tests/scripts/test-ensure-discovery-item.cjs tests/scripts/test-legacy-research-split.cjs tests/scripts/test-knowledge-chunker.cjs tests/scripts/test-knowledge-config.cjs tests/scripts/test-knowledge-discovery.cjs tests/scripts/test-knowledge-embeddings.cjs tests/scripts/test-knowledge-integration.cjs tests/scripts/test-knowledge-openai.cjs tests/scripts/test-knowledge-openai-compatible.cjs tests/scripts/test-knowledge-openai-integration.cjs tests/scripts/test-knowledge-progress.cjs tests/scripts/test-knowledge-rerank.cjs tests/scripts/test-knowledge-retry.cjs tests/scripts/test-knowledge-setup-forms.cjs tests/scripts/test-knowledge-store.cjs tests/scripts/test-conventions-lint.cjs tests/scripts/test-pipeline-simulation.cjs tests/scripts/test-engine-agent-state.cjs tests/scripts/test-prose-record-action.cjs tests/scripts/test-prose-cases.cjs tests/scripts/test-prose-snapshots.cjs", + "test": "node --test tests/scripts/test-render.cjs tests/scripts/test-engine-gateway.cjs tests/scripts/test-engine-epic-projections.cjs tests/scripts/test-engine-start-projections.cjs tests/scripts/test-engine-workunit-projections.cjs tests/scripts/test-engine-discussion-map.cjs tests/scripts/test-engine-discovery-map.cjs tests/scripts/test-engine-tasks.cjs tests/scripts/test-engine-render-surfaces.cjs tests/scripts/test-engine-transactions.cjs tests/scripts/test-engine-workunit-create.cjs tests/scripts/test-engine-workunit-pivot.cjs tests/scripts/test-engine-workunit-absorb.cjs tests/scripts/test-engine-workunit-promote.cjs tests/scripts/test-engine-discovery-session.cjs tests/scripts/test-engine-cache.cjs tests/scripts/test-engine-manifest-io.cjs tests/scripts/test-engine-manifest-fields.cjs tests/scripts/test-engine-boot.cjs tests/scripts/test-migration-orchestrator.cjs tests/scripts/test-migration-047.cjs tests/scripts/test-migration-048.cjs tests/scripts/test-migration-049.cjs tests/scripts/test-migration-050.cjs tests/scripts/test-migration-051.cjs tests/scripts/test-migration-052.cjs tests/scripts/test-engine-specification-projections.cjs tests/scripts/test-engine-discovery-projections.cjs tests/scripts/test-gateway-for-start.cjs tests/scripts/test-gateway-for-specification.cjs tests/scripts/test-gateway-for-workflow-continue-epic.cjs tests/scripts/test-gateway-for-workflow-continue-feature.cjs tests/scripts/test-gateway-for-workflow-continue-bugfix.cjs tests/scripts/test-gateway-for-workflow-continue-quickfix.cjs tests/scripts/test-gateway-for-workflow-continue-cross-cutting.cjs tests/scripts/test-gateway-for-discovery-skill.cjs tests/scripts/test-gateway-for-bridge.cjs tests/scripts/test-reads-derivations.cjs tests/scripts/test-ensure-discovery-item.cjs tests/scripts/test-legacy-research-split.cjs tests/scripts/test-knowledge-chunker.cjs tests/scripts/test-knowledge-config.cjs tests/scripts/test-knowledge-discovery.cjs tests/scripts/test-knowledge-embeddings.cjs tests/scripts/test-knowledge-integration.cjs tests/scripts/test-knowledge-openai.cjs tests/scripts/test-knowledge-openai-compatible.cjs tests/scripts/test-knowledge-openai-integration.cjs tests/scripts/test-knowledge-progress.cjs tests/scripts/test-knowledge-rerank.cjs tests/scripts/test-knowledge-retry.cjs tests/scripts/test-knowledge-setup-forms.cjs tests/scripts/test-knowledge-store.cjs tests/scripts/test-conventions-lint.cjs tests/scripts/test-pipeline-simulation.cjs tests/scripts/test-engine-agent-state.cjs tests/scripts/test-prose-invariants.cjs tests/scripts/test-prose-record-action.cjs tests/scripts/test-prose-cases.cjs tests/scripts/test-prose-snapshots.cjs", "test:migrations": "for t in tests/scripts/test-migration-*.sh; do bash $t || exit 1; done", "test:cli": "for t in tests/scripts/test-engine-manifest.sh tests/scripts/test-inbox-promotion.sh tests/scripts/test-knowledge-cli.sh tests/scripts/test-knowledge-build.sh; do bash $t || exit 1; done" }, diff --git a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json index f5bc80e4b..85b404f92 100644 --- a/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/discussion-entry-seeds-from-carrier/case.json @@ -1,7 +1,10 @@ { - "origin": "feature mainline — single-phase work seeds from its durable carrier", + "origin": "feature mainline \u2014 single-phase work seeds from its durable carrier", "files": [ "skills/workflow-discussion-entry/SKILL.md", "skills/workflow-shared/references/ensure-discovery-item.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/implementation-picks-first-task/assert.md b/tests/prose/cases/implementation-picks-first-task/assert.md index 97e435949..0445421ef 100644 --- a/tests/prose/cases/implementation-picks-first-task/assert.md +++ b/tests/prose/cases/implementation-picks-first-task/assert.md @@ -16,4 +16,3 @@ The prose should have taken this path: Further claims: - the setup document is written once, by the setup step and nowhere else -- no task is started: the walk stops before any task is picked up diff --git a/tests/prose/cases/implementation-picks-first-task/case.json b/tests/prose/cases/implementation-picks-first-task/case.json index 20268377d..cf3393720 100644 --- a/tests/prose/cases/implementation-picks-first-task/case.json +++ b/tests/prose/cases/implementation-picks-first-task/case.json @@ -1,5 +1,5 @@ { - "origin": "feature mainline — entry through resume detection to the first task", + "origin": "feature mainline \u2014 entry through resume detection to the first task", "files": [ "skills/workflow-implementation-entry/SKILL.md", "skills/workflow-implementation-entry/references/validate-phase.md", @@ -8,6 +8,16 @@ "skills/workflow-implementation-process/references/environment-setup.md" ], "answers": [ - "none — no special setup is needed" - ] + "none \u2014 no special setup is needed" + ], + "invariants": { + "engine_before_write": true, + "calls_include": [ + "task init" + ], + "calls_exclude": [ + "task start", + "task next" + ] + } } diff --git a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json index 3eb224b9c..71758ea1b 100644 --- a/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json +++ b/tests/prose/cases/investigation-entry-seeds-from-carrier/case.json @@ -1,7 +1,10 @@ { - "origin": "bugfix mainline — a bug shaped in discovery is never re-interrogated", + "origin": "bugfix mainline \u2014 a bug shaped in discovery is never re-interrogated", "files": [ "skills/workflow-investigation-entry/SKILL.md", "skills/workflow-investigation-entry/references/invoke-skill.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/planning-entry-asks-for-late-context/case.json b/tests/prose/cases/planning-entry-asks-for-late-context/case.json index e5e2d1d46..83b632fd8 100644 --- a/tests/prose/cases/planning-entry-asks-for-late-context/case.json +++ b/tests/prose/cases/planning-entry-asks-for-late-context/case.json @@ -1,5 +1,5 @@ { - "origin": "feature mainline — planning offers a last chance to add context", + "origin": "feature mainline \u2014 planning offers a last chance to add context", "files": [ "skills/workflow-planning-entry/SKILL.md", "skills/workflow-planning-entry/references/validate-spec.md", @@ -7,6 +7,9 @@ "skills/workflow-planning-entry/references/cross-cutting-context.md" ], "answers": [ - "continue — nothing has changed since the specification" - ] + "continue \u2014 nothing has changed since the specification" + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/review-entry-fresh/case.json b/tests/prose/cases/review-entry-fresh/case.json index c67cada98..e19d4dfa4 100644 --- a/tests/prose/cases/review-entry-fresh/case.json +++ b/tests/prose/cases/review-entry-fresh/case.json @@ -1,7 +1,10 @@ { - "origin": "feature mainline — review refuses to start until delivery is complete", + "origin": "feature mainline \u2014 review refuses to start until delivery is complete", "files": [ "skills/workflow-review-entry/SKILL.md", "skills/workflow-review-entry/references/validate-phase.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/spec-entry-from-discussion/case.json b/tests/prose/cases/spec-entry-from-discussion/case.json index f380e57ce..f7f4aa938 100644 --- a/tests/prose/cases/spec-entry-from-discussion/case.json +++ b/tests/prose/cases/spec-entry-from-discussion/case.json @@ -1,9 +1,12 @@ { - "origin": "feature mainline — a feature's spec draws on its discussion", + "origin": "feature mainline \u2014 a feature's spec draws on its discussion", "files": [ "skills/workflow-specification-entry/SKILL.md", "skills/workflow-specification-entry/references/validate-source.md", "skills/workflow-specification-entry/references/validate-phase.md", "skills/workflow-specification-entry/references/invoke-skill.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/spec-entry-from-investigation/case.json b/tests/prose/cases/spec-entry-from-investigation/case.json index 3eed6746c..e0849f435 100644 --- a/tests/prose/cases/spec-entry-from-investigation/case.json +++ b/tests/prose/cases/spec-entry-from-investigation/case.json @@ -1,8 +1,11 @@ { - "origin": "bugfix mainline — the same entry skill, sourced from an investigation", + "origin": "bugfix mainline \u2014 the same entry skill, sourced from an investigation", "files": [ "skills/workflow-specification-entry/SKILL.md", "skills/workflow-specification-entry/references/validate-source.md", "skills/workflow-specification-entry/references/invoke-skill.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/cases/start-lists-active-work/case.json b/tests/prose/cases/start-lists-active-work/case.json index 59711da9f..c4b7667c2 100644 --- a/tests/prose/cases/start-lists-active-work/case.json +++ b/tests/prose/cases/start-lists-active-work/case.json @@ -1,6 +1,9 @@ { - "origin": "feature mainline — the entry skill surfaces work without touching it", + "origin": "feature mainline \u2014 the entry skill surfaces work without touching it", "files": [ "skills/workflow-start/SKILL.md" - ] + ], + "invariants": { + "engine_before_write": true + } } diff --git a/tests/prose/lib/cases.cjs b/tests/prose/lib/cases.cjs index a4b3d79df..e26d82620 100644 --- a/tests/prose/lib/cases.cjs +++ b/tests/prose/lib/cases.cjs @@ -9,7 +9,8 @@ // // case.json the values code branches on: // { origin, files[], answers[], stubs{name: trigger}, -// world: "claims" (optional) } +// world: "claims" (optional), +// invariants{} (optional) — see lib/invariants.cjs } // fixture.md optional. Prose describing the starting world — // what has already happened, where the session // stands. Given to the walker. @@ -45,6 +46,7 @@ const fs = require('fs'); const path = require('path'); +const invariants = require('./invariants.cjs'); const ROOT = path.join(__dirname, '../../..'); const PROSE_DIR = path.join(ROOT, 'tests/prose'); @@ -104,6 +106,7 @@ function loadCase(id) { }), answers: meta.answers || [], worldMode: meta.world || null, + invariants: meta.invariants || null, stubs: Object.entries(meta.stubs || {}).map(([name, trigger]) => ({ name, trigger })), situation: readIf(dir, FILES.situation), act: readIf(dir, FILES.act), @@ -189,6 +192,12 @@ function validateCorpus(cases) { if (c.worldMode === 'claims' && !c.hasFixtureState) { errors.push(`${at}: world "claims" needs a world to mutate`); } + for (const e of invariants.declarationErrors(c.invariants)) { + errors.push(`${at}: ${FILES.meta} ${e}`); + } + if (c.invariants && !c.hasFixtureState) { + errors.push(`${at}: invariants without ${FILES.fixtureState} — no walk to check`); + } if (c.hasAssertionState && !c.hasFixtureState) { errors.push(`${at}: ${FILES.assertionState} without ${FILES.fixtureState} — nothing to act on`); } diff --git a/tests/prose/lib/invariants.cjs b/tests/prose/lib/invariants.cjs new file mode 100644 index 000000000..d7243e458 --- /dev/null +++ b/tests/prose/lib/invariants.cjs @@ -0,0 +1,143 @@ +'use strict'; + +// Deterministic checks over the recorded actions. +// +// The world delta proves the outcome. It cannot prove the outcome was +// reached by following the prose: a walker that ignored every instruction +// and wrote the expected files directly lands the same delta. Only the +// order of what it actually did separates the two, and that is recorded. +// +// These checks read that record in code and decide before any agent sees +// the case. A judgement an agent never makes is a judgement that cannot +// drift — which is the whole reason they exist, and why they are worth +// having even though they can only ever cover the coarse shape of a walk. +// +// Declared per case in case.json, hand-written like every other part of a +// case: +// +// "invariants": { +// "engine_before_write": true, // no .workflows write out of nowhere +// "calls_include": ["task init"], // these commands must have run +// "calls_exclude": ["task start"] // these must not have +// } +// +// Deliberately not a pinned call sequence. A recorded-and-replayed +// sequence would freeze whatever the walker happened to do, which is how +// a test starts certifying broken prose instead of catching it. + +const ENGINE_CALL = /\/(engine|gateway)\.cjs\b/; +const WRITE_TOOLS = new Set(['Write', 'Edit', 'NotebookEdit']); +const WORKFLOWS = '.workflows/'; +// A file is not only written through the write tools. A walker reaches +// for the shell as readily — `printf … > .workflows/…` creates state +// just as surely, and an Opus walk was observed doing exactly that while +// a tool-only check reported that nothing had been written at all. +// Segment separators bound the match so a redirect elsewhere in a +// compound command is not mistaken for one into the workflow directory. +const SHELL_WRITE = /(>>?|\btee\b|\bcp\b|\bmv\b|\binstall\b)[^|;&]*\.workflows\//; + +function isWrite(row) { + if (WRITE_TOOLS.has(row.tool)) return row.detail.includes(WORKFLOWS); + return row.tool === 'Bash' && SHELL_WRITE.test(row.detail); +} + +const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude']; + +/** The commands the walk ran, in order. */ +function commands(rows) { + return rows.filter((r) => r.tool === 'Bash' && r.event === 'PreToolUse').map((r) => r.detail); +} + +/** + * A write the prose could only have reached by consulting state first. + * Reading a file is not enough — the engine is how a workflow skill + * learns anything, so a walk that writes workflow state having never + * called it did not get there by following the prose. + */ +function engineBeforeWrite(rows) { + const firstEngine = rows.findIndex( + (r) => r.tool === 'Bash' && ENGINE_CALL.test(r.detail), + ); + const firstWrite = rows.findIndex(isWrite); + if (firstWrite === -1) { + return { ok: true, detail: 'no workflow state was written' }; + } + if (firstEngine === -1) { + return { ok: false, detail: `wrote ${rows[firstWrite].detail} having never called the engine` }; + } + if (firstEngine > firstWrite) { + return { + ok: false, + detail: `wrote ${rows[firstWrite].detail} before any engine call`, + }; + } + return { ok: true, detail: 'every workflow write followed an engine call' }; +} + +function callsInclude(rows, wanted) { + const ran = commands(rows); + const missing = wanted.filter((w) => !ran.some((c) => c.includes(w))); + return missing.length + ? { ok: false, detail: `never ran: ${missing.join(', ')}` } + : { ok: true, detail: `ran all of: ${wanted.join(', ')}` }; +} + +function callsExclude(rows, forbidden) { + const ran = commands(rows); + const found = forbidden.filter((f) => ran.some((c) => c.includes(f))); + return found.length + ? { ok: false, detail: `ran what it should not have: ${found.join(', ')}` } + : { ok: true, detail: `ran none of: ${forbidden.join(', ')}` }; +} + +/** + * Run a case's declared invariants against its recorded actions. + * Returns one result per declared check, in declaration order. + */ +function check(rows, declared) { + if (!declared) return []; + const results = []; + if (declared.engine_before_write) { + results.push({ name: 'engine_before_write', ...engineBeforeWrite(rows) }); + } + if (declared.calls_include && declared.calls_include.length) { + results.push({ name: 'calls_include', ...callsInclude(rows, declared.calls_include) }); + } + if (declared.calls_exclude && declared.calls_exclude.length) { + results.push({ name: 'calls_exclude', ...callsExclude(rows, declared.calls_exclude) }); + } + return results; +} + +/** The checks as the asserter reads them — verdict first, computed by code. */ +function format(results) { + if (!results.length) return null; + return results + .map((r) => `${r.ok ? 'PASS' : 'FAIL'} ${r.name} — ${r.detail}`) + .join('\n'); +} + +/** Shape errors in a case's declaration, for corpus validation. */ +function declarationErrors(declared) { + if (declared === undefined || declared === null) return []; + const errors = []; + if (typeof declared !== 'object' || Array.isArray(declared)) { + return ['invariants must be an object']; + } + for (const key of Object.keys(declared)) { + if (!NAMES.includes(key)) errors.push(`unknown invariant "${key}" (known: ${NAMES.join(', ')})`); + } + if ('engine_before_write' in declared && typeof declared.engine_before_write !== 'boolean') { + errors.push('engine_before_write must be true or false'); + } + for (const key of ['calls_include', 'calls_exclude']) { + if (!(key in declared)) continue; + const value = declared[key]; + if (!Array.isArray(value) || value.some((v) => typeof v !== 'string' || !v.trim())) { + errors.push(`${key} must be an array of non-empty strings`); + } + } + return errors; +} + +module.exports = { check, format, declarationErrors, NAMES }; diff --git a/tests/prose/lib/prompts.cjs b/tests/prose/lib/prompts.cjs index b6a7cb1a4..c15a26f11 100644 --- a/tests/prose/lib/prompts.cjs +++ b/tests/prose/lib/prompts.cjs @@ -75,11 +75,12 @@ function walkerPrompt({ worldDir, root, situation, task, scope, stubs, answers } return `${parts.join('\n\n')}\n`; } -function asserterPrompt({ expected, world, actions, walk, substitutions }) { +function asserterPrompt({ expected, world, actions, checks, walk, substitutions }) { const t = loadTemplate('asserter'); const parts = [fill(t.main, { expected })]; if (world) parts.push(fill(t.world, { expecting: world.expecting, delta: world.delta })); if (actions) parts.push(fill(t.actions, { actions })); + if (checks) parts.push(fill(t.checks, { checks })); if (substitutions) parts.push(fill(t.substitutions, { substitutions })); if (walk) parts.push(fill(t.walk, { walk })); return `${parts.join('\n\n')}\n`; diff --git a/tests/prose/lib/worlds.cjs b/tests/prose/lib/worlds.cjs index 6cc652a77..14b570076 100644 --- a/tests/prose/lib/worlds.cjs +++ b/tests/prose/lib/worlds.cjs @@ -331,6 +331,19 @@ function destroyWorld(dir) { fs.rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } +/** The recorded actions as rows, for checks that run in code. */ +function readActionRows(worldDir) { + const file = path.join(worldDir, ACTION_LOG); + if (!fs.existsSync(file)) return []; + return fs.readFileSync(file, 'utf8') + .split('\n') + .filter(Boolean) + .map((l) => { + const [event, tool, detail = '', outcome = null, output = null] = l.split('\t'); + return { event, tool, detail, outcome, output }; + }); +} + /** What the walk actually did, as recorded by the walker's hook. */ function readActionLog(worldDir) { const file = path.join(worldDir, ACTION_LOG); @@ -356,7 +369,7 @@ function readWalkLog(worldDir) { module.exports = { ROOT, ENGINE, KNOWLEDGE, MAINLINES_DIR, WORLD_PREFIX, - ACTION_LOG, readActionLog, WALK_LOG, readWalkLog, + ACTION_LOG, readActionLog, readActionRows, WALK_LOG, readWalkLog, runRecipe, collectTree, readSnapshot, snapshotDir, recipeHash, storedHash, writeSnapshot, verifySnapshot, diffWorld, buildWorld, destroyWorld, }; diff --git a/tests/prose/prompts/asserter.md b/tests/prose/prompts/asserter.md index ff8f54bc6..a012670c5 100644 --- a/tests/prose/prompts/asserter.md +++ b/tests/prose/prompts/asserter.md @@ -36,6 +36,23 @@ wins and the discrepancy is itself worth reporting. {{actions}} +=== checks === + +DETERMINISTIC CHECKS — run by code over the recorded actions, before you +saw this case. The verdicts below are computed, not judged. + +They exist because a world can be reached without following the prose: a +walk that ignored the instructions and wrote the expected files directly +lands the same state as one that earned it. Only the order of what was +done separates them. + +Take each verdict as given. A FAIL here is a failure of the case, and you +may not overturn it, explain it away, or weigh it against the walk — the +walk is the walker's account, and these are not. Report every one in your +verdict. + +{{checks}} + === substitutions === HARNESS SUBSTITUTIONS the case armed for this walk. Where the recorded diff --git a/tests/prose/run.cjs b/tests/prose/run.cjs index bc6c4b251..166f0b115 100644 --- a/tests/prose/run.cjs +++ b/tests/prose/run.cjs @@ -22,6 +22,7 @@ const { execFileSync } = require('child_process'); const cases = require('./lib/cases.cjs'); const worlds = require('./lib/worlds.cjs'); const prompts = require('./lib/prompts.cjs'); +const invariants = require('./lib/invariants.cjs'); const ROOT = cases.ROOT; @@ -145,6 +146,7 @@ function cmdAssert(argv) { let world = null; let actions = null; let walk = null; + let checks = null; if (c.hasFixtureState) { const dir = requireWorld(argv, c); const delta = worlds.diffWorld(c.id, dir, c.worldMode === 'claims'); @@ -167,6 +169,7 @@ function cmdAssert(argv) { + 'registry has reloaded since it changed, and that this project is trusted ' + '(hasTrustDialogAccepted). Do not judge this run.'); } + checks = invariants.format(invariants.check(worlds.readActionRows(dir), c.invariants)); walk = worlds.readWalkLog(dir); // Same stance as the action log: the walk is harness-captured, so its // absence is a broken hook, not a quiet walk. Judging without it would @@ -186,7 +189,7 @@ function cmdAssert(argv) { }).join('\n') : null; process.stdout.write( - prompts.asserterPrompt({ expected: c.assert, world, actions, walk, substitutions }), + prompts.asserterPrompt({ expected: c.assert, world, actions, checks, walk, substitutions }), ); } diff --git a/tests/scripts/test-prose-invariants.cjs b/tests/scripts/test-prose-invariants.cjs new file mode 100644 index 000000000..4d79cd855 --- /dev/null +++ b/tests/scripts/test-prose-invariants.cjs @@ -0,0 +1,191 @@ +'use strict'; + +// Deterministic checks over a walk's recorded actions. +// +// These are the assertions no agent makes, so they are the ones that +// cannot drift between runs. What they mainly buy is skip-detection: a +// walker that ignored the prose and wrote the expected files directly +// produces the right world, and only the order of what it did gives it +// away. + +const { describe, it } = require('node:test'); +const assert = require('node:assert'); + +const invariants = require('../prose/lib/invariants.cjs'); + +const ENGINE = 'cd . && node .claude/skills/workflow-engine/scripts/engine.cjs'; + +/** Rows in the shape worlds.readActionRows produces. */ +const bash = (detail) => ({ event: 'PreToolUse', tool: 'Bash', detail }); +const wrote = (detail) => ({ event: 'PreToolUse', tool: 'Write', detail }); +const read = (detail) => ({ event: 'PreToolUse', tool: 'Read', detail }); + +const verdicts = (results) => Object.fromEntries(results.map((r) => [r.name, r.ok])); + +describe('engine_before_write — the skip-to-the-end detector', () => { + const declared = { engine_before_write: true }; + + it('passes a walk that consulted the engine before writing state', () => { + const rows = [ + read('./.claude/skills/workflow-implementation-process/SKILL.md'), + bash(`${ENGINE} task init pay pay`), + wrote('./.workflows/.state/environment-setup.md'), + ]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); + + it('fails a walk that wrote workflow state having never called the engine', () => { + const rows = [ + read('./.claude/skills/workflow-implementation-process/SKILL.md'), + wrote('./.workflows/pay/discussion/pay.md'), + ]; + const [result] = invariants.check(rows, declared); + assert.equal(result.ok, false); + assert.match(result.detail, /never called the engine/); + }); + + it('fails a walk that wrote first and called the engine afterwards', () => { + const rows = [ + wrote('./.workflows/pay/discussion/pay.md'), + bash(`${ENGINE} discussion-map set pay pay x decided`), + ]; + const [result] = invariants.check(rows, declared); + assert.equal(result.ok, false); + assert.match(result.detail, /before any engine call/); + }); + + it('passes a read-only walk, which writes no state to justify', () => { + const rows = [read('./.claude/skills/workflow-start/SKILL.md'), bash(`${ENGINE} boot`)]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); + + it('ignores writes outside .workflows — a walk may touch a scratch file', () => { + const rows = [wrote('./notes.txt')]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); + + it('counts a shell redirect as a write — a walker reaches for it readily', () => { + // Observed live: an Opus walk created the setup document with printf + // rather than the Write tool, and a tool-only check reported that + // nothing had been written at all. + const rows = [ + bash("cd . && mkdir -p .workflows/.state && printf 'No special setup required.\n' > .workflows/.state/environment-setup.md"), + ]; + const [result] = invariants.check(rows, declared); + assert.equal(result.ok, false); + assert.match(result.detail, /never called the engine/); + }); + + it('counts tee, cp and mv into the workflow directory', () => { + for (const command of [ + 'cd . && echo x | tee .workflows/a.md', + 'cd . && cp /tmp/a.md .workflows/a.md', + 'cd . && mv /tmp/a.md .workflows/a.md', + ]) { + assert.equal(invariants.check([bash(command)], declared)[0].ok, false, command); + } + }); + + it('does not mistake reading workflow state for writing it', () => { + const rows = [ + bash('cd . && cat .workflows/pay/manifest.json 2>&1'), + bash('cd . && grep -r pay .workflows/ | head -5'), + ]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); + + it('does not let a redirect elsewhere in a compound command count', () => { + const rows = [bash('cd . && cat notes.md > /tmp/out.txt && ls .workflows/')]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); + + it('counts a gateway call as consulting state, as the prose does', () => { + const rows = [ + bash('cd . && node .claude/skills/workflow-start/scripts/gateway.cjs view'), + wrote('./.workflows/pay/manifest.json'), + ]; + assert.equal(invariants.check(rows, declared)[0].ok, true); + }); +}); + +describe('calls_include / calls_exclude', () => { + it('fails when a command the case requires never ran', () => { + const rows = [bash(`${ENGINE} boot`)]; + const [result] = invariants.check(rows, { calls_include: ['task init', 'boot'] }); + assert.equal(result.ok, false); + assert.match(result.detail, /never ran: task init/); + assert.ok(!result.detail.includes('boot,'), 'and names only what is missing'); + }); + + it('passes when every required command ran', () => { + const rows = [bash(`${ENGINE} task init pay pay`), bash(`${ENGINE} boot`)]; + assert.equal(invariants.check(rows, { calls_include: ['task init', 'boot'] })[0].ok, true); + }); + + it('fails when a forbidden command ran — the walk went too far', () => { + const rows = [bash(`${ENGINE} task init pay pay`), bash(`${ENGINE} task start pay pay`)]; + const [result] = invariants.check(rows, { calls_exclude: ['task start'] }); + assert.equal(result.ok, false); + assert.match(result.detail, /task start/); + }); + + it('passes when no forbidden command ran', () => { + const rows = [bash(`${ENGINE} task init pay pay`)]; + assert.equal(invariants.check(rows, { calls_exclude: ['task start'] })[0].ok, true); + }); + + it('reads commands only — a file whose name matches is not a call', () => { + const rows = [read('./.claude/skills/workflow-engine/task-start-notes.md')]; + assert.equal(invariants.check(rows, { calls_exclude: ['task start'] })[0].ok, true); + assert.equal(invariants.check(rows, { calls_include: ['task start'] })[0].ok, false); + }); +}); + +describe('declaration', () => { + it('runs every declared check, and only those', () => { + const rows = [bash(`${ENGINE} boot`)]; + const results = invariants.check(rows, { engine_before_write: true, calls_include: ['boot'] }); + assert.deepEqual(verdicts(results), { engine_before_write: true, calls_include: true }); + }); + + it('runs nothing when a case declares nothing', () => { + assert.deepEqual(invariants.check([bash('x')], null), []); + assert.deepEqual(invariants.check([bash('x')], {}), []); + }); + + it('skips a check switched off rather than treating it as declared', () => { + assert.deepEqual(invariants.check([bash('x')], { engine_before_write: false }), []); + }); + + it('formats verdicts computed-first, so they read as facts', () => { + const out = invariants.format(invariants.check([wrote('./.workflows/a.md')], { engine_before_write: true })); + assert.match(out, /^FAIL {2}engine_before_write — /); + }); + + it('formats nothing when there is nothing to report', () => { + assert.equal(invariants.format([]), null); + }); +}); + +describe('declaration validation', () => { + it('accepts an absent declaration', () => { + assert.deepEqual(invariants.declarationErrors(undefined), []); + assert.deepEqual(invariants.declarationErrors(null), []); + }); + + it('rejects an unknown check, which would otherwise pass silently', () => { + const errors = invariants.declarationErrors({ calls_includes: ['x'] }); + assert.equal(errors.length, 1); + assert.match(errors[0], /unknown invariant "calls_includes"/); + }); + + it('rejects the wrong type for each known check', () => { + assert.match(invariants.declarationErrors({ engine_before_write: 'yes' })[0], /true or false/); + assert.match(invariants.declarationErrors({ calls_include: 'task init' })[0], /array of non-empty strings/); + assert.match(invariants.declarationErrors({ calls_exclude: [''] })[0], /array of non-empty strings/); + }); + + it('rejects a declaration that is not an object', () => { + assert.match(invariants.declarationErrors(['engine_before_write'])[0], /must be an object/); + }); +});