diff --git a/.claude/agents/prose-orchestrator.md b/.claude/agents/prose-orchestrator.md index fee116010..f38e81fad 100644 --- a/.claude/agents/prose-orchestrator.md +++ b/.claude/agents/prose-orchestrator.md @@ -44,8 +44,7 @@ a walk. ## Steps 1. **Build the world** — `node tests/prose/run.cjs world `. The - response carries the path, or `world: null` for a structure-only case - (skip `--world` everywhere below when so). + response carries the path to the world it built. 2. **Walk** — `node tests/prose/run.cjs prompt --world `. Dispatch the **prose-walker** agent with that output as its prompt, @@ -104,6 +103,10 @@ CASE: MODEL: VERDICT: PASS | FAIL | FLAKY | INVALID | HARNESS ERROR +CHECKS: PATH: / steps passed WORLD: PASS | FAIL MARKERS: diff --git a/.claude/skills/prose-test/SKILL.md b/.claude/skills/prose-test/SKILL.md index 54b9f812b..bbc3178c6 100644 --- a/.claude/skills/prose-test/SKILL.md +++ b/.claude/skills/prose-test/SKILL.md @@ -38,8 +38,13 @@ about an expected result can leak into a later dispatch. ## Step 3: Collate Report a verdict table from what the orchestrators returned: case id, -model, verdict, path steps passed, world, markers. Quote the evidence -line for every failure. +model, verdict, deterministic checks, path steps passed, world, markers. +Quote the evidence line for every failure. + +Carry the checks through as reported. They were decided in code before +any agent saw the case, so they are the one part of a verdict that rests +on nothing's judgement — summarising them, or taking a green overall +verdict as proof they passed, puts an assumption back where a fact was. The model column comes from the harness record, not from any agent's say-so. An edited agent definition does not reach a running session until diff --git a/tests/prose/cases/smoke-start-boot-structure/act.md b/tests/prose/cases/smoke-start-boot-structure/act.md deleted file mode 100644 index cc3f6a902..000000000 --- a/tests/prose/cases/smoke-start-boot-structure/act.md +++ /dev/null @@ -1,5 +0,0 @@ -Structure-only: read the initialisation portion of -skills/workflow-start/SKILL.md — everything the skill does before any -work is shown or routed. Establish what initialisation consists of and -in what order its parts run. Stop there; do not trace the rest of the -skill. diff --git a/tests/prose/cases/smoke-start-boot-structure/assert.md b/tests/prose/cases/smoke-start-boot-structure/assert.md deleted file mode 100644 index 058974b50..000000000 --- a/tests/prose/cases/smoke-start-boot-structure/assert.md +++ /dev/null @@ -1,7 +0,0 @@ -The prose should have taken this path: - -1. casing conventions are loaded before the boot pipeline runs -2. the boot pipeline is declared mandatory — it must complete before the - skill proceeds -3. the knowledge gate is reached only after boot, and branches on what - boot reported rather than on its own checks diff --git a/tests/prose/cases/smoke-start-boot-structure/case.json b/tests/prose/cases/smoke-start-boot-structure/case.json deleted file mode 100644 index c47d0f93f..000000000 --- a/tests/prose/cases/smoke-start-boot-structure/case.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "origin": "framework smoke \u2014 a structure-only walk, no world", - "entry": "workflow-start", - "files": [ - "skills/workflow-start/SKILL.md#Boot" - ] -} diff --git a/tests/prose/lib/cases.cjs b/tests/prose/lib/cases.cjs index 371f8cc37..1293e452c 100644 --- a/tests/prose/lib/cases.cjs +++ b/tests/prose/lib/cases.cjs @@ -216,6 +216,10 @@ function validateCorpus(cases) { } } + if (!c.hasFixtureState) { + errors.push(`${at}: no ${FILES.fixtureState} — a case with no world records no ` + + 'actions and produces no delta, so nothing it claims can be answered'); + } if (!c.act) errors.push(`${at}: no ${FILES.act}`); if (!c.assert) errors.push(`${at}: no ${FILES.assert} — the expected trace is what catches a silent repair`); @@ -226,21 +230,9 @@ function validateCorpus(cases) { errors.push(`${at}: world "claims" and ${FILES.assertionState} are exclusive — ` + 'a world a recipe can build must be built'); } - if (c.worldMode === 'claims' && !c.hasFixtureState) { - errors.push(`${at}: world "claims" needs a world to mutate`); - } for (const e of invariants.declarationErrors(c.invariants)) { errors.push(`${at}: ${FILES.meta} ${e}`); } - if (c.invariants && !c.hasFixtureState) { - errors.push(`${at}: invariants without ${FILES.fixtureState} — no walk to check`); - } - if (c.hasAssertionState && !c.hasFixtureState) { - errors.push(`${at}: ${FILES.assertionState} without ${FILES.fixtureState} — nothing to act on`); - } - if (c.situation && !c.hasFixtureState) { - errors.push(`${at}: ${FILES.situation} describes a world this case does not build`); - } for (const which of ['fixtureState', 'assertionState']) { try { requireState(c.id, which); @@ -254,7 +246,6 @@ function validateCorpus(cases) { if (!s.trigger || !s.trigger.trim()) { errors.push(`${at}: stub "${s.name}" declares no trigger — the case owns the moment`); } - if (!c.hasFixtureState) errors.push(`${at}: stub "${s.name}" needs a world to fire in`); } } return errors; diff --git a/tests/prose/lib/prompts.cjs b/tests/prose/lib/prompts.cjs index c15a26f11..c1132c34c 100644 --- a/tests/prose/lib/prompts.cjs +++ b/tests/prose/lib/prompts.cjs @@ -55,11 +55,9 @@ function indent(text, by = ' ') { return text.split('\n').map((l) => (l.length ? by + l : l)).join('\n'); } -function walkerPrompt({ worldDir, root, situation, task, scope, stubs, answers }) { +function walkerPrompt({ worldDir, situation, task, scope, stubs, answers }) { const t = loadTemplate('walker'); - const parts = [ - worldDir ? fill(t.world, { world_dir: worldDir }) : fill(t.structural, { root }), - ]; + const parts = [fill(t.world, { world_dir: worldDir })]; if (situation) parts.push(fill(t.situation, { situation })); parts.push(fill(t.task, { task, scope })); if (answers) parts.push(fill(t.answers, { answers })); diff --git a/tests/prose/prompts/walker.md b/tests/prose/prompts/walker.md index 693c2d4ac..f1017f38d 100644 --- a/tests/prose/prompts/walker.md +++ b/tests/prose/prompts/walker.md @@ -15,12 +15,6 @@ Project directory — your cwd for EVERY command: {{world_dir}} The workflow skills are installed at .claude/skills/ inside that project. Mutations are expected and safe: the project is a disposable test world. -=== structural === -Structure-only walk: read the named prose and trace the logic. Execute -nothing — no commands, no writes. - -Repository root: {{root}} - === situation === SITUATION — where the project stands as you begin: diff --git a/tests/prose/run.cjs b/tests/prose/run.cjs index 166f0b115..fc3732ac7 100644 --- a/tests/prose/run.cjs +++ b/tests/prose/run.cjs @@ -96,10 +96,6 @@ function cmdSelect(argv) { function cmdWorld(argv) { const c = getCase(argv[0]); - if (!c.hasFixtureState) { - process.stdout.write(`${JSON.stringify({ world: null, note: 'structure-only case — no world' })}\n`); - return; - } process.stdout.write(`${JSON.stringify({ world: worlds.buildWorld(c.id), case: c.id })}\n`); } @@ -113,11 +109,10 @@ function cmdDestroy(argv) { function cmdPrompt(argv) { const c = getCase(argv[0]); - const worldDir = c.hasFixtureState ? requireWorld(argv, c) : null; + const worldDir = requireWorld(argv, c); process.stdout.write(prompts.walkerPrompt({ worldDir, - root: ROOT, situation: c.situation, task: c.act, scope: c.files @@ -132,10 +127,6 @@ function cmdPrompt(argv) { function cmdDiff(argv) { const c = getCase(argv[0]); - if (!c.hasFixtureState) { - process.stdout.write(`${JSON.stringify({ note: 'structure-only case — no world to diff' }, null, 2)}\n`); - return; - } process.stdout.write(`${JSON.stringify(worlds.diffWorld(c.id, requireWorld(argv, c)), null, 2)}\n`); } @@ -143,45 +134,40 @@ function cmdDiff(argv) { function cmdAssert(argv) { const c = getCase(argv[0]); - let world = null; - let actions = null; - let walk = null; - let checks = null; - if (c.hasFixtureState) { - const dir = requireWorld(argv, c); - const delta = worlds.diffWorld(c.id, dir, c.worldMode === 'claims'); - world = { - expecting: delta.expecting, - delta: [ - JSON.stringify({ added: delta.added, removed: delta.removed, identical: delta.identical }, null, 2), - ...delta.changed, - ].join('\n'), - }; - actions = worlds.readActionLog(dir); - // No log is the harness failing, not the walk: a walk in a world - // always runs commands, and the walker's hook records every one. - // Refuse loudly rather than let an agent judge on the narrative - // alone — which is the thing the recording exists to replace. - if (!actions) { - die(`no action log at ${path.join(dir, worlds.ACTION_LOG)} — the prose-walker ` - + 'PostToolUse hook did not fire, so there is no record of what the walk did.\n' - + 'Check the hooks block in .claude/agents/prose-walker.md, that the agent ' - + 'registry has reloaded since it changed, and that this project is trusted ' - + '(hasTrustDialogAccepted). Do not judge this run.'); - } - checks = invariants.format(invariants.check(worlds.readActionRows(dir), c.invariants)); - walk = worlds.readWalkLog(dir); - // Same stance as the action log: the walk is harness-captured, so its - // absence is a broken hook, not a quiet walk. Judging without it would - // fall back to whatever the walker chose to say at the end — the very - // summary this record exists to replace. - if (!walk) { - die(`no walk log at ${path.join(dir, worlds.WALK_LOG)} — the prose-walker ` - + 'SubagentStop hook did not fire, so there is no turn-by-turn record of ' - + 'the walk.\nCheck the hooks block in .claude/agents/prose-walker.md and ' - + 'that the agent registry has reloaded since it changed. Do not judge this run.'); - } + const dir = requireWorld(argv, c); + const delta = worlds.diffWorld(c.id, dir, c.worldMode === 'claims'); + const world = { + expecting: delta.expecting, + delta: [ + JSON.stringify({ added: delta.added, removed: delta.removed, identical: delta.identical }, null, 2), + ...delta.changed, + ].join('\n'), + }; + const actions = worlds.readActionLog(dir); + // No log is the harness failing, not the walk: a walk in a world + // always runs commands, and the walker's hook records every one. + // Refuse loudly rather than let an agent judge on the narrative + // alone — which is the thing the recording exists to replace. + if (!actions) { + die(`no action log at ${path.join(dir, worlds.ACTION_LOG)} — the prose-walker ` + + 'PostToolUse hook did not fire, so there is no record of what the walk did.\n' + + 'Check the hooks block in .claude/agents/prose-walker.md, that the agent ' + + 'registry has reloaded since it changed, and that this project is trusted ' + + '(hasTrustDialogAccepted). Do not judge this run.'); + } + const checks = invariants.format(invariants.check(worlds.readActionRows(dir), c.invariants)); + const walk = worlds.readWalkLog(dir); + // Same stance as the action log: the walk is harness-captured, so its + // absence is a broken hook, not a quiet walk. Judging without it would + // fall back to whatever the walker chose to say at the end — the very + // summary this record exists to replace. + if (!walk) { + die(`no walk log at ${path.join(dir, worlds.WALK_LOG)} — the prose-walker ` + + 'SubagentStop hook did not fire, so there is no turn-by-turn record of ' + + 'the walk.\nCheck the hooks block in .claude/agents/prose-walker.md and ' + + 'that the agent registry has reloaded since it changed. Do not judge this run.'); } + const substitutions = c.stubs.length ? c.stubs.map((s) => { const stub = cases.readStub(s.name); @@ -197,7 +183,7 @@ function cmdAssert(argv) { function statesOf(c) { return [ - c.hasFixtureState ? 'fixture' : null, + 'fixture', c.hasAssertionState ? 'assertion' : null, ].filter(Boolean); } @@ -245,7 +231,7 @@ function cmdList() { for (const c of all) { const stubs = c.stubs.length ? ` stubs=${c.stubs.map((s) => s.name).join(',')}` : ''; const expects = c.hasAssertionState ? 'assertion state' : 'no change'; - process.stdout.write(`${c.id}\n ${c.hasFixtureState ? 'fixture' : 'no world'} → ${expects}${stubs}\n`); + process.stdout.write(`${c.id}\n fixture → ${expects}${stubs}\n`); } process.stdout.write(`\n${all.length} cases, ${cases.listStubs().length} stubs`); process.stdout.write(errors.length ? `, ${errors.length} VALIDATION ERRORS:\n` : ', corpus valid\n');