Skip to content

feat(prose-tests): decide in code what an agent should not be deciding #562

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
leeovery wants to merge 2 commits into
base: fix/prose-tests-capture-the-walk
Choose a base branch
from
Open

feat(prose-tests): decide in code what an agent should not be deciding #562

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5f7f193
feat(prose-tests): decide in code what an agent should not be deciding
leeovery Jul 26, 2026
e659940
fix(prose-tests): a shell redirect writes state too
leeovery Jul 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .claude/agents/prose-asserter.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ Return exactly this and nothing else:
as recorded. If no such line exists, say `unrecorded`. A verdict is
only as trustworthy as the model that produced the walk, so it is
stated before anything is judged, never inferred, and never omitted.
0b. **Checks** — every deterministic check and its computed verdict,
repeated as given. Omit the section only when the prompt carried none.
A FAILing check fails the case on its own, whatever the path and world
show; it was decided in code and is not yours to revisit.
1. **Path** — one line per expected step: PASS or FAIL, each PASS quoting
the line of the walk that shows it, each FAIL stating what is missing
or contradicting.
Expand All @@ -97,4 +101,5 @@ Return exactly this and nothing else:
`DEVIATION` in the walk. Each is a finding in its own right,
even when everything else passed.
4. **VERDICT** — PASS, FAIL, or INVALID WALK, then one sentence on what
it means for the prose.
it means for the prose. Any FAILing deterministic check makes this
FAIL.
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"build": "node build/knowledge.build.js",
"knowledge:repl": "node scripts/knowledge-repl.js",
"typecheck": "tsc -p tsconfig.json",
"test": "node --test tests/scripts/test-render.cjs tests/scripts/test-engine-gateway.cjs tests/scripts/test-engine-epic-projections.cjs tests/scripts/test-engine-start-projections.cjs tests/scripts/test-engine-workunit-projections.cjs tests/scripts/test-engine-discussion-map.cjs tests/scripts/test-engine-discovery-map.cjs tests/scripts/test-engine-tasks.cjs tests/scripts/test-engine-render-surfaces.cjs tests/scripts/test-engine-transactions.cjs tests/scripts/test-engine-workunit-create.cjs tests/scripts/test-engine-workunit-pivot.cjs tests/scripts/test-engine-workunit-absorb.cjs tests/scripts/test-engine-workunit-promote.cjs tests/scripts/test-engine-discovery-session.cjs tests/scripts/test-engine-cache.cjs tests/scripts/test-engine-manifest-io.cjs tests/scripts/test-engine-manifest-fields.cjs tests/scripts/test-engine-boot.cjs tests/scripts/test-migration-orchestrator.cjs tests/scripts/test-migration-047.cjs tests/scripts/test-migration-048.cjs tests/scripts/test-migration-049.cjs tests/scripts/test-migration-050.cjs tests/scripts/test-migration-051.cjs tests/scripts/test-migration-052.cjs tests/scripts/test-engine-specification-projections.cjs tests/scripts/test-engine-discovery-projections.cjs tests/scripts/test-gateway-for-start.cjs tests/scripts/test-gateway-for-specification.cjs tests/scripts/test-gateway-for-workflow-continue-epic.cjs tests/scripts/test-gateway-for-workflow-continue-feature.cjs tests/scripts/test-gateway-for-workflow-continue-bugfix.cjs tests/scripts/test-gateway-for-workflow-continue-quickfix.cjs tests/scripts/test-gateway-for-workflow-continue-cross-cutting.cjs tests/scripts/test-gateway-for-discovery-skill.cjs tests/scripts/test-gateway-for-bridge.cjs tests/scripts/test-reads-derivations.cjs tests/scripts/test-ensure-discovery-item.cjs tests/scripts/test-legacy-research-split.cjs tests/scripts/test-knowledge-chunker.cjs tests/scripts/test-knowledge-config.cjs tests/scripts/test-knowledge-discovery.cjs tests/scripts/test-knowledge-embeddings.cjs tests/scripts/test-knowledge-integration.cjs tests/scripts/test-knowledge-openai.cjs tests/scripts/test-knowledge-openai-compatible.cjs tests/scripts/test-knowledge-openai-integration.cjs tests/scripts/test-knowledge-progress.cjs tests/scripts/test-knowledge-rerank.cjs tests/scripts/test-knowledge-retry.cjs tests/scripts/test-knowledge-setup-forms.cjs tests/scripts/test-knowledge-store.cjs tests/scripts/test-conventions-lint.cjs tests/scripts/test-pipeline-simulation.cjs tests/scripts/test-engine-agent-state.cjs tests/scripts/test-prose-record-action.cjs tests/scripts/test-prose-cases.cjs tests/scripts/test-prose-snapshots.cjs",
"test": "node --test tests/scripts/test-render.cjs tests/scripts/test-engine-gateway.cjs tests/scripts/test-engine-epic-projections.cjs tests/scripts/test-engine-start-projections.cjs tests/scripts/test-engine-workunit-projections.cjs tests/scripts/test-engine-discussion-map.cjs tests/scripts/test-engine-discovery-map.cjs tests/scripts/test-engine-tasks.cjs tests/scripts/test-engine-render-surfaces.cjs tests/scripts/test-engine-transactions.cjs tests/scripts/test-engine-workunit-create.cjs tests/scripts/test-engine-workunit-pivot.cjs tests/scripts/test-engine-workunit-absorb.cjs tests/scripts/test-engine-workunit-promote.cjs tests/scripts/test-engine-discovery-session.cjs tests/scripts/test-engine-cache.cjs tests/scripts/test-engine-manifest-io.cjs tests/scripts/test-engine-manifest-fields.cjs tests/scripts/test-engine-boot.cjs tests/scripts/test-migration-orchestrator.cjs tests/scripts/test-migration-047.cjs tests/scripts/test-migration-048.cjs tests/scripts/test-migration-049.cjs tests/scripts/test-migration-050.cjs tests/scripts/test-migration-051.cjs tests/scripts/test-migration-052.cjs tests/scripts/test-engine-specification-projections.cjs tests/scripts/test-engine-discovery-projections.cjs tests/scripts/test-gateway-for-start.cjs tests/scripts/test-gateway-for-specification.cjs tests/scripts/test-gateway-for-workflow-continue-epic.cjs tests/scripts/test-gateway-for-workflow-continue-feature.cjs tests/scripts/test-gateway-for-workflow-continue-bugfix.cjs tests/scripts/test-gateway-for-workflow-continue-quickfix.cjs tests/scripts/test-gateway-for-workflow-continue-cross-cutting.cjs tests/scripts/test-gateway-for-discovery-skill.cjs tests/scripts/test-gateway-for-bridge.cjs tests/scripts/test-reads-derivations.cjs tests/scripts/test-ensure-discovery-item.cjs tests/scripts/test-legacy-research-split.cjs tests/scripts/test-knowledge-chunker.cjs tests/scripts/test-knowledge-config.cjs tests/scripts/test-knowledge-discovery.cjs tests/scripts/test-knowledge-embeddings.cjs tests/scripts/test-knowledge-integration.cjs tests/scripts/test-knowledge-openai.cjs tests/scripts/test-knowledge-openai-compatible.cjs tests/scripts/test-knowledge-openai-integration.cjs tests/scripts/test-knowledge-progress.cjs tests/scripts/test-knowledge-rerank.cjs tests/scripts/test-knowledge-retry.cjs tests/scripts/test-knowledge-setup-forms.cjs tests/scripts/test-knowledge-store.cjs tests/scripts/test-conventions-lint.cjs tests/scripts/test-pipeline-simulation.cjs tests/scripts/test-engine-agent-state.cjs tests/scripts/test-prose-invariants.cjs tests/scripts/test-prose-record-action.cjs tests/scripts/test-prose-cases.cjs tests/scripts/test-prose-snapshots.cjs",
"test:migrations": "for t in tests/scripts/test-migration-*.sh; do bash $t || exit 1; done",
"test:cli": "for t in tests/scripts/test-engine-manifest.sh tests/scripts/test-inbox-promotion.sh tests/scripts/test-knowledge-cli.sh tests/scripts/test-knowledge-build.sh; do bash $t || exit 1; done"
},
Expand Down
7 changes: 5 additions & 2 deletions tests/prose/cases/discussion-entry-seeds-from-carrier/case.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
{
"origin": "feature mainline — single-phase work seeds from its durable carrier",
"origin": "feature mainline \u2014 single-phase work seeds from its durable carrier",
"files": [
"skills/workflow-discussion-entry/SKILL.md",
"skills/workflow-shared/references/ensure-discovery-item.md"
]
],
"invariants": {
"engine_before_write": true
}
}
1 change: 0 additions & 1 deletion tests/prose/cases/implementation-picks-first-task/assert.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,3 @@ The prose should have taken this path:
Further claims:

- the setup document is written once, by the setup step and nowhere else
- no task is started: the walk stops before any task is picked up
16 changes: 13 additions & 3 deletions tests/prose/cases/implementation-picks-first-task/case.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"origin": "feature mainline — entry through resume detection to the first task",
"origin": "feature mainline \u2014 entry through resume detection to the first task",
"files": [
"skills/workflow-implementation-entry/SKILL.md",
"skills/workflow-implementation-entry/references/validate-phase.md",
Expand All @@ -8,6 +8,16 @@
"skills/workflow-implementation-process/references/environment-setup.md"
],
"answers": [
"none — no special setup is needed"
]
"none \u2014 no special setup is needed"
],
"invariants": {
"engine_before_write": true,
"calls_include": [
"task init"
],
"calls_exclude": [
"task start",
"task next"
]
}
}
7 changes: 5 additions & 2 deletions tests/prose/cases/investigation-entry-seeds-from-carrier/case.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
{
"origin": "bugfix mainline — a bug shaped in discovery is never re-interrogated",
"origin": "bugfix mainline \u2014 a bug shaped in discovery is never re-interrogated",
"files": [
"skills/workflow-investigation-entry/SKILL.md",
"skills/workflow-investigation-entry/references/invoke-skill.md"
]
],
"invariants": {
"engine_before_write": true
}
}
9 changes: 6 additions & 3 deletions tests/prose/cases/planning-entry-asks-for-late-context/case.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
{
"origin": "feature mainline — planning offers a last chance to add context",
"origin": "feature mainline \u2014 planning offers a last chance to add context",
"files": [
"skills/workflow-planning-entry/SKILL.md",
"skills/workflow-planning-entry/references/validate-spec.md",
"skills/workflow-planning-entry/references/validate-phase.md",
"skills/workflow-planning-entry/references/cross-cutting-context.md"
],
"answers": [
"continue — nothing has changed since the specification"
]
"continue \u2014 nothing has changed since the specification"
],
"invariants": {
"engine_before_write": true
}
}
7 changes: 5 additions & 2 deletions tests/prose/cases/review-entry-fresh/case.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
{
"origin": "feature mainline — review refuses to start until delivery is complete",
"origin": "feature mainline \u2014 review refuses to start until delivery is complete",
"files": [
"skills/workflow-review-entry/SKILL.md",
"skills/workflow-review-entry/references/validate-phase.md"
]
],
"invariants": {
"engine_before_write": true
}
}
7 changes: 5 additions & 2 deletions tests/prose/cases/spec-entry-from-discussion/case.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
{
"origin": "feature mainline — a feature's spec draws on its discussion",
"origin": "feature mainline \u2014 a feature's spec draws on its discussion",
"files": [
"skills/workflow-specification-entry/SKILL.md",
"skills/workflow-specification-entry/references/validate-source.md",
"skills/workflow-specification-entry/references/validate-phase.md",
"skills/workflow-specification-entry/references/invoke-skill.md"
]
],
"invariants": {
"engine_before_write": true
}
}
7 changes: 5 additions & 2 deletions tests/prose/cases/spec-entry-from-investigation/case.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
{
"origin": "bugfix mainline — the same entry skill, sourced from an investigation",
"origin": "bugfix mainline \u2014 the same entry skill, sourced from an investigation",
"files": [
"skills/workflow-specification-entry/SKILL.md",
"skills/workflow-specification-entry/references/validate-source.md",
"skills/workflow-specification-entry/references/invoke-skill.md"
]
],
"invariants": {
"engine_before_write": true
}
}
7 changes: 5 additions & 2 deletions tests/prose/cases/start-lists-active-work/case.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
{
"origin": "feature mainline — the entry skill surfaces work without touching it",
"origin": "feature mainline \u2014 the entry skill surfaces work without touching it",
"files": [
"skills/workflow-start/SKILL.md"
]
],
"invariants": {
"engine_before_write": true
}
}
11 changes: 10 additions & 1 deletion tests/prose/lib/cases.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@
//
// case.json the values code branches on:
// { origin, files[], answers[], stubs{name: trigger},
// world: "claims" (optional) }
// world: "claims" (optional),
// invariants{} (optional) — see lib/invariants.cjs }
// fixture.md optional. Prose describing the starting world —
// what has already happened, where the session
// stands. Given to the walker.
Expand Down Expand Up @@ -45,6 +46,7 @@

const fs = require('fs');
const path = require('path');
const invariants = require('./invariants.cjs');

const ROOT = path.join(__dirname, '../../..');
const PROSE_DIR = path.join(ROOT, 'tests/prose');
Expand Down Expand Up @@ -104,6 +106,7 @@ function loadCase(id) {
}),
answers: meta.answers || [],
worldMode: meta.world || null,
invariants: meta.invariants || null,
stubs: Object.entries(meta.stubs || {}).map(([name, trigger]) => ({ name, trigger })),
situation: readIf(dir, FILES.situation),
act: readIf(dir, FILES.act),
Expand Down Expand Up @@ -189,6 +192,12 @@ function validateCorpus(cases) {
if (c.worldMode === 'claims' && !c.hasFixtureState) {
errors.push(`${at}: world "claims" needs a world to mutate`);
}
for (const e of invariants.declarationErrors(c.invariants)) {
errors.push(`${at}: ${FILES.meta} ${e}`);
}
if (c.invariants && !c.hasFixtureState) {
errors.push(`${at}: invariants without ${FILES.fixtureState} — no walk to check`);
}
if (c.hasAssertionState && !c.hasFixtureState) {
errors.push(`${at}: ${FILES.assertionState} without ${FILES.fixtureState} — nothing to act on`);
}
Expand Down
143 changes: 143 additions & 0 deletions tests/prose/lib/invariants.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
'use strict';

// Deterministic checks over the recorded actions.
//
// The world delta proves the outcome. It cannot prove the outcome was
// reached by following the prose: a walker that ignored every instruction
// and wrote the expected files directly lands the same delta. Only the
// order of what it actually did separates the two, and that is recorded.
//
// These checks read that record in code and decide before any agent sees
// the case. A judgement an agent never makes is a judgement that cannot
// drift — which is the whole reason they exist, and why they are worth
// having even though they can only ever cover the coarse shape of a walk.
//
// Declared per case in case.json, hand-written like every other part of a
// case:
//
// "invariants": {
// "engine_before_write": true, // no .workflows write out of nowhere
// "calls_include": ["task init"], // these commands must have run
// "calls_exclude": ["task start"] // these must not have
// }
//
// Deliberately not a pinned call sequence. A recorded-and-replayed
// sequence would freeze whatever the walker happened to do, which is how
// a test starts certifying broken prose instead of catching it.

const ENGINE_CALL = /\/(engine|gateway)\.cjs\b/;
const WRITE_TOOLS = new Set(['Write', 'Edit', 'NotebookEdit']);
const WORKFLOWS = '.workflows/';
// A file is not only written through the write tools. A walker reaches
// for the shell as readily — `printf … > .workflows/…` creates state
// just as surely, and an Opus walk was observed doing exactly that while
// a tool-only check reported that nothing had been written at all.
// Segment separators bound the match so a redirect elsewhere in a
// compound command is not mistaken for one into the workflow directory.
const SHELL_WRITE = /(>>?|\btee\b|\bcp\b|\bmv\b|\binstall\b)[^|;&]*\.workflows\//;

function isWrite(row) {
if (WRITE_TOOLS.has(row.tool)) return row.detail.includes(WORKFLOWS);
return row.tool === 'Bash' && SHELL_WRITE.test(row.detail);
}

const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude'];

/** The commands the walk ran, in order. */
function commands(rows) {
return rows.filter((r) => r.tool === 'Bash' && r.event === 'PreToolUse').map((r) => r.detail);
}

/**
* A write the prose could only have reached by consulting state first.
* Reading a file is not enough — the engine is how a workflow skill
* learns anything, so a walk that writes workflow state having never
* called it did not get there by following the prose.
*/
function engineBeforeWrite(rows) {
const firstEngine = rows.findIndex(
(r) => r.tool === 'Bash' && ENGINE_CALL.test(r.detail),
);
const firstWrite = rows.findIndex(isWrite);
if (firstWrite === -1) {
return { ok: true, detail: 'no workflow state was written' };
}
if (firstEngine === -1) {
return { ok: false, detail: `wrote ${rows[firstWrite].detail} having never called the engine` };
}
if (firstEngine > firstWrite) {
return {
ok: false,
detail: `wrote ${rows[firstWrite].detail} before any engine call`,
};
}
return { ok: true, detail: 'every workflow write followed an engine call' };
}

function callsInclude(rows, wanted) {
const ran = commands(rows);
const missing = wanted.filter((w) => !ran.some((c) => c.includes(w)));
return missing.length
? { ok: false, detail: `never ran: ${missing.join(', ')}` }
: { ok: true, detail: `ran all of: ${wanted.join(', ')}` };
}

function callsExclude(rows, forbidden) {
const ran = commands(rows);
const found = forbidden.filter((f) => ran.some((c) => c.includes(f)));
return found.length
? { ok: false, detail: `ran what it should not have: ${found.join(', ')}` }
: { ok: true, detail: `ran none of: ${forbidden.join(', ')}` };
}

/**
* Run a case's declared invariants against its recorded actions.
* Returns one result per declared check, in declaration order.
*/
function check(rows, declared) {
if (!declared) return [];
const results = [];
if (declared.engine_before_write) {
results.push({ name: 'engine_before_write', ...engineBeforeWrite(rows) });
}
if (declared.calls_include && declared.calls_include.length) {
results.push({ name: 'calls_include', ...callsInclude(rows, declared.calls_include) });
}
if (declared.calls_exclude && declared.calls_exclude.length) {
results.push({ name: 'calls_exclude', ...callsExclude(rows, declared.calls_exclude) });
}
return results;
}

/** The checks as the asserter reads them — verdict first, computed by code. */
function format(results) {
if (!results.length) return null;
return results
.map((r) => `${r.ok ? 'PASS' : 'FAIL'} ${r.name} — ${r.detail}`)
.join('\n');
}

/** Shape errors in a case's declaration, for corpus validation. */
function declarationErrors(declared) {
if (declared === undefined || declared === null) return [];
const errors = [];
if (typeof declared !== 'object' || Array.isArray(declared)) {
return ['invariants must be an object'];
}
for (const key of Object.keys(declared)) {
if (!NAMES.includes(key)) errors.push(`unknown invariant "${key}" (known: ${NAMES.join(', ')})`);
}
if ('engine_before_write' in declared && typeof declared.engine_before_write !== 'boolean') {
errors.push('engine_before_write must be true or false');
}
for (const key of ['calls_include', 'calls_exclude']) {
if (!(key in declared)) continue;
const value = declared[key];
if (!Array.isArray(value) || value.some((v) => typeof v !== 'string' || !v.trim())) {
errors.push(`${key} must be an array of non-empty strings`);
}
}
return errors;
}

module.exports = { check, format, declarationErrors, NAMES };
3 changes: 2 additions & 1 deletion tests/prose/lib/prompts.cjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,12 @@ function walkerPrompt({ worldDir, root, situation, task, scope, stubs, answers }
return `${parts.join('\n\n')}\n`;
}

function asserterPrompt({ expected, world, actions, walk, substitutions }) {
function asserterPrompt({ expected, world, actions, checks, walk, substitutions }) {
const t = loadTemplate('asserter');
const parts = [fill(t.main, { expected })];
if (world) parts.push(fill(t.world, { expecting: world.expecting, delta: world.delta }));
if (actions) parts.push(fill(t.actions, { actions }));
if (checks) parts.push(fill(t.checks, { checks }));
if (substitutions) parts.push(fill(t.substitutions, { substitutions }));
if (walk) parts.push(fill(t.walk, { walk }));
return `${parts.join('\n\n')}\n`;
Expand Down
Loading