Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions .claude/agents/prose-asserter.md
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,23 @@ closes with an incorporate") is evidenced from the recorded actions. A
step about reasoning or output is evidenced from the walk. Where the two
disagree about what was done, the recorded actions win.

**Read them against each other, always, before ruling on anything.** Two
divergences matter and both are reported under markers:

- **Narrated but not recorded** — the walk claims a command or a file
write the record does not contain. Treat the record as right. A step
resting on that claim fails, and the discrepancy is a finding in its
own right, because it means the walker described something it did not
do.
- **Recorded but not narrated** — the record holds a call the walk never
mentions. Usually harmless orientation, and not a failure by itself.
Report it anyway: a call that touched state and went unmentioned is the
shape a silent repair takes.

This comparison is required, not a courtesy. The two streams exist so
that neither has to be taken on trust, and that only holds if they are
actually set against each other every time.

The record settles *that* a call was made, never *why*. A call the walker
made during a wrong turn of its own, and then abandoned, appears in the
record exactly like one the prose asked for — only the walk says which it
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
"skills/workflow-shared/references/ensure-discovery-item.md"
],
"invariants": {
"engine_before_write": true,
"calls_include": [
"manifest get pay.discussion.pay status"
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
"skills/workflow-investigation-entry/references/invoke-skill.md"
],
"invariants": {
"engine_before_write": true,
"calls_include": [
"manifest get crash-fix.investigation.crash-fix status",
"render phase-note"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
"continue \u2014 nothing has changed since the specification"
],
"invariants": {
"engine_before_write": true,
"calls_include": [
"render entry-gate pay.planning.pay",
"manifest list",
Expand Down
1 change: 0 additions & 1 deletion tests/prose/cases/review-entry-fresh/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
"skills/workflow-review-entry/references/validate-phase.md"
],
"invariants": {
"engine_before_write": true,
"calls_include": [
"render entry-gate pay.review.pay"
],
Expand Down
1 change: 0 additions & 1 deletion tests/prose/cases/spec-entry-from-discussion/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
"skills/workflow-specification-entry/references/invoke-skill.md"
],
"invariants": {
"engine_before_write": true,
"calls_include": [
"render entry-gate pay.specification.pay"
],
Expand Down
1 change: 0 additions & 1 deletion tests/prose/cases/spec-entry-from-investigation/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
"skills/workflow-specification-entry/references/invoke-skill.md"
],
"invariants": {
"engine_before_write": true,
"calls_include": [
"render entry-gate crash-fix.specification.crash-fix"
],
Expand Down
1 change: 0 additions & 1 deletion tests/prose/cases/start-lists-active-work/case.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
"skills/workflow-start/SKILL.md"
],
"invariants": {
"engine_before_write": true,
"calls_include": [
"engine.cjs boot",
"gateway.cjs"
Expand Down
34 changes: 33 additions & 1 deletion tests/prose/lib/invariants.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,38 @@ function isWrite(row) {

const NAMES = ['engine_before_write', 'calls_include', 'calls_exclude', 'calls_in_order'];

/**
* Prose the walk actually opened, as repo-relative paths.
*
* A world holds the skills at `.claude/skills/`; the corpus names them at
* `skills/`. Same file, two addresses, so one is translated to the other.
*/
function proseRead(rows) {
const seen = new Set();
for (const row of rows) {
if (row.tool !== 'Read') continue;
const m = row.detail.match(/\.claude\/skills\/(.+\.md)$/);
if (m) seen.add(`skills/${m[1]}`);
}
return seen;
}

/**
* Prose a walk went through but the case never declared.
*
* `files` is not documentation: it decides whether a change to a file
* selects this case for a run. A file the walk traverses and the case
* omits is prose that can be edited without ever re-running the test that
* covers it. Reported, never corrected — which files belong is an
* authoring judgement, and the reachable set is far wider than the walked
* one, so a list assembled by following every link would select this case
* for branches it deliberately stops before.
*/
function undeclaredProse(rows, declared) {
const listed = new Set(declared || []);
return [...proseRead(rows)].filter((f) => !listed.has(f)).sort();
}

/** The commands the walk ran, in order. */
function commands(rows) {
return rows.filter((r) => r.tool === 'Bash' && r.event === 'PreToolUse').map((r) => r.detail);
Expand Down Expand Up @@ -184,4 +216,4 @@ function declarationErrors(declared) {
return errors;
}

module.exports = { check, format, declarationErrors, NAMES };
module.exports = { check, format, declarationErrors, undeclaredProse, NAMES };
3 changes: 2 additions & 1 deletion tests/prose/lib/prompts.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -73,12 +73,13 @@ function walkerPrompt({ worldDir, situation, task, scope, stubs, answers }) {
return `${parts.join('\n\n')}\n`;
}

function asserterPrompt({ expected, world, actions, checks, walk, substitutions }) {
function asserterPrompt({ expected, world, actions, checks, walk, substitutions, scope }) {
const t = loadTemplate('asserter');
const parts = [fill(t.main, { expected })];
if (world) parts.push(fill(t.world, { expecting: world.expecting, delta: world.delta }));
if (actions) parts.push(fill(t.actions, { actions }));
if (checks) parts.push(fill(t.checks, { checks }));
if (scope) parts.push(fill(t.scope, { scope }));
if (substitutions) parts.push(fill(t.substitutions, { substitutions }));
if (walk) parts.push(fill(t.walk, { walk }));
return `${parts.join('\n\n')}\n`;
Expand Down
13 changes: 13 additions & 0 deletions tests/prose/prompts/asserter.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,19 @@ verdict.

{{checks}}

=== scope ===

UNDECLARED PROSE — files this walk opened that the case does not list,
computed from the record.

The case's file list decides whether editing a file selects this case for
a run, so prose missing from it can be changed without the test that
covers it ever running again. This is a finding about the case, never
about the prose or the walk: report it under markers and let it affect
nothing else.

{{scope}}

=== substitutions ===

HARNESS SUBSTITUTIONS the case armed for this walk. Where the recorded
Expand Down
9 changes: 7 additions & 2 deletions tests/prose/run.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,9 @@ function cmdAssert(argv) {
+ 'registry has reloaded since it changed, and that this project is trusted '
+ '(hasTrustDialogAccepted). Do not judge this run.');
}
const checks = invariants.format(invariants.check(worlds.readActionRows(dir), c.invariants));
const rows = worlds.readActionRows(dir);
const checks = invariants.format(invariants.check(rows, c.invariants));
const undeclared = invariants.undeclaredProse(rows, c.files.map((f) => f.path));
const walk = worlds.readWalkLog(dir);
// Same stance as the action log: the walk is harness-captured, so its
// absence is a broken hook, not a quiet walk. Judging without it would
Expand All @@ -175,7 +177,10 @@ function cmdAssert(argv) {
}).join('\n')
: null;
process.stdout.write(
prompts.asserterPrompt({ expected: c.assert, world, actions, checks, walk, substitutions }),
prompts.asserterPrompt({
expected: c.assert, world, actions, checks, walk, substitutions,
scope: undeclared.length ? undeclared.map((f) => `- ${f}`).join('\n') : null,
}),
);
}

Expand Down
36 changes: 36 additions & 0 deletions tests/scripts/test-prose-invariants.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -289,3 +289,39 @@ describe('entry points — where a walk may begin', () => {
assert.deepEqual(cases.validateCorpus(cases.loadAllCases()), []);
});
});

describe('undeclared prose — the case list against what the walk opened', () => {
const rd = (detail) => ({ event: 'PreToolUse', tool: 'Read', detail });

it('names prose the walk opened that the case never declared', () => {
const rows = [
rd('./.claude/skills/workflow-specification-entry/SKILL.md'),
rd('./.claude/skills/workflow-specification-entry/references/validate-phase.md'),
];
assert.deepEqual(
invariants.undeclaredProse(rows, ['skills/workflow-specification-entry/SKILL.md']),
['skills/workflow-specification-entry/references/validate-phase.md'],
);
});

it('is quiet when the list already covers the walk', () => {
const rows = [rd('./.claude/skills/workflow-review-entry/SKILL.md')];
assert.deepEqual(invariants.undeclaredProse(rows, ['skills/workflow-review-entry/SKILL.md']), []);
});

it('ignores files that are not prose — a walk reads world state too', () => {
const rows = [rd('./.workflows/pay/manifest.json'), rd('./.workflows/pay/discussion/pay.md')];
assert.deepEqual(invariants.undeclaredProse(rows, []), []);
});

it('ignores commands — only what was opened counts', () => {
const rows = [bash('cd . && cat .claude/skills/workflow-start/SKILL.md')];
assert.deepEqual(invariants.undeclaredProse(rows, []), []);
});

it('reports each file once however often it was reopened', () => {
const f = './.claude/skills/workflow-start/references/active-work.md';
assert.deepEqual(invariants.undeclaredProse([rd(f), rd(f)], []),
['skills/workflow-start/references/active-work.md']);
});
});