From 96ed05f2bed41beb253abd131122e6ea566c3439 Mon Sep 17 00:00:00 2001 From: Lee Overy Date: Mon, 27 Jul 2026 14:09:04 +0100 Subject: [PATCH] fix(prose-tests): what a write put in a file is evidence, not incident MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The symptom-gathering case failed on a claim that could not be verified: that the user's ignorance was recorded in the investigation file rather than left as template placeholders. The asserter was right to refuse — nothing in its prompt showed the file's post-interview content. The Edit record truncated the written text at 400 characters, and the world delta names added paths without carrying content. That cap was a read's rule applied to the wrong thing. Only a read is incidental: nothing is ever claimed about the bytes a walker read back out of a file, so those are trimmed hard and stay trimmed. But what a write put in a file settles every claim about the artifact a phase leaves behind — and in claims mode the response is the only copy of that content the asserter will ever see. The discovery cases escaped this by luck alone: their content lands via workunit create, a Bash call, which already kept 10,000 characters. Write, Edit and NotebookEdit responses now get the produced-output ceiling alongside Bash. Reads keep the hard trim. Co-Authored-By: Claude Opus 5 --- tests/prose/lib/record-action.cjs | 27 ++++++++++++-------- tests/scripts/test-prose-record-action.cjs | 29 ++++++++++++++++++++++ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/tests/prose/lib/record-action.cjs b/tests/prose/lib/record-action.cjs index 89684a383..ceb311a59 100644 --- a/tests/prose/lib/record-action.cjs +++ b/tests/prose/lib/record-action.cjs @@ -44,14 +44,19 @@ const VIOLATIONS = 'tests/prose/.agent-tool-use.log'; const WORLD = /(^|[\s"'`])(\/[^\s"'`]*\/prose-world-[A-Za-z0-9]+)/; // These caps exist to protect the asserter's prompt, never to save disk — // every recorded action is read into it, and a walk makes twenty-odd file -// reads whose bodies are whole skill files. They apply only where the -// content is incidental. A file a walker read back is incidental: no claim -// ever rests on those bytes. A command's output is not — it settles what -// the prose was shown, whether a gate rendered empty, whether a menu had -// entries — so it gets a ceiling generous enough that real evidence is -// never trimmed, bounding only a runaway that would swamp the prompt. +// reads whose bodies are whole skill files. +// +// Only a read is incidental. Nothing is ever claimed about the bytes a +// walker read back out of a file, so those are trimmed hard. Everything a +// walk *produces* is evidence and is kept: what a command returned settles +// whether a gate rendered empty or a menu had entries, and what a write +// put in a file settles every claim about the artifact a phase leaves +// behind. Trimming a write was a read's rule applied to the wrong thing — +// it left a claim about a written file unprovable, because the only copy +// of that content lived in the response being cut. const MAX_OUTPUT = 400; -const MAX_COMMAND_OUTPUT = 10000; +const MAX_PRODUCED_OUTPUT = 10000; +const WRITE_RESPONSE_TOOLS = new Set(['Bash', 'Write', 'Edit', 'NotebookEdit']); function read() { try { @@ -213,14 +218,16 @@ function main() { // instead — so reaching this point is itself the success signal. parts.push('ok'); parts.push( - responseText(payload.tool_response, tool === 'Bash' ? MAX_COMMAND_OUTPUT : MAX_OUTPUT) - .split(world).join('.'), + responseText( + payload.tool_response, + WRITE_RESPONSE_TOOLS.has(tool) ? MAX_PRODUCED_OUTPUT : MAX_OUTPUT, + ).split(world).join('.'), ); } else if (event === 'PostToolUseFailure') { parts.push('FAILED'); parts.push( responseText(payload.tool_response ?? payload.tool_output ?? payload.error, - MAX_COMMAND_OUTPUT).split(world).join('.'), + MAX_PRODUCED_OUTPUT).split(world).join('.'), ); } else if (stop) { parts.push(flatten(payload.last_assistant_message, MAX_OUTPUT).split(world).join('.')); diff --git a/tests/scripts/test-prose-record-action.cjs b/tests/scripts/test-prose-record-action.cjs index c3823ccdb..172a8cd2a 100644 --- a/tests/scripts/test-prose-record-action.cjs +++ b/tests/scripts/test-prose-record-action.cjs @@ -89,6 +89,35 @@ describe('prose recorder — tool events', () => { assert.ok(row.includes('warning: store is stale')); }); + it('keeps what a write put in a file — claims rest on it', () => { + // A read is incidental; a write is the artifact a phase leaves behind. + // Trimming this left a claim about a written file unprovable, because + // the response was the only copy of that content. + const written = `## Symptoms\n${'detail '.repeat(400)}`; + fire({ + hook_event_name: 'PostToolUse', + tool_name: 'Edit', + agent_type: 'prose-walker', + tool_input: { file_path: `${world}/.workflows/crash-fix/investigation/crash-fix.md` }, + tool_response: { filePath: 'crash-fix.md', newString: written }, + }); + const [row] = logLines(); + assert.ok(!row.includes('[truncated]'), 'the written content survives'); + assert.ok(row.includes('detail detail'), 'and is legible in the record'); + }); + + it('keeps a Write the same way', () => { + const written = 'x'.repeat(3000); + fire({ + hook_event_name: 'PostToolUse', + tool_name: 'Write', + agent_type: 'prose-walker', + tool_input: { file_path: `${world}/.workflows/notes.md` }, + tool_response: { type: 'create', content: written }, + }); + assert.ok(!logLines()[0].includes('[truncated]')); + }); + it('gives a command far more room than a file read', () => { const long = 'x'.repeat(1800); fire({