diff --git a/tests/prose/lib/record-action.cjs b/tests/prose/lib/record-action.cjs index e228b4064..66e452d03 100644 --- a/tests/prose/lib/record-action.cjs +++ b/tests/prose/lib/record-action.cjs @@ -42,6 +42,11 @@ const LOG = '.walk-actions.log'; const VIOLATIONS = 'tests/prose/.agent-tool-use.log'; const WORLD = /(^|[\s"'`])(\/[^\s"'`]*\/prose-world-[A-Za-z0-9]+)/; const MAX_OUTPUT = 400; +// A command's output is the evidence that settles what the prose was +// shown — whether a gate rendered empty, whether a menu had entries. It +// gets the room a whole rendered section needs; a file read does not, +// since no claim turns on the bytes a walker read back. +const MAX_COMMAND_OUTPUT = 2000; function read() { try { @@ -65,6 +70,24 @@ function summarise(input) { return flatten(raw, 200); } +/** + * What a tool actually gave back. The payload field is `tool_response`, + * and its shape is the tool's own: a shell reports `{stdout, stderr}`, + * everything else answers in whatever form suits it. Reading a shell's + * streams directly keeps a command's output legible as output rather + * than as a JSON envelope around it. + */ +function responseText(response, limit) { + if (response === null || response === undefined) return ''; + if (typeof response === 'object') { + const { stdout, stderr } = response; + if (typeof stdout === 'string' || typeof stderr === 'string') { + return flatten([stdout, stderr].filter(Boolean).join('\n'), limit); + } + } + return flatten(response, limit); +} + /** * The agent's own harness transcript — written by the runtime, not the * agent. Authority on the model it ran on, and on which world it walked, @@ -134,11 +157,19 @@ function main() { ]; if (event === 'PostToolUse') { - parts.push(payload.tool_output_is_error ? 'ERROR' : 'ok'); - parts.push(flatten(payload.tool_output, MAX_OUTPUT).split(world).join('.')); + // A call that failed never arrives here — it raises PostToolUseFailure + // instead — so reaching this point is itself the success signal. + parts.push('ok'); + parts.push( + responseText(payload.tool_response, tool === 'Bash' ? MAX_COMMAND_OUTPUT : MAX_OUTPUT) + .split(world).join('.'), + ); } else if (event === 'PostToolUseFailure') { parts.push('FAILED'); - parts.push(flatten(payload.tool_output ?? payload.error, MAX_OUTPUT)); + parts.push( + responseText(payload.tool_response ?? payload.tool_output ?? payload.error, + MAX_COMMAND_OUTPUT).split(world).join('.'), + ); } else if (stop) { parts.push(flatten(payload.last_assistant_message, MAX_OUTPUT).split(world).join('.')); } diff --git a/tests/scripts/test-prose-record-action.cjs b/tests/scripts/test-prose-record-action.cjs index 756aef4ee..06f11da9e 100644 --- a/tests/scripts/test-prose-record-action.cjs +++ b/tests/scripts/test-prose-record-action.cjs @@ -63,29 +63,76 @@ describe('prose recorder — tool events', () => { assert.match(row, /^PreToolUse\tBash\t/); }); - it('records the output of a call, so a claim about it can be checked', () => { + it('records a command output, so a claim about it can be checked', () => { fire({ hook_event_name: 'PostToolUse', tool_name: 'Bash', agent_type: 'prose-walker', tool_input: { command: `cd ${world} && engine view` }, - tool_output: 'MENU: 1. Continue "Pay"', + tool_response: { stdout: 'MENU: 1. Continue "Pay"', stderr: '' }, }); const [row] = logLines(); assert.ok(row.includes('\tok\t'), 'a successful call is marked ok'); - assert.ok(row.includes('MENU: 1. Continue "Pay"'), 'output is recorded verbatim'); + assert.ok(row.includes('MENU: 1. Continue "Pay"'), 'output is recorded, not just its status'); }); - it('marks a failing call so a walk cannot look cleaner than it was', () => { + it('keeps stderr, where a command often says what went wrong', () => { + fire({ + hook_event_name: 'PostToolUse', + tool_name: 'Bash', + agent_type: 'prose-walker', + tool_input: { command: `cd ${world} && engine view` }, + tool_response: { stdout: 'partial output', stderr: 'warning: store is stale' }, + }); + const [row] = logLines(); + assert.ok(row.includes('partial output')); + assert.ok(row.includes('warning: store is stale')); + }); + + it('gives a command far more room than a file read', () => { + const long = 'x'.repeat(1800); + fire({ + hook_event_name: 'PostToolUse', + tool_name: 'Bash', + agent_type: 'prose-walker', + tool_input: { command: `cd ${world} && engine view` }, + tool_response: { stdout: long }, + }); + assert.ok(!logLines()[0].includes('[truncated]'), 'a rendered section survives whole'); + + fs.rmSync(path.join(world, LOG)); + fire({ + hook_event_name: 'PostToolUse', + tool_name: 'Read', + agent_type: 'prose-walker', + tool_input: { file_path: `${world}/big.md` }, + tool_response: { file: { content: long } }, + }); + assert.ok(logLines()[0].includes('[truncated]'), 'a file read is trimmed — no claim rests on it'); + }); + + it('records a tool that answers in its own shape, not a shell’s', () => { fire({ hook_event_name: 'PostToolUse', + tool_name: 'Read', + agent_type: 'prose-walker', + tool_input: { file_path: `${world}/notes.md` }, + tool_response: { file: { filePath: 'notes.md', numLines: 3 } }, + }); + assert.ok(logLines()[0].includes('numLines')); + }); + + it('marks a failing call so a walk cannot look cleaner than it was', () => { + fire({ + hook_event_name: 'PostToolUseFailure', tool_name: 'Bash', agent_type: 'prose-walker', tool_input: { command: `cd ${world} && engine nope` }, - tool_output: 'unknown command', - tool_output_is_error: true, + tool_response: { stdout: 'Exit code 1\nUsage: engine ', stderr: '' }, }); - assert.ok(logLines()[0].includes('\tERROR\t')); + const [row] = logLines(); + assert.ok(row.includes('\tFAILED\t')); + assert.ok(row.includes('Usage: engine '), 'and says what came back'); }); it('ignores anything happening outside a world', () => {