diff --git a/docs/replay-verify.md b/docs/replay-verify.md index 5738707..4814c34 100644 --- a/docs/replay-verify.md +++ b/docs/replay-verify.md @@ -77,6 +77,17 @@ A corpus is a gold label file plus a prepared directory (`normalized//s Fix generation (`--fix generate`) runs ONE chat completion per arm-A-reproduced case against an OpenAI-compatible endpoint (default: glm-5.2 on z.ai). The prompt carries the gold step's action and observation, ±3 surrounding steps, and the task statement; the reply must be a single corrected shell command. Cases beyond `--max-fix-cases` are excluded by a seeded random sample and marked `sampled-out` in the report. Arm B replays the prefix in its own fresh sandbox and executes the corrected command; `failureVanished` = exit 0 with the failure signature absent. +### Iterative fix loop (`--fix loop`) + +`--fix loop` (in `src/replay-fix-loop.ts`) replaces the single shot with an execution-feedback loop of up to `--fix-attempts` attempts (default 3) per case: + +- Attempt 1 is byte-identical to the one-shot prompt, so the report's `fixFlipAttempt1` stays directly comparable to `--fix generate`. +- When an arm fails (nonzero exit or the failure signature persists) or the model call itself fails, the next prompt carries every prior command with its REAL executed stdout/stderr (clipped tails), never a paraphrase. +- Retries may answer with a short script — at most 5 commands in one fenced block — executed as ONE `/bin/sh` unit; a longer script is rejected as a failed attempt and the rejection feeds the next prompt. +- Every attempt runs in its own fresh sandbox with the same replayed prefix. A used sandbox is never mutated mid-arm, so a flip always proves the corrected step against the recorded prefix state, not against debris from an earlier attempt. Wall-time cost is one extra prefix replay per attempt. + +The report adds `fixFlipAttempt1` (flips at attempt 1 over cases whose attempt 1 executed), `flipsByAttempt`, the full per-attempt trail on each case row (`fix.attempts`), and one `armB-attempt-result.json` per executed attempt (`armB-result.json` keeps the last executed attempt — the flipped one when the loop flips). + Honest-reporting notes baked into the report: - A gold step whose recorded observation carries **no returncode** can never satisfy "arm A reproduces the recorded returncode": it counts against the replayability rate and the per-case table says why. diff --git a/src/index.ts b/src/index.ts index 625ad67..789538c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -121,6 +121,7 @@ export * from './replay-verify.js' // sandbox-backed counterfactual replay + ver export * from './replay-corpus.js' // corpus enumeration: replayable cases + exclusion reasons export * from './replay-batch.js' // batch runner: replayability + fix-flip rates export * from './replay-fix.js' // counterfactual patch synthesis (one LLM call per case) +export * from './replay-fix-loop.js' // iterative fix loop: real-output feedback, fresh sandbox per attempt export * from './replay-wire.js' // analyst finding → replay-verify invocation export * from './upload.js' // planUpload / executeUpload({ backend? }) export * from './upload-state.js' // dedup state diff --git a/src/replay-batch.ts b/src/replay-batch.ts index b5b732b..1e363e2 100644 --- a/src/replay-batch.ts +++ b/src/replay-batch.ts @@ -45,6 +45,7 @@ import { generateFixCommand, zaiChatCaller, } from './replay-fix.js' +import { type FixLoopAttemptRecord, runFixLoop } from './replay-fix-loop.js' import { deriveFailureSignature, ingestCodeTraceBenchSteps, @@ -164,8 +165,12 @@ export interface ReplayBatchOptions { readonly out: string readonly baseUrl?: string readonly apiKey?: string - /** 'generate' runs one LLM call per arm-A-reproduced case, then arm B. */ - readonly fix: 'none' | 'generate' + /** 'generate' = one LLM call per arm-A-reproduced case, then arm B. + * 'loop' = iterative: failed arms feed their real output into up to + * --fix-attempts prompts, each executed in its own fresh sandbox. */ + readonly fix: 'none' | 'generate' | 'loop' + /** Attempt budget per case in loop mode (default 3). */ + readonly fixAttempts?: number readonly fixCaller?: ChatCompletionCaller readonly fixModelLabel?: string /** Cap on LLM fix calls; eligible cases beyond it are seeded-sampled out. */ @@ -191,12 +196,19 @@ export interface ReplayBatchFixResult { readonly sampledOut: boolean readonly command: string | null readonly llmError: string | null + /** Loop mode: token totals summed across every attempt (null when the + * provider reported no usage). */ readonly usage: ChatUsage | null readonly armBExit: number | null readonly armBPrefixExecuted: number | null readonly armBPrefixDivergences: number | null readonly failureVanished: boolean | null readonly armBError: string | null + /** Loop mode only: the full per-attempt trail. Null in generate mode. */ + readonly attempts: readonly FixLoopAttemptRecord[] | null + /** 1-based attempt that flipped the failure; null when none did. + * Generate mode: 1 when the single attempt flipped. */ + readonly flippedAtAttempt: number | null } export interface ReplayBatchCaseRow { @@ -255,6 +267,15 @@ export interface ReplayBatchReport { denominator: number value: number | null } | null + /** Loop mode only: flips at attempt 1 over cases whose attempt 1 executed — + * the number directly comparable to the one-shot fixFlipRate. */ + readonly fixFlipAttempt1: { + numerator: number + denominator: number + value: number | null + } | null + /** Loop mode only: flip count keyed by the attempt number that flipped. */ + readonly flipsByAttempt: Record | null } readonly llm: { readonly model: string @@ -509,9 +530,10 @@ export async function runReplayBatch(options: ReplayBatchOptions): Promise r.status === 'ok' && r.replayed) @@ -544,19 +566,128 @@ export async function runReplayBatch(options: ReplayBatchOptions): Promise c.trajId === row.trajId && c.corpus === row.corpus)! const label = `fix ${row.corpus}/${row.trajId}` - onProgress(`${label}: generating corrected command`) - calls += 1 - const generated = await generateFixCommand(caller, { + const signature = verdictByTraj.get(row.trajId)?.signature ?? null + const stepTimeoutMs = options.stepTimeoutMs ?? replayCase.recordedStepTimeoutMs ?? 120_000 + const promptInput = { taskStatement: replayCase.taskStatement, steps: replayCase.steps, k: replayCase.k, - }) + } + const caseOut = join(options.out, caseOutDirName(row)) + + if (options.fix === 'loop') { + onProgress(`${label}: fix loop (budget ${fixAttempts} attempts)`) + const result = await runFixLoop( + caller, + promptInput, + async (command, attempt) => { + onProgress( + `${label}: attempt ${attempt} arm B — ${command.split('\n')[0]!.slice(0, 120)}`, + ) + const armB = await executeArmB( + replayCase, + command, + backendFactory(row.derivedImage!), + signature, + stepTimeoutMs, + options.prefixLimit, + (message) => onProgress(`${label}: attempt ${attempt}: ${message}`), + ) + mkdirSync(caseOut, { recursive: true }) + writeFileSync( + join(caseOut, `armB-attempt${attempt}-result.json`), + `${JSON.stringify({ command, ...armB }, null, 2)}\n`, + ) + // armB-result.json always holds the LAST executed attempt — the + // flipped one when the loop flips (it stops there). + writeFileSync( + join(caseOut, 'armB-result.json'), + `${JSON.stringify({ command, attempt, ...armB }, null, 2)}\n`, + ) + return armB + }, + { maxAttempts: fixAttempts, onProgress: (message) => onProgress(`${label}: ${message}`) }, + ) + calls += result.llmCalls + failures += result.llmFailures + promptTokens += result.promptTokens + completionTokens += result.completionTokens + const usage = + result.promptTokens + result.completionTokens > 0 + ? { promptTokens: result.promptTokens, completionTokens: result.completionTokens } + : null + const flippedRecord = + result.flippedAtAttempt !== null + ? result.attempts.find((a) => a.attempt === result.flippedAtAttempt)! + : null + const summary = flippedRecord ?? [...result.attempts].reverse().find((a) => a.executed) ?? null + const lastRecord = result.attempts.at(-1) ?? null + rows[index] = { + ...row, + fix: summary + ? { + attempted: true, + sampledOut: false, + command: summary.command, + llmError: null, + usage, + armBExit: summary.exitCode, + armBPrefixExecuted: summary.prefixExecuted, + armBPrefixDivergences: summary.prefixDivergences, + failureVanished: summary.failureVanished, + armBError: null, + attempts: result.attempts, + flippedAtAttempt: result.flippedAtAttempt, + } + : result.aborted + ? { + attempted: true, + sampledOut: false, + command: lastRecord?.command ?? null, + llmError: null, + usage, + armBExit: null, + armBPrefixExecuted: null, + armBPrefixDivergences: null, + failureVanished: null, + armBError: lastRecord?.armBError ?? 'sandbox error', + attempts: result.attempts, + flippedAtAttempt: null, + } + : { + attempted: true, + sampledOut: false, + command: null, + llmError: lastRecord?.llmError ?? 'no attempt produced a runnable fix', + usage, + armBExit: null, + armBPrefixExecuted: null, + armBPrefixDivergences: null, + failureVanished: null, + armBError: null, + attempts: result.attempts, + flippedAtAttempt: null, + }, + } + onProgress( + `${label}: loop done — flipped=${result.flipped}` + + (result.flippedAtAttempt !== null ? ` at attempt ${result.flippedAtAttempt}` : '') + + ` (${result.llmCalls} calls, ${result.attempts.filter((a) => a.executed).length} arms)`, + ) + continue + } + + onProgress(`${label}: generating corrected command`) + calls += 1 + const generated = await generateFixCommand(caller, promptInput) if (!generated.succeeded) { failures += 1 rows[index] = { @@ -572,6 +703,8 @@ export async function runReplayBatch(options: ReplayBatchOptions): Promise r.recordedReturncodeAtK !== null && r.recordedReturncodeAtK !== 0, ) const flippedNonzeroRc = armBNonzeroRc.filter((r) => r.fix!.failureVanished === true) + const attempt1Executed = rows.filter( + (r) => r.fix?.attempts?.find((a) => a.attempt === 1)?.executed === true, + ) + const flippedAt1 = rows.filter((r) => r.fix?.flippedAtAttempt === 1) + const flipsByAttempt: Record = {} + for (const row of rows) { + const at = row.fix?.flippedAtAttempt + if (typeof at === 'number') flipsByAttempt[String(at)] = (flipsByAttempt[String(at)] ?? 0) + 1 + } const excludedByReason: Record = {} for (const excluded of enumeration.excluded) { excludedByReason[excluded.reason] = (excludedByReason[excluded.reason] ?? 0) + 1 @@ -699,7 +842,7 @@ export async function runReplayBatch(options: ReplayBatchOptions): Promise 0) { + const parts = Object.entries(headline.flipsByAttempt) + .sort((a, b) => Number(a[0]) - Number(b[0])) + .map(([attempt, count]) => `attempt ${attempt}: ${count}`) + lines.push(`- Flips by attempt: ${parts.join(', ')}.`) + } lines.push('') lines.push('## Enumeration') lines.push('') @@ -824,7 +988,11 @@ export function renderBatchReport(report: ReplayBatchReport): string { ? 'llm-failed' : fix.armBError ? 'armB-error' - : 'generated' + : fix.attempts + ? fix.flippedAtAttempt !== null + ? `flip@${fix.flippedAtAttempt}` + : `exhausted(${fix.attempts.length})` + : 'generated' lines.push( [ row.corpus, @@ -863,7 +1031,8 @@ export function renderBatchReport(report: ReplayBatchReport): string { export interface ReplayBatchCliArgs { readonly corpora: CorpusSpec[] readonly out: string - readonly fix: 'none' | 'generate' + readonly fix: 'none' | 'generate' | 'loop' + readonly fixAttempts: number readonly fixModel: string readonly fixBaseUrl: string readonly fixApiKeyEnv: string @@ -909,13 +1078,20 @@ export function parseReplayBatchArgs(argv: readonly string[]): ReplayBatchCliArg return n } const fix = values.get('--fix') ?? 'none' - if (fix !== 'none' && fix !== 'generate') { - throw new Error(`replay-verify-batch: --fix must be none or generate, got ${fix}`) + if (fix !== 'none' && fix !== 'generate' && fix !== 'loop') { + throw new Error(`replay-verify-batch: --fix must be none, generate, or loop, got ${fix}`) + } + const fixAttempts = optionalNumber('--fix-attempts') ?? 3 + if (!Number.isInteger(fixAttempts) || fixAttempts < 1) { + throw new Error( + `replay-verify-batch: --fix-attempts must be a positive integer, got ${values.get('--fix-attempts')}`, + ) } return { corpora, out: out ?? '.', fix, + fixAttempts, fixModel: values.get('--fix-model') ?? 'glm-5.2', fixBaseUrl: values.get('--fix-base-url') ?? 'https://api.z.ai/api/coding/paas/v4', fixApiKeyEnv: values.get('--fix-api-key-env') ?? 'ZAI_GLM_API_KEY', @@ -938,7 +1114,7 @@ export function replayBatchUsage(): string { Usage: traces replay-verify-batch \\ --corpus NAME=:: [--corpus ...] \\ - --out DIR [--fix none|generate] [--enumerate-only] \\ + --out DIR [--fix none|generate|loop] [--fix-attempts 3] [--enumerate-only] \\ [--fix-model glm-5.2] [--fix-base-url URL] [--fix-api-key-env ZAI_GLM_API_KEY] \\ [--max-fix-cases 30] [--seed 17] [--step-timeout MS] [--prefix-limit N] \\ [--case-filter SUBSTRING] [--case-limit N] \\ @@ -957,6 +1133,13 @@ Usage: --max-fix-cases, seeded sample beyond it) and executes the corrected command as arm B in a fresh sandbox. + --fix loop iterates: attempt 1 is the one-shot prompt; when an arm fails + (nonzero exit or the failure signature persists) or the model call fails, + the next prompt carries every prior command with its REAL executed + stdout/stderr, up to --fix-attempts attempts (default 3). Retries may + answer with a short script (<=5 commands) executed as one /bin/sh unit. + Every attempt runs in its own fresh sandbox with the same replayed prefix. + Outputs batch-report.json, batch-report.md, cases.jsonl (incremental), and one directory per case with the full replay-verify artifacts. See docs/replay-verify.md for orchestrator setup and honest limits. @@ -1001,7 +1184,7 @@ export async function cmdReplayVerifyBatch(argv: readonly string[]): Promise Promise + +export interface FixLoopOptions { + /** Total LLM attempts per case (≥1); 1 degenerates to the one-shot path. */ + readonly maxAttempts: number + /** Command-line cap on retry scripts (default 5). */ + readonly maxScriptCommands?: number + /** Chars kept per stdout/stderr tail in records and retry prompts. */ + readonly outputTailChars?: number + readonly onProgress?: (message: string) => void +} + +export interface FixLoopAttemptRecord { + readonly attempt: number + readonly command: string | null + readonly llmError: string | null + readonly usage: { readonly promptTokens: number; readonly completionTokens: number } | null + readonly executed: boolean + readonly exitCode: number | null + readonly prefixExecuted: number | null + readonly prefixDivergences: number | null + readonly failureVanished: boolean | null + readonly stdoutTail: string | null + readonly stderrTail: string | null + readonly armBError: string | null + readonly wallMs: number +} + +export interface FixLoopResult { + readonly flipped: boolean + readonly flippedAtAttempt: number | null + readonly attempts: readonly FixLoopAttemptRecord[] + /** True when a sandbox error ended the loop before the attempt budget. */ + readonly aborted: boolean + readonly llmCalls: number + /** Calls that produced no runnable fix: transport errors, empty replies, + * and retry scripts over the command cap. */ + readonly llmFailures: number + readonly promptTokens: number + readonly completionTokens: number +} + +function toFailedAttempt(record: FixLoopAttemptRecord): FailedFixAttempt { + return { + attempt: record.attempt, + command: record.command, + exitCode: record.exitCode, + stdoutTail: record.stdoutTail, + stderrTail: record.stderrTail, + llmError: record.llmError ?? record.armBError, + } +} + +export async function runFixLoop( + caller: ChatCompletionCaller, + input: FixPromptInput, + executor: FixArmExecutor, + options: FixLoopOptions, +): Promise { + if (!Number.isInteger(options.maxAttempts) || options.maxAttempts < 1) { + throw new Error( + `replay-fix-loop: maxAttempts must be a positive integer, got ${options.maxAttempts}`, + ) + } + const scriptCap = options.maxScriptCommands ?? 5 + const tailChars = options.outputTailChars ?? 1600 + const attempts: FixLoopAttemptRecord[] = [] + let llmCalls = 0 + let llmFailures = 0 + let promptTokens = 0 + let completionTokens = 0 + + const unexecuted = ( + attempt: number, + llmError: string, + usage: FixLoopAttemptRecord['usage'], + command: string | null, + started: number, + ): FixLoopAttemptRecord => ({ + attempt, + command, + llmError, + usage, + executed: false, + exitCode: null, + prefixExecuted: null, + prefixDivergences: null, + failureVanished: null, + stdoutTail: null, + stderrTail: null, + armBError: null, + wallMs: Date.now() - started, + }) + + for (let attempt = 1; attempt <= options.maxAttempts; attempt++) { + const started = Date.now() + const prompt = + attempt === 1 + ? buildFixPrompt(input) + : buildRetryFixPrompt(input, attempts.map(toFailedAttempt), scriptCap) + llmCalls += 1 + const outcome = await caller.complete(prompt.system, prompt.user) + if (!outcome.succeeded) { + llmFailures += 1 + attempts.push(unexecuted(attempt, outcome.error, null, null, started)) + options.onProgress?.( + `fix-loop attempt ${attempt}: LLM failed — ${outcome.error.slice(0, 160)}`, + ) + continue + } + const usage = outcome.value.usage + promptTokens += usage?.promptTokens ?? 0 + completionTokens += usage?.completionTokens ?? 0 + const command = extractFixCommand(outcome.value.content) + if (command === null) { + llmFailures += 1 + attempts.push( + unexecuted( + attempt, + `completion carried no usable command: ${clipText(outcome.value.content, 300)}`, + usage, + null, + started, + ), + ) + continue + } + // Attempt 1 mirrors one-shot exactly, so only retries enforce the cap. + if (attempt > 1) { + const commandLines = countScriptCommands(command) + if (commandLines > scriptCap) { + llmFailures += 1 + attempts.push( + unexecuted( + attempt, + `script exceeds ${scriptCap} command lines (${commandLines})`, + usage, + null, + started, + ), + ) + options.onProgress?.( + `fix-loop attempt ${attempt}: rejected script with ${commandLines} command lines`, + ) + continue + } + } + let execution: FixArmExecution + try { + execution = await executor(command, attempt) + } catch (err) { + const message = err instanceof Error ? err.message : String(err) + attempts.push({ + ...unexecuted(attempt, '', usage, command, started), + llmError: null, + armBError: message.slice(0, 500), + }) + options.onProgress?.( + `fix-loop attempt ${attempt}: sandbox error — ${message.slice(0, 160)}`, + ) + return { + flipped: false, + flippedAtAttempt: null, + attempts, + aborted: true, + llmCalls, + llmFailures, + promptTokens, + completionTokens, + } + } + attempts.push({ + attempt, + command, + llmError: null, + usage, + executed: true, + exitCode: execution.exitCode, + prefixExecuted: execution.prefixExecuted, + prefixDivergences: execution.prefixDivergences, + failureVanished: execution.failureVanished, + stdoutTail: clipText(execution.stdout, tailChars), + stderrTail: clipText(execution.stderr, tailChars), + armBError: null, + wallMs: Date.now() - started, + }) + options.onProgress?.( + `fix-loop attempt ${attempt}: exit=${execution.exitCode} failureVanished=${execution.failureVanished}`, + ) + if (execution.failureVanished) { + return { + flipped: true, + flippedAtAttempt: attempt, + attempts, + aborted: false, + llmCalls, + llmFailures, + promptTokens, + completionTokens, + } + } + } + return { + flipped: false, + flippedAtAttempt: null, + attempts, + aborted: false, + llmCalls, + llmFailures, + promptTokens, + completionTokens, + } +} diff --git a/src/replay-fix.ts b/src/replay-fix.ts index 80b7a52..4b4d3da 100644 --- a/src/replay-fix.ts +++ b/src/replay-fix.ts @@ -116,7 +116,8 @@ export interface FixPromptInput { readonly contextRadius?: number } -function clip(text: string, limit: number): string { +/** Head+tail excerpt with an elision marker; identity below the limit. */ +export function clipText(text: string, limit: number): string { if (text.length <= limit) return text const half = Math.floor(limit / 2) return `${text.slice(0, half)}\n… [${text.length - limit} chars elided] …\n${text.slice(-half)}` @@ -124,24 +125,38 @@ function clip(text: string, limit: number): string { function renderStep(step: CodeTraceBenchStep, marker: string): string { const rc = parseRecordedReturncode(step.observation) - const output = clip(parseObservationOutput(step.observation).trim(), 1600) + const output = clipText(parseObservationOutput(step.observation).trim(), 1600) return [ `### step ${step.step_id}${marker}`, '```sh', - clip(step.action, 2000), + clipText(step.action, 2000), '```', `returncode: ${rc ?? 'none recorded'}`, output.length > 0 ? `output:\n\`\`\`\n${output}\n\`\`\`` : 'output: (empty)', ].join('\n') } -export function buildFixPrompt(input: FixPromptInput): { system: string; user: string } { +/** Shared user-prompt sections: task statement, ±radius context, failing step. */ +function promptBody(input: FixPromptInput): string[] { const radius = input.contextRadius ?? 3 const target = input.steps.find((s) => s.step_id === input.k) if (!target) throw new Error(`replay-fix: no step with step_id ${input.k}`) const context = input.steps.filter( (s) => s.step_id !== input.k && Math.abs(s.step_id - input.k) <= radius, ) + return [ + '## Task the agent was solving', + input.taskStatement ? clipText(input.taskStatement, 4000) : '(no task statement recorded)', + '', + '## Surrounding steps', + ...context.map((s) => renderStep(s, '')), + '', + '## Failing step to correct', + renderStep(target, ' (INCORRECT — correct this one)'), + ] +} + +export function buildFixPrompt(input: FixPromptInput): { system: string; user: string } { const system = [ 'You repair one failed shell command from a recorded coding-agent trajectory.', 'The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.', @@ -150,20 +165,83 @@ export function buildFixPrompt(input: FixPromptInput): { system: string; user: s 'The corrected command must accomplish the failing step\'s intent and exit 0. No prose outside the fenced block.', ].join('\n') const user = [ - '## Task the agent was solving', - input.taskStatement ? clip(input.taskStatement, 4000) : '(no task statement recorded)', + ...promptBody(input), '', - '## Surrounding steps', - ...context.map((s) => renderStep(s, '')), + 'Output the single corrected replacement for the failing step now.', + ].join('\n') + return { system, user } +} + +/** One prior attempt of the fix loop, rendered into the retry prompt. */ +export interface FailedFixAttempt { + readonly attempt: number + /** Null when the model call itself failed before producing a command. */ + readonly command: string | null + readonly exitCode: number | null + readonly stdoutTail: string | null + readonly stderrTail: string | null + readonly llmError: string | null +} + +function renderFailedAttempt(prior: FailedFixAttempt): string { + if (prior.command === null) { + return [ + `### attempt ${prior.attempt}`, + `model call failed before producing a command: ${prior.llmError ?? 'unknown error'}`, + ].join('\n') + } + const stdout = (prior.stdoutTail ?? '').trim() + const stderr = (prior.stderrTail ?? '').trim() + return [ + `### attempt ${prior.attempt}`, + '```sh', + clipText(prior.command, 2000), + '```', + `exit code: ${prior.exitCode ?? 'not executed'}`, + stdout.length > 0 ? `stdout:\n\`\`\`\n${stdout}\n\`\`\`` : 'stdout: (empty)', + stderr.length > 0 ? `stderr:\n\`\`\`\n${stderr}\n\`\`\`` : 'stderr: (empty)', + ].join('\n') +} + +/** + * Retry prompt for fix-loop attempts ≥2: the original context plus every prior + * attempt with its REAL executed output, and permission to answer with a short + * script (the block still executes as one /bin/sh unit). + */ +export function buildRetryFixPrompt( + input: FixPromptInput, + priorAttempts: readonly FailedFixAttempt[], + maxScriptCommands = 5, +): { system: string; user: string } { + if (priorAttempts.length === 0) { + throw new Error('replay-fix: buildRetryFixPrompt requires at least one prior attempt') + } + const system = [ + 'You repair one failed shell command from a recorded coding-agent trajectory.', + 'The trajectory replays inside the original docker image; every command runs as a fresh /bin/sh subshell from a fixed working directory.', + 'Earlier corrected commands were executed for real and failed; their actual output is included below.', + `Reply with a corrected fix inside a single \`\`\`sh fenced block: either one command, or a short script of at most ${maxScriptCommands} commands (one per line).`, + 'The whole block executes as ONE /bin/sh unit from the fixed working directory and must exit 0.', + 'Do not repeat a command that already failed. Keep reasoning brief. No prose outside the fenced block.', + ].join('\n') + const user = [ + ...promptBody(input), '', - '## Failing step to correct', - renderStep(target, ' (INCORRECT — correct this one)'), + '## Previous fix attempts (executed for real — all failed)', + ...priorAttempts.map((prior) => renderFailedAttempt(prior)), '', - 'Output the single corrected replacement for the failing step now.', + `Output a corrected fix now — one \`\`\`sh block, at most ${maxScriptCommands} commands.`, ].join('\n') return { system, user } } +/** Non-empty, non-comment lines of a fix script — the loop's script-size cap. */ +export function countScriptCommands(script: string): number { + return script + .split('\n') + .filter((line) => line.trim().length > 0 && !line.trim().startsWith('#')).length +} + /** Last fenced code block, else the whole trimmed content; null when empty. */ export function extractFixCommand(content: string): string | null { const blocks = [...content.matchAll(/```(?:sh|bash|shell)?\n([\s\S]*?)```/g)] @@ -188,7 +266,7 @@ export async function generateFixCommand( if (command === null) { return { succeeded: false, - error: `completion carried no usable command: ${clip(outcome.value.content, 300)}`, + error: `completion carried no usable command: ${clipText(outcome.value.content, 300)}`, } } return { succeeded: true, value: { command, usage: outcome.value.usage } } diff --git a/tests/replay-batch.test.ts b/tests/replay-batch.test.ts index 78b3102..a59842b 100644 --- a/tests/replay-batch.test.ts +++ b/tests/replay-batch.test.ts @@ -448,6 +448,128 @@ describe('runReplayBatch', () => { expect(report.headline.fixFlipRate!.denominator).toBe(2) }) + it('fix=loop retries with feedback, opens a fresh sandbox per attempt, and reports @1 vs final', async () => { + const dir = makeRoot() + const corpus = writeFixtureCorpus(dir, 'loop', [ + { + trajId: 'traj-flips-at-2', + steps: failingSteps(), + goldIncorrectSteps: [3], + raw: { baseImage: 'example/ok:1', runConfigCwd: '/repo' }, + taskMd: 'Fix the build.', + }, + { + trajId: 'traj-never-flips', + steps: failingSteps(), + goldIncorrectSteps: [3], + raw: { baseImage: 'example/ok:2', runConfigCwd: '/repo' }, + }, + ]) + const out = join(dir, 'out') + const prompts: string[] = [] + const callsByTraj = new Map() + const caller: ChatCompletionCaller = { + complete: async (_system, user) => { + prompts.push(user) + // The retry prompt carries the failed command; use that to key replies. + const isRetry = user.includes('## Previous fix attempts') + return { + succeeded: true, + value: { + content: isRetry + ? '```sh\nfix file.c && make target\n```' + : '```sh\nfirst-guess && make target\n```', + usage: { promptTokens: 100, completionTokens: 20 }, + }, + } + }, + } + let opens = 0 + const report = await runReplayBatch({ + corpora: [corpus], + out, + fix: 'loop', + fixAttempts: 3, + fixCaller: caller, + fixModelLabel: 'fake-model', + preparer: fakePreparer(), + backendFactory: (derivedImage) => { + const inner = scriptedBackend((action) => { + if (action === 'make target') { + return { exitCode: 2, stdout: 'stopped', stderr: 'file.c:9:2: error: broken build' } + } + // Only the corrected command on the ok:1 image ever flips. + if (action === 'fix file.c && make target' && derivedImage === 'derived-example/ok:1') { + return { exitCode: 0, stdout: 'built ok', stderr: '' } + } + if (action.includes('make target')) { + return { exitCode: 2, stdout: '', stderr: 'file.c:9:2: error: broken build' } + } + return { exitCode: 0, stdout: '', stderr: '' } + }) + return { + async open() { + opens += 1 + return inner.open() + }, + } + }, + }) + + const flips = report.cases.find((c) => c.trajId === 'traj-flips-at-2')! + expect(flips.fix).toMatchObject({ + attempted: true, + command: 'fix file.c && make target', + armBExit: 0, + failureVanished: true, + flippedAtAttempt: 2, + llmError: null, + armBError: null, + }) + expect(flips.fix!.attempts).toHaveLength(2) + expect(flips.fix!.attempts![0]).toMatchObject({ + attempt: 1, + executed: true, + exitCode: 2, + failureVanished: false, + }) + expect(flips.fix!.usage).toEqual({ promptTokens: 200, completionTokens: 40 }) + + const never = report.cases.find((c) => c.trajId === 'traj-never-flips')! + expect(never.fix).toMatchObject({ failureVanished: false, flippedAtAttempt: null }) + expect(never.fix!.attempts).toHaveLength(3) + + // 2 arm-A sessions + 2 attempts (flip case) + 3 attempts (exhausted case), + // each in its own fresh sandbox. + expect(opens).toBe(7) + + expect(report.headline.fixFlipRate).toEqual({ numerator: 1, denominator: 2, value: 0.5 }) + expect(report.headline.fixFlipAttempt1).toEqual({ numerator: 0, denominator: 2, value: 0 }) + expect(report.headline.flipsByAttempt).toEqual({ '2': 1 }) + expect(report.llm).toEqual({ + model: 'fake-model', + calls: 5, + failures: 0, + promptTokens: 500, + completionTokens: 100, + }) + + // The retry prompt fed back the failed command and its REAL output. + const retryPrompt = prompts.find((p) => p.includes('## Previous fix attempts'))! + expect(retryPrompt).toContain('first-guess && make target') + expect(retryPrompt).toContain('file.c:9:2: error: broken build') + + const caseDir = join(out, 'loop--traj-flips-at-2') + expect(existsSync(join(caseDir, 'armB-attempt1-result.json'))).toBe(true) + expect(existsSync(join(caseDir, 'armB-attempt2-result.json'))).toBe(true) + const final = JSON.parse(readFileSync(join(caseDir, 'armB-result.json'), 'utf8')) + expect(final).toMatchObject({ attempt: 2, exitCode: 0 }) + const markdown = readFileSync(join(out, 'batch-report.md'), 'utf8') + expect(markdown).toContain('flip@2') + expect(markdown).toContain('exhausted(3)') + expect(markdown).toContain('Fix-flip@1: 0.0%') + }) + it('records an LLM failure as a row, not an abort', async () => { const dir = makeRoot() const corpus = writeFixtureCorpus(dir, 'llmfail', [ @@ -515,4 +637,19 @@ describe('parseReplayBatchArgs', () => { ).toThrow(/--fix/) expect(parseReplayBatchArgs(['--help'])).toBe('help') }) + + it('parses --fix loop with --fix-attempts and validates the budget', () => { + const args = parseReplayBatchArgs([ + '--corpus', 'a=/l::/p', + '--out', '/o', + '--fix', 'loop', + '--fix-attempts', '4', + ]) + expect(args).toMatchObject({ fix: 'loop', fixAttempts: 4 }) + const defaulted = parseReplayBatchArgs(['--corpus', 'a=/l::/p', '--out', '/o', '--fix', 'loop']) + expect(defaulted).toMatchObject({ fix: 'loop', fixAttempts: 3 }) + expect(() => + parseReplayBatchArgs(['--corpus', 'a=/l::/p', '--out', '/o', '--fix', 'loop', '--fix-attempts', '0']), + ).toThrow(/--fix-attempts/) + }) }) diff --git a/tests/replay-fix-loop.test.ts b/tests/replay-fix-loop.test.ts new file mode 100644 index 0000000..8649ed8 --- /dev/null +++ b/tests/replay-fix-loop.test.ts @@ -0,0 +1,256 @@ +import { describe, expect, it } from 'vitest' +import { + buildFixPrompt, + buildRetryFixPrompt, + type ChatCompletionCaller, + countScriptCommands, +} from '../src/replay-fix.js' +import { + type FixArmExecution, + runFixLoop, +} from '../src/replay-fix-loop.js' +import { fixtureStep } from './replay-corpus-fixture.js' + +const steps = () => [ + fixtureStep(1, 'ls', 0, 'files'), + fixtureStep(2, 'sed -i broken file.c', 0), + fixtureStep(3, 'make target', 2, 'file.c:9:2: error: broken build\nstopped'), + fixtureStep(4, 'echo done', 0), +] + +const input = () => ({ taskStatement: 'Fix the build.', steps: steps(), k: 3 }) + +function fenced(command: string): string { + return `\`\`\`sh\n${command}\n\`\`\`` +} + +/** Caller that replies with the queued contents in order and records prompts. */ +function queuedCaller(replies: readonly ({ content: string } | { error: string })[]): { + caller: ChatCompletionCaller + prompts: { system: string; user: string }[] +} { + const prompts: { system: string; user: string }[] = [] + let call = 0 + return { + prompts, + caller: { + complete: async (system, user) => { + prompts.push({ system, user }) + const reply = replies[call++] + if (!reply) throw new Error(`caller exhausted after ${call - 1} replies`) + if ('error' in reply) return { succeeded: false, error: reply.error } + return { + succeeded: true, + value: { content: reply.content, usage: { promptTokens: 10, completionTokens: 5 } }, + } + }, + }, + } +} + +/** Executor that fails until `flipOn`, recording every command and attempt. */ +function scriptedExecutor(flipOn: (command: string, attempt: number) => boolean) { + const executions: { command: string; attempt: number }[] = [] + const executor = async (command: string, attempt: number): Promise => { + executions.push({ command, attempt }) + if (flipOn(command, attempt)) { + return { + exitCode: 0, + prefixExecuted: 2, + prefixDivergences: 0, + failureVanished: true, + stdout: 'built ok', + stderr: '', + } + } + return { + exitCode: 2, + prefixExecuted: 2, + prefixDivergences: 0, + failureVanished: false, + stdout: 'stopped', + stderr: 'file.c:9:2: error: broken build', + } + } + return { executor, executions } +} + +describe('countScriptCommands', () => { + it('counts non-empty non-comment lines', () => { + expect(countScriptCommands('make -j2')).toBe(1) + expect(countScriptCommands('# fix\nsed -i x f\n\nmake target\n')).toBe(2) + expect(countScriptCommands('a\nb\nc\nd\ne\nf')).toBe(6) + }) +}) + +describe('buildRetryFixPrompt', () => { + it('carries the failed command, its real output, and the script allowance', () => { + const { system, user } = buildRetryFixPrompt(input(), [ + { + attempt: 1, + command: 'make -j2 target', + exitCode: 2, + stdoutTail: 'stopped', + stderrTail: 'file.c:9:2: error: broken build', + llmError: null, + }, + ]) + expect(system).toContain('at most 5 commands') + expect(system).toContain('ONE /bin/sh unit') + expect(user).toContain('## Previous fix attempts') + expect(user).toContain('make -j2 target') + expect(user).toContain('exit code: 2') + expect(user).toContain('file.c:9:2: error: broken build') + expect(user).toContain('Fix the build.') + expect(user).toContain('step 3 (INCORRECT — correct this one)') + }) + + it('renders a model-call failure as an attempt without a command', () => { + const { user } = buildRetryFixPrompt(input(), [ + { + attempt: 1, + command: null, + exitCode: null, + stdoutTail: null, + stderrTail: null, + llmError: 'This operation was aborted', + }, + ]) + expect(user).toContain('model call failed before producing a command: This operation was aborted') + }) + + it('requires at least one prior attempt', () => { + expect(() => buildRetryFixPrompt(input(), [])).toThrow(/prior attempt/) + }) +}) + +describe('runFixLoop', () => { + it('degenerates to one-shot when attempt 1 flips: one call, one arm, one-shot prompt', async () => { + const { caller, prompts } = queuedCaller([{ content: fenced('fix file.c && make target') }]) + const { executor, executions } = scriptedExecutor(() => true) + const result = await runFixLoop(caller, input(), executor, { maxAttempts: 3 }) + expect(result).toMatchObject({ + flipped: true, + flippedAtAttempt: 1, + aborted: false, + llmCalls: 1, + llmFailures: 0, + promptTokens: 10, + completionTokens: 5, + }) + expect(result.attempts).toHaveLength(1) + expect(executions).toEqual([{ command: 'fix file.c && make target', attempt: 1 }]) + // Attempt 1 must be byte-identical to the one-shot prompt. + expect(prompts[0]).toEqual(buildFixPrompt(input())) + }) + + it('flips on attempt 3 and feeds each failure back into the next prompt', async () => { + const { caller, prompts } = queuedCaller([ + { content: fenced('fix-v1 && make target') }, + { content: fenced('fix-v2 && make target') }, + { content: fenced('fix-v3 && make target') }, + ]) + const { executor, executions } = scriptedExecutor((command) => command.startsWith('fix-v3')) + const result = await runFixLoop(caller, input(), executor, { maxAttempts: 3 }) + expect(result).toMatchObject({ flipped: true, flippedAtAttempt: 3, llmCalls: 3, llmFailures: 0 }) + expect(result.attempts.map((a) => a.executed)).toEqual([true, true, true]) + expect(executions.map((e) => e.attempt)).toEqual([1, 2, 3]) + // Attempt 2 sees attempt 1's command and real output; attempt 3 sees both. + expect(prompts[1]!.user).toContain('fix-v1 && make target') + expect(prompts[1]!.user).toContain('file.c:9:2: error: broken build') + expect(prompts[2]!.user).toContain('fix-v1 && make target') + expect(prompts[2]!.user).toContain('fix-v2 && make target') + expect(result.promptTokens).toBe(30) + expect(result.completionTokens).toBe(15) + }) + + it('exhausts the attempt budget without a flip', async () => { + const { caller } = queuedCaller([ + { content: fenced('fix-v1') }, + { content: fenced('fix-v2') }, + { content: fenced('fix-v3') }, + ]) + const { executor, executions } = scriptedExecutor(() => false) + const result = await runFixLoop(caller, input(), executor, { maxAttempts: 3 }) + expect(result).toMatchObject({ flipped: false, flippedAtAttempt: null, aborted: false }) + expect(result.attempts).toHaveLength(3) + expect(executions).toHaveLength(3) + expect(result.attempts.every((a) => a.failureVanished === false)).toBe(true) + }) + + it('retries after a model-call failure and can flip on the retry', async () => { + const { caller, prompts } = queuedCaller([ + { error: 'This operation was aborted' }, + { content: fenced('fix file.c && make target') }, + ]) + const { executor } = scriptedExecutor(() => true) + const result = await runFixLoop(caller, input(), executor, { maxAttempts: 3 }) + expect(result).toMatchObject({ + flipped: true, + flippedAtAttempt: 2, + llmCalls: 2, + llmFailures: 1, + }) + expect(result.attempts[0]).toMatchObject({ + executed: false, + llmError: 'This operation was aborted', + command: null, + }) + expect(prompts[1]!.user).toContain('model call failed before producing a command') + }) + + it('executes a retry script as one arm and rejects scripts over the cap', async () => { + const script = 'apt-get install -y jq\nsed -i x file.c\nmake target' + const tooLong = 'a\nb\nc\nd\ne\nf' + const { caller } = queuedCaller([ + { content: fenced('fix-v1') }, + { content: fenced(tooLong) }, + { content: fenced(script) }, + ]) + const { executor, executions } = scriptedExecutor((command) => command === script) + const result = await runFixLoop(caller, input(), executor, { maxAttempts: 3 }) + expect(result).toMatchObject({ flipped: true, flippedAtAttempt: 3, llmFailures: 1 }) + expect(result.attempts[1]).toMatchObject({ + executed: false, + llmError: 'script exceeds 5 command lines (6)', + }) + // The whole script arrives at the executor as ONE arm. + expect(executions).toEqual([ + { command: 'fix-v1', attempt: 1 }, + { command: script, attempt: 3 }, + ]) + }) + + it('does not enforce the script cap on attempt 1 (one-shot parity)', async () => { + const sixLines = 'a\nb\nc\nd\ne\nf' + const { caller } = queuedCaller([{ content: fenced(sixLines) }]) + const { executor, executions } = scriptedExecutor(() => true) + const result = await runFixLoop(caller, input(), executor, { maxAttempts: 1 }) + expect(result.flipped).toBe(true) + expect(executions).toEqual([{ command: sixLines, attempt: 1 }]) + }) + + it('aborts on a sandbox error and records it on the attempt', async () => { + const { caller } = queuedCaller([{ content: fenced('fix-v1') }]) + const executor = async () => { + throw new Error('sandbox create timed out') + } + const result = await runFixLoop(caller, input(), executor, { maxAttempts: 3 }) + expect(result).toMatchObject({ flipped: false, aborted: true, llmCalls: 1 }) + expect(result.attempts).toHaveLength(1) + expect(result.attempts[0]).toMatchObject({ + executed: false, + command: 'fix-v1', + armBError: 'sandbox create timed out', + llmError: null, + }) + }) + + it('rejects a non-positive attempt budget', async () => { + const { caller } = queuedCaller([]) + const { executor } = scriptedExecutor(() => true) + await expect(runFixLoop(caller, input(), executor, { maxAttempts: 0 })).rejects.toThrow( + /maxAttempts/, + ) + }) +})