diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 6c19178..c7cc305 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -6,14 +6,14 @@ "email": "hi@okis.dev" }, "description": "Multi-model orchestration marketplace for Claude Code.", - "version": "0.0.49", + "version": "0.0.50", "plugins": [ { "name": "grok", "source": "./plugins/grok", "displayName": "Grok Companion", "description": "Local Grok CLI delegation: task, review, resumable history, best-of-n tournaments, background jobs, stats, and setup health checks.", - "version": "0.0.49", + "version": "0.0.50", "author": { "name": "Harry Yep" }, @@ -34,7 +34,7 @@ "source": "./plugins/codex", "displayName": "Codex Companion", "description": "First party local Codex CLI delegation for tasks, reviews, resumable threads, and durable background jobs.", - "version": "0.0.49", + "version": "0.0.50", "author": { "name": "Harry Yep" }, @@ -55,7 +55,7 @@ "source": "./plugins/fusion", "displayName": "Fusion Orchestrator", "description": "Multi-model orchestration: tier agents, routing rules, blind panel, ultra fleet, model config, and drift doctor.", - "version": "0.0.49", + "version": "0.0.50", "author": { "name": "Harry Yep" }, diff --git a/CHANGELOG.md b/CHANGELOG.md index b955e18..35d7699 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # changelog +## 0.0.50 + +- codex failure attribution stops collapsing three distinct deaths into one bucket: a seven day read of 175 engine jobs found 34 foreground wall clock kills, 41 percent of every `gpt-5.6-sol` write run against 15 percent for terra and near zero for consult, and their event streams split them into 19 genuine overruns, 12 runs stuck in an `apply_patch verification failed` retry loop and 3 that lost their exec child to `UnknownProcessId`. only the first 19 are a sizing problem, so the standing advice to split the brief was the wrong remedy for 44 percent of the family. neither failing string ever reaches the structured event stream, since events carry only `command_execution` and `file_change` items and a `file_change` is emitted only on success, so detection rides the child stderr that was already streamed live, counted on the arriving chunk with a carried partial line so a pattern split across two reads counts exactly once: `patch_thrash` fires on the third consecutive patch verification failure and a completed `file_change` resets the count, `exec_lost` on the third lost process, both terminate early through the same path a collaboration violation uses, and both join the repeated class in the breaker so one is a retry and two open it. what is saved is the wall clock rather than the label, so a run that genuinely reaches the deadline still reports `timeout` +- a deliberate kill stops losing its reason on the way to a terminal record: a collaboration policy violation exits through `cleanupRequired`, which leaves the record non terminal at `phase: cleanup-required`, and the reconciliation that finalizes it hardcoded `failureKind: "died"` over whatever was already recorded, so the attribution was destroyed exactly when it mattered and the resumable patch on the terminal path never saw a resumable kind. reconciliation now preserves a failure kind that is in the resumable set and falls back to `died` only when none was recorded, leaving the identity replaced and checkpoint pending paths converging on `died` as before. this is the same defect as the timeout bucket one entry above, a real cause overwritten by a generic one, and it surfaced only because the assertion meant to prove the resume footer could never pass +- three mechanical companion defects that each cost real work in the same seven days are fixed at their seams: codex refused to start three times in two days because the working directory had no ancestor `.git`, so a read only consult now auto passes `--skip-git-repo-check`, where the check carries no safety meaning, while a write fails fast naming the flag, and the recorded `request` keeps the user's declared intent rather than the inferred value so the inference never propagates down a resumed thread; a single flight bounce destroyed the staged brief twice, because the raw transport was consumed and unlinked at dispatch entry while the already running guard threw later from inside the task path, and the payload now survives until a job record is actually reserved; and the `gpt-5.6-sol` foreground write warning, which rode 58 records of which 34 succeeded, stops being written after `executeRecord` returns and becomes a preflight on the exact risk combination, carrying the measured 41 percent instead of a p90 claim +- a policy killed run becomes salvageable the way a timed out one is: `timeout`, `policy` and `patch_thrash` with a live thread all receive the resume footer and the wrapper contract's single scripted wind down matches the same set, after two runs lost 413 and 443 seconds of completed work with zero output because resumability was keyed to `timeout` alone. resumability and the breaker stay orthogonal, so `policy` still opens the circuit +- `/fusion:stats` stops reporting a structurally false unverified rate: every one of the 34 wall clock deaths was salvaged by a wind down resume that settled cleanly, 27 accepted and 6 rejected, but not one of the dead attempts is any worker record's `peerJobId`, since the wrapper binds only to the resume, so they could never receive a verdict by any existing path and sat at `unverified` forever, making one package read as two engine records. a terminal error record carrying a resumable failure kind, a thread id, no recorded verdict and a later same workspace job resuming that thread is now derived as `superseded` and counted in its own bucket; a recorded verdict always wins, and `superseded` is never accepted by `--record` nor written to disk +- the verification list lesson recurs one layer up, at the brief: a peer package reported `76 pass, 0 fail` while 64 process dependent tests, including every test it had just written, were skipped in its sandbox, because `node --test` exits 0 on a skip and the brief's done criterion accepted the exit code. the package was honest about the skip and two dispatches were still spent before the mandated full environment rerun showed `137 pass, 3 fail`. an implementation brief's acceptance criterion has to read `# fail 0` and `# skipped 0`, or name the titles that must appear passing, so that the work is done and the work is verified cannot both be claimed by one vacuous green + ## 0.0.49 - the judgment posture unverified ceiling is removed entirely, by user directive, after its first week fired almost solely on false positives: the ceiling denied writes in sessions that were verifying constantly, because the 0.0.48 sensor's output evidence regex (`fail 0`) is node:test reporter grammar while the sessions that hit the stop live in vitest repositories (`Tests 1635 passed`), and the exit status channel requires the runner as the final unpiped segment, a shape the standing discipline of piping every run through `tail` or `grep` guarantees never occurs. three days of audit ground truth: 1126 counted writes, 18 verification resets, 25 ceiling denies across six sessions, every denied session demonstrably green on its own suite. the deny branch, `FUSION_INLINE_UNVERIFIED_CEILING`, the approaching ceiling advisory note, the `unverified-ceiling` audit reason and the stats `unverifiedCeilingStops` counter all leave; judgment posture counts and advises and never denies a main loop write, `strict` keeps every floor verbatim, and the two posture independent denials (no-op heartbeat Bash, reaped worker probe) stay. the vitest blind sensor itself is deliberately not repaired here, removal rather than recalibration was the directive, and it remains the open candidate since advisory counts still overstate in vitest repositories diff --git a/plugins/codex/.claude-plugin/plugin.json b/plugins/codex/.claude-plugin/plugin.json index fd863c3..f66a598 100644 --- a/plugins/codex/.claude-plugin/plugin.json +++ b/plugins/codex/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "codex", "displayName": "Codex Companion", - "version": "0.0.49", + "version": "0.0.50", "description": "First party local Codex CLI delegation for tasks, reviews, resumable threads, and durable background jobs.", "author": { "name": "Harry Yep" diff --git a/plugins/codex/agents/codex-rescue.md b/plugins/codex/agents/codex-rescue.md index d0c9c0d..2cf9173 100644 --- a/plugins/codex/agents/codex-rescue.md +++ b/plugins/codex/agents/codex-rescue.md @@ -24,7 +24,7 @@ Forwarding rules: - If the Read call fails, the file is not empty, or the Write call fails, use a foreground Bash call to run `node "${CLAUDE_PLUGIN_ROOT}/scripts/codex-companion.mjs" transport-discard --raw-args-token TOKEN` with the validated token before returning the failure. Do not expose the token or transport file to the user. - Do not use Read for any path except the newly allocated empty transport file, and do not read it after writing. Do not search, run Git, execute tests, inspect job state, or perform any check, collection, cancellation, or companion operation beyond the fixed task operation. - When an explicit background request returns a receipt, return it unchanged. A direct slash command user inspects progress through status and collects the deliverable through result; when Fusion is installed, its monitor can notify them of completion. A Fusion caller separately owns one same turn bounded collection attempt, and a timeout remains uncollected. -- When a task operation's companion output ends with `state: error` and `failure: timeout` and its body contains a line beginning `Resume Codex job`, run the command printed on that line exactly once, unchanged except for appending a space, `--`, a space, and this double quoted wind down prompt: "Wind down: do not start new work. Finish the smallest coherent deliverable from the work already completed and report the files changed and the verification output." Run it as one additional foreground Bash call and relay the second companion output verbatim in place of the first. This is the single authorized exception to the one operation rule: exactly one resume per task operation, never chained; any second timeout, any other failure, or any output without that line is relayed as received. +- When a task operation's companion output ends with `state: error` and a `failure:` value of `timeout`, `policy`, or `patch_thrash` and its body contains a line beginning `Resume Codex job`, run the command printed on that line exactly once, unchanged except for appending a space, `--`, a space, and this double quoted wind down prompt: "Wind down: do not start new work. Finish the smallest coherent deliverable from the work already completed and report the files changed and the verification output." Run it as one additional foreground Bash call and relay the second companion output verbatim in place of the first. This is the single authorized exception to the one operation rule: exactly one resume per task operation, never chained; any second failure, any other failure, or any output without that line is relayed as received. - Return the companion stdout exactly as received. Do not summarize, paraphrase, prefix, suffix, or continue the work. - Relay the companion's stdout verbatim inside a fenced block. Never retype, summarize, or re-spell any part of it, including footers. Put commentary outside the fence. - If the companion invocation fails, return the failure exactly as Bash reports it. Do not generate a substitute answer. diff --git a/plugins/codex/scripts/codex-companion.mjs b/plugins/codex/scripts/codex-companion.mjs index 023d17d..ad309ee 100644 --- a/plugins/codex/scripts/codex-companion.mjs +++ b/plugins/codex/scripts/codex-companion.mjs @@ -99,7 +99,8 @@ const RECORD_ACCEPTANCE_JOB_ID_PATTERN = /^[a-f0-9]{32}$/; const RECORD_ACCEPTANCE_VALUES = new Set(["accepted", "rejected", "unverified"]); const RECORD_ACCEPTANCE_SOURCES = new Set(["collector", "main-loop", "stats"]); const SEMANTIC_FAILURE_KINDS = new Set(["intent_override", "scope_rewrite", "wrong_approach", "style_mismatch"]); -const SOL_FOREGROUND_WRITE_WARNING = "warning: sol p90 wall clock exceeds the 600s foreground cap. Split the brief or name gpt-5.6-terra."; +const RESUMABLE_FAILURE_KINDS = new Set(["timeout", "policy", "patch_thrash"]); +const SOL_FOREGROUND_WRITE_WARNING = "warning: 41% of foreground gpt-5.6-sol write tasks timed out in the last 7 days. Split the package or use gpt-5.6-terra."; let activeCommandArgv = null; let cachedCompanionVersion = null; let companionVersionRead = false; @@ -237,6 +238,35 @@ function rejectImplicitSubdirectoryCwd(cwd, options) { } } +function hasGitAncestor(cwd) { + let current = cwd; + for (;;) { + try { + fs.lstatSync(path.join(current, ".git")); + return true; + } catch (error) { + if (error?.code !== "ENOENT") { + return true; + } + } + const parent = path.dirname(current); + if (parent === current) { + return false; + } + current = parent; + } +} + +function taskGitPreflight(cwd, write, skipGitRepoCheck) { + if (skipGitRepoCheck || hasGitAncestor(cwd)) { + return Boolean(skipGitRepoCheck); + } + if (write) { + throw new CompanionError("Codex write tasks outside a Git repository require --skip-git-repo-check. Pass --skip-git-repo-check to run this write task.", "input"); + } + return true; +} + function resolveOutputSchemaFile(cwd, value) { if (typeof value !== "string" || !value.trim()) { throw new CompanionError("--output-schema requires a non-empty path.", "input"); @@ -626,12 +656,14 @@ function consumeRawCommandStdin() { } } -function consumeRawCommandTransport(token, { allowUnwritten = false } = {}) { +function consumeRawCommandTransport(token, { allowUnwritten = false, retain = false } = {}) { const { directory, file, ownerFile } = transportPaths(token); let descriptor; let directoryIdentity = null; let fileIdentity = null; let ownerIdentity = null; + let retained = false; + const release = () => cleanupVerifiedTransport(directory, file, ownerFile, directoryIdentity, fileIdentity, ownerIdentity); try { const directoryStats = fs.lstatSync(directory); if (!directoryStats.isDirectory() || directoryStats.isSymbolicLink()) { @@ -674,17 +706,25 @@ function consumeRawCommandTransport(token, { allowUnwritten = false } = {}) { throw new CompanionError("The raw command transport file is not private.", "permission"); } const bytes = readBoundedDescriptor(descriptor, MAX_RAW_ARGUMENT_BYTES); + let value; if (bytes.length === 0) { if (allowUnwritten) { - return ""; + value = ""; + } else { + throw new CompanionError("The raw command transport is unwritten.", "input"); + } + } else { + try { + value = new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + throw new CompanionError("Raw command arguments must be valid UTF-8.", "input"); } - throw new CompanionError("The raw command transport is unwritten.", "input"); } - try { - return new TextDecoder("utf-8", { fatal: true }).decode(bytes); - } catch { - throw new CompanionError("Raw command arguments must be valid UTF-8.", "input"); + if (retain) { + retained = true; + return { release, value }; } + return value; } catch (error) { if (error instanceof CompanionError) { throw error; @@ -694,11 +734,13 @@ function consumeRawCommandTransport(token, { allowUnwritten = false } = {}) { if (descriptor !== undefined) { fs.closeSync(descriptor); } - cleanupVerifiedTransport(directory, file, ownerFile, directoryIdentity, fileIdentity, ownerIdentity); + if (!retained) { + release(); + } } } -function resolveCommandTransport(rawArgv) { +function resolveCommandTransport(rawArgv, { retainStaged = false } = {}) { const defaultWrite = rawArgv[0] === "--transport-default-write"; const transportArgv = defaultWrite ? rawArgv.slice(1) : rawArgv; if (transportArgv[0] === "--request-stdin") { @@ -716,7 +758,11 @@ function resolveCommandTransport(rawArgv) { if (transportArgv.length !== 2) { throw new CompanionError("Raw command transport requires exactly one token.", "input"); } - return { argv: [consumeRawCommandTransport(transportArgv[1])], defaultWrite, ingress: "staged_file" }; + const consumed = consumeRawCommandTransport(transportArgv[1], { retain: retainStaged }); + if (retainStaged) { + return { argv: [consumed.value], defaultWrite, ingress: "staged_file", release: consumed.release }; + } + return { argv: [consumed], defaultWrite, ingress: "staged_file" }; } function readTaskPrompt(cwd, options, positionals, positionalText) { @@ -953,6 +999,20 @@ function signalRecorded(record, prefix) { } } +function deadOwnerPatch(record) { + const message = deadOwnerMessage(record); + const recorded = RESUMABLE_FAILURE_KINDS.has(record.failureKind) ? record.failureKind : null; + if (!recorded) { + return { status: "error", failureKind: "died", errorMessage: message, errorTail: message }; + } + return { + status: "error", + failureKind: recorded, + errorMessage: record.errorMessage ?? message, + errorTail: record.errorTail ?? message + }; +} + function cleanupRequired(file, failureKind, message) { return updateJobRecordFile(file, { errorMessage: message, @@ -1002,14 +1062,8 @@ function currentProcessIdentity() { return identity; } -function isSolForegroundWriteTask(record) { - return ( - record.jobClass === "task" && - record.delivery === "foreground" && - record.mode === "write" && - typeof record.resolvedModel === "string" && - record.resolvedModel.toLowerCase().includes("sol") - ); +function isSolForegroundWriteRequest({ background, model, write }) { + return !background && write && typeof model === "string" && model.trim().toLowerCase() === "gpt-5.6-sol"; } function shellArgument(value) { @@ -1025,14 +1079,15 @@ function appendResumeFooter(text, footer) { return value.split("\n").includes(footer) ? value : value ? `${value}\n\n${footer}` : footer; } -function timeoutResumePatch(record, patch) { - const failureKind = patch.failureKind ?? record.failureKind; - const threadId = patch.threadId ?? record.threadId ?? record.request?.resumeThreadId; - if (failureKind !== "timeout" || typeof threadId !== "string" || !threadId.trim()) { +function resumableFailurePatch(record, patch) { + const next = { ...record, ...patch }; + const failureKind = next.failureKind; + const threadId = next.threadId ?? next.request?.resumeThreadId; + if (!RESUMABLE_FAILURE_KINDS.has(failureKind) || typeof threadId !== "string" || !threadId.trim()) { return patch; } - const resumeCommand = timeoutResumeCommand(record, threadId); - const footer = `Resume Codex job ${record.id}: ${resumeCommand}`; + const resumeCommand = timeoutResumeCommand(next, threadId); + const footer = `Resume Codex job ${next.id}: ${resumeCommand}`; const resultText = Object.hasOwn(patch, "resultText") ? patch.resultText : record.resultText; const partialResultText = Object.hasOwn(patch, "partialResultText") ? patch.partialResultText : record.partialResultText; if (typeof resultText === "string" && resultText.trim()) { @@ -1043,7 +1098,7 @@ function timeoutResumePatch(record, patch) { function finishJob(file, patch, env = process.env) { const existing = readJobRecordFile(file); - const record = finishJobRecordFile(file, existing ? timeoutResumePatch(existing, patch) : patch); + const record = finishJobRecordFile(file, existing ? resumableFailurePatch(existing, patch) : patch); pruneJobState(resolveDataDir(env), { claudeSessionId: record.claudeSessionId, protectedJobFiles: [file], resumeWorkspaceJobFiles: [file] }); return record; } @@ -1104,13 +1159,7 @@ export function repairRunningRecordSync({ record, file }, env = process.env) { if (!cleanupComplete(refreshed)) { return cleanupRequired(file, "died", "The companion exited without a terminal outcome, but verified process cleanup did not complete."); } - const message = deadOwnerMessage(current); - return finishJob(file, { - status: "error", - failureKind: "died", - errorMessage: message, - errorTail: message - }); + return finishJob(file, deadOwnerPatch(current)); } export async function refreshRunningJobRecord(found, env = process.env) { @@ -1155,13 +1204,7 @@ export async function refreshRunningJobRecord(found, env = process.env) { if (!cleanupComplete(refreshed)) { return cleanupRequired(found.file, "died", "The companion exited without a terminal outcome, but verified process cleanup did not complete."); } - const message = deadOwnerMessage(current); - return finishJob(found.file, { - status: "error", - failureKind: "died", - errorMessage: message, - errorTail: message - }); + return finishJob(found.file, deadOwnerPatch(current)); } function latestResumeRecord(dataDir, cwd, claudeSessionId) { @@ -1233,7 +1276,7 @@ function inheritedRouting(record, options, defaultServiceTier) { return { effort, inherited, model, serviceTier }; } -function createReservedJob({ background, brief, codexVersion, companionVersion, cwd, dataDir, jobClass, mode, request, timeoutMs }) { +function createReservedJob({ background, brief, codexVersion, companionVersion, cwd, dataDir, diagnostics = [], jobClass, mode, request, timeoutMs }) { brief = boundedPrompt(brief); const ownerIdentity = background ? null : currentProcessIdentity(); const reserveWorkspace = (reservedRequest) => withWorkspaceLock(dataDir, cwd, () => { @@ -1256,6 +1299,7 @@ function createReservedJob({ background, brief, codexVersion, companionVersion, companionVersion, cwd, delivery: background ? backgroundDelivery() : "foreground", + diagnostics, eventsFile: jobEventsPath(dataDir, cwd, id), jobClass, kind: jobClass, @@ -1305,7 +1349,7 @@ function executionArgs(record) { model: request.model, serviceTier: Object.hasOwn(request, "serviceTier") ? request.serviceTier : "priority", network: Boolean(request.network), - skipGitRepoCheck: Boolean(request.skipGitRepoCheck), + skipGitRepoCheck: request.transport === "task" ? taskGitPreflight(record.cwd, request.write === true, Boolean(request.skipGitRepoCheck)) : Boolean(request.skipGitRepoCheck), web: Boolean(request.web) }; if (request.transport === "native-review") { @@ -1486,11 +1530,7 @@ async function executeRecord(found) { const structured = structuredOutputOutcome(record.request ?? {}, outcome); const resolvedModel = outcome.resolvedModel ?? record.resolvedModel; const modelDrift = taskModelDrift(record, prompt, resolvedModel); - const completedRecord = { ...record, resolvedModel }; - const diagnostics = outcome.diagnostics.map((diagnostic) => ({ ...diagnostic, message: redactDiagnostic(diagnostic.message) })); - if (isSolForegroundWriteTask(completedRecord)) { - diagnostics.push({ type: "warning", message: SOL_FOREGROUND_WRITE_WARNING }); - } + const diagnostics = [...record.diagnostics, ...outcome.diagnostics.map((diagnostic) => ({ ...diagnostic, message: redactDiagnostic(diagnostic.message) }))]; return finishJob(found.file, { cumulativeTokenUsage: outcome.cumulativeTokenUsage, diagnostics, @@ -1710,9 +1750,6 @@ async function dispatchJob(found, asJson) { return; } const completed = await executeRecord(found); - if (isSolForegroundWriteTask(completed)) { - process.stderr.write(`${SOL_FOREGROUND_WRITE_WARNING}\n`); - } renderRecord(completed, asJson); collectRenderedJob(found.file, completed); if (completed.status !== "done") { @@ -1721,60 +1758,78 @@ async function dispatchJob(found, asJson) { } async function handleTask(rawArgv, transport = {}) { - const { options, positionals, positionalText } = commandArgs(rawArgv, { - booleanOptions: ["background", "fresh", "json", "network", "resume-last", "skip-git-repo-check", "web", "write"], - optionsBeforePositionals: true, - valueOptions: ["cwd", "effort", "model", "output-schema", "prompt-file", "resume", "service-tier"] - }); - const serviceTier = serviceTierOption(options["service-tier"]); - const cwd = resolveCwd(options); - rejectImplicitSubdirectoryCwd(cwd, options); - const dataDir = resolveDataDir(); - const resume = resolveResume(dataDir, cwd, options); - const routing = resume.threadId ? inheritedRouting(latestJobRecordForThread(dataDir, resume.threadId), options, serviceTier) : inheritedRouting(null, options, serviceTier); - const write = options.write ?? Boolean(transport.defaultWrite); - const outputSchemaFile = options["output-schema"] ? resolveOutputSchemaFile(cwd, options["output-schema"]) : null; - let prompt = readTaskPrompt(cwd, options, positionals, positionalText); - if (!prompt.trim() && resume.threadId) { - prompt = CONTINUE_PROMPT; - } - if (!prompt.trim()) { - throw new CompanionError("Provide a Codex task prompt or --prompt-file.", "input"); - } - if (options.network && !write) { - throw new CompanionError("--network requires --write because network access applies to the workspace-write sandbox.", "input"); + let releaseTransport = transport.release; + try { + const { options, positionals, positionalText } = commandArgs(rawArgv, { + booleanOptions: ["background", "fresh", "json", "network", "resume-last", "skip-git-repo-check", "web", "write"], + optionsBeforePositionals: true, + valueOptions: ["cwd", "effort", "model", "output-schema", "prompt-file", "resume", "service-tier"] + }); + const serviceTier = serviceTierOption(options["service-tier"]); + const cwd = resolveCwd(options); + rejectImplicitSubdirectoryCwd(cwd, options); + const dataDir = resolveDataDir(); + const resume = resolveResume(dataDir, cwd, options); + const routing = resume.threadId ? inheritedRouting(latestJobRecordForThread(dataDir, resume.threadId), options, serviceTier) : inheritedRouting(null, options, serviceTier); + const write = options.write ?? Boolean(transport.defaultWrite); + const background = Boolean(options.background); + const outputSchemaFile = options["output-schema"] ? resolveOutputSchemaFile(cwd, options["output-schema"]) : null; + let prompt = readTaskPrompt(cwd, options, positionals, positionalText); + if (!prompt.trim() && resume.threadId) { + prompt = CONTINUE_PROMPT; + } + if (!prompt.trim()) { + throw new CompanionError("Provide a Codex task prompt or --prompt-file.", "input"); + } + if (options.network && !write) { + throw new CompanionError("--network requires --write because network access applies to the workspace-write sandbox.", "input"); + } + const skipGitRepoCheck = Boolean(options["skip-git-repo-check"]); + taskGitPreflight(cwd, write, skipGitRepoCheck); + const probe = preflightCodex(cwd); + const request = { + effort: routing.effort, + fresh: Boolean(options.fresh), + ingress: transport.ingress ?? "argv", + model: routing.model, + network: Boolean(options.network), + outputSchemaFile, + resumeSourceJobId: resume.sourceJobId, + resumeThreadId: resume.threadId, + skipGitRepoCheck, + serviceTier: routing.serviceTier, + transport: "task", + web: Boolean(options.web), + write: Boolean(write), + ...(routing.inherited ? { inheritedFromThread: true } : {}) + }; + validateExecutionRequest(cwd, request); + const diagnostics = isSolForegroundWriteRequest({ background, model: request.model, write: request.write }) ? [{ type: "warning", message: SOL_FOREGROUND_WRITE_WARNING }] : []; + if (diagnostics.length > 0) { + process.stderr.write(`${SOL_FOREGROUND_WRITE_WARNING}\n`); + } + const found = createReservedJob({ + background, + brief: prompt, + codexVersion: probe.version, + companionVersion: companionVersion(), + cwd, + dataDir, + diagnostics, + jobClass: "task", + mode: write ? "write" : "consult", + request, + timeoutMs: resolveExecutionTimeout(background) + }); + releaseTransport?.(); + releaseTransport = null; + await dispatchJob(found, Boolean(options.json)); + } catch (error) { + if (!/^Codex (?:job|thread) .+ is already running/.test(error?.message ?? "")) { + releaseTransport?.(); + } + throw error; } - const probe = preflightCodex(cwd); - const request = { - effort: routing.effort, - fresh: Boolean(options.fresh), - ingress: transport.ingress ?? "argv", - model: routing.model, - network: Boolean(options.network), - outputSchemaFile, - resumeSourceJobId: resume.sourceJobId, - resumeThreadId: resume.threadId, - skipGitRepoCheck: Boolean(options["skip-git-repo-check"]), - serviceTier: routing.serviceTier, - transport: "task", - web: Boolean(options.web), - write: Boolean(write), - ...(routing.inherited ? { inheritedFromThread: true } : {}) - }; - validateExecutionRequest(cwd, request); - const found = createReservedJob({ - background: Boolean(options.background), - brief: prompt, - codexVersion: probe.version, - companionVersion: companionVersion(), - cwd, - dataDir, - jobClass: "task", - mode: write ? "write" : "consult", - request, - timeoutMs: resolveExecutionTimeout(Boolean(options.background)) - }); - await dispatchJob(found, Boolean(options.json)); } function reviewTarget(options) { @@ -2295,7 +2350,7 @@ async function main() { await handleRecordAcceptance(receivedArgv); return; } - const transport = resolveCommandTransport(receivedArgv); + const transport = resolveCommandTransport(receivedArgv, { retainStaged: subcommand === "task" }); if (transport.defaultWrite && subcommand !== "task") { throw new CompanionError("The raw command transport write default is valid only for task.", "input"); } diff --git a/plugins/codex/scripts/lib/codex-exec.mjs b/plugins/codex/scripts/lib/codex-exec.mjs index 6930e8b..e284629 100644 --- a/plugins/codex/scripts/lib/codex-exec.mjs +++ b/plugins/codex/scripts/lib/codex-exec.mjs @@ -28,6 +28,9 @@ const DEFAULT_ROLLOUT_TAIL_MAX_BYTES = 16 * 1024 * 1024; const MAX_ROLLOUT_SCAN_ENTRIES = 50000; const MAX_DIAGNOSTICS = 64; const MAX_DIAGNOSTIC_MESSAGE_BYTES = 4096; +const PATCH_VERIFICATION_FAILURE = "apply_patch verification failed"; +const UNKNOWN_PROCESS_ID = "unknownprocessid"; +const STDERR_PARTIAL_LINE_TAIL_LENGTH = Math.max(PATCH_VERIFICATION_FAILURE.length, UNKNOWN_PROCESS_ID.length) - 1; const EFFORT_PATTERN = /^[a-z][a-z0-9_-]{0,31}$/; const SERVICE_TIER_PATTERN = /^[a-z][a-z0-9_-]{0,31}$/; const THREAD_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; @@ -902,6 +905,34 @@ function boundedBuffer(current, chunk, maxBytes) { return combined.length <= maxBytes ? combined : combined.subarray(combined.length - maxBytes); } +function stderrPatternOccurrences(text, pattern, previousTailLength) { + let count = 0; + let offset = 0; + for (;;) { + const index = text.indexOf(pattern, offset); + if (index === -1) { + return count; + } + if (index + pattern.length > previousTailLength) { + count += 1; + } + offset = index + pattern.length; + } +} + +function observeStderrPatterns(state, chunk) { + const previousTail = state.stderrPartialLine; + const text = `${previousTail}${Buffer.from(chunk).toString("utf8")}`; + const lowerCaseText = text.toLowerCase(); + const previousTailLength = previousTail.length; + const patchFailures = stderrPatternOccurrences(lowerCaseText, PATCH_VERIFICATION_FAILURE, previousTailLength); + const lostProcesses = stderrPatternOccurrences(lowerCaseText, UNKNOWN_PROCESS_ID, previousTailLength); + const lastNewline = text.lastIndexOf("\n"); + const partialLine = lastNewline === -1 ? text : text.slice(lastNewline + 1); + state.stderrPartialLine = partialLine.slice(-STDERR_PARTIAL_LINE_TAIL_LENGTH); + return { lostProcesses, patchFailures }; +} + function boundedMessage(message) { const buffer = Buffer.from(String(message ?? "")); return (buffer.length <= MAX_DIAGNOSTIC_MESSAGE_BYTES ? buffer : buffer.subarray(0, MAX_DIAGNOSTIC_MESSAGE_BYTES)).toString("utf8"); @@ -1067,6 +1098,9 @@ function validateEvent(state, event) { const tool = observedString(event.item.tool) ?? "unknown collaboration tool"; state.collaborationViolation = `Delegated Codex execution attempted the disabled ${tool} tool.`; } + if (event.type === "item.completed" && event.item.type === "file_change") { + state.patchFailureCount = 0; + } if (event.type === "item.completed" && event.item.type === "agent_message" && typeof event.item.text === "string") { if (Buffer.byteLength(event.item.text) > state.resultMaxBytes) { recordResourceError(state, `Codex final response exceeded the ${state.resultMaxBytes} byte limit.`); @@ -1154,6 +1188,20 @@ function failureOutcome(state, processOutcome, stderrTail) { if (state.collaborationViolation) { return { status: "error", failureKind: "policy", errorMessage: state.collaborationViolation }; } + if (state.patchThrash) { + return { + status: "error", + failureKind: "patch_thrash", + errorMessage: `Codex could not land a patch because its context lines did not match the file on disk after ${state.patchFailureCount} consecutive apply_patch verification failures.` + }; + } + if (state.execLost) { + return { + status: "error", + failureKind: "exec_lost", + errorMessage: `The sandbox lost the spawned process after ${state.lostProcessCount} UnknownProcessId errors.` + }; + } if (state.cancelled) { return { status: "cancelled", failureKind: "cancelled", errorMessage: "Codex execution was cancelled." }; } @@ -1242,9 +1290,13 @@ export async function runCodex(options = {}) { diagnostics: [], eventCount: 0, eventsTruncated: false, + execLost: false, expectedResumeThreadId: typeof options.resumeThreadId === "string" && options.resumeThreadId.trim() ? options.resumeThreadId.trim() : null, finalResponse: "", + lostProcessCount: 0, oversizedEventsSkipped: 0, + patchFailureCount: 0, + patchThrash: false, protocolError: null, resourceError: null, resumeThreadMismatch: false, @@ -1257,6 +1309,7 @@ export async function runCodex(options = {}) { threadId: null, threadStarted: false, stdinError: null, + stderrPartialLine: "", timedOut: false, timeoutMs, turnFailureMessage: null, @@ -1397,6 +1450,23 @@ export async function runCodex(options = {}) { }, timeoutMs); timer.unref?.(); child.stderr?.on("data", (chunk) => { + if (!state.timedOut) { + const { lostProcesses, patchFailures } = observeStderrPatterns(state, chunk); + if (!state.patchThrash && patchFailures > 0) { + state.patchFailureCount = Math.min(3, state.patchFailureCount + patchFailures); + if (state.patchFailureCount === 3) { + state.patchThrash = true; + void requestTermination(); + } + } + if (!state.execLost && lostProcesses > 0) { + state.lostProcessCount = Math.min(3, state.lostProcessCount + lostProcesses); + if (state.lostProcessCount === 3) { + state.execLost = true; + void requestTermination(); + } + } + } stderr = boundedBuffer(stderr, chunk, stderrMaxBytes); if (options.logFile) { try { @@ -1590,7 +1660,7 @@ export async function runCodex(options = {}) { usage.tokenUsageUnavailableReason = null; } const stderrTail = stderr.toString("utf8"); - const failure = state.timedOut + const failure = state.timedOut && !state.patchThrash && !state.execLost ? timeoutOutcome(state) : failureOutcome(state, { ...processOutcome, spawnError }, stderrTail); const rawErrorMessage = failure.errorMessage; diff --git a/plugins/codex/skills/codex-cli-runtime/SKILL.md b/plugins/codex/skills/codex-cli-runtime/SKILL.md index 5413a25..101921a 100644 --- a/plugins/codex/skills/codex-cli-runtime/SKILL.md +++ b/plugins/codex/skills/codex-cli-runtime/SKILL.md @@ -28,14 +28,15 @@ Execution rules: - Invoke the helper only through foreground `Bash` with `timeout: 600000`. - Place every task option before the first prompt token. Use `--` before a prompt that begins with an option shaped token. - Task, review, and adversarial review reject an implicit working directory below its repository top level before job creation. Pass `--cwd` with either the repository root or the intended subdirectory to choose the sandbox root explicitly. -- The companion forwards `--skip-git-repo-check` to Codex exec. Without that flag or the dangerous bypass flag, Codex refuses to start unless the working directory has an ancestor `.git` entry; the projects trust map in `config.toml` is not consulted by exec. A gate failure names `--skip-git-repo-check` as the remedy. +- The companion forwards `--skip-git-repo-check` to Codex exec. For a task with no ancestor `.git` entry, a read only consult automatically receives that flag because it cannot write. A write task fails before job creation unless the request explicitly includes it. The projects trust map in `config.toml` is not consulted by exec. A gate failure names `--skip-git-repo-check` as the remedy. - Never use Bash background mode. Complexity, duration, and model choice never justify implicit background execution. - Pass `--background` only when the received request explicitly contains it. The companion owns detachment and returns a durable job receipt. - A direct slash command invocation returns that receipt without automatic collection. Direct users inspect progress through status and collect the deliverable through result; when Fusion is installed, its monitor can notify them of completion. Fusion orchestration separately owns one same turn collection attempt capped at 540000ms for jobs it creates. A timeout remains explicitly uncollected. - A task is read only unless `--write` is present. Review commands are always read only. - Leave `--model` and `--effort` unset unless explicitly requested so Codex configuration remains authoritative. - Use `--resume ` only with a real thread identifier returned by Codex. Use `--resume-last` for the newest eligible task thread launched by the current Claude session, or the newest eligible workspace task when no Claude session id is available. -- A foreground timeout with a resumable thread is salvaged by the wrapper's single scripted wind down resume; the resumed job links through `request.resumeThreadId`, and a second timeout terminalizes the package. +- A foreground task failure of `timeout`, `policy`, or `patch_thrash` with a resumable thread is salvaged by the wrapper's single scripted wind down resume; the resumed job links through `request.resumeThreadId`, and a second failure terminalizes the package. +- A foreground `--write --model gpt-5.6-sol` task emits a nonblocking warning before execution and records it as a diagnostic: 41% of those tasks timed out in the last 7 days. Split the package or use `gpt-5.6-terra`. - Use `--fresh` only when the caller explicitly requests a new thread. It cannot be combined with either resume form. - Use `--web` only when explicitly requested. Use `--network` only with `--write` and only when explicitly requested. - Use `--output-schema ` only for task mode. The companion resolves the path, requires a regular JSON file at most 256 KiB, and records one JSON parsing result without retrying the task. Native `review` ignores output schemas on tested CLI versions. Review shaped task briefs can use `${CLAUDE_PLUGIN_ROOT}/schemas/adversarial-review-verdict.schema.json`; adversarial review uses that schema automatically. diff --git a/plugins/fusion/.claude-plugin/plugin.json b/plugins/fusion/.claude-plugin/plugin.json index 635cb66..27d2c33 100644 --- a/plugins/fusion/.claude-plugin/plugin.json +++ b/plugins/fusion/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "fusion", "displayName": "Fusion Orchestrator", - "version": "0.0.49", + "version": "0.0.50", "description": "Multi-model orchestration: tier agents, routing rules, blind panel, ultra fleet, model config, and drift doctor.", "author": { "name": "Harry Yep" diff --git a/plugins/fusion/scripts/breaker-check.mjs b/plugins/fusion/scripts/breaker-check.mjs index f598089..25232bc 100644 --- a/plugins/fusion/scripts/breaker-check.mjs +++ b/plugins/fusion/scripts/breaker-check.mjs @@ -14,7 +14,7 @@ const LOOKBACK_ENV = "FUSION_BREAKER_LOOKBACK_HOURS"; const DEFAULT_LOOKBACK_HOURS = 12; const WORKER_BREAKER_LANES = ["fusion:claude-worker", "fusion:trivial-worker"]; const HARD_FAILURE_KINDS = new Set(["quota", "auth", "missing_cli", "protocol", "transport", "sandbox"]); -const REPEATED_FAILURE_KINDS = new Set(["rate_limited", "timeout", "stall", "process", "died"]); +const REPEATED_FAILURE_KINDS = new Set(["rate_limited", "timeout", "stall", "process", "died", "patch_thrash", "exec_lost"]); const BREAKER_FAILURE_KINDS = new Set([...HARD_FAILURE_KINDS, ...REPEATED_FAILURE_KINDS, "permission"]); const GROK_FAILURE_STATUSES = new Set(["error", "failed"]); const CODEX_FAILURE_STATUSES = new Set(["error", "failed"]); diff --git a/plugins/fusion/scripts/fusion-stats.mjs b/plugins/fusion/scripts/fusion-stats.mjs index 21f4ec9..1d987c9 100644 --- a/plugins/fusion/scripts/fusion-stats.mjs +++ b/plugins/fusion/scripts/fusion-stats.mjs @@ -28,6 +28,7 @@ const PRUNE_EVIDENCE = Symbol("pruneEvidence"); const FUSION_TASK_ID_PATTERN = /^fusion-[0-9a-f]{24}$/; const ENGINE_JOB_ID_PATTERN = /^[0-9a-f]{32}$/; const RECORD_VERDICTS = new Set(["accepted", "rejected", "unverified"]); +const RESUMABLE_CODEX_FAILURE_KINDS = new Set(["timeout", "policy", "patch_thrash"]); const SEMANTIC_FAILURE_KINDS = new Set(["intent_override", "scope_rewrite", "wrong_approach", "style_mismatch"]); const RECORD_SOURCES = new Set(["collector", "main-loop"]); const FUSION_ACCEPTANCE_EPOCH_ENV = "FUSION_ACCEPTANCE_EPOCH"; @@ -124,9 +125,12 @@ export function normalizeCollectionMethod(value) { return normalized; } -function semanticAcceptance(raw) { +function semanticAcceptance(raw, superseded = false) { const recorded = raw?.semanticStatus ?? raw?.acceptance; - return recorded === "accepted" || recorded === "rejected" ? recorded : "unverified"; + if (recorded === "accepted" || recorded === "rejected") { + return recorded; + } + return superseded ? "superseded" : "unverified"; } function acceptanceProvenance(raw) { @@ -471,11 +475,13 @@ function terminalLedgerRecord(raw, { workspaceRoot = null, workspaceKey = null, } const model = nonEmptyString(raw?.model ?? raw?.request?.model); const effort = nonEmptyString(raw?.effort ?? raw?.request?.effort); + const resumeThreadId = nonEmptyString(raw?.request?.resumeThreadId); return { id: jobId, status, workspaceRoot: nonEmptyString(raw?.workspaceRoot) ?? workspaceRoot, sessionId: nonEmptyString(raw?.sessionId), + threadId: nonEmptyString(raw?.threadId), jobClass: nonEmptyString(raw?.kind ?? raw?.jobClass) ?? "unknown", createdAt: raw?.createdAt ?? null, startedAt: raw?.startedAt ?? null, @@ -483,7 +489,7 @@ function terminalLedgerRecord(raw, { workspaceRoot = null, workspaceKey = null, completedAt: raw?.finishedAt ?? raw?.completedAt ?? raw?.observedAt ?? null, timeoutMs: raw?.timeoutMs ?? null, failureKind: raw?.failureKind ?? null, - request: model || effort ? { model, effort } : null, + request: model || effort || resumeThreadId ? { model, effort, ...(resumeThreadId ? { resumeThreadId } : {}) } : null, _fusionObservedModel: raw?.modelSource === "rollout-turn-context" ? model : null, _fusionObservedEffort: raw?.effortSource === "rollout-turn-context" ? effort : null, tokenUsage: raw?.tokenUsage ?? null, @@ -620,14 +626,14 @@ function mergeJobEvidence(preferred, records) { const merged = { ...preferred, request: preferred?.request && typeof preferred.request === "object" ? { ...preferred.request } : preferred?.request }; merged._fusionWorkspaceRoots = [...new Set(records.flatMap((record) => Array.isArray(record?._fusionWorkspaceRoots) ? record._fusionWorkspaceRoots : [record?.workspaceRoot]).filter((value) => typeof value === "string" && value.trim()).map((value) => path.resolve(value)))]; for (const record of records) { - for (const field of ["workspaceRoot", "sessionId", "jobClass", "kind", "createdAt", "startedAt", "finishedAt", "completedAt", "updatedAt", "tokenUsage", "tokenUsageAvailability", "_fusionObservedModel", "_fusionObservedEffort", "_fusionScopeKey", "_fusionRepositoryKey"]) { + for (const field of ["workspaceRoot", "sessionId", "threadId", "jobClass", "kind", "createdAt", "startedAt", "finishedAt", "completedAt", "updatedAt", "tokenUsage", "tokenUsageAvailability", "_fusionObservedModel", "_fusionObservedEffort", "_fusionScopeKey", "_fusionRepositoryKey"]) { if (merged[field] == null && record?.[field] != null) { merged[field] = record[field]; } } if (record?.request && typeof record.request === "object") { merged.request = merged.request && typeof merged.request === "object" ? merged.request : {}; - for (const field of ["model", "effort"]) { + for (const field of ["model", "effort", "resumeThreadId"]) { if (merged.request[field] == null && record.request[field] != null) { merged.request[field] = record.request[field]; } @@ -778,6 +784,28 @@ function workspaceRootsForJob(raw) { return [...new Set([...(Array.isArray(raw?._fusionWorkspaceRoots) ? raw._fusionWorkspaceRoots : []), raw?.workspaceRoot].filter((value) => typeof value === "string" && value.trim()).map((value) => path.resolve(value)))]; } +function sameCodexWorkspace(left, right) { + const rightRoots = new Set(workspaceRootsForJob(right)); + return workspaceRootsForJob(left).some((workspaceRoot) => rightRoots.has(workspaceRoot)); +} + +function isSupersededCodexJob(raw, jobs) { + if (raw?.status !== "error" || !RESUMABLE_CODEX_FAILURE_KINDS.has(nonEmptyString(raw?.failureKind))) { + return false; + } + const threadId = nonEmptyString(raw?.threadId); + const createdAt = Date.parse(raw?.createdAt ?? ""); + const recorded = raw?.semanticStatus ?? raw?.acceptance; + if (!threadId || !Number.isFinite(createdAt) || recorded === "accepted" || recorded === "rejected") { + return false; + } + return jobs.some((candidate) => { + const resumeThreadId = nonEmptyString(candidate?.request?.resumeThreadId); + const resumedAt = Date.parse(candidate?.createdAt ?? ""); + return candidate !== raw && resumeThreadId === threadId && Number.isFinite(resumedAt) && resumedAt > createdAt && sameCodexWorkspace(raw, candidate); + }); +} + function observationForJob(raw, env, auditCache) { const workspaceRoots = workspaceRootsForJob(raw); if (workspaceRoots.length === 0) { @@ -1165,6 +1193,7 @@ export function fileBasedEngineStats(descriptor, { all = false, env = process.en let latest = null; for (const raw of scoped) { const job = descriptor.normalizeJob(raw); + const acceptance = descriptor.id === "codex" ? semanticAcceptance(raw, isSupersededCodexJob(raw, scoped)) : null; let model = null; let effort = null; bump(byStatus, job.status); @@ -1196,7 +1225,7 @@ export function fileBasedEngineStats(descriptor, { all = false, env = process.en sku: codexSku(model ?? "unknown", effort), createdAtMs, terminal: CODEX_TERMINAL_STATUSES.has(job.status), - acceptance: semanticAcceptance(raw), + acceptance, failureKind: nonEmptyString(raw?.failureKind), outputTokens: codexUsage.usage?.outputTokens ?? null, durationSeconds: job.durationSeconds, @@ -1207,19 +1236,19 @@ export function fileBasedEngineStats(descriptor, { all = false, env = process.en if (descriptor.id === "codex") { const jobId = nonEmptyString(raw?.id); if (CODEX_TERMINAL_STATUSES.has(job.status)) { - bump(byAcceptance, semanticAcceptance(raw)); + bump(byAcceptance, acceptance); } else { pendingTransportJobs += 1; } const historical = Number.isFinite(Date.parse(raw?.finishedAt ?? "")) && Date.parse(raw.finishedAt) < acceptanceEpoch.timestamp; - if (jobId && job.status === "error" && semanticAcceptance(raw) === "accepted" && nonEmptyString(raw?.failureKind) !== "timeout") { + if (jobId && job.status === "error" && acceptance === "accepted" && !RESUMABLE_CODEX_FAILURE_KINDS.has(nonEmptyString(raw?.failureKind))) { if (historical) { historicalAcceptanceAnomalies += 1; } else { acceptedWithErrorTransport.push(jobId); } } - if (jobId && job.status === "done" && semanticAcceptance(raw) === "unverified") { + if (jobId && job.status === "done" && acceptance === "unverified") { if (historical) { historicalAcceptanceAnomalies += 1; } else { @@ -1851,6 +1880,7 @@ function sessionScopedEngineStats(descriptor, sessionId, env) { } } } + const codexJobs = descriptor.id === "codex" ? selectPreferredJobs(jobs, descriptor, () => true) : []; const scoped = selectPreferredJobs(jobs, descriptor, (raw) => descriptor.sessionOf(raw) === sessionId); totalJobs = scoped.length; for (const raw of scoped) { @@ -1858,7 +1888,7 @@ function sessionScopedEngineStats(descriptor, sessionId, env) { bump(byStatus, status); if (descriptor.id === "codex") { if (CODEX_TERMINAL_STATUSES.has(status)) { - bump(byAcceptance, semanticAcceptance(raw)); + bump(byAcceptance, semanticAcceptance(raw, isSupersededCodexJob(raw, codexJobs))); } else { pendingTransportJobs += 1; } @@ -2671,7 +2701,8 @@ function unsettledEngineEntries(descriptor, env) { if (!descriptor.isTerminal(raw)) { return []; } - if (semanticAcceptance(raw) !== "unverified") { + const acceptance = descriptor.id === "codex" ? semanticAcceptance(raw, isSupersededCodexJob(raw, scoped)) : semanticAcceptance(raw); + if (acceptance !== "unverified") { return []; } const id = nonEmptyString(raw?.id); diff --git a/plugins/grok/.claude-plugin/plugin.json b/plugins/grok/.claude-plugin/plugin.json index bf502b7..c4d0f71 100644 --- a/plugins/grok/.claude-plugin/plugin.json +++ b/plugins/grok/.claude-plugin/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "grok", "displayName": "Grok Companion", - "version": "0.0.49", + "version": "0.0.50", "description": "Local Grok CLI delegation: task, review, resumable history, background jobs, stats, and setup health checks.", "author": { "name": "Harry Yep" diff --git a/tests/breaker-check.test.mjs b/tests/breaker-check.test.mjs index aa821b6..02cddb3 100644 --- a/tests/breaker-check.test.mjs +++ b/tests/breaker-check.test.mjs @@ -394,6 +394,24 @@ test("two consecutive timeouts open the breaker and a later success closes it", assert.strictEqual(run(sandbox).stdout, ""); }); +for (const failureKind of ["patch_thrash", "exec_lost"]) { + test(`one ${failureKind} keeps the Codex breaker closed, while a consecutive second occurrence opens it`, (t) => { + const sandbox = makeSandbox(t); + writeRecord(jobFile(sandbox.codexState, "workspace", `${failureKind}-first`), { + status: "error", + failureKind, + finishedAt: new Date(Date.now() - 3 * 60000).toISOString() + }); + assert.strictEqual(run(sandbox).stdout, ""); + writeRecord(jobFile(sandbox.codexState, "workspace", `${failureKind}-second`), { + status: "error", + failureKind, + finishedAt: new Date(Date.now() - 2 * 60000).toISOString() + }); + assert.match(run(sandbox).stdout, new RegExp(`codex breaker.*last failure ${failureKind}`)); + }); +} + test("a successful terminal job closes a previously hard breaker", (t) => { const sandbox = makeSandbox(t); writeRecord(jobFile(sandbox.codexState, "workspace", "auth-failure"), { diff --git a/tests/codex-companion.test.mjs b/tests/codex-companion.test.mjs index 1446890..e0eb9d2 100644 --- a/tests/codex-companion.test.mjs +++ b/tests/codex-companion.test.mjs @@ -33,11 +33,11 @@ import { gitIsolation } from "./lib/git-fixture.mjs"; const processInspectionTests = new Set([ "task accepts an explicit cwd below a repository top level", - "task accepts an implicit cwd outside a Git repository", + "task preflight auto passes the Git bypass for consults and rejects writes outside Git", "task accepts an explicit repository top level from a subdirectory", "identity-replaced owners are terminalized without signaling the live replacement", "task stays foreground by default and persists the complete terminal record", - "sol foreground write tasks add one diagnostic warning without changing the prompt", + "sol foreground write warning is emitted before execution and recorded", "sol warning is limited to foreground write tasks", "task resolves an output schema, forwards it, and records parsed structured output", "task retains a non-JSON agent message and records the structured parsing error", @@ -69,12 +69,14 @@ const processInspectionTests = new Set([ "record-acceptance permits explicit acceptance of failed transport", "foreground timeout persists recovered partial delivery and incomplete cumulative usage", "timeout jobs without a thread do not advertise resumability", + "policy failures with a live thread are resumable while unrelated failures are not", "history lists canonical jobs across workspaces with local thread and delivery metadata", "explicit background launch returns a durable receipt and result wait collects it", "managed background delivery remains pending until result collection", "tight history quotas retain a completed background review until result collects it", "result wait leaves a live background job running when its bounded wait expires", "workspace reservation makes simultaneous task launches single flight", + "a single-flight bounce retains staged raw arguments for one retry", "main workspace and a sibling worktree admit concurrent tasks for the same repository", "resume-last uses the current Claude session without claiming an unverifiable usage delta", "resume-last inherits model, effort, and service tier from its thread", @@ -147,6 +149,10 @@ function runGit(sandbox, args, options = {}) { }); } +function initializeGitRepository(sandbox, cwd = sandbox.workDir) { + runGit(sandbox, ["init", "--quiet"], { cwd }); +} + test("task header parsing captures model and effort only from a pipe-separated header", () => { assert.deepEqual( parseTaskHeader("\n lane: codex | MODEL: gpt-5.6-terra | Effort: xhigh | verification: node --test\nImplement the change."), @@ -183,16 +189,26 @@ test("task accepts an explicit cwd below a repository top level", (t) => { assert.equal(jobRecords(sandbox)[0].diagnostics.some((diagnostic) => diagnostic.type === "warning"), false); }); -test("task accepts an implicit cwd outside a Git repository", (t) => { +test("task preflight auto passes the Git bypass for consults and rejects writes outside Git", (t) => { const sandbox = makeSandbox(t); const subdirectory = path.join(sandbox.workDir, "apps", "api"); fs.mkdirSync(subdirectory, { recursive: true }); - const result = runCompanion(["task", "inspect the API"], { cwd: subdirectory, env: envFor(sandbox) }); + const consult = runCompanion(["task", "inspect the API"], { cwd: subdirectory, env: envFor(sandbox) }); - assert.equal(result.status, 0, result.stderr); - assert.equal(result.stderr, ""); - assert.equal(jobRecords(sandbox)[0].diagnostics.some((diagnostic) => diagnostic.type === "warning"), false); + assert.equal(consult.status, 0, consult.stderr); + assert.equal(consult.stderr, ""); + assert.ok(readArgs(sandbox).includes("--skip-git-repo-check")); + assert.equal(jobRecords(sandbox)[0].request.skipGitRepoCheck, false); + fs.rmSync(sandbox.argsFile); + + const write = runCompanion(["task", "--write", "implement the API"], { cwd: subdirectory, env: envFor(sandbox) }); + + assert.equal(write.status, 1); + assert.match(write.stderr, /--skip-git-repo-check/); + assert.match(write.stderr, /failure: input/); + assert.equal(fs.existsSync(sandbox.argsFile), false); + assert.equal(jobRecords(sandbox).length, 1); }); test("task accepts an explicit repository top level from a subdirectory", (t) => { @@ -344,6 +360,7 @@ setInterval(() => {}, 1000);`, test("task stays foreground by default and persists the complete terminal record", (t) => { const sandbox = makeSandbox(t); + initializeGitRepository(sandbox); const env = envFor(sandbox); const result = runCompanion(["task", "--write --model gpt-test --effort max --web --network implement this safely"], { cwd: sandbox.workDir, @@ -387,40 +404,50 @@ test("task stays foreground by default and persists the complete terminal record assert.equal(fs.statSync(path.dirname(entry.file)).mode & 0o777, 0o700); }); -test("sol foreground write tasks add one diagnostic warning without changing the prompt", (t) => { +test("sol foreground write warning is emitted before execution and recorded", (t) => { const sandbox = makeSandbox(t); - const result = runCompanion(["task", "--write", "--model", "requested-model", "implement the sol-safe change"], { + initializeGitRepository(sandbox); + const traceCodex = path.join(sandbox.root, "trace-codex"); + fs.writeFileSync( + traceCodex, + [ + "#!/usr/bin/env node", + "if (process.argv.includes('--version')) {", + " process.stdout.write('codex-cli 0.146.0\\n');", + " require('node:fs').unlinkSync(process.argv[1]);", + "}" + ].join("\n"), + { mode: 0o755 } + ); + const result = runCompanion(["task", "--write", "--model", "gpt-5.6-sol", "implement the sol-safe change"], { cwd: sandbox.workDir, env: envFor(sandbox, { - CODEX_HOME: path.join(sandbox.root, "codex-home"), - FAKE_CODEX_MODE: "rollout-completed", - FAKE_CODEX_RESOLVED_MODEL: "gpt-5.6-sol" + CODEX_BIN: traceCodex }) }); - const warning = "warning: sol p90 wall clock exceeds the 600s foreground cap. Split the brief or name gpt-5.6-terra."; + const warning = "warning: 41% of foreground gpt-5.6-sol write tasks timed out in the last 7 days. Split the package or use gpt-5.6-terra."; - assert.equal(result.status, 0, result.stderr); + assert.equal(result.status, 1, result.stderr); assert.equal(result.stderr, `${warning}\n`); - assert.equal(fs.readFileSync(sandbox.stdinFile, "utf8").trim(), "implement the sol-safe change"); const [record] = jobRecords(sandbox); + assert.equal(record.failureKind, "missing_cli"); assert.equal(record.diagnostics.filter((diagnostic) => diagnostic.type === "warning" && diagnostic.message === warning).length, 1); }); test("sol warning is limited to foreground write tasks", async (t) => { - const warning = "warning: sol p90 wall clock exceeds the 600s foreground cap. Split the brief or name gpt-5.6-terra."; + const warning = "warning: 41% of foreground gpt-5.6-sol write tasks timed out in the last 7 days. Split the package or use gpt-5.6-terra."; const cases = [ - { args: ["task", "--write", "use terra"], model: "gpt-5.6-terra" }, - { args: ["task", "use sol in consult mode"], model: "gpt-5.6-sol" } + { args: ["task", "--write", "--model", "gpt-5.6-terra", "use terra"], write: true }, + { args: ["task", "--model", "gpt-5.6-sol", "use sol in consult mode"], write: false } ]; for (const entry of cases) { const sandbox = makeSandbox(t); + if (entry.write) { + initializeGitRepository(sandbox); + } const result = runCompanion(entry.args, { cwd: sandbox.workDir, - env: envFor(sandbox, { - CODEX_HOME: path.join(sandbox.root, "codex-home"), - FAKE_CODEX_MODE: "rollout-completed", - FAKE_CODEX_RESOLVED_MODEL: entry.model - }) + env: envFor(sandbox) }); assert.equal(result.status, 0, result.stderr); assert.equal(result.stderr, ""); @@ -428,12 +455,9 @@ test("sol warning is limited to foreground write tasks", async (t) => { } const background = makeSandbox(t); - const env = envFor(background, { - CODEX_HOME: path.join(background.root, "codex-home"), - FAKE_CODEX_MODE: "rollout-completed", - FAKE_CODEX_RESOLVED_MODEL: "gpt-5.6-sol" - }); - const launched = runCompanion(["task", "--background", "--write", "use sol in the background"], { cwd: background.workDir, env }); + initializeGitRepository(background); + const env = envFor(background); + const launched = runCompanion(["task", "--background", "--write", "--model", "gpt-5.6-sol", "use sol in the background"], { cwd: background.workDir, env }); assert.equal(launched.status, 0, launched.stderr); assert.equal(launched.stderr, ""); const completed = await waitFor(() => { @@ -518,6 +542,7 @@ test("option shaped text after a task prompt cannot enable background or change test("single raw task arguments preserve prompt whitespace byte for byte", (t) => { const sandbox = makeSandbox(t); + initializeGitRepository(sandbox); const env = envFor(sandbox); const prompt = String.raw`preserve repeated spaces indentation, "quotes", and \d+`; @@ -531,6 +556,7 @@ test("single raw task arguments preserve prompt whitespace byte for byte", (t) = test("structured raw transport preserves shell syntax without evaluating it", (t) => { const sandbox = makeSandbox(t); + initializeGitRepository(sandbox); const markerOne = path.join(sandbox.root, "command-substitution-ran"); const markerTwo = path.join(sandbox.root, "backtick-ran"); const prompt = ` inspect $(touch ${markerOne}) and !\`touch ${markerTwo}\`\nEOF\n'outer \"inner\nkeep \\\\ exactly `; @@ -673,6 +699,7 @@ test("raw transport rejects expired input and removes the verified transport", ( test("programmatic stdin ingress preserves opaque raw arguments without a staging file", (t) => { const sandbox = makeSandbox(t); + initializeGitRepository(sandbox); const marker = path.join(sandbox.root, "stdin-command-substitution-ran"); const prompt = ` inspect $(touch ${marker})\nkeep quotes ' " and \\ exactly `; const result = runCompanion(["task", "--transport-default-write", "--request-stdin"], { @@ -733,6 +760,7 @@ test("transport allocation prunes verified inputs older than the retention windo test("rescue transport write defaults remain overridable by raw arguments", (t) => { const sandbox = makeSandbox(t); + initializeGitRepository(sandbox); const defaulted = createTransport(sandbox, "modify the file"); const writeResult = runCompanion(["task", "--transport-default-write", "--raw-args-token", defaulted.token], { cwd: sandbox.workDir, @@ -1250,6 +1278,40 @@ test("timeout jobs without a thread do not advertise resumability", (t) => { assert.equal(Object.hasOwn(record, "resumeCommand"), false); }); +test("policy failures with a live thread are resumable while unrelated failures are not", (t) => { + const policySandbox = makeSandbox(t); + const policy = runCompanion(["task", "--json", "stop the collaboration violation"], { + cwd: policySandbox.workDir, + env: envFor(policySandbox, { FAKE_CODEX_MODE: "collab-completed" }) + }); + + assert.equal(policy.status, 1, policy.stderr); + const policyRecord = JSON.parse(policy.stdout); + assert.equal(policyRecord.failureKind, "policy"); + assert.equal(policyRecord.threadId, "thread-123"); + + const reconciled = runCompanion(["status", "--json"], { cwd: policySandbox.workDir, env: envFor(policySandbox, {}) }); + assert.equal(reconciled.status, 0, reconciled.stderr); + const [settledPolicy] = jobRecords(policySandbox); + assert.equal(settledPolicy.status, "error"); + assert.equal(settledPolicy.failureKind, "policy"); + assert.equal(settledPolicy.resumable, true); + assert.match(settledPolicy.partialResultText, new RegExp(`Resume Codex job ${settledPolicy.id}:`)); + + const errorSandbox = makeSandbox(t); + const error = runCompanion(["task", "--json", "fail after the thread starts"], { + cwd: errorSandbox.workDir, + env: envFor(errorSandbox, { FAKE_CODEX_MODE: "failed" }) + }); + + assert.equal(error.status, 1, error.stderr); + const errorRecord = JSON.parse(error.stdout); + assert.equal(errorRecord.failureKind, "error"); + assert.equal(errorRecord.threadId, "thread-123"); + assert.equal(Object.hasOwn(errorRecord, "resumable"), false); + assert.equal(Object.hasOwn(errorRecord, "resumeCommand"), false); +}); + test("history lists canonical jobs across workspaces with local thread and delivery metadata", (t) => { const sandbox = makeSandbox(t); const sibling = path.join(sandbox.root, "sibling"); @@ -1384,6 +1446,31 @@ test("workspace reservation makes simultaneous task launches single flight", asy assert.equal(jobRecords(sandbox)[0].status, "done"); }); +test("a single-flight bounce retains staged raw arguments for one retry", async (t) => { + const sandbox = makeSandbox(t); + const env = envFor(sandbox, { FAKE_CODEX_DELAY_MS: "250" }); + const first = spawnCompanion(["task", "first task"], { cwd: sandbox.workDir, env }); + await waitFor(() => jobRecords(sandbox).some((record) => record.status === "running")); + const transport = createTransport(sandbox, "retry this staged task", env); + + const bounced = runCompanion(["task", "--raw-args-token", transport.token], { cwd: sandbox.workDir, env }); + + assert.equal(bounced.status, 1); + assert.match(bounced.stderr, /already running in this workspace/); + assert.equal(fs.existsSync(transport.file), true); + assert.equal(fs.existsSync(transport.ownerFile), true); + const firstResult = await childResult(first); + assert.equal(firstResult.code, 0, firstResult.stderr); + + const retried = runCompanion(["task", "--raw-args-token", transport.token], { cwd: sandbox.workDir, env }); + + assert.equal(retried.status, 0, retried.stderr); + assert.equal(fs.readFileSync(sandbox.stdinFile, "utf8"), "retry this staged task"); + assert.equal(fs.existsSync(transport.file), false); + assert.equal(fs.existsSync(transport.ownerFile), false); + assert.equal(fs.existsSync(path.dirname(transport.file)), false); +}); + test("main workspace and a sibling worktree admit concurrent tasks for the same repository", async (t) => { const sandbox = makeSandbox(t); const sibling = path.join(sandbox.root, "sibling-worktree"); diff --git a/tests/codex-exec.test.mjs b/tests/codex-exec.test.mjs index 312023a..4de4eed 100644 --- a/tests/codex-exec.test.mjs +++ b/tests/codex-exec.test.mjs @@ -41,6 +41,7 @@ const LOAD_TOLERANT_POLL_INTERVAL_MS = 25; const TIMEOUT_TEST_TIMEOUT_MS = 8000; const TIMEOUT_TEST_TERMINATION_GRACE_MS = 8000; const COMPANION_WATCHDOG_TIMEOUT_MS = 30000; +const PATCH_VERIFICATION_FAILURE = "apply_patch verification failed: Failed to find expected lines in target-file"; function fixture(t) { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "fusion-codex-exec-")); @@ -61,9 +62,13 @@ function fixture(t) { async function runFixture(t, mode, options = {}) { const files = fixture(t); + const bin = options.script ? path.join(files.dir, "scripted-codex.mjs") : fakeCodex; + if (options.script) { + fs.writeFileSync(bin, options.script, { mode: 0o700 }); + } const env = { ...process.env, - CODEX_BIN: fakeCodex, + CODEX_BIN: bin, FAKE_CODEX_ARGS_FILE: files.argsFile, FAKE_CODEX_CHILD_PID_FILE: files.childPidFile, FAKE_CODEX_INT_FILE: files.interruptFile, @@ -112,6 +117,18 @@ async function runFixture(t, mode, options = {}) { return { files, outcome, args }; } +function scriptedCodex(body) { + return `#!/usr/bin/env node + +for await (const _chunk of process.stdin) {} +const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const emit = (event) => process.stdout.write(JSON.stringify(event) + "\\n"); +const usage = { input_tokens: 1, cached_input_tokens: 0, output_tokens: 1, reasoning_output_tokens: 0 }; +emit({ type: "thread.started", thread_id: "thread-123" }); +emit({ type: "turn.started" }); +${body}`; +} + async function waitUntil(predicate, timeoutMs = LOAD_TOLERANT_WAIT_TIMEOUT_MS) { const deadline = Date.now() + timeoutMs; for (;;) { @@ -637,6 +654,67 @@ test("a nonzero exit invalidates an otherwise completed turn", async (t) => { assert.equal(outcome.exitCode, 7); }); +test("three consecutive apply_patch verification failures terminate as patch_thrash", async (t) => { + const { outcome } = await runFixture(t, "hang", { + env: { FAKE_CODEX_STDERR: `${PATCH_VERIFICATION_FAILURE}\n`.repeat(3) }, + terminationGraceMs: 100, + timeoutMs: TIMEOUT_TEST_TIMEOUT_MS + }); + assert.equal(outcome.status, "error"); + assert.equal(outcome.failureKind, "patch_thrash"); + assert.equal(outcome.timedOut, false); + assert.match(outcome.errorMessage, /could not land a patch because its context lines did not match the file on disk after 3 consecutive apply_patch verification failures/i); +}); + +test("a completed file_change resets the apply_patch verification failure counter", async (t) => { + const { outcome } = await runFixture(t, "scripted", { + script: scriptedCodex(`process.stderr.write(${JSON.stringify(`${PATCH_VERIFICATION_FAILURE}\n`.repeat(2))}); +await delay(100); +emit({ type: "item.completed", item: { id: "patch_1", type: "file_change" } }); +await delay(100); +process.stderr.write(${JSON.stringify(`${PATCH_VERIFICATION_FAILURE}\n`.repeat(2))}); +await delay(100); +emit({ type: "turn.completed", usage });`), + timeoutMs: TIMEOUT_TEST_TIMEOUT_MS + }); + assert.equal(outcome.status, "done"); + assert.equal(outcome.failureKind, null); + assert.equal(outcome.timedOut, false); +}); + +test("a split apply_patch verification failure is counted once", async (t) => { + const splitAt = PATCH_VERIFICATION_FAILURE.indexOf("verification"); + const { outcome } = await runFixture(t, "scripted", { + script: scriptedCodex(`process.stderr.write(${JSON.stringify(PATCH_VERIFICATION_FAILURE.slice(0, splitAt))}); +await delay(100); +process.stderr.write(${JSON.stringify(`${PATCH_VERIFICATION_FAILURE.slice(splitAt)}\n`)}); +await delay(100); +process.stderr.write(${JSON.stringify(`${PATCH_VERIFICATION_FAILURE}\n`)}); +await delay(100); +emit({ type: "item.completed", item: { id: "patch_1", type: "file_change" } }); +await delay(100); +process.stderr.write(${JSON.stringify(`${PATCH_VERIFICATION_FAILURE}\n`.repeat(2))}); +await delay(100); +emit({ type: "turn.completed", usage });`), + timeoutMs: TIMEOUT_TEST_TIMEOUT_MS + }); + assert.equal(outcome.status, "done"); + assert.equal(outcome.failureKind, null); + assert.equal(outcome.timedOut, false); +}); + +test("three UnknownProcessId errors terminate as exec_lost", async (t) => { + const { outcome } = await runFixture(t, "hang", { + env: { FAKE_CODEX_STDERR: "exec_command failed for tool: UnknownProcessId { process_id: 1 }\n".repeat(3) }, + terminationGraceMs: 100, + timeoutMs: TIMEOUT_TEST_TIMEOUT_MS + }); + assert.equal(outcome.status, "error"); + assert.equal(outcome.failureKind, "exec_lost"); + assert.equal(outcome.timedOut, false); + assert.match(outcome.errorMessage, /sandbox lost the spawned process after 3 UnknownProcessId errors/i); +}); + test("timeout terminates the process group", async (t) => { const { outcome } = await runFixture(t, "hang", { timeoutMs: TIMEOUT_TEST_TIMEOUT_MS, diff --git a/tests/codex-wrapper-contract.test.mjs b/tests/codex-wrapper-contract.test.mjs index d4d3ac2..0663418 100644 --- a/tests/codex-wrapper-contract.test.mjs +++ b/tests/codex-wrapper-contract.test.mjs @@ -211,3 +211,12 @@ test("the SubagentStop matcher matches all seven peer and Claude worker agent na } assert.doesNotMatch("general-purpose", matcher); }); + +test("the Codex rescue contract resumes every resumable failure kind exactly once", () => { + const contract = fs.readFileSync(path.join(repoRoot, "plugins", "codex", "agents", "codex-rescue.md"), "utf8"); + + assert.match(contract, /`failure:` value of `timeout`, `policy`, or `patch_thrash`/); + assert.match(contract, /exactly one resume per task operation, never chained/); + assert.match(contract, /Wind down: do not start new work\. Finish the smallest coherent deliverable from the work already completed and report the files changed and the verification output\./); + assert.match(contract, /any second failure, any other failure, or any output without that line is relayed as received/); +}); diff --git a/tests/fusion-stats.test.mjs b/tests/fusion-stats.test.mjs index 5b5c152..4cef8c6 100644 --- a/tests/fusion-stats.test.mjs +++ b/tests/fusion-stats.test.mjs @@ -1488,6 +1488,71 @@ test("semantic acceptance reads the job record and excludes non-terminal transpo assert.deepStrictEqual(rejected.byAcceptance, { rejected: 1 }); }); +test("Codex classifies resumable failed attempts as superseded without losing terminal totals", (t) => { + const dir = sandbox(t); + const stateRoot = path.join(dir, "state"); + const env = { FUSION_CODEX_STATE: stateRoot, FUSION_DATA_DIR: path.join(dir, "fusion") }; + writeCodexJob(stateRoot, dir, "timed-out", { + status: "error", + jobClass: "task", + createdAt: "2026-07-14T00:00:00.000Z", + threadId: "timeout-thread", + failureKind: "timeout" + }); + writeCodexJob(stateRoot, dir, "timed-out-resume", { + status: "done", + jobClass: "task", + createdAt: "2026-07-14T00:01:00.000Z", + semanticStatus: "accepted", + request: { resumeThreadId: "timeout-thread" } + }); + writeCodexJob(stateRoot, dir, "rejected-timeout", { + status: "error", + jobClass: "task", + createdAt: "2026-07-14T00:02:00.000Z", + threadId: "rejected-thread", + failureKind: "timeout", + semanticStatus: "rejected" + }); + writeCodexJob(stateRoot, dir, "rejected-timeout-resume", { + status: "done", + jobClass: "task", + createdAt: "2026-07-14T00:03:00.000Z", + semanticStatus: "accepted", + request: { resumeThreadId: "rejected-thread" } + }); + writeCodexJob(stateRoot, dir, "unresumed-timeout", { + status: "error", + jobClass: "task", + createdAt: "2026-07-14T00:04:00.000Z", + threadId: "unresumed-thread", + failureKind: "timeout" + }); + writeCodexJob(stateRoot, dir, "policy-failure", { + status: "error", + jobClass: "task", + createdAt: "2026-07-14T00:05:00.000Z", + threadId: "policy-thread", + failureKind: "policy" + }); + writeCodexJob(stateRoot, dir, "policy-resume", { + status: "done", + jobClass: "task", + createdAt: "2026-07-14T00:06:00.000Z", + semanticStatus: "accepted", + request: { resumeThreadId: "policy-thread" } + }); + + const stats = codexStats({ env, cwd: dir }); + assert.deepStrictEqual(stats.byAcceptance, { superseded: 2, accepted: 3, rejected: 1, unverified: 1 }); + assert.deepStrictEqual(stats.acceptanceAnomalies.doneWithoutAcceptance, []); + const terminalCount = Object.entries(stats.byTransportStatus) + .filter(([status]) => ["done", "error", "cancelled"].includes(status)) + .reduce((total, [, count]) => total + count, 0); + assert.strictEqual(Object.values(stats.byAcceptance).reduce((total, count) => total + count, 0), terminalCount); + assert.match(renderFusionStats({ scope: dir, codex: stats }), /^- superseded: 2$/m); +}); + test("Codex acceptance and token observations stay scoped when unrelated repositories reuse a job id", (t) => { const dir = sandbox(t); const firstRoot = path.join(dir, "first"); @@ -2190,6 +2255,23 @@ test("--record validates every batch pair before writing", (t) => { assert.strictEqual(readWorkerRecord(presentTaskId, env).acceptance, "unverified"); }); +test("--record refuses the derived superseded classification without changing a Codex job", (t) => { + const dir = sandbox(t); + const stateRoot = path.join(dir, "state"); + const jobId = "a".repeat(32); + const jobFile = writeCodexJob(stateRoot, dir, jobId, { status: "done", jobClass: "task" }); + const before = fs.readFileSync(jobFile, "utf8"); + + const result = run( + { cwd: dir, codexState: stateRoot }, + ["--record", `${jobId}=superseded`] + ); + + assert.notStrictEqual(result.status, 0); + assert.match(result.stderr, /--record verdict must be accepted, rejected, or unverified\./); + assert.strictEqual(fs.readFileSync(jobFile, "utf8"), before); +}); + test("Codex reports acceptance anomalies only when the job record and transport state diverge", (t) => { const dir = sandbox(t); const stateRoot = path.join(dir, "state"); @@ -2208,12 +2290,14 @@ test("Codex reports acceptance anomalies only when the job record and transport assert.match(rendered, /Acceptance anomalies:\n- Accepted ledger entries with error transport: 1 \(eeeeeeee\)\n- Done jobs without acceptance records: 1 \(ffffffffffffffffffffffffffffffff\)/); }); -test("Codex acceptedWithErrorTransport excludes salvaged timeouts but keeps other error kinds", (t) => { +test("Codex acceptedWithErrorTransport excludes resumable failures but keeps other error kinds", (t) => { const dir = sandbox(t); const stateRoot = path.join(dir, "state"); const fusionData = path.join(dir, "fusion"); const nonTimeoutAcceptedId = "1".repeat(32); const salvagedTimeoutId = "2".repeat(32); + const salvagedPolicyId = "3".repeat(32); + const salvagedPatchThrashId = "4".repeat(32); writeCodexJob(stateRoot, dir, nonTimeoutAcceptedId, { status: "error", jobClass: "task", @@ -2226,6 +2310,18 @@ test("Codex acceptedWithErrorTransport excludes salvaged timeouts but keeps othe semanticStatus: "accepted", failureKind: "timeout" }); + writeCodexJob(stateRoot, dir, salvagedPolicyId, { + status: "error", + jobClass: "task", + semanticStatus: "accepted", + failureKind: "policy" + }); + writeCodexJob(stateRoot, dir, salvagedPatchThrashId, { + status: "error", + jobClass: "task", + semanticStatus: "accepted", + failureKind: "patch_thrash" + }); const stats = codexStats({ env: { FUSION_CODEX_STATE: stateRoot, FUSION_DATA_DIR: fusionData }, cwd: dir }); assert.deepStrictEqual(stats.acceptanceAnomalies, { @@ -2233,6 +2329,8 @@ test("Codex acceptedWithErrorTransport excludes salvaged timeouts but keeps othe doneWithoutAcceptance: [] }); assert.ok(!stats.acceptanceAnomalies.acceptedWithErrorTransport.includes(salvagedTimeoutId)); + assert.ok(!stats.acceptanceAnomalies.acceptedWithErrorTransport.includes(salvagedPolicyId)); + assert.ok(!stats.acceptanceAnomalies.acceptedWithErrorTransport.includes(salvagedPatchThrashId)); }); test("Codex groups pre-epoch acceptance anomalies and falls back from invalid epochs", (t) => { @@ -2784,6 +2882,7 @@ test("--unsettled lists terminal unverified jobs and preserves Codex acceptance const codexUnsettledId = "a".repeat(32); const codexNoProvenanceId = "b".repeat(32); const codexSettledId = "c".repeat(32); + const codexSupersededId = "d".repeat(32); writeCodexJob(stateRoot, dir, codexUnsettledId, { status: "done", @@ -2805,6 +2904,20 @@ test("--unsettled lists terminal unverified jobs and preserves Codex acceptance acceptance: "accepted", finishedAt: "2026-07-21T00:04:00.000Z" }); + writeCodexJob(stateRoot, dir, codexSupersededId, { + status: "error", + jobClass: "task", + createdAt: "2026-07-21T00:04:00.000Z", + threadId: "superseded-thread", + failureKind: "timeout" + }); + writeCodexJob(stateRoot, dir, "e".repeat(32), { + status: "done", + jobClass: "task", + createdAt: "2026-07-21T00:05:00.000Z", + acceptance: "accepted", + request: { resumeThreadId: "superseded-thread" } + }); writeGrokJob(grokData, dir, "grok-unsettled", { status: "done", mode: "consult", @@ -2838,6 +2951,7 @@ test("--unsettled lists terminal unverified jobs and preserves Codex acceptance const withoutProvenance = report.unsettled.find((entry) => entry.id === codexNoProvenanceId); assert.strictEqual(Object.hasOwn(withoutProvenance, "acceptanceSource"), false); assert.strictEqual(Object.hasOwn(withoutProvenance, "acceptanceRecordedAt"), false); + assert.ok(!report.unsettled.some((entry) => entry.id === codexSupersededId)); const rendered = run({ cwd: dir, codexState: stateRoot }, ["--unsettled"], extraEnv); assert.strictEqual(rendered.status, 0, rendered.stderr);