diff --git a/src/server/request-log.ts b/src/server/request-log.ts index 2bbd881954..2a1117a4ea 100644 --- a/src/server/request-log.ts +++ b/src/server/request-log.ts @@ -254,7 +254,7 @@ export function requestLogEntryFromPersistedUsage(entry: PersistedUsageEntry): R usageStatus: entry.usageStatus, ...(entry.usage ? { usage: entry.usage } : {}), ...(entry.totalTokens !== undefined ? { totalTokens: entry.totalTokens } : {}), - ...(entry.attempts?.length ? { attempts: entry.attempts } : {}), + ...(entry.attempts !== undefined ? { attempts: entry.attempts } : {}), ...(routeDecision ? { routeDecision } : {}), }; } @@ -348,7 +348,7 @@ export function addRequestLog(entry: RequestLogEntry) { usageStatus: entry.usageStatus, ...(entry.usage ? { usage: entry.usage } : {}), ...(entry.totalTokens !== undefined ? { totalTokens: entry.totalTokens } : {}), - ...(entry.attempts?.length ? { attempts: entry.attempts } : {}), + ...(entry.attempts !== undefined ? { attempts: entry.attempts } : {}), ...failureDiagnostics, ...(entry.routeDecision ? { routeDecision: entry.routeDecision } : {}), }); @@ -838,7 +838,7 @@ export function addFinalRequestLog( usageStatus, ...(loggedUsage ? { usage: loggedUsage } : {}), ...(totalTokens !== undefined ? { totalTokens } : {}), - ...(attempts?.length ? { attempts } : {}), + ...(attempts !== undefined ? { attempts } : {}), ...(logCtx.affinity ? { affinity: logCtx.affinity } : {}), ...(logCtx.transportPhase ? { transportPhase: logCtx.transportPhase } : {}), ...(logCtx.terminalSource ? { terminalSource: logCtx.terminalSource } : {}), diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index c0d9bf8373..25bf843194 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -1649,6 +1649,19 @@ async function handleResponsesInner( const adapterProvider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, inboundWire); const adapter = resolveAdapter(adapterProvider, config.cacheRetention); logCtx.providerAdapter = adapter.name; + // Ordinary requests receive one durable attempt only after their final initial + // adapter is resolved. Combo children own their attempt and retries keep it. + if (!options.comboAttempt && !logCtx.activeAttempt) { + const attempt = beginRequestAttempt( + (logCtx.attempts?.length ?? 0) + 1, + logCtx.provider, + route.modelId, + adapter.name, + ); + logCtx.activeAttempt = attempt; + logCtx.activeAttemptStartedAt = Date.now(); + (logCtx.attempts ??= []).push(attempt); + } sealRequestAttemptIdentity(logCtx.activeAttempt, logCtx.provider, adapter.name); const isPassthrough = "passthrough" in adapter && !!adapter.passthrough; diff --git a/src/usage/log.ts b/src/usage/log.ts index b0e8778ab4..23432c0764 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -385,7 +385,7 @@ function normalizeUsageEntry(entry: PersistedUsageEntry): PersistedUsageEntry { usageStatus: entry.usageStatus, ...(entry.usage ? { usage: normalizeUsageValue(entry.usage) } : {}), ...(typeof entry.totalTokens === "number" ? { totalTokens: entry.totalTokens } : {}), - ...(attempts.length > 0 ? { attempts } : {}), + ...(Array.isArray(entry.attempts) ? { attempts } : {}), ...(entry.errorCode ? { errorCode: entry.errorCode } : {}), ...(entry.terminalStatus ? { terminalStatus: entry.terminalStatus } : {}), ...(entry.closeReason ? { closeReason: entry.closeReason } : {}), diff --git a/src/vision/index.ts b/src/vision/index.ts index 37dacc5962..f44b713ec4 100644 --- a/src/vision/index.ts +++ b/src/vision/index.ts @@ -275,6 +275,72 @@ function renderDescription(out: { text: string; error?: string }): OcxTextConten }; } +const IMAGE_OMITTED_TEXT = "[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]"; + +function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * Keep the native Responses passthrough body aligned with image replacements made in the parsed + * message graph. The passthrough adapter serializes `_rawBody`, while translated adapters serialize + * `context.messages`; updating only the latter would send the original pixels to a text-only + * Responses upstream even after the vision sidecar produced a caption. + * + * Rewrites only image-bearing user/developer messages and tool outputs. All other native Responses + * items (reasoning, calls, ids, compaction, and provider-specific metadata) remain byte-structurally + * untouched. + */ +function syncRawBodyImageDescriptions(parsed: OcxParsedRequest, descriptions: readonly string[]): void { + const rawBody = parsed._rawBody; + if (!isPlainRecord(rawBody) || !Array.isArray(rawBody.input)) return; + + let nextDescription = 0; + const rewriteImages = (value: unknown, nonEmptyImageUrlsOnly: boolean): unknown => { + if (Array.isArray(value)) { + let changed = false; + const rewritten = value.map(entry => { + const next = rewriteImages(entry, nonEmptyImageUrlsOnly); + if (next !== entry) changed = true; + return next; + }); + return changed ? rewritten : value; + } + if (!isPlainRecord(value)) return value; + if (value.type === "input_image" && typeof value.image_url === "string") { + if (nonEmptyImageUrlsOnly && value.image_url.length === 0) { + return { type: "input_text", text: IMAGE_OMITTED_TEXT }; + } + const description = descriptions[nextDescription++]; + return { type: "input_text", text: description ?? IMAGE_OMITTED_TEXT }; + } + return value; + }; + + let changed = false; + const input = rawBody.input.map(item => { + if (!isPlainRecord(item)) return item; + const type = typeof item.type === "string" ? item.type : (typeof item.role === "string" ? "message" : ""); + const role = typeof item.role === "string" ? item.role : ""; + const isMessageContent = ( + (type === "message" && (role === "user" || role === "developer")) + || type === "agent_message" + ); + const field = isMessageContent + ? "content" + : (type === "function_call_output" || type === "custom_tool_call_output") + ? "output" + : undefined; + if (!field) return item; + const rewritten = rewriteImages(item[field], isMessageContent); + if (rewritten === item[field]) return item; + changed = true; + return { ...item, [field]: rewritten }; + }); + + if (changed) rawBody.input = input; +} + function sha256(value: string | Uint8Array): string { return createHash("sha256").update(value).digest("hex"); } @@ -369,7 +435,10 @@ export async function describeImagesInPlace( } targets.push({ msg, parts }); } - if (jobs.length === 0) return; + if (jobs.length === 0) { + syncRawBodyImageDescriptions(parsed, []); + return; + } // 2. Admit misses in source order. Cache hits and same-turn waiters do not consume the cap. const inFlight = new Map>(); @@ -425,6 +494,7 @@ export async function describeImagesInPlace( // 3. Rebuild each message, replacing image parts with their descriptions in order. let oi = 0; + const descriptions: string[] = []; for (const { msg, parts } of targets) { const newParts: OcxContentPart[] = []; for (const p of parts) { @@ -433,6 +503,7 @@ export async function describeImagesInPlace( continue; } const replacement = renderDescription(outcomes[oi++]); + descriptions.push(replacement.text); const reservation = translatorBudget?.reserveTransient( descriptionEncoder.encode(replacement.text).byteLength, { kind: "request_copies" }, @@ -442,6 +513,7 @@ export async function describeImagesInPlace( } msg.content = newParts; } + syncRawBodyImageDescriptions(parsed, descriptions); } /** @@ -452,13 +524,15 @@ export async function describeImagesInPlace( */ export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?: TranslatorBudget): boolean { let stripped = false; + const descriptions: string[] = []; for (const msg of parsed.context.messages) { if (!carriesImages(msg.role) || !Array.isArray(msg.content)) continue; const parts = msg.content as OcxContentPart[]; if (!parts.some(p => p.type === "image")) continue; msg.content = parts.map(p => { if (p.type !== "image") return p; - const replacement = { type: "text", text: "[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]" } as OcxContentPart; + const replacement = { type: "text", text: IMAGE_OMITTED_TEXT } as OcxContentPart; + descriptions.push((replacement as OcxTextContent).text); const reservation = translatorBudget?.reserveTransient( descriptionEncoder.encode((replacement as OcxTextContent).text).byteLength, { kind: "request_copies" }, @@ -468,5 +542,6 @@ export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?: }); stripped = true; } + syncRawBodyImageDescriptions(parsed, descriptions); return stripped; } diff --git a/tests/request-log.test.ts b/tests/request-log.test.ts index f668b460cf..347724dcc4 100644 --- a/tests/request-log.test.ts +++ b/tests/request-log.test.ts @@ -23,8 +23,9 @@ import { sealRequestAttemptIdentity, type RequestLogContext, } from "../src/server/request-log"; +import { handleResponses } from "../src/server/responses"; import { bridgeToResponsesSSE } from "../src/bridge"; -import type { AdapterEvent, OcxUsage } from "../src/types"; +import type { AdapterEvent, OcxConfig, OcxUsage } from "../src/types"; import { appendUsageEntry, readUsageEntries, @@ -53,6 +54,64 @@ function log(overrides: Partial): RequestLogEntry { } describe("request log metadata", () => { + test("creates one ordinary attempt after the final adapter is resolved", async () => { + const originalFetch = globalThis.fetch; + globalThis.fetch = (async () => Response.json({ + id: "resp_attempt", + object: "response", + status: "completed", + output: [], + usage: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + })) as typeof fetch; + const logCtx: RequestLogContext = { model: "unknown", provider: "unknown" }; + const config = { + defaultProvider: "gateway", + providers: { + gateway: { + adapter: "openai-responses", + authMode: "key", + apiKey: "test-key", + baseUrl: "https://gateway.example/v1", + }, + }, + } as OcxConfig; + + try { + const response = await handleResponses(new Request("http://localhost/v1/responses", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model: "gateway/test-model", input: "hello", stream: false }), + }), config, logCtx); + + expect(response.status).toBe(200); + expect(logCtx.providerAdapter).toBe("openai-responses"); + expect(logCtx.attempts).toEqual([expect.objectContaining({ + ordinal: 1, + provider: "gateway", + model: "test-model", + adapter: "openai-responses", + sendCount: 1, + })]); + } finally { + globalThis.fetch = originalFetch; + } + }); + + test("projects explicitly empty attempts from persisted usage", () => { + const projected = requestLogEntryFromPersistedUsage({ + requestId: "ocx-empty-attempts", + timestamp: 1, + provider: "openai", + model: "gpt-test", + status: 200, + durationMs: 1, + usageStatus: "unreported", + attempts: [], + }); + + expect(projected.attempts).toEqual([]); + }); + test("records the adapter's exact outbound reasoning parameter", () => { const attempt = beginRequestAttempt(1, "xai", "grok-4.5", "openai-chat"); const logCtx: RequestLogContext = { diff --git a/tests/usage-log.test.ts b/tests/usage-log.test.ts index 8e5c8687f4..b80c2cef36 100644 --- a/tests/usage-log.test.ts +++ b/tests/usage-log.test.ts @@ -38,6 +38,21 @@ afterEach(() => { }); describe("usage log", () => { + test("preserves explicitly empty attempts through normalization", () => { + const normalized = normalizeUsageEntryForTest({ + requestId: "ocx-empty-attempts", + timestamp: 1, + provider: "openai", + model: "gpt-test", + status: 200, + durationMs: 1, + usageStatus: "unreported", + attempts: [], + }); + + expect(normalized.attempts).toEqual([]); + }); + test("persists the rate-limit-429 recovery kind on attempts", () => { const entry: PersistedUsageEntry = { requestId: "ocx-ratelimit-kind", diff --git a/tests/vision-sidecar-e2e.test.ts b/tests/vision-sidecar-e2e.test.ts index 1069fd22df..a504b91c4d 100644 --- a/tests/vision-sidecar-e2e.test.ts +++ b/tests/vision-sidecar-e2e.test.ts @@ -6,6 +6,8 @@ import { saveConfig } from "../src/config"; import { startServer } from "../src/server"; import { PROVIDER_REGISTRY } from "../src/providers/registry"; import type { OcxConfig } from "../src/types"; +import { parseRequest } from "../src/responses/parser"; +import { resetVisionDescriptionCache, stripImagesInPlace } from "../src/vision"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; import { fakeChatGptJwt } from "./helpers/fake-chatgpt-jwt"; @@ -27,6 +29,7 @@ beforeEach(() => { testDir = mkdtempSync(join(tmpdir(), "ocx-vision-e2e-")); process.env.OPENCODEX_HOME = testDir; globalThis.fetch = originalFetch; + resetVisionDescriptionCache(); }); afterEach(() => { @@ -78,6 +81,29 @@ function serveUpstream(record: (bodyText: string) => void) { }); } +/** Fake text-only upstream (openai-responses passthrough wire): records the forwarded body. */ +function serveResponsesUpstream(record: (bodyText: string) => void) { + return Bun.serve({ + hostname: "127.0.0.1", port: 0, + async fetch(req) { + record(await req.text()); + return Response.json({ + id: "resp_vision_1", + object: "response", + status: "completed", + output: [{ + id: "msg_vision_1", + type: "message", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "I see a red logo.", annotations: [] }], + }], + usage: { input_tokens: 3, output_tokens: 2, total_tokens: 5 }, + }); + }, + }); +} + function baseRequest(model: string) { return { model, stream: false, @@ -88,7 +114,41 @@ function baseRequest(model: string) { }; } +function toolImageRequest(model: string) { + return { + model, stream: false, + input: [ + { + type: "function_call", call_id: "call_view_image", name: "view_image", + arguments: JSON.stringify({ path: "/tmp/screenshot.png" }), + }, + { + type: "function_call_output", call_id: "call_view_image", + output: [ + { type: "input_text", text: "Image loaded." }, + { type: "input_image", image_url: PNG_DATA_URL, detail: "high" }, + ], + }, + ], + }; +} + describe("vision sidecar fallback (issue #88, end-to-end)", () => { + test("raw-body normalization removes an image when parsing produces zero captions", () => { + const parsed = parseRequest({ + model: "textonly/blind-model", + input: [{ + type: "message", + role: "user", + content: [{ type: "input_image", image_url: "" }], + }], + }); + + expect(stripImagesInPlace(parsed)).toBe(false); + expect(JSON.stringify(parsed._rawBody)).not.toContain("input_image"); + expect(JSON.stringify(parsed._rawBody)).toContain("[image omitted:"); + }); + test("noVisionModels request fires the sidecar and forwards the caption instead of the image", async () => { let upstreamBody = ""; let sidecarBody = ""; @@ -161,6 +221,158 @@ describe("vision sidecar fallback (issue #88, end-to-end)", () => { } }); + test("Responses passthrough removes every raw image when fewer captions than images are produced", async () => { + let upstreamBody = ""; + let sidecarHits = 0; + upstream = serveResponsesUpstream(b => { upstreamBody = b; }); + sidecar = serveSidecar(() => { sidecarHits += 1; }); + globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + const requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + const url = new URL(requestUrl); + const prefix = "/backend-api/codex"; + if (url.hostname === "chatgpt.com" && url.pathname.startsWith(prefix)) { + return originalFetch(new URL(`${url.pathname.slice(prefix.length)}${url.search}`, sidecar!.url), init); + } + return originalFetch(input, init); + }) as typeof fetch; + + const config: OcxConfig = { + port: 0, hostname: "127.0.0.1", defaultProvider: "textonly", openaiProviderTierVersion: 2, + providers: { + textonly: { + adapter: "openai-responses", + authMode: "key", + baseUrl: upstream.url.toString().replace(/\/$/, ""), + responsesPath: "/responses", + allowPrivateNetwork: true, + apiKey: "key-alpha-000111222333", + noVisionModels: ["blind-model"], + }, + openai: { + adapter: "openai-responses", + authMode: "forward", + baseUrl: "https://chatgpt.com/backend-api/codex", + codexAccountMode: "direct", + }, + }, + } as OcxConfig; + saveConfig(config); + const server = startServer(0); + try { + const token = fakeChatGptJwt({ chatgpt_account_id: "acct-vision-sidecar" }); + const request = baseRequest("textonly/blind-model"); + request.input[0].content.splice(1, 0, { type: "input_image", image_url: "" }); + const res = await fetch(new URL("/v1/responses", server.url), { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${token}`, + "chatgpt-account-id": "acct-vision-sidecar", + }, + body: JSON.stringify(request), + }); + expect(res.status).toBe(200); + expect(sidecarHits).toBe(1); + expect(upstreamBody).toContain(CAPTION); + expect(upstreamBody).not.toContain("aGVsbG8taW1hZ2UtYnl0ZXM="); + expect(upstreamBody).not.toContain("input_image"); + } finally { + await server.stop(true); + } + }); + + test("Responses passthrough replaces an image returned by a client tool", async () => { + let upstreamBody = ""; + let sidecarHits = 0; + upstream = serveResponsesUpstream(b => { upstreamBody = b; }); + sidecar = serveSidecar(() => { sidecarHits += 1; }); + globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + const requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + const url = new URL(requestUrl); + const prefix = "/backend-api/codex"; + if (url.hostname === "chatgpt.com" && url.pathname.startsWith(prefix)) { + return originalFetch(new URL(`${url.pathname.slice(prefix.length)}${url.search}`, sidecar!.url), init); + } + return originalFetch(input, init); + }) as typeof fetch; + + const config: OcxConfig = { + port: 0, hostname: "127.0.0.1", defaultProvider: "textonly", openaiProviderTierVersion: 2, + providers: { + textonly: { + adapter: "openai-responses", + authMode: "key", + baseUrl: upstream.url.toString().replace(/\/$/, ""), + responsesPath: "/responses", + allowPrivateNetwork: true, + apiKey: "key-alpha-000111222333", + noVisionModels: ["blind-model"], + }, + openai: { + adapter: "openai-responses", + authMode: "forward", + baseUrl: "https://chatgpt.com/backend-api/codex", + codexAccountMode: "direct", + }, + }, + } as OcxConfig; + saveConfig(config); + const server = startServer(0); + try { + const token = fakeChatGptJwt({ chatgpt_account_id: "acct-vision-sidecar" }); + const res = await fetch(new URL("/v1/responses", server.url), { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${token}`, + "chatgpt-account-id": "acct-vision-sidecar", + }, + body: JSON.stringify(toolImageRequest("textonly/blind-model")), + }); + expect(res.status).toBe(200); + expect(sidecarHits).toBe(1); + expect(upstreamBody).toContain(CAPTION); + expect(upstreamBody).not.toContain("aGVsbG8taW1hZ2UtYnl0ZXM="); + expect(upstreamBody).not.toContain("image_url"); + } finally { + await server.stop(true); + } + }); + + test("Responses passthrough strips images when no vision sidecar is available", async () => { + let upstreamBody = ""; + upstream = serveResponsesUpstream(b => { upstreamBody = b; }); + const config: OcxConfig = { + port: 0, hostname: "127.0.0.1", defaultProvider: "textonly", + providers: { + textonly: { + adapter: "openai-responses", + authMode: "key", + baseUrl: upstream.url.toString().replace(/\/$/, ""), + responsesPath: "/responses", + allowPrivateNetwork: true, + apiKey: "key-alpha-000111222333", + noVisionModels: ["blind-model"], + }, + }, + } as OcxConfig; + saveConfig(config); + const server = startServer(0); + try { + const res = await fetch(new URL("/v1/responses", server.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(baseRequest("textonly/blind-model")), + }); + expect(res.status).toBe(200); + expect(upstreamBody).toContain("[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]"); + expect(upstreamBody).not.toContain("aGVsbG8taW1hZ2UtYnl0ZXM="); + expect(upstreamBody).not.toContain("image_url"); + } finally { + await server.stop(true); + } + }); + test("models outside noVisionModels keep their image untouched (no sidecar call)", async () => { let upstreamBody = ""; let sidecarHits = 0;