diff --git a/src/vision/index.ts b/src/vision/index.ts index 37dacc5962..4bc53a66b3 100644 --- a/src/vision/index.ts +++ b/src/vision/index.ts @@ -275,6 +275,68 @@ function renderDescription(out: { text: string; error?: string }): OcxTextConten }; } +function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +/** + * Keep the native Responses passthrough body aligned with image replacements made in the parsed + * message graph. The passthrough adapter serializes `_rawBody`, while translated adapters serialize + * `context.messages`; updating only the latter would send the original pixels to a text-only + * Responses upstream even after the vision sidecar produced a caption. + * + * Rewrites only image-bearing user/developer messages and tool outputs. All other native Responses + * items (reasoning, calls, ids, compaction, and provider-specific metadata) remain byte-structurally + * untouched. + */ +function syncRawBodyImageDescriptions(parsed: OcxParsedRequest, descriptions: readonly string[]): void { + const rawBody = parsed._rawBody; + if (!isPlainRecord(rawBody) || !Array.isArray(rawBody.input) || descriptions.length === 0) return; + + let nextDescription = 0; + const rewriteImages = (value: unknown, nonEmptyImageUrlsOnly: boolean): unknown => { + if (Array.isArray(value)) { + let changed = false; + const rewritten = value.map(entry => { + const next = rewriteImages(entry, nonEmptyImageUrlsOnly); + if (next !== entry) changed = true; + return next; + }); + return changed ? rewritten : value; + } + if (!isPlainRecord(value)) return value; + if (value.type === "input_image" && typeof value.image_url === "string") { + if (nonEmptyImageUrlsOnly && value.image_url.length === 0) return value; + const description = descriptions[nextDescription++]; + return description === undefined ? value : { type: "input_text", text: description }; + } + return value; + }; + + let changed = false; + const input = rawBody.input.map(item => { + if (!isPlainRecord(item)) return item; + const type = typeof item.type === "string" ? item.type : (typeof item.role === "string" ? "message" : ""); + const role = typeof item.role === "string" ? item.role : ""; + const isMessageContent = ( + (type === "message" && (role === "user" || role === "developer")) + || type === "agent_message" + ); + const field = isMessageContent + ? "content" + : (type === "function_call_output" || type === "custom_tool_call_output") + ? "output" + : undefined; + if (!field) return item; + const rewritten = rewriteImages(item[field], isMessageContent); + if (rewritten === item[field]) return item; + changed = true; + return { ...item, [field]: rewritten }; + }); + + if (changed) rawBody.input = input; +} + function sha256(value: string | Uint8Array): string { return createHash("sha256").update(value).digest("hex"); } @@ -425,6 +487,7 @@ export async function describeImagesInPlace( // 3. Rebuild each message, replacing image parts with their descriptions in order. let oi = 0; + const descriptions: string[] = []; for (const { msg, parts } of targets) { const newParts: OcxContentPart[] = []; for (const p of parts) { @@ -433,6 +496,7 @@ export async function describeImagesInPlace( continue; } const replacement = renderDescription(outcomes[oi++]); + descriptions.push(replacement.text); const reservation = translatorBudget?.reserveTransient( descriptionEncoder.encode(replacement.text).byteLength, { kind: "request_copies" }, @@ -442,6 +506,7 @@ export async function describeImagesInPlace( } msg.content = newParts; } + syncRawBodyImageDescriptions(parsed, descriptions); } /** @@ -452,6 +517,7 @@ export async function describeImagesInPlace( */ export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?: TranslatorBudget): boolean { let stripped = false; + const descriptions: string[] = []; for (const msg of parsed.context.messages) { if (!carriesImages(msg.role) || !Array.isArray(msg.content)) continue; const parts = msg.content as OcxContentPart[]; @@ -459,6 +525,7 @@ export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?: msg.content = parts.map(p => { if (p.type !== "image") return p; const replacement = { type: "text", text: "[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]" } as OcxContentPart; + descriptions.push((replacement as OcxTextContent).text); const reservation = translatorBudget?.reserveTransient( descriptionEncoder.encode((replacement as OcxTextContent).text).byteLength, { kind: "request_copies" }, @@ -468,5 +535,6 @@ export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?: }); stripped = true; } + if (stripped) syncRawBodyImageDescriptions(parsed, descriptions); return stripped; } diff --git a/tests/vision-sidecar-e2e.test.ts b/tests/vision-sidecar-e2e.test.ts index 2e44ec3fb2..b4d5737071 100644 --- a/tests/vision-sidecar-e2e.test.ts +++ b/tests/vision-sidecar-e2e.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { saveConfig } from "../src/config"; import { startServer } from "../src/server"; import type { OcxConfig } from "../src/types"; +import { resetVisionDescriptionCache } from "../src/vision"; import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home"; import { fakeChatGptJwt } from "./helpers/fake-chatgpt-jwt"; @@ -26,6 +27,7 @@ beforeEach(() => { testDir = mkdtempSync(join(tmpdir(), "ocx-vision-e2e-")); process.env.OPENCODEX_HOME = testDir; globalThis.fetch = originalFetch; + resetVisionDescriptionCache(); }); afterEach(() => { @@ -77,6 +79,29 @@ function serveUpstream(record: (bodyText: string) => void) { }); } +/** Fake text-only upstream (openai-responses passthrough wire): records the forwarded body. */ +function serveResponsesUpstream(record: (bodyText: string) => void) { + return Bun.serve({ + hostname: "127.0.0.1", port: 0, + async fetch(req) { + record(await req.text()); + return Response.json({ + id: "resp_vision_1", + object: "response", + status: "completed", + output: [{ + id: "msg_vision_1", + type: "message", + role: "assistant", + status: "completed", + content: [{ type: "output_text", text: "I see a red logo.", annotations: [] }], + }], + usage: { input_tokens: 3, output_tokens: 2, total_tokens: 5 }, + }); + }, + }); +} + function baseRequest(model: string) { return { model, stream: false, @@ -87,6 +112,25 @@ function baseRequest(model: string) { }; } +function toolImageRequest(model: string) { + return { + model, stream: false, + input: [ + { + type: "function_call", call_id: "call_view_image", name: "view_image", + arguments: JSON.stringify({ path: "/tmp/screenshot.png" }), + }, + { + type: "function_call_output", call_id: "call_view_image", + output: [ + { type: "input_text", text: "Image loaded." }, + { type: "input_image", image_url: PNG_DATA_URL, detail: "high" }, + ], + }, + ], + }; +} + describe("vision sidecar fallback (issue #88, end-to-end)", () => { test("noVisionModels request fires the sidecar and forwards the caption instead of the image", async () => { let upstreamBody = ""; @@ -160,6 +204,157 @@ describe("vision sidecar fallback (issue #88, end-to-end)", () => { } }); + test("Responses passthrough does not let an empty image consume the real image caption", async () => { + let upstreamBody = ""; + let sidecarHits = 0; + upstream = serveResponsesUpstream(b => { upstreamBody = b; }); + sidecar = serveSidecar(() => { sidecarHits += 1; }); + globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + const requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + const url = new URL(requestUrl); + const prefix = "/backend-api/codex"; + if (url.hostname === "chatgpt.com" && url.pathname.startsWith(prefix)) { + return originalFetch(new URL(`${url.pathname.slice(prefix.length)}${url.search}`, sidecar!.url), init); + } + return originalFetch(input, init); + }) as typeof fetch; + + const config: OcxConfig = { + port: 0, hostname: "127.0.0.1", defaultProvider: "textonly", openaiProviderTierVersion: 2, + providers: { + textonly: { + adapter: "openai-responses", + authMode: "key", + baseUrl: upstream.url.toString().replace(/\/$/, ""), + responsesPath: "/responses", + allowPrivateNetwork: true, + apiKey: "key-alpha-000111222333", + noVisionModels: ["blind-model"], + }, + openai: { + adapter: "openai-responses", + authMode: "forward", + baseUrl: "https://chatgpt.com/backend-api/codex", + codexAccountMode: "direct", + }, + }, + } as OcxConfig; + saveConfig(config); + const server = startServer(0); + try { + const token = fakeChatGptJwt({ chatgpt_account_id: "acct-vision-sidecar" }); + const request = baseRequest("textonly/blind-model"); + request.input[0].content.splice(1, 0, { type: "input_image", image_url: "" }); + const res = await fetch(new URL("/v1/responses", server.url), { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${token}`, + "chatgpt-account-id": "acct-vision-sidecar", + }, + body: JSON.stringify(request), + }); + expect(res.status).toBe(200); + expect(sidecarHits).toBe(1); + expect(upstreamBody).toContain(CAPTION); + expect(upstreamBody).not.toContain("aGVsbG8taW1hZ2UtYnl0ZXM="); + } finally { + await server.stop(true); + } + }); + + test("Responses passthrough replaces an image returned by a client tool", async () => { + let upstreamBody = ""; + let sidecarHits = 0; + upstream = serveResponsesUpstream(b => { upstreamBody = b; }); + sidecar = serveSidecar(() => { sidecarHits += 1; }); + globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => { + const requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url; + const url = new URL(requestUrl); + const prefix = "/backend-api/codex"; + if (url.hostname === "chatgpt.com" && url.pathname.startsWith(prefix)) { + return originalFetch(new URL(`${url.pathname.slice(prefix.length)}${url.search}`, sidecar!.url), init); + } + return originalFetch(input, init); + }) as typeof fetch; + + const config: OcxConfig = { + port: 0, hostname: "127.0.0.1", defaultProvider: "textonly", openaiProviderTierVersion: 2, + providers: { + textonly: { + adapter: "openai-responses", + authMode: "key", + baseUrl: upstream.url.toString().replace(/\/$/, ""), + responsesPath: "/responses", + allowPrivateNetwork: true, + apiKey: "key-alpha-000111222333", + noVisionModels: ["blind-model"], + }, + openai: { + adapter: "openai-responses", + authMode: "forward", + baseUrl: "https://chatgpt.com/backend-api/codex", + codexAccountMode: "direct", + }, + }, + } as OcxConfig; + saveConfig(config); + const server = startServer(0); + try { + const token = fakeChatGptJwt({ chatgpt_account_id: "acct-vision-sidecar" }); + const res = await fetch(new URL("/v1/responses", server.url), { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${token}`, + "chatgpt-account-id": "acct-vision-sidecar", + }, + body: JSON.stringify(toolImageRequest("textonly/blind-model")), + }); + expect(res.status).toBe(200); + expect(sidecarHits).toBe(1); + expect(upstreamBody).toContain(CAPTION); + expect(upstreamBody).not.toContain("aGVsbG8taW1hZ2UtYnl0ZXM="); + expect(upstreamBody).not.toContain("image_url"); + } finally { + await server.stop(true); + } + }); + + test("Responses passthrough strips images when no vision sidecar is available", async () => { + let upstreamBody = ""; + upstream = serveResponsesUpstream(b => { upstreamBody = b; }); + const config: OcxConfig = { + port: 0, hostname: "127.0.0.1", defaultProvider: "textonly", + providers: { + textonly: { + adapter: "openai-responses", + authMode: "key", + baseUrl: upstream.url.toString().replace(/\/$/, ""), + responsesPath: "/responses", + allowPrivateNetwork: true, + apiKey: "key-alpha-000111222333", + noVisionModels: ["blind-model"], + }, + }, + } as OcxConfig; + saveConfig(config); + const server = startServer(0); + try { + const res = await fetch(new URL("/v1/responses", server.url), { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(baseRequest("textonly/blind-model")), + }); + expect(res.status).toBe(200); + expect(upstreamBody).toContain("[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]"); + expect(upstreamBody).not.toContain("aGVsbG8taW1hZ2UtYnl0ZXM="); + expect(upstreamBody).not.toContain("image_url"); + } finally { + await server.stop(true); + } + }); + test("models outside noVisionModels keep their image untouched (no sidecar call)", async () => { let upstreamBody = ""; let sidecarHits = 0;