Skip to content

fix(vision): sync captions into Responses passthrough #1047

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
baileyh8 wants to merge 2 commits into from
+263 −0
Closed

fix(vision): sync captions into Responses passthrough #1047

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0f46fdc
fix(vision): sync captions into Responses passthrough
baileyh8 Aug 5, 2026
18784cd
fix(vision): skip empty message image references
baileyh8 Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions src/vision/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,68 @@ function renderDescription(out: { text: string; error?: string }): OcxTextConten
};
}

function isPlainRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}

/**
* Keep the native Responses passthrough body aligned with image replacements made in the parsed
* message graph. The passthrough adapter serializes `_rawBody`, while translated adapters serialize
* `context.messages`; updating only the latter would send the original pixels to a text-only
* Responses upstream even after the vision sidecar produced a caption.
*
* Rewrites only image-bearing user/developer messages and tool outputs. All other native Responses
* items (reasoning, calls, ids, compaction, and provider-specific metadata) remain byte-structurally
* untouched.
*/
function syncRawBodyImageDescriptions(parsed: OcxParsedRequest, descriptions: readonly string[]): void {
const rawBody = parsed._rawBody;
if (!isPlainRecord(rawBody) || !Array.isArray(rawBody.input) || descriptions.length === 0) return;

let nextDescription = 0;
const rewriteImages = (value: unknown, nonEmptyImageUrlsOnly: boolean): unknown => {
if (Array.isArray(value)) {
let changed = false;
const rewritten = value.map(entry => {
const next = rewriteImages(entry, nonEmptyImageUrlsOnly);
if (next !== entry) changed = true;
return next;
});
return changed ? rewritten : value;
}
if (!isPlainRecord(value)) return value;
if (value.type === "input_image" && typeof value.image_url === "string") {
if (nonEmptyImageUrlsOnly && value.image_url.length === 0) return value;
const description = descriptions[nextDescription++];
return description === undefined ? value : { type: "input_text", text: description };
Comment thread
baileyh8 marked this conversation as resolved.
Comment on lines +308 to +311

@coderabbitai coderabbitai Bot Aug 5, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🎯 Functional Correctness | 🟠 Major | ⚡ Quick win

Remove empty input_image parts from _rawBody.

On Line 309, an empty image_url returns the original input_image part. A text-only Responses provider can still reject this unsupported part.

Remove the empty part or replace it with the omission marker. Apply the same result when descriptions.length === 0. Add an end-to-end assertion that forwarded input contains no input_image parts for this case.

As per path instructions, flag provider/adapter contract drift in src/**.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/vision/index.ts` around lines 308 - 311, Update the input-image
normalization logic in the visible transformation function so empty image_url
values are omitted or replaced with the existing omission marker instead of
returning the original input_image part; apply the same behavior when
descriptions.length is zero. Add an end-to-end assertion verifying the forwarded
input contains no input_image parts for this case, and flag the provider/adapter
contract drift under src/**.

Source: Path instructions

}
return value;
};

let changed = false;
const input = rawBody.input.map(item => {
if (!isPlainRecord(item)) return item;
const type = typeof item.type === "string" ? item.type : (typeof item.role === "string" ? "message" : "");
const role = typeof item.role === "string" ? item.role : "";
const isMessageContent = (
(type === "message" && (role === "user" || role === "developer"))
|| type === "agent_message"
);
const field = isMessageContent
? "content"
: (type === "function_call_output" || type === "custom_tool_call_output")
? "output"
: undefined;
if (!field) return item;
const rewritten = rewriteImages(item[field], isMessageContent);
if (rewritten === item[field]) return item;
changed = true;
return { ...item, [field]: rewritten };
});

if (changed) rawBody.input = input;
}

function sha256(value: string | Uint8Array): string {
return createHash("sha256").update(value).digest("hex");
}
Expand Down Expand Up @@ -425,6 +487,7 @@ export async function describeImagesInPlace(

// 3. Rebuild each message, replacing image parts with their descriptions in order.
let oi = 0;
const descriptions: string[] = [];
for (const { msg, parts } of targets) {
const newParts: OcxContentPart[] = [];
for (const p of parts) {
Expand All @@ -433,6 +496,7 @@ export async function describeImagesInPlace(
continue;
}
const replacement = renderDescription(outcomes[oi++]);
descriptions.push(replacement.text);
const reservation = translatorBudget?.reserveTransient(
descriptionEncoder.encode(replacement.text).byteLength,
{ kind: "request_copies" },
Expand All @@ -442,6 +506,7 @@ export async function describeImagesInPlace(
}
msg.content = newParts;
}
syncRawBodyImageDescriptions(parsed, descriptions);
}

/**
Expand All @@ -452,13 +517,15 @@ export async function describeImagesInPlace(
*/
export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?: TranslatorBudget): boolean {
let stripped = false;
const descriptions: string[] = [];
for (const msg of parsed.context.messages) {
if (!carriesImages(msg.role) || !Array.isArray(msg.content)) continue;
const parts = msg.content as OcxContentPart[];
if (!parts.some(p => p.type === "image")) continue;
msg.content = parts.map(p => {
if (p.type !== "image") return p;
const replacement = { type: "text", text: "[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]" } as OcxContentPart;
descriptions.push((replacement as OcxTextContent).text);
const reservation = translatorBudget?.reserveTransient(
descriptionEncoder.encode((replacement as OcxTextContent).text).byteLength,
{ kind: "request_copies" },
Expand All @@ -468,5 +535,6 @@ export function stripImagesInPlace(parsed: OcxParsedRequest, translatorBudget?:
});
stripped = true;
}
if (stripped) syncRawBodyImageDescriptions(parsed, descriptions);
return stripped;
}
195 changes: 195 additions & 0 deletions tests/vision-sidecar-e2e.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { join } from "node:path";
import { saveConfig } from "../src/config";
import { startServer } from "../src/server";
import type { OcxConfig } from "../src/types";
import { resetVisionDescriptionCache } from "../src/vision";
import { installIsolatedCodexHome, type IsolatedCodexHome } from "./helpers/isolated-codex-home";
import { fakeChatGptJwt } from "./helpers/fake-chatgpt-jwt";

Expand All @@ -26,6 +27,7 @@ beforeEach(() => {
testDir = mkdtempSync(join(tmpdir(), "ocx-vision-e2e-"));
process.env.OPENCODEX_HOME = testDir;
globalThis.fetch = originalFetch;
resetVisionDescriptionCache();
});

afterEach(() => {
Expand Down Expand Up @@ -77,6 +79,29 @@ function serveUpstream(record: (bodyText: string) => void) {
});
}

/** Fake text-only upstream (openai-responses passthrough wire): records the forwarded body. */
function serveResponsesUpstream(record: (bodyText: string) => void) {
return Bun.serve({
hostname: "127.0.0.1", port: 0,
async fetch(req) {
record(await req.text());
return Response.json({
id: "resp_vision_1",
object: "response",
status: "completed",
output: [{
id: "msg_vision_1",
type: "message",
role: "assistant",
status: "completed",
content: [{ type: "output_text", text: "I see a red logo.", annotations: [] }],
}],
usage: { input_tokens: 3, output_tokens: 2, total_tokens: 5 },
});
},
});
}

function baseRequest(model: string) {
return {
model, stream: false,
Expand All @@ -87,6 +112,25 @@ function baseRequest(model: string) {
};
}

function toolImageRequest(model: string) {
return {
model, stream: false,
input: [
{
type: "function_call", call_id: "call_view_image", name: "view_image",
arguments: JSON.stringify({ path: "/tmp/screenshot.png" }),
},
{
type: "function_call_output", call_id: "call_view_image",
output: [
{ type: "input_text", text: "Image loaded." },
{ type: "input_image", image_url: PNG_DATA_URL, detail: "high" },
],
},
],
};
}

describe("vision sidecar fallback (issue #88, end-to-end)", () => {
test("noVisionModels request fires the sidecar and forwards the caption instead of the image", async () => {
let upstreamBody = "";
Expand Down Expand Up @@ -160,6 +204,157 @@ describe("vision sidecar fallback (issue #88, end-to-end)", () => {
}
});

test("Responses passthrough does not let an empty image consume the real image caption", async () => {
let upstreamBody = "";
let sidecarHits = 0;
upstream = serveResponsesUpstream(b => { upstreamBody = b; });
sidecar = serveSidecar(() => { sidecarHits += 1; });
globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => {
const requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
const url = new URL(requestUrl);
const prefix = "/backend-api/codex";
if (url.hostname === "chatgpt.com" && url.pathname.startsWith(prefix)) {
return originalFetch(new URL(`${url.pathname.slice(prefix.length)}${url.search}`, sidecar!.url), init);
}
return originalFetch(input, init);
}) as typeof fetch;

const config: OcxConfig = {
port: 0, hostname: "127.0.0.1", defaultProvider: "textonly", openaiProviderTierVersion: 2,
providers: {
textonly: {
adapter: "openai-responses",
authMode: "key",
baseUrl: upstream.url.toString().replace(/\/$/, ""),
responsesPath: "/responses",
allowPrivateNetwork: true,
apiKey: "key-alpha-000111222333",
noVisionModels: ["blind-model"],
},
openai: {
adapter: "openai-responses",
authMode: "forward",
baseUrl: "https://chatgpt.com/backend-api/codex",
codexAccountMode: "direct",
},
},
} as OcxConfig;
saveConfig(config);
const server = startServer(0);
try {
const token = fakeChatGptJwt({ chatgpt_account_id: "acct-vision-sidecar" });
const request = baseRequest("textonly/blind-model");
request.input[0].content.splice(1, 0, { type: "input_image", image_url: "" });
const res = await fetch(new URL("/v1/responses", server.url), {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${token}`,
"chatgpt-account-id": "acct-vision-sidecar",
},
body: JSON.stringify(request),
});
expect(res.status).toBe(200);
expect(sidecarHits).toBe(1);
expect(upstreamBody).toContain(CAPTION);
expect(upstreamBody).not.toContain("aGVsbG8taW1hZ2UtYnl0ZXM=");
} finally {
await server.stop(true);
}
});

test("Responses passthrough replaces an image returned by a client tool", async () => {
let upstreamBody = "";
let sidecarHits = 0;
upstream = serveResponsesUpstream(b => { upstreamBody = b; });
sidecar = serveSidecar(() => { sidecarHits += 1; });
globalThis.fetch = ((input: RequestInfo | URL, init?: RequestInit) => {
const requestUrl = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
const url = new URL(requestUrl);
const prefix = "/backend-api/codex";
if (url.hostname === "chatgpt.com" && url.pathname.startsWith(prefix)) {
return originalFetch(new URL(`${url.pathname.slice(prefix.length)}${url.search}`, sidecar!.url), init);
}
return originalFetch(input, init);
}) as typeof fetch;

const config: OcxConfig = {
port: 0, hostname: "127.0.0.1", defaultProvider: "textonly", openaiProviderTierVersion: 2,
providers: {
textonly: {
adapter: "openai-responses",
authMode: "key",
baseUrl: upstream.url.toString().replace(/\/$/, ""),
responsesPath: "/responses",
allowPrivateNetwork: true,
apiKey: "key-alpha-000111222333",
noVisionModels: ["blind-model"],
},
openai: {
adapter: "openai-responses",
authMode: "forward",
baseUrl: "https://chatgpt.com/backend-api/codex",
codexAccountMode: "direct",
},
},
} as OcxConfig;
saveConfig(config);
const server = startServer(0);
try {
const token = fakeChatGptJwt({ chatgpt_account_id: "acct-vision-sidecar" });
const res = await fetch(new URL("/v1/responses", server.url), {
method: "POST",
headers: {
"content-type": "application/json",
authorization: `Bearer ${token}`,
"chatgpt-account-id": "acct-vision-sidecar",
},
body: JSON.stringify(toolImageRequest("textonly/blind-model")),
});
expect(res.status).toBe(200);
expect(sidecarHits).toBe(1);
expect(upstreamBody).toContain(CAPTION);
expect(upstreamBody).not.toContain("aGVsbG8taW1hZ2UtYnl0ZXM=");
expect(upstreamBody).not.toContain("image_url");
} finally {
await server.stop(true);
}
});

test("Responses passthrough strips images when no vision sidecar is available", async () => {
let upstreamBody = "";
upstream = serveResponsesUpstream(b => { upstreamBody = b; });
const config: OcxConfig = {
port: 0, hostname: "127.0.0.1", defaultProvider: "textonly",
providers: {
textonly: {
adapter: "openai-responses",
authMode: "key",
baseUrl: upstream.url.toString().replace(/\/$/, ""),
responsesPath: "/responses",
allowPrivateNetwork: true,
apiKey: "key-alpha-000111222333",
noVisionModels: ["blind-model"],
},
},
} as OcxConfig;
saveConfig(config);
const server = startServer(0);
try {
const res = await fetch(new URL("/v1/responses", server.url), {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify(baseRequest("textonly/blind-model")),
});
expect(res.status).toBe(200);
expect(upstreamBody).toContain("[image omitted: this model is text-only and the vision sidecar is unavailable (no ChatGPT login)]");
expect(upstreamBody).not.toContain("aGVsbG8taW1hZ2UtYnl0ZXM=");
expect(upstreamBody).not.toContain("image_url");
} finally {
await server.stop(true);
}
});

test("models outside noVisionModels keep their image untouched (no sidecar call)", async () => {
let upstreamBody = "";
let sidecarHits = 0;
Expand Down
Loading