diff --git a/README.md b/README.md index a23e577..b830351 100644 --- a/README.md +++ b/README.md @@ -93,6 +93,26 @@ model should have `"input": ["text", "image"]`. Other `/vision` subcommands: Config is stored at `~/.pi/agent/vision.json` (not `vision-tool.json`, so it doesn't collide with the community package during transition). +### Supported provider APIs + +The vision call speaks the wire format of the model's `api` in `models.json`: + +| `api` | Request | Auth header | +|---|---|---| +| `openai-completions` | `POST {baseUrl}/chat/completions` (`image_url` data URL) | `Authorization: Bearer` | +| `openai-responses` | `POST {baseUrl}/responses` (`input_image` part; system prompt as `instructions`) — `baseUrl` carries the version, e.g. `…/v1` | `Authorization: Bearer` | +| `google-generative-ai` | `POST {baseUrl}/models/{id}:generateContent` (`inlineData`) — `baseUrl` already carries the version, e.g. `…/v1beta` | `x-goog-api-key` | +| `anthropic-messages` | `POST {baseUrl}/v1/messages` (base64 `image` block) — `baseUrl` has no `/v1` | `x-api-key` + `anthropic-version` | + +`reasoning-effort` is forwarded as `reasoning_effort` (chat/completions), `reasoning.effort` +(Responses), `thinkingConfig.thinkingLevel` +(Gemini, honouring the model's `thinkingLevelMap`) or an extended-thinking budget (Anthropic). +`off` sends nothing. `temperature` is never sent to Anthropic or Responses (newer Claude and reasoning models reject it). + +Any other `api` (e.g. `google-vertex`, `bedrock-converse-stream`) is rejected +up front with an `unsupported_api` error naming the supported ones, instead of an opaque 404. +`/vision model` only lists models on a supported API. + ## Resilience (v0.2.0) DELEGATE mode (text-only primary) is resilient + cheap: @@ -292,7 +312,8 @@ cache miss. Every local-only event is audited with `local_only: true` ### Workflow-fit defaults (auto-detect) On a fresh install (or after `/vision clear`), the extension auto-detects the -vision model at session start from `~/.pi/agent/models.json` — aligned to the +vision model (only models on a [supported API](#supported-provider-apis)) at +session start from `~/.pi/agent/models.json` — aligned to the [AGENTS.md LLM-backend policy](https://github.com/earendil-works) of "Ollama Cloud primary + frontier escalation": diff --git a/extensions/vision.ts b/extensions/vision.ts index 14bff73..27d0277 100644 --- a/extensions/vision.ts +++ b/extensions/vision.ts @@ -58,6 +58,7 @@ import { loadImage } from "../lib/image.ts"; import { mapWithConcurrency } from "../lib/batch.ts"; import { buildBatchToolResult } from "../lib/marker.ts"; import { autoDetectDefaults } from "../lib/defaults.ts"; +import { isSupportedVisionApi, unsupportedApiMessage } from "../lib/wire.ts"; import { clearAuditLog, countAuditLog, resolveAuditPath, tailAuditLog } from "../lib/audit.ts"; /** Current config. Loaded on session_start, mutated by /vision, saved to disk. */ @@ -185,6 +186,16 @@ function resync(pi: ExtensionAPI, ctx: ExtensionCommandContext): void { syncToolAvailability(pi, ctx.model, { enabled: config.enabled }); } +/** Warn when the configured vision model is registered but uses an API the + * vision call cannot speak (it would only fail later, on the first image). */ +function warnIfUnsupportedApi(ctx: ExtensionContext): void { + if (!config.provider || !config.model) return; + const model = ctx.modelRegistry.find(config.provider, config.model); + if (model && !isSupportedVisionApi(model.api)) { + ctx.ui.notify(unsupportedApiMessage(model), "warning"); + } +} + /** Apply a setting edit, persist, re-sync visibility if needed, and rebuild * the cache when cache-shape fields change. */ function applyAndSave(id: string, value: string, pi: ExtensionAPI, ctx: ExtensionCommandContext): void { @@ -192,12 +203,16 @@ function applyAndSave(id: string, value: string, pi: ExtensionAPI, ctx: Extensio saveConfig(config, getAgentDir()); setSharedState(config, cache); if (id === "enabled" || id === "model") resync(pi, ctx); + if (id === "model") warnIfUnsupportedApi(ctx); if (id === "cachePersist" || id === "cacheMaxEntries") rebuildCache(); } -/** Vision-capable authed models from the registry (input includes "image"). */ +/** Vision-capable authed models from the registry (input includes "image") on + * an API the vision call can speak — others would 404 on the first image. */ function visionCapableModels(ctx: ExtensionContext): Model[] { - return ctx.modelRegistry.getAvailable().filter((m) => m.input.includes("image")); + return ctx.modelRegistry + .getAvailable() + .filter((m) => m.input.includes("image") && isSupportedVisionApi(m.api)); } /** Open pi's native select picker over vision-capable models. Sets provider + @@ -853,6 +868,7 @@ export default function visionExtension(pi: ExtensionAPI): void { saveConfig(config, agentDir); resync(pi, ctx); ctx.ui.notify(`Vision model set to ${value}.`, "info"); + warnIfUnsupportedApi(ctx); return; } case "max-dim": { diff --git a/lib/defaults.ts b/lib/defaults.ts index 01ac7fa..f259da4 100644 --- a/lib/defaults.ts +++ b/lib/defaults.ts @@ -20,6 +20,7 @@ * (auto-detect only fires when both provider + model are unset). */ import type { Api, Model } from "@earendil-works/pi-ai"; +import { isSupportedVisionApi } from "./wire.ts"; export interface DetectedDefaults { provider: string | undefined; @@ -36,7 +37,9 @@ export const PREFERRED_PRIMARY_PROVIDER = "Ollama"; * via /vision fallback if they want frontier escalation). * * Algorithm: - * 1. Filter to vision-capable models (`input` includes "image"). + * 1. Filter to vision-capable models (`input` includes "image") whose `api` the + * vision call can speak (`isSupportedVisionApi`) — otherwise the pick would + * 404 on the first describe_image. * 2. If none → undefined (no-op; the existing not-configured error guides * the user). * 3. Primary: prefer the `Ollama` provider; among those, first by sorted @@ -46,7 +49,7 @@ export const PREFERRED_PRIMARY_PROVIDER = "Ollama"; * Pure + deterministic (same input in any order → same output). */ export function autoDetectDefaults(models: Model[]): DetectedDefaults { - const visionModels = models.filter((m) => m.input?.includes("image")); + const visionModels = models.filter((m) => m.input?.includes("image") && isSupportedVisionApi(m.api)); if (visionModels.length === 0) { return { provider: undefined, model: undefined }; } diff --git a/lib/delegate.ts b/lib/delegate.ts index 58338e3..0d716df 100644 --- a/lib/delegate.ts +++ b/lib/delegate.ts @@ -14,14 +14,16 @@ * secondary vision model on failure. * 4. Abort-aware — `ctx.signal` stops retry + skips fallback. * - * Clean-room: the OpenAI-compatible `/chat/completions` request shape with a - * base64 data-URL image is standard API usage, not copied from pi-vision-tool. + * Request shapes (OpenAI chat/completions, Gemini generateContent, Anthropic + * messages) live in `lib/wire.ts`; the call dispatches on `model.api`. They use + * standard base64 image payloads, not copied from pi-vision-tool. */ import type { Model, Api, ProviderHeaders } from "@earendil-works/pi-ai"; import type { ExtensionContext } from "@earendil-works/pi-coding-agent"; import { getAgentDir } from "@earendil-works/pi-coding-agent"; import { isConfiguredForDelegation, loadConfig, type ReasoningLevel, type VisionConfig } from "./config.ts"; import { loadImage, type LoadedImage } from "./image.ts"; +import { buildVisionRequest, extractVisionText, isSupportedVisionApi, unsupportedApiMessage } from "./wire.ts"; import { cacheKey, type VisionCache } from "./cache.ts"; import { AbortError, classifyError, withRetry } from "./resilience.ts"; import { appendAuditEntry, resolveAuditPath, truncateImagePathForLog, type AuditEntry } from "./audit.ts"; @@ -59,20 +61,11 @@ export interface DelegateFailure { export type DelegateResult = DelegateSuccess | DelegateFailure; -/** Build provider-specific reasoning params for the request body, if any. */ -function buildReasoningParams( - visionModel: Model, - level: ReasoningLevel, -): Record | undefined { - if (!visionModel.reasoning || level === "off") return undefined; - return { reasoning_effort: level }; -} - /** - * Call the vision model's OpenAI-compat chat/completions endpoint with the - * image as a data URL + the user's prompt (and an optional system prompt). - * Returns the model's text response. Exported + fetch-based so tests can mock - * `globalThis.fetch`. + * Call the vision model with the image as base64 + the user's prompt (and an + * optional system prompt), in the wire format of `visionModel.api` (see + * `lib/wire.ts`). Returns the model's text response. Exported + fetch-based so + * tests can mock `globalThis.fetch`. */ export async function callVisionModel( visionModel: Model, @@ -84,44 +77,20 @@ export async function callVisionModel( reasoning: ReasoningLevel, systemPrompt?: string, ): Promise { - const baseUrl = visionModel.baseUrl.replace(/\/+$/, ""); - const messages: unknown[] = []; - if (systemPrompt && systemPrompt.length > 0) { - messages.push({ role: "system", content: systemPrompt }); - } - messages.push({ - role: "user", - content: [ - { - type: "image_url", - image_url: { url: `data:${image.mimeType};base64,${image.data}` }, - }, - { type: "text", text: prompt }, - ], + const request = buildVisionRequest({ + model: visionModel, + apiKey, + providerHeaders, + image, + prompt, + reasoning, + systemPrompt, }); - const body: Record = { - model: visionModel.id, - messages, - max_tokens: 4096, - temperature: 0, - }; - const reasoningParams = buildReasoningParams(visionModel, reasoning); - if (reasoningParams) Object.assign(body, reasoningParams); - - const headers: Record = { "Content-Type": "application/json" }; - if (apiKey) headers.Authorization = `Bearer ${apiKey}`; - // pi-ai 0.84 widened ProviderHeaders to Record — null means - // "unset"; a null value must never reach fetch's HeadersInit (typed Record). - if (providerHeaders) { - for (const [key, value] of Object.entries(providerHeaders)) { - if (value !== null) headers[key] = value; - } - } - const response = await fetch(`${baseUrl}/chat/completions`, { + const response = await fetch(request.url, { method: "POST", - headers, - body: JSON.stringify(body), + headers: request.headers, + body: JSON.stringify(request.body), signal, }); @@ -132,13 +101,9 @@ export async function callVisionModel( ); } - const json = (await response.json()) as { - choices?: Array<{ message?: { content?: string; reasoning_content?: string } }>; - }; - const msg = json.choices?.[0]?.message; - const text = msg?.content || msg?.reasoning_content; + const { text, reason } = extractVisionText(visionModel.api, await response.json()); if (!text) { - throw new Error("Vision model returned no content in the response"); + throw new Error(`Vision model returned no content in the response${reason ? ` (${reason})` : ""}`); } return text; } @@ -256,6 +221,12 @@ export async function delegateToVisionModel( }; } + // Fail before reading the image or touching the network: a model on an API + // the call cannot speak would only come back as an opaque 404. + if (!isSupportedVisionApi(visionModel.api)) { + return { ok: false, error: { code: "unsupported_api", message: unsupportedApiMessage(visionModel) } }; + } + const auth = await ctx.modelRegistry.getApiKeyAndHeaders(visionModel); if (!auth.ok) { return { @@ -425,6 +396,13 @@ async function runFallback( details: { primaryError: primaryErrorTag(primaryErr), fallbackModel: fallbackId }, }; } + if (!isSupportedVisionApi(fbModel.api)) { + return { + ok: false, + error: { code: "unsupported_api", message: unsupportedApiMessage(fbModel) }, + details: { primaryError: primaryErrorTag(primaryErr), fallbackModel: fallbackId }, + }; + } const fbAuth = await ctx.modelRegistry.getApiKeyAndHeaders(fbModel); if (!fbAuth.ok) { return { diff --git a/lib/wire.ts b/lib/wire.ts new file mode 100644 index 0000000..156d02e --- /dev/null +++ b/lib/wire.ts @@ -0,0 +1,325 @@ +/** + * Wire formats for the vision call. + * + * `callVisionModel` used to POST an OpenAI-style `/chat/completions` request + * to every provider, whatever its `api`. A model registered with + * `google-generative-ai` or `anthropic-messages` therefore got a 404 from a + * route that does not exist on that API. This module owns the per-API request + * shape and response parsing so the call dispatches on `model.api` instead. + * + * Pure: no I/O, no pi runtime. Types only. + * + * Conventions follow pi-ai's own adapters: + * - `google-generative-ai`: `baseUrl` already carries the version segment + * (e.g. `https://generativelanguage.googleapis.com/v1beta`). + * - `anthropic-messages`: `baseUrl` has no version segment; the Anthropic SDK + * appends `/v1/messages`. + * - `openai-responses`: `baseUrl` carries the version segment (e.g. + * `https://api.openai.com/v1`); `/responses` is appended. + */ +import type { Api, Model, ProviderHeaders } from "@earendil-works/pi-ai"; +import type { ReasoningLevel } from "./config.ts"; +import type { LoadedImage } from "./image.ts"; + +/** APIs the vision call can speak. Anything else is rejected before the network. */ +export const SUPPORTED_VISION_APIS = [ + "openai-completions", + "openai-responses", + "google-generative-ai", + "anthropic-messages", +] as const; + +export type SupportedVisionApi = (typeof SUPPORTED_VISION_APIS)[number]; + +export function isSupportedVisionApi(api: unknown): api is SupportedVisionApi { + return typeof api === "string" && (SUPPORTED_VISION_APIS as readonly string[]).includes(api); +} + +/** Output cap shared by every API (matches the pre-existing OpenAI request). */ +const MAX_OUTPUT_TOKENS = 4096; + +const ANTHROPIC_VERSION = "2023-06-01"; + +/** Anthropic extended-thinking budgets per pi reasoning level (tokens). */ +const ANTHROPIC_THINKING_BUDGETS: Record, number> = { + minimal: 1024, + low: 2048, + medium: 8192, + high: 16384, + xhigh: 32768, +}; + +export interface VisionWireInput { + model: Model; + apiKey: string | undefined; + providerHeaders: ProviderHeaders | undefined; + image: LoadedImage; + prompt: string; + reasoning: ReasoningLevel; + systemPrompt?: string | undefined; +} + +export interface VisionWireRequest { + url: string; + headers: Record; + body: Record; +} + +export interface VisionWireText { + /** The model's answer, or undefined when the response carried none. */ + text: string | undefined; + /** Why there is no text (block reason / finish reason), for the error message. */ + reason?: string | undefined; +} + +/** + * Actionable message for a model whose API this extension cannot call. Must not + * match `/returned \d{3}:/` — `classifyError` would read it as an HTTP status. + */ +export function unsupportedApiMessage(model: Pick, "provider" | "id" | "api">): string { + return [ + `Vision model "${model.provider}/${model.id}" uses the "${model.api}" API, which the vision tool cannot call.`, + `Supported APIs: ${SUPPORTED_VISION_APIS.join(", ")}.`, + "Pick a model from a provider that uses one of them (/vision model), or register", + "the same model under such a provider in ~/.pi/agent/models.json.", + ].join("\n"); +} + +export class UnsupportedVisionApiError extends Error { + constructor(model: Pick, "provider" | "id" | "api">) { + super(unsupportedApiMessage(model)); + this.name = "UnsupportedVisionApiError"; + } +} + +function trimTrailingSlashes(url: string): string { + return url.replace(/\/+$/, ""); +} + +/** Merge provider headers over the API's own; a null value means "unset". */ +function withProviderHeaders( + base: Record, + providerHeaders: ProviderHeaders | undefined, +): Record { + const headers = { ...base }; + if (providerHeaders) { + for (const [key, value] of Object.entries(providerHeaders)) { + if (value !== null) headers[key] = value; + } + } + return headers; +} + +function openAiRequest(input: VisionWireInput): VisionWireRequest { + const { model, apiKey, image, prompt, systemPrompt, reasoning } = input; + const messages: unknown[] = []; + if (systemPrompt && systemPrompt.length > 0) { + messages.push({ role: "system", content: systemPrompt }); + } + messages.push({ + role: "user", + content: [ + { type: "image_url", image_url: { url: `data:${image.mimeType};base64,${image.data}` } }, + { type: "text", text: prompt }, + ], + }); + const body: Record = { + model: model.id, + messages, + max_tokens: MAX_OUTPUT_TOKENS, + temperature: 0, + }; + if (model.reasoning && reasoning !== "off") body.reasoning_effort = reasoning; + + const headers: Record = { "Content-Type": "application/json" }; + if (apiKey) headers.Authorization = `Bearer ${apiKey}`; + return { + url: `${trimTrailingSlashes(model.baseUrl)}/chat/completions`, + headers: withProviderHeaders(headers, input.providerHeaders), + body, + }; +} + +/** + * OpenAI Responses API: `{baseUrl}/responses` with `input` items. The image and + * prompt are `input_image` / `input_text` parts, the system prompt goes in + * `instructions`, and reasoning is the nested `reasoning.effort`. + */ +function responsesRequest(input: VisionWireInput): VisionWireRequest { + const { model, apiKey, image, prompt, systemPrompt, reasoning } = input; + const body: Record = { + model: model.id, + input: [ + { + role: "user", + content: [ + { type: "input_image", image_url: `data:${image.mimeType};base64,${image.data}` }, + { type: "input_text", text: prompt }, + ], + }, + ], + max_output_tokens: MAX_OUTPUT_TOKENS, + }; + if (systemPrompt && systemPrompt.length > 0) body.instructions = systemPrompt; + // `temperature` is deliberately omitted: reasoning models reject it. + if (model.reasoning && reasoning !== "off") body.reasoning = { effort: reasoning }; + + const headers: Record = { "Content-Type": "application/json" }; + if (apiKey) headers.Authorization = `Bearer ${apiKey}`; + return { + url: `${trimTrailingSlashes(model.baseUrl)}/responses`, + headers: withProviderHeaders(headers, input.providerHeaders), + body, + }; +} + +/** Gemini `thinkingLevel` for a pi level, honouring the model's own map. */ +function googleThinkingLevel(model: Model, level: ReasoningLevel): string | undefined { + if (!model.reasoning || level === "off") return undefined; + const mapped = model.thinkingLevelMap?.[level]; + if (mapped === null) return undefined; // the model marks this level unsupported + if (typeof mapped === "string") return mapped; + return level === "xhigh" ? "high" : level; +} + +function googleRequest(input: VisionWireInput): VisionWireRequest { + const { model, apiKey, image, prompt, systemPrompt, reasoning } = input; + const generationConfig: Record = { + temperature: 0, + maxOutputTokens: MAX_OUTPUT_TOKENS, + }; + const thinkingLevel = googleThinkingLevel(model, reasoning); + if (thinkingLevel) generationConfig.thinkingConfig = { thinkingLevel }; + + const body: Record = { + contents: [ + { + role: "user", + parts: [{ inlineData: { mimeType: image.mimeType, data: image.data } }, { text: prompt }], + }, + ], + generationConfig, + }; + if (systemPrompt && systemPrompt.length > 0) { + body.systemInstruction = { parts: [{ text: systemPrompt }] }; + } + + const headers: Record = { "Content-Type": "application/json" }; + if (apiKey) headers["x-goog-api-key"] = apiKey; + const id = encodeURIComponent(model.id.replace(/^models\//, "")); + return { + url: `${trimTrailingSlashes(model.baseUrl)}/models/${id}:generateContent`, + headers: withProviderHeaders(headers, input.providerHeaders), + body, + }; +} + +function anthropicRequest(input: VisionWireInput): VisionWireRequest { + const { model, apiKey, image, prompt, systemPrompt, reasoning } = input; + const body: Record = { + model: model.id, + max_tokens: MAX_OUTPUT_TOKENS, + messages: [ + { + role: "user", + content: [ + { type: "image", source: { type: "base64", media_type: image.mimeType, data: image.data } }, + { type: "text", text: prompt }, + ], + }, + ], + }; + if (systemPrompt && systemPrompt.length > 0) body.system = systemPrompt; + // `temperature` is deliberately omitted: newer Claude models reject it, and + // extended thinking requires it unset or 1. + if (model.reasoning && reasoning !== "off") { + const budget = ANTHROPIC_THINKING_BUDGETS[reasoning]; + body.thinking = { type: "enabled", budget_tokens: budget }; + body.max_tokens = budget + MAX_OUTPUT_TOKENS; // must exceed the thinking budget + } + + const headers: Record = { + "Content-Type": "application/json", + "anthropic-version": ANTHROPIC_VERSION, + }; + if (apiKey) headers["x-api-key"] = apiKey; + return { + url: `${trimTrailingSlashes(model.baseUrl)}/v1/messages`, + headers: withProviderHeaders(headers, input.providerHeaders), + body, + }; +} + +/** Build the HTTP request for `input.model.api`. Throws on an unsupported API. */ +export function buildVisionRequest(input: VisionWireInput): VisionWireRequest { + switch (input.model.api) { + case "openai-completions": + return openAiRequest(input); + case "openai-responses": + return responsesRequest(input); + case "google-generative-ai": + return googleRequest(input); + case "anthropic-messages": + return anthropicRequest(input); + default: + throw new UnsupportedVisionApiError(input.model); + } +} + +type Json = Record; + +function openAiText(json: Json): VisionWireText { + const choice = json.choices?.[0]; + const msg = choice?.message; + const text = msg?.content || msg?.reasoning_content; + return { text: text || undefined, reason: choice?.finish_reason }; +} + +function responsesText(json: Json): VisionWireText { + const output: Json[] = Array.isArray(json.output) ? json.output : []; + const text = output + .filter((item) => item.type === "message" && Array.isArray(item.content)) + .flatMap((item) => item.content as Json[]) + .filter((part) => part.type === "output_text" && typeof part.text === "string") + .map((part) => part.text as string) + .join(""); + return { text: text || undefined, reason: json.incomplete_details?.reason }; +} + +function googleText(json: Json): VisionWireText { + const candidate = json.candidates?.[0]; + const parts: Json[] = Array.isArray(candidate?.content?.parts) ? candidate.content.parts : []; + const joined = (wantThought: boolean) => + parts + .filter((p) => typeof p.text === "string" && Boolean(p.thought) === wantThought) + .map((p) => p.text as string) + .join(""); + // Prefer the visible answer; fall back to thought text like the OpenAI + // path falls back to `reasoning_content`. + const text = joined(false) || joined(true); + return { text: text || undefined, reason: json.promptFeedback?.blockReason ?? candidate?.finishReason }; +} + +function anthropicText(json: Json): VisionWireText { + const blocks: Json[] = Array.isArray(json.content) ? json.content : []; + const text = blocks + .filter((b) => b.type === "text" && typeof b.text === "string") + .map((b) => b.text as string) + .join(""); + return { text: text || undefined, reason: json.stop_reason }; +} + +/** Pull the answer text out of a parsed response body for `api`. */ +export function extractVisionText(api: Api, json: unknown): VisionWireText { + const body: Json = json && typeof json === "object" ? (json as Json) : {}; + switch (api) { + case "google-generative-ai": + return googleText(body); + case "anthropic-messages": + return anthropicText(body); + case "openai-responses": + return responsesText(body); + default: + return openAiText(body); + } +} diff --git a/tests/defaults.test.ts b/tests/defaults.test.ts index 75a7e63..4a80760 100644 --- a/tests/defaults.test.ts +++ b/tests/defaults.test.ts @@ -133,4 +133,34 @@ test("primary model without a provider field → handled (sort key empty string) const result = autoDetectDefaults([vision("", "lonely")]); assert.equal(result.provider, ""); assert.equal(result.model, "lonely"); -}); \ No newline at end of file +}); + +test("models on an api the vision call cannot speak are never picked (would 404)", () => { + const onApi = (api: string, p: string, id: string) => + ({ ...vision(p, id), api: api as Api }) as Model; + // Both sort before "quantforge-openai" — either used to win and 404 on the first call. + const result = autoDetectDefaults([ + onApi("bedrock-converse-stream", "aws-bedrock", "claude-x"), + onApi("google-vertex", "aws-vertex", "gem-x"), + onApi("openai-completions", "quantforge-openai", "gemini-x"), + ]); + assert.equal(result.provider, "quantforge-openai"); + assert.equal(result.model, "gemini-x"); +}); + +test("supported non-openai apis (google, anthropic) remain eligible", () => { + const google = { ...vision("g", "gem"), api: "google-generative-ai" as Api } as Model; + assert.deepEqual(autoDetectDefaults([google]), { provider: "g", model: "gem" }); + const anthropic = { ...vision("a", "claude"), api: "anthropic-messages" as Api } as Model; + assert.deepEqual(autoDetectDefaults([anthropic]), { provider: "a", model: "claude" }); +}); + +test("openai-responses models remain eligible", () => { + const responses = { ...vision("r", "gpt-x"), api: "openai-responses" as Api } as Model; + assert.deepEqual(autoDetectDefaults([responses]), { provider: "r", model: "gpt-x" }); +}); + +test("only unsupported-api vision models → all undefined (no-op)", () => { + const unsupported = { ...vision("p", "m"), api: "bedrock-converse-stream" as Api } as Model; + assert.deepEqual(autoDetectDefaults([unsupported]), { provider: undefined, model: undefined }); +}); diff --git a/tests/delegate.test.ts b/tests/delegate.test.ts index 1505704..d859900 100644 --- a/tests/delegate.test.ts +++ b/tests/delegate.test.ts @@ -906,4 +906,198 @@ test("T47 regression: single-image success with audit on → v0.4.0 behavior pre } }); +// ── api-aware dispatch (regression: google/anthropic models used to 404) ── + +test("callVisionModel: google-generative-ai → generateContent with x-goog-api-key + inlineData", async () => { + const m = mockFetch({ status: 200, body: { candidates: [{ content: { parts: [{ text: "a blue circle" }] } }] } }); + try { + const text = await callVisionModel( + makeVisionModel({ api: "google-generative-ai" as Api, id: "gemini-x", baseUrl: "https://gw.test/v1beta" }), + "key-g", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "describe this", + undefined, + "off", + ); + assert.equal(text, "a blue circle"); + assert.equal(m.calls.length, 1); + assert.equal(m.calls[0]!.url, "https://gw.test/v1beta/models/gemini-x:generateContent"); + const headers = m.calls[0]!.init.headers as Record; + assert.equal(headers["x-goog-api-key"], "key-g"); + assert.equal("Authorization" in headers, false); + const body = JSON.parse(m.calls[0]!.init.body as string); + assert.equal(body.contents[0].parts[0].inlineData.mimeType, "image/png"); + assert.equal(body.contents[0].parts[0].inlineData.data, PNG_1x1_B64); + assert.equal(body.contents[0].parts[1].text, "describe this"); + } finally { + m.restore(); + } +}); + +test("callVisionModel: anthropic-messages → /v1/messages with x-api-key + base64 image block", async () => { + const m = mockFetch({ status: 200, body: { content: [{ type: "text", text: "a green square" }], stop_reason: "end_turn" } }); + try { + const text = await callVisionModel( + makeVisionModel({ api: "anthropic-messages" as Api, id: "claude-x", baseUrl: "https://gw.test" }), + "key-a", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "describe this", + undefined, + "off", + ); + assert.equal(text, "a green square"); + assert.equal(m.calls[0]!.url, "https://gw.test/v1/messages"); + const headers = m.calls[0]!.init.headers as Record; + assert.equal(headers["x-api-key"], "key-a"); + assert.equal(headers["anthropic-version"], "2023-06-01"); + const body = JSON.parse(m.calls[0]!.init.body as string); + assert.equal(body.messages[0].content[0].source.media_type, "image/png"); + assert.equal(body.messages[0].content[1].text, "describe this"); + } finally { + m.restore(); + } +}); + +test("callVisionModel: unsupported api rejects before any network call", async () => { + const m = mockFetch({ status: 200, body: {} }); + try { + await assert.rejects( + callVisionModel( + makeVisionModel({ api: "bedrock-converse-stream" as Api }), + "k", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "p", + undefined, + "off", + ), + /"bedrock-converse-stream" API/, + ); + assert.equal(m.calls.length, 0, "no request was sent"); + } finally { + m.restore(); + } +}); + +test("callVisionModel: empty response reports why (finish/block reason) and stays a no-content error", async () => { + const m = mockFetch({ status: 200, body: { promptFeedback: { blockReason: "SAFETY" } } }); + try { + await assert.rejects( + callVisionModel( + makeVisionModel({ api: "google-generative-ai" as Api, baseUrl: "https://gw.test/v1beta" }), + "k", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "p", + undefined, + "off", + ), + /returned no content in the response \(SAFETY\)/, + ); + } finally { + m.restore(); + } +}); + +test("delegateToVisionModel: google-generative-ai primary works end to end", async () => { + const dir = tmpDir(); + const m = mockFetch({ status: 200, body: { candidates: [{ content: { parts: [{ text: "desc via gemini" }] } }] } }); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ + cwd: dir, + model: makeVisionModel({ api: "google-generative-ai" as Api, id: "gemini-x", baseUrl: "https://gw.test/v1beta" }), + }); + const cfg = { ...DEFAULT_CONFIG, provider: "gw", model: "gemini-x" }; + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + assert.equal(r.ok, true); + if (r.ok) assert.equal(r.text, "desc via gemini"); + assert.match(m.calls[0]!.url, /\/models\/gemini-x:generateContent$/); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: openai-responses primary works end to end", async () => { + const dir = tmpDir(); + const m = mockFetch({ + status: 200, + body: { output: [{ type: "message", content: [{ type: "output_text", text: "desc via responses" }] }] }, + }); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = makeCtx({ + cwd: dir, + model: makeVisionModel({ api: "openai-responses" as Api, id: "gpt-x", baseUrl: "https://gw.test/v1" }), + }); + const cfg = { ...DEFAULT_CONFIG, provider: "gw", model: "gpt-x" }; + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + assert.equal(r.ok, true); + if (r.ok) assert.equal(r.text, "desc via responses"); + assert.equal(m.calls[0]!.url, "https://gw.test/v1/responses"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: unsupported api → unsupported_api pre-flight (no image read, no network, not audited)", async () => { + const { dir, file, auditPath, cfg } = setupAuditTest(); + const m = mockFetch({ status: 200, body: {} }); + try { + const ctx = makeCtx({ cwd: dir, model: makeVisionModel({ api: "bedrock-converse-stream" as Api }) }); + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + assert.equal(r.ok, false); + if (!r.ok) { + assert.equal(r.error.code, "unsupported_api"); + assert.match(r.error.message, /bedrock-converse-stream/); + assert.match(r.error.message, /google-generative-ai/); + } + assert.equal(m.calls.length, 0); + assert.equal(countAuditLog(auditPath), 0, "pre-flight errors are not routing events"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("delegateToVisionModel: fallback on an unsupported api → unsupported_api with primary-error traceability", async () => { + const dir = tmpDir(); + const primary = makeVisionModel(); + const fallback = makeVisionModel({ api: "bedrock-converse-stream" as Api, id: "fb-model", provider: "fb" }); + const m = mockFetchSeq([{ status: 400, body: "bad request" }]); + try { + const file = join(dir, "pixel.png"); + writeFileSync(file, PNG_BYTES); + const ctx = { + cwd: dir, + modelRegistry: { + find: (provider: string) => (provider === "ollama" ? primary : fallback), + getApiKeyAndHeaders: async () => ({ ok: true, apiKey: "k" }), + } as unknown as ExtensionContext["modelRegistry"], + } as unknown as ExtensionContext; + const cfg = { + ...DEFAULT_CONFIG, provider: "ollama", model: "minimax-m3:cloud", + retryAttempts: 1, retryBackoffMs: 1, + fallbackProvider: "fb", fallbackModel: "fb-model", + }; + const r = await delegateToVisionModel(ctx, cfg, { image_path: file, prompt: "p", compress: false, reasoning: "off" }, undefined); + assert.equal(r.ok, false); + if (!r.ok) { + assert.equal(r.error.code, "unsupported_api"); + assert.equal(r.details?.fallbackModel, "fb/fb-model"); + assert.match(r.details?.primaryError ?? "", /400/); + } + assert.equal(m.calls.length, 1, "only the primary hit the network"); + } finally { + m.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + diff --git a/tests/wire.test.ts b/tests/wire.test.ts new file mode 100644 index 0000000..8f533e2 --- /dev/null +++ b/tests/wire.test.ts @@ -0,0 +1,308 @@ +/** + * Unit tests for `lib/wire.ts` — the per-API request/response shapes the + * vision call dispatches on. Pure: no fetch, no pi runtime. + * + * Regression context: `callVisionModel` once POSTed `/chat/completions` to + * every provider, so a model on `google-generative-ai` or `anthropic-messages` + * came back as an opaque 404. + */ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import type { Api, Model } from "@earendil-works/pi-ai"; +import { + SUPPORTED_VISION_APIS, + UnsupportedVisionApiError, + buildVisionRequest, + extractVisionText, + isSupportedVisionApi, + unsupportedApiMessage, + type VisionWireInput, +} from "../lib/wire.ts"; + +function makeModel(api: string, overrides: Partial> = {}): Model { + return { + id: "m-1", + name: "M 1", + api: api as Api, + provider: "prov", + baseUrl: "https://example.test/base", + reasoning: false, + input: ["text", "image"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 100000, + maxTokens: 4096, + ...overrides, + }; +} + +function makeInput(model: Model, overrides: Partial = {}): VisionWireInput { + return { + model, + apiKey: "key-1", + providerHeaders: undefined, + image: { data: "QUJD", mimeType: "image/png" }, + prompt: "describe", + reasoning: "off", + ...overrides, + }; +} + +// ── api support ────────────────────────────────────────────────────────── + +test("isSupportedVisionApi: accepts the four supported APIs only", () => { + for (const api of SUPPORTED_VISION_APIS) assert.equal(isSupportedVisionApi(api), true); + assert.deepEqual([...SUPPORTED_VISION_APIS], [ + "openai-completions", + "openai-responses", + "google-generative-ai", + "anthropic-messages", + ]); + for (const api of ["openai-codex-responses", "google-vertex", "bedrock-converse-stream", "", undefined, null, 5]) { + assert.equal(isSupportedVisionApi(api), false, String(api)); + } +}); + +test("buildVisionRequest: unsupported api throws an actionable error (no HTTP status shape)", () => { + const model = makeModel("bedrock-converse-stream"); + assert.throws(() => buildVisionRequest(makeInput(model)), UnsupportedVisionApiError); + const msg = unsupportedApiMessage(model); + assert.match(msg, /prov\/m-1/); + assert.match(msg, /"bedrock-converse-stream"/); + for (const api of SUPPORTED_VISION_APIS) assert.ok(msg.includes(api), api); + // classifyError reads /returned (\d{3}):/ as an HTTP status — must not match. + assert.doesNotMatch(msg, /returned \d{3}:/); +}); + +// ── openai-completions ─────────────────────────────────────────────────── + +test("openai: /chat/completions, Bearer auth, image_url data URL, temperature 0", () => { + const req = buildVisionRequest(makeInput(makeModel("openai-completions", { baseUrl: "https://example.test/v1///" }))); + assert.equal(req.url, "https://example.test/v1/chat/completions"); + assert.equal(req.headers.Authorization, "Bearer key-1"); + assert.equal(req.headers["Content-Type"], "application/json"); + const body = req.body as any; + assert.equal(body.model, "m-1"); + assert.equal(body.max_tokens, 4096); + assert.equal(body.temperature, 0); + assert.equal(body.messages[0].content[0].image_url.url, "data:image/png;base64,QUJD"); + assert.equal(body.messages[0].content[1].text, "describe"); + assert.equal("reasoning_effort" in body, false); +}); + +test("openai: system prompt first; reasoning_effort only when model.reasoning and level != off", () => { + const withSys = buildVisionRequest(makeInput(makeModel("openai-completions"), { systemPrompt: "be terse" })).body as any; + assert.deepEqual(withSys.messages[0], { role: "system", content: "be terse" }); + + const on = buildVisionRequest(makeInput(makeModel("openai-completions", { reasoning: true }), { reasoning: "high" })).body as any; + assert.equal(on.reasoning_effort, "high"); + const off = buildVisionRequest(makeInput(makeModel("openai-completions", { reasoning: true }), { reasoning: "off" })).body as any; + assert.equal("reasoning_effort" in off, false); + const notReasoning = buildVisionRequest(makeInput(makeModel("openai-completions"), { reasoning: "high" })).body as any; + assert.equal("reasoning_effort" in notReasoning, false); +}); + +test("openai: no api key → no Authorization header; null provider headers are dropped, others override", () => { + const req = buildVisionRequest( + makeInput(makeModel("openai-completions"), { + apiKey: undefined, + providerHeaders: { "X-Org": "acme", "X-Unset": null, "Content-Type": "application/vnd.test+json" } as any, + }), + ); + assert.equal("Authorization" in req.headers, false); + assert.equal(req.headers["X-Org"], "acme"); + assert.equal("X-Unset" in req.headers, false); + assert.equal(req.headers["Content-Type"], "application/vnd.test+json"); +}); + +// ── openai-responses ───────────────────────────────────────────────────── + +const RESPONSES_USER_ITEM = { + role: "user", + content: [ + { type: "input_image", image_url: "data:image/png;base64,QUJD" }, + { type: "input_text", text: "describe" }, + ], +}; + +test("responses: {baseUrl}/responses, Bearer auth, input_image + input_text parts, max_output_tokens", () => { + const req = buildVisionRequest(makeInput(makeModel("openai-responses", { baseUrl: "https://example.test/v1///" }))); + assert.equal(req.url, "https://example.test/v1/responses"); + assert.equal(req.headers.Authorization, "Bearer key-1"); + assert.equal(req.headers["Content-Type"], "application/json"); + assert.equal(req.body.model, "m-1"); + assert.equal(req.body.max_output_tokens, 4096); + assert.deepEqual(req.body.input, [RESPONSES_USER_ITEM]); + assert.equal("messages" in req.body, false); + assert.equal("temperature" in req.body, false); +}); + +test("responses: system prompt → instructions, not an input item; reasoning is nested reasoning.effort only when model.reasoning and level != off", () => { + const withSys = buildVisionRequest(makeInput(makeModel("openai-responses"), { systemPrompt: "be terse" })).body; + assert.equal(withSys.instructions, "be terse"); + assert.deepEqual(withSys.input, [RESPONSES_USER_ITEM]); + + const on = buildVisionRequest(makeInput(makeModel("openai-responses", { reasoning: true }), { reasoning: "high" })).body; + assert.deepEqual(on.reasoning, { effort: "high" }); + assert.equal("reasoning_effort" in on, false); + const off = buildVisionRequest(makeInput(makeModel("openai-responses", { reasoning: true }), { reasoning: "off" })).body; + assert.equal("reasoning" in off, false); + const notReasoning = buildVisionRequest(makeInput(makeModel("openai-responses"), { reasoning: "high" })).body; + assert.equal("reasoning" in notReasoning, false); +}); + +// ── google-generative-ai ───────────────────────────────────────────────── + +test("google: {baseUrl}/models/{id}:generateContent, x-goog-api-key, inlineData part", () => { + const model = makeModel("google-generative-ai", { id: "gemini-x", baseUrl: "https://gw.test/v1beta/" }); + const req = buildVisionRequest(makeInput(model)); + assert.equal(req.url, "https://gw.test/v1beta/models/gemini-x:generateContent"); + assert.equal(req.headers["x-goog-api-key"], "key-1"); + assert.equal("Authorization" in req.headers, false); + const body = req.body as any; + assert.deepEqual(body.contents, [ + { role: "user", parts: [{ inlineData: { mimeType: "image/png", data: "QUJD" } }, { text: "describe" }] }, + ]); + assert.equal(body.generationConfig.temperature, 0); + assert.equal(body.generationConfig.maxOutputTokens, 4096); + assert.equal("thinkingConfig" in body.generationConfig, false); + assert.equal("systemInstruction" in body, false); +}); + +test("google: strips a leading models/ from the id; system prompt → systemInstruction", () => { + const model = makeModel("google-generative-ai", { id: "models/gemini-x", baseUrl: "https://gw.test/v1beta" }); + const req = buildVisionRequest(makeInput(model, { systemPrompt: "be terse" })); + assert.equal(req.url, "https://gw.test/v1beta/models/gemini-x:generateContent"); + assert.deepEqual((req.body as any).systemInstruction, { parts: [{ text: "be terse" }] }); +}); + +test("google: thinkingLevel honours thinkingLevelMap, xhigh → high, null → omitted, off → omitted", () => { + const level = (model: Model, reasoning: VisionWireInput["reasoning"]) => + ((buildVisionRequest(makeInput(model, { reasoning })).body as any).generationConfig.thinkingConfig ?? {}).thinkingLevel; + + const plain = makeModel("google-generative-ai", { reasoning: true }); + assert.equal(level(plain, "low"), "low"); + assert.equal(level(plain, "xhigh"), "high"); + assert.equal(level(plain, "off"), undefined); + + const mapped = makeModel("google-generative-ai", { reasoning: true, thinkingLevelMap: { low: "minimal", high: null } }); + assert.equal(level(mapped, "low"), "minimal"); + assert.equal(level(mapped, "high"), undefined); + + const notReasoning = makeModel("google-generative-ai"); + assert.equal(level(notReasoning, "high"), undefined); +}); + +// ── anthropic-messages ─────────────────────────────────────────────────── + +test("anthropic: {baseUrl}/v1/messages, x-api-key + anthropic-version, base64 image block, no temperature", () => { + const model = makeModel("anthropic-messages", { id: "claude-x", baseUrl: "https://gw.test/" }); + const req = buildVisionRequest(makeInput(model)); + assert.equal(req.url, "https://gw.test/v1/messages"); + assert.equal(req.headers["x-api-key"], "key-1"); + assert.equal(req.headers["anthropic-version"], "2023-06-01"); + assert.equal("Authorization" in req.headers, false); + const body = req.body as any; + assert.equal(body.model, "claude-x"); + assert.equal(body.max_tokens, 4096); + assert.equal("temperature" in body, false); + assert.equal("thinking" in body, false); + assert.equal("system" in body, false); + assert.deepEqual(body.messages[0].content, [ + { type: "image", source: { type: "base64", media_type: "image/png", data: "QUJD" } }, + { type: "text", text: "describe" }, + ]); +}); + +test("anthropic: system prompt → top-level system", () => { + const body = buildVisionRequest(makeInput(makeModel("anthropic-messages"), { systemPrompt: "be terse" })).body as any; + assert.equal(body.system, "be terse"); +}); + +test("anthropic: reasoning on → enabled thinking, max_tokens above the budget; off/non-reasoning → none", () => { + const on = buildVisionRequest(makeInput(makeModel("anthropic-messages", { reasoning: true }), { reasoning: "low" })).body as any; + assert.deepEqual(on.thinking, { type: "enabled", budget_tokens: 2048 }); + assert.ok(on.max_tokens > on.thinking.budget_tokens); + + const off = buildVisionRequest(makeInput(makeModel("anthropic-messages", { reasoning: true }), { reasoning: "off" })).body as any; + assert.equal("thinking" in off, false); + assert.equal(off.max_tokens, 4096); + + const notReasoning = buildVisionRequest(makeInput(makeModel("anthropic-messages"), { reasoning: "high" })).body as any; + assert.equal("thinking" in notReasoning, false); +}); + +// ── extractVisionText ──────────────────────────────────────────────────── + +test("extract openai: content, then reasoning_content, else undefined + finish_reason", () => { + assert.equal(extractVisionText("openai-completions" as Api, { choices: [{ message: { content: "hi" } }] }).text, "hi"); + assert.equal( + extractVisionText("openai-completions" as Api, { choices: [{ message: { content: "", reasoning_content: "think" } }] }).text, + "think", + ); + const none = extractVisionText("openai-completions" as Api, { choices: [{ message: {}, finish_reason: "length" }] }); + assert.equal(none.text, undefined); + assert.equal(none.reason, "length"); +}); + +test("extract responses: joins output_text of message items, skipping reasoning and refusal parts; incomplete → reason", () => { + const api = "openai-responses" as Api; + const ok = extractVisionText(api, { + status: "completed", + output: [ + { type: "reasoning", summary: [] }, + { + type: "message", + content: [ + { type: "output_text", text: "a" }, + { type: "refusal", refusal: "no" }, + { type: "output_text", text: "b" }, + ], + }, + ], + }); + assert.equal(ok.text, "ab"); + const incomplete = extractVisionText(api, { + status: "incomplete", + incomplete_details: { reason: "max_output_tokens" }, + output: [{ type: "reasoning", summary: [] }], + }); + assert.equal(incomplete.text, undefined); + assert.equal(incomplete.reason, "max_output_tokens"); +}); + +test("extract google: joins visible parts, ignores thought parts, falls back to thoughts", () => { + const api = "google-generative-ai" as Api; + const parts = (...p: unknown[]) => ({ candidates: [{ content: { parts: p } }] }); + assert.equal(extractVisionText(api, parts({ text: "a" }, { text: "b" })).text, "ab"); + assert.equal(extractVisionText(api, parts({ text: "secret", thought: true }, { text: "answer" })).text, "answer"); + assert.equal(extractVisionText(api, parts({ text: "only thought", thought: true })).text, "only thought"); +}); + +test("extract google: blocked prompt / empty candidate → undefined with the reason", () => { + const api = "google-generative-ai" as Api; + const blocked = extractVisionText(api, { promptFeedback: { blockReason: "SAFETY" } }); + assert.equal(blocked.text, undefined); + assert.equal(blocked.reason, "SAFETY"); + const maxTokens = extractVisionText(api, { candidates: [{ finishReason: "MAX_TOKENS", content: { parts: [] } }] }); + assert.equal(maxTokens.text, undefined); + assert.equal(maxTokens.reason, "MAX_TOKENS"); +}); + +test("extract anthropic: joins text blocks, skips thinking blocks; none → undefined + stop_reason", () => { + const api = "anthropic-messages" as Api; + const ok = extractVisionText(api, { + content: [{ type: "thinking", thinking: "hmm" }, { type: "text", text: "a" }, { type: "text", text: "b" }], + stop_reason: "end_turn", + }); + assert.equal(ok.text, "ab"); + const none = extractVisionText(api, { content: [{ type: "thinking", thinking: "hmm" }], stop_reason: "max_tokens" }); + assert.equal(none.text, undefined); + assert.equal(none.reason, "max_tokens"); +}); + +test("extract: non-object body never throws", () => { + for (const api of SUPPORTED_VISION_APIS) { + assert.equal(extractVisionText(api as Api, null).text, undefined); + assert.equal(extractVisionText(api as Api, "oops").text, undefined); + } +});