feat(delegate): dispatch the vision call on model.api (openai-responses, google-generative-ai, anthropic-messages)

README.md

@@ -93,6 +93,26 @@ model should have `"input": ["text", "image"]`. Other `/vision` subcommands:
 Config is stored at `~/.pi/agent/vision.json` (not `vision-tool.json`, so it
 doesn't collide with the community package during transition).
 
+### Supported provider APIs
+
+The vision call speaks the wire format of the model's `api` in `models.json`:
+
+| `api` | Request | Auth header |
+|---|---|---|
+| `openai-completions` | `POST {baseUrl}/chat/completions` (`image_url` data URL) | `Authorization: Bearer` |
+| `openai-responses` | `POST {baseUrl}/responses` (`input_image` part; system prompt as `instructions`) — `baseUrl` carries the version, e.g. `…/v1` | `Authorization: Bearer` |
+| `google-generative-ai` | `POST {baseUrl}/models/{id}:generateContent` (`inlineData`) — `baseUrl` already carries the version, e.g. `…/v1beta` | `x-goog-api-key` |
+| `anthropic-messages` | `POST {baseUrl}/v1/messages` (base64 `image` block) — `baseUrl` has no `/v1` | `x-api-key` + `anthropic-version` |
+
+`reasoning-effort` is forwarded as `reasoning_effort` (chat/completions), `reasoning.effort`
+(Responses), `thinkingConfig.thinkingLevel`
+(Gemini, honouring the model's `thinkingLevelMap`) or an extended-thinking budget (Anthropic).
+`off` sends nothing. `temperature` is never sent to Anthropic or Responses (newer Claude and reasoning models reject it).
+
+Any other `api` (e.g. `google-vertex`, `bedrock-converse-stream`) is rejected
+up front with an `unsupported_api` error naming the supported ones, instead of an opaque 404.
+`/vision model` only lists models on a supported API.
+
 ## Resilience (v0.2.0)
 
 DELEGATE mode (text-only primary) is resilient + cheap:
@@ -292,7 +312,8 @@ cache miss. Every local-only event is audited with `local_only: true`
 ### Workflow-fit defaults (auto-detect)
 
 On a fresh install (or after `/vision clear`), the extension auto-detects the
-vision model at session start from `~/.pi/agent/models.json` — aligned to the
+vision model (only models on a [supported API](#supported-provider-apis)) at
+session start from `~/.pi/agent/models.json` — aligned to the
 [AGENTS.md LLM-backend policy](https://github.com/earendil-works) of "Ollama
 Cloud primary + frontier escalation":
 

extensions/vision.ts

@@ -58,6 +58,7 @@ import { loadImage } from "../lib/image.ts";
 import { mapWithConcurrency } from "../lib/batch.ts";
 import { buildBatchToolResult } from "../lib/marker.ts";
 import { autoDetectDefaults } from "../lib/defaults.ts";
+import { isSupportedVisionApi, unsupportedApiMessage } from "../lib/wire.ts";
 import { clearAuditLog, countAuditLog, resolveAuditPath, tailAuditLog } from "../lib/audit.ts";
 
 /** Current config. Loaded on session_start, mutated by /vision, saved to disk. */
@@ -185,19 +186,33 @@ function resync(pi: ExtensionAPI, ctx: ExtensionCommandContext): void {
   syncToolAvailability(pi, ctx.model, { enabled: config.enabled });
 }
 
+/** Warn when the configured vision model is registered but uses an API the
+ *  vision call cannot speak (it would only fail later, on the first image). */
+function warnIfUnsupportedApi(ctx: ExtensionContext): void {
+  if (!config.provider || !config.model) return;
+  const model = ctx.modelRegistry.find(config.provider, config.model);
+  if (model && !isSupportedVisionApi(model.api)) {
+    ctx.ui.notify(unsupportedApiMessage(model), "warning");
+  }
+}
+
 /** Apply a setting edit, persist, re-sync visibility if needed, and rebuild
  *  the cache when cache-shape fields change. */
 function applyAndSave(id: string, value: string, pi: ExtensionAPI, ctx: ExtensionCommandContext): void {
   config = applySettingChange(config, id, value);
   saveConfig(config, getAgentDir());
   setSharedState(config, cache);
   if (id === "enabled" || id === "model") resync(pi, ctx);
+  if (id === "model") warnIfUnsupportedApi(ctx);
   if (id === "cachePersist" || id === "cacheMaxEntries") rebuildCache();
 }
 
-/** Vision-capable authed models from the registry (input includes "image"). */
+/** Vision-capable authed models from the registry (input includes "image") on
+ *  an API the vision call can speak — others would 404 on the first image. */
 function visionCapableModels(ctx: ExtensionContext): Model<Api>[] {
-  return ctx.modelRegistry.getAvailable().filter((m) => m.input.includes("image"));
+  return ctx.modelRegistry
+    .getAvailable()
+    .filter((m) => m.input.includes("image") && isSupportedVisionApi(m.api));
 }
 
 /** Open pi's native select picker over vision-capable models. Sets provider +
@@ -853,6 +868,7 @@ export default function visionExtension(pi: ExtensionAPI): void {
           saveConfig(config, agentDir);
           resync(pi, ctx);
           ctx.ui.notify(`Vision model set to ${value}.`, "info");
+          warnIfUnsupportedApi(ctx);
           return;
         }
         case "max-dim": {

lib/defaults.ts

@@ -20,6 +20,7 @@
  * (auto-detect only fires when both provider + model are unset).
  */
 import type { Api, Model } from "@earendil-works/pi-ai";
+import { isSupportedVisionApi } from "./wire.ts";
 
 export interface DetectedDefaults {
   provider: string | undefined;
@@ -36,7 +37,9 @@ export const PREFERRED_PRIMARY_PROVIDER = "Ollama";
  * via /vision fallback if they want frontier escalation).
  *
  * Algorithm:
- * 1. Filter to vision-capable models (`input` includes "image").
+ * 1. Filter to vision-capable models (`input` includes "image") whose `api` the
+ *    vision call can speak (`isSupportedVisionApi`) — otherwise the pick would
+ *    404 on the first describe_image.
  * 2. If none → undefined (no-op; the existing not-configured error guides
  *    the user).
  * 3. Primary: prefer the `Ollama` provider; among those, first by sorted
@@ -46,7 +49,7 @@ export const PREFERRED_PRIMARY_PROVIDER = "Ollama";
  * Pure + deterministic (same input in any order → same output).
  */
 export function autoDetectDefaults(models: Model<Api>[]): DetectedDefaults {
-  const visionModels = models.filter((m) => m.input?.includes("image"));
+  const visionModels = models.filter((m) => m.input?.includes("image") && isSupportedVisionApi(m.api));
   if (visionModels.length === 0) {
     return { provider: undefined, model: undefined };
   }

lib/delegate.ts

@@ -14,14 +14,16 @@
  *    secondary vision model on failure.
  * 4. Abort-aware — `ctx.signal` stops retry + skips fallback.
  *
- * Clean-room: the OpenAI-compatible `/chat/completions` request shape with a
- * base64 data-URL image is standard API usage, not copied from pi-vision-tool.
+ * Request shapes (OpenAI chat/completions, Gemini generateContent, Anthropic
+ * messages) live in `lib/wire.ts`; the call dispatches on `model.api`. They use
+ * standard base64 image payloads, not copied from pi-vision-tool.
  */
 import type { Model, Api, ProviderHeaders } from "@earendil-works/pi-ai";
 import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
 import { getAgentDir } from "@earendil-works/pi-coding-agent";
 import { isConfiguredForDelegation, loadConfig, type ReasoningLevel, type VisionConfig } from "./config.ts";
 import { loadImage, type LoadedImage } from "./image.ts";
+import { buildVisionRequest, extractVisionText, isSupportedVisionApi, unsupportedApiMessage } from "./wire.ts";
 import { cacheKey, type VisionCache } from "./cache.ts";
 import { AbortError, classifyError, withRetry } from "./resilience.ts";
 import { appendAuditEntry, resolveAuditPath, truncateImagePathForLog, type AuditEntry } from "./audit.ts";
@@ -59,20 +61,11 @@ export interface DelegateFailure {
 
 export type DelegateResult = DelegateSuccess | DelegateFailure;
 
-/** Build provider-specific reasoning params for the request body, if any. */
-function buildReasoningParams(
-  visionModel: Model<Api>,
-  level: ReasoningLevel,
-): Record<string, unknown> | undefined {
-  if (!visionModel.reasoning || level === "off") return undefined;
-  return { reasoning_effort: level };
-}
-
 /**
- * Call the vision model's OpenAI-compat chat/completions endpoint with the
- * image as a data URL + the user's prompt (and an optional system prompt).
- * Returns the model's text response. Exported + fetch-based so tests can mock
- * `globalThis.fetch`.
+ * Call the vision model with the image as base64 + the user's prompt (and an
+ * optional system prompt), in the wire format of `visionModel.api` (see
+ * `lib/wire.ts`). Returns the model's text response. Exported + fetch-based so
+ * tests can mock `globalThis.fetch`.
  */
 export async function callVisionModel(
   visionModel: Model<Api>,
@@ -84,44 +77,20 @@ export async function callVisionModel(
   reasoning: ReasoningLevel,
   systemPrompt?: string,
 ): Promise<string> {
-  const baseUrl = visionModel.baseUrl.replace(/\/+$/, "");
-  const messages: unknown[] = [];
-  if (systemPrompt && systemPrompt.length > 0) {
-    messages.push({ role: "system", content: systemPrompt });
-  }
-  messages.push({
-    role: "user",
-    content: [
-      {
-        type: "image_url",
-        image_url: { url: `data:${image.mimeType};base64,${image.data}` },
-      },
-      { type: "text", text: prompt },
-    ],
+  const request = buildVisionRequest({
+    model: visionModel,
+    apiKey,
+    providerHeaders,
+    image,
+    prompt,
+    reasoning,
+    systemPrompt,
   });
-  const body: Record<string, unknown> = {
-    model: visionModel.id,
-    messages,
-    max_tokens: 4096,
-    temperature: 0,
-  };
-  const reasoningParams = buildReasoningParams(visionModel, reasoning);
-  if (reasoningParams) Object.assign(body, reasoningParams);
-
-  const headers: Record<string, string> = { "Content-Type": "application/json" };
-  if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
-  // pi-ai 0.84 widened ProviderHeaders to Record<string, string | null> — null means
-  // "unset"; a null value must never reach fetch's HeadersInit (typed Record<string, string>).
-  if (providerHeaders) {
-    for (const [key, value] of Object.entries(providerHeaders)) {
-      if (value !== null) headers[key] = value;
-    }
-  }
 
-  const response = await fetch(`${baseUrl}/chat/completions`, {
+  const response = await fetch(request.url, {
     method: "POST",
-    headers,
-    body: JSON.stringify(body),
+    headers: request.headers,
+    body: JSON.stringify(request.body),
     signal,
   });
 
@@ -132,13 +101,9 @@ export async function callVisionModel(
     );
   }
 
-  const json = (await response.json()) as {
-    choices?: Array<{ message?: { content?: string; reasoning_content?: string } }>;
-  };
-  const msg = json.choices?.[0]?.message;
-  const text = msg?.content || msg?.reasoning_content;
+  const { text, reason } = extractVisionText(visionModel.api, await response.json());
   if (!text) {
-    throw new Error("Vision model returned no content in the response");
+    throw new Error(`Vision model returned no content in the response${reason ? ` (${reason})` : ""}`);
   }
   return text;
 }
@@ -256,6 +221,12 @@ export async function delegateToVisionModel(
     };
   }
 
+  // Fail before reading the image or touching the network: a model on an API
+  // the call cannot speak would only come back as an opaque 404.
+  if (!isSupportedVisionApi(visionModel.api)) {
+    return { ok: false, error: { code: "unsupported_api", message: unsupportedApiMessage(visionModel) } };
+  }
+
   const auth = await ctx.modelRegistry.getApiKeyAndHeaders(visionModel);
   if (!auth.ok) {
     return {
@@ -425,6 +396,13 @@ async function runFallback(
       details: { primaryError: primaryErrorTag(primaryErr), fallbackModel: fallbackId },
     };
   }
+  if (!isSupportedVisionApi(fbModel.api)) {
+    return {
+      ok: false,
+      error: { code: "unsupported_api", message: unsupportedApiMessage(fbModel) },
+      details: { primaryError: primaryErrorTag(primaryErr), fallbackModel: fallbackId },
+    };
+  }
   const fbAuth = await ctx.modelRegistry.getApiKeyAndHeaders(fbModel);
   if (!fbAuth.ok) {
     return {