From 220b4f728a15bf04e60e8be1f84b1e6225662337 Mon Sep 17 00:00:00 2001 From: Yuxin Qiao <104957188+Yuxin-Qiao@users.noreply.github.com> Date: Tue, 4 Aug 2026 04:05:56 +0800 Subject: [PATCH] fix(providers): activate vision sidecar for NVIDIA NIM text-only models Add noVisionModels to the nvidia registry entry (issue #956). The NIM /v1/models catalog carries no input-modality metadata, so the registry is the only source of truth: without the list, the vision sidecar never runs and text-only NIM models either 400 on raw image parts upstream or are advertised text-only so the app blocks attachments client-side. Covers the text-only chat/code families from the live catalog (2026-08-04) plus the documented kimi family. Vision-capable NIM models (llama-3.2-*- vision-instruct, phi-3-vision, neva-22b, vila, nemotron-*-vl, ...) and non-chat endpoints stay out. Regression tests: registry list contents/exclusions, bare-config router inheritance, planVisionSidecar activation, and catalog inputModalities advertisement. --- src/providers/registry.ts | 76 +++++++++++++++++++++++++ tests/nvidia-nim-hardening.test.ts | 91 ++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+) diff --git a/src/providers/registry.ts b/src/providers/registry.ts index 91d3ad07d..931171a2e 100644 --- a/src/providers/registry.ts +++ b/src/providers/registry.ts @@ -520,6 +520,79 @@ const NVIDIA_NIM_KIMI_MODELS = [ ...NVIDIA_NIM_KIMI_THINKING_MODELS, "moonshotai/kimi-k2-instruct", "moonshotai/kimi-k2-instruct-0905", ]; +// 260804 NVIDIA NIM text-only chat/code families (issue #956): the NIM /v1/models catalog +// carries no input-modality metadata, so the registry is the only source of truth. Snapshot +// from the live catalog (2026-08-04) plus the documented kimi family above. Vision-capable +// NIM models (llama-3.2-*-vision-instruct, phi-3-vision, kosmos-2, fuyu-8b, deplot, neva-22b, +// vila, nemotron-*-vl, nemotron-nano-12b-v2-vl, nemotron-3-nano-omni) and non-chat endpoints +// (embeddings, guards, translators, detectors, image/video) stay out. New text-only ids must +// be appended individually. +const NVIDIA_NIM_NO_VISION_MODELS = [ + "01-ai/yi-large", + "ai21labs/jamba-1.5-large-instruct", + "aisingapore/sea-lion-7b-instruct", + "bigcode/starcoder2-15b", + "databricks/dbrx-instruct", + "deepseek-ai/deepseek-coder-6.7b-instruct", + "deepseek-ai/deepseek-v4-flash", + "deepseek-ai/deepseek-v4-pro", + "google/codegemma-1.1-7b", + "google/codegemma-7b", + "google/gemma-2b", + "google/recurrentgemma-2b", + "ibm/granite-3.0-3b-a800m-instruct", + "ibm/granite-3.0-8b-instruct", + "ibm/granite-34b-code-instruct", + "ibm/granite-8b-code-instruct", + "meta/codellama-70b", + "meta/llama-3.1-70b-instruct", + "meta/llama-3.1-8b-instruct", + "meta/llama-3.2-1b-instruct", + "meta/llama-3.2-3b-instruct", + "meta/llama-3.3-70b-instruct", + "meta/llama2-70b", + "microsoft/phi-3.5-moe-instruct", + "minimaxai/minimax-m3", + "mistralai/codestral-22b-instruct-v0.1", + "mistralai/mistral-7b-instruct-v0.3", + "mistralai/mistral-large", + "mistralai/mistral-large-2-instruct", + "mistralai/mistral-medium-3.5-128b", + "mistralai/mistral-nemotron", + "mistralai/mixtral-8x22b-v0.1", + "moonshotai/kimi-k2.6", + "moonshotai/kimi-k2.5", + "moonshotai/kimi-k2-thinking", + "moonshotai/kimi-k2-instruct", + "moonshotai/kimi-k2-instruct-0905", + "nv-mistralai/mistral-nemo-12b-instruct", + "nvidia/llama-3.1-nemotron-51b-instruct", + "nvidia/llama-3.1-nemotron-70b-instruct", + "nvidia/llama-3.1-nemotron-nano-8b-v1", + "nvidia/llama-3.1-nemotron-ultra-253b-v1", + "nvidia/llama-3.3-nemotron-super-49b-v1", + "nvidia/llama-3.3-nemotron-super-49b-v1.5", + "nvidia/llama3-chatqa-1.5-70b", + "nvidia/mistral-nemo-minitron-8b-8k-instruct", + "nvidia/nemotron-3-nano-30b-a3b", + "nvidia/nemotron-3-super-120b-a12b", + "nvidia/nemotron-3-ultra-550b-a55b", + "nvidia/nemotron-4-340b-instruct", + "nvidia/nemotron-mini-4b-instruct", + "nvidia/nemotron-nano-3-30b-a3b", + "nvidia/nvidia-nemotron-nano-9b-v2", + "openai/gpt-oss-120b", + "openai/gpt-oss-20b", + "poolside/laguna-xs-2.1", + "stepfun-ai/step-3.7-flash", + "thinkingmachines/inkling", + "writer/palmyra-creative-122b", + "writer/palmyra-fin-70b-32k", + "writer/palmyra-med-70b", + "writer/palmyra-med-70b-32k", + "z-ai/glm-5.2", + "zyphra/zamba2-7b-instruct", +]; const KIMI_CODING_MODEL_CONTEXT_WINDOWS: Record = Object.fromEntries( KIMI_CODING_MODELS.map(id => [id, id === "k3[1m]" ? KIMI_K3_1M_CONTEXT_WINDOW : KIMI_K3_STANDARD_CONTEXT_WINDOW]), ); @@ -1230,11 +1303,14 @@ export const PROVIDER_REGISTRY: readonly ProviderRegistryEntry[] = [ // family is live-discovered with no capability metadata, so Codex would otherwise send // reasoning_effort=medium. Exact-id lists per modelInList semantics; gpt-oss on NIM keeps // its working reasoning_effort. Future kimi ids must be appended individually. + // - NIM exposes no input modalities, so text-only chat/code families are listed in + // noVisionModels (issue #956) to activate the vision sidecar and advertise image input. { id: "nvidia", label: "NVIDIA NIM", baseUrl: "https://integrate.api.nvidia.com/v1", adapter: "openai-chat", authKind: "key", dashboardUrl: "https://build.nvidia.com", // Free pricing, but an API key is still required (free key from build.nvidia.com). freeTier: true, parallelToolCalls: false, + noVisionModels: NVIDIA_NIM_NO_VISION_MODELS, noReasoningModels: NVIDIA_NIM_KIMI_MODELS, modelReasoningEfforts: Object.fromEntries(NVIDIA_NIM_KIMI_MODELS.map(id => [id, []])), preserveReasoningContentModels: NVIDIA_NIM_KIMI_THINKING_MODELS, diff --git a/tests/nvidia-nim-hardening.test.ts b/tests/nvidia-nim-hardening.test.ts index c757617ae..a67d4dbfa 100644 --- a/tests/nvidia-nim-hardening.test.ts +++ b/tests/nvidia-nim-hardening.test.ts @@ -1,11 +1,15 @@ // 260715 issue #126: NVIDIA NIM hardening — parallel_tool_calls opt-out, kimi // reasoning_effort suppression, and openai-chat formatErrorBody detail surfacing. +// 260804 issue #956: NIM text-only families get noVisionModels so the vision sidecar runs. // Plan/evidence: devlog/_plan/260715_issue126_nim_kimi. import { describe, expect, test } from "bun:test"; import { createOpenAIChatAdapter, formatOpenAIChatErrorBody } from "../src/adapters/openai-chat"; import { applyProviderConfigHints, normalizeRoutedCatalogEntry } from "../src/codex/catalog"; +import { PROVIDER_REGISTRY } from "../src/providers/registry"; +import { parseRequest } from "../src/responses/parser"; import { routeModel } from "../src/router"; import type { OcxConfig, OcxParsedRequest, OcxTool } from "../src/types"; +import { planVisionSidecar } from "../src/vision"; const tools: OcxTool[] = [{ name: "shell", description: "run", parameters: { type: "object" } }]; @@ -86,6 +90,93 @@ describe("nvidia NIM registry hardening (issue #126)", () => { const entry = normalizeRoutedCatalogEntry({ slug: "nvidia/moonshotai/kimi-k2.6" }, hinted.parallelToolCalls); expect(entry.supports_parallel_tool_calls).toBe(false); }); + + test("registry nvidia entry declares text-only families and excludes vision-capable models", () => { + const nvidia = PROVIDER_REGISTRY.find(entry => entry.id === "nvidia")!; + for (const id of [ + "deepseek-ai/deepseek-v4-flash", + "deepseek-ai/deepseek-v4-pro", + "z-ai/glm-5.2", + "minimaxai/minimax-m3", + "moonshotai/kimi-k2.6", + "nvidia/nemotron-3-ultra-550b-a55b", + "nvidia/nemotron-3-super-120b-a12b", + "openai/gpt-oss-120b", + ]) { + expect(nvidia.noVisionModels).toContain(id); + } + for (const id of [ + "meta/llama-3.2-11b-vision-instruct", + "meta/llama-3.2-90b-vision-instruct", + "microsoft/phi-3-vision-128k-instruct", + "adept/fuyu-8b", + "nvidia/llama-3.1-nemotron-nano-vl-8b-v1", + "nvidia/nemotron-nano-12b-v2-vl", + "nvidia/neva-22b", + "nvidia/vila", + ]) { + expect(nvidia.noVisionModels).not.toContain(id); + } + }); + + test("bare persisted nvidia config inherits noVisionModels from the registry", () => { + const route = routeModel(nvidiaConfig(), "nvidia/deepseek-ai/deepseek-v4-flash"); + expect(route.provider.noVisionModels).toContain("deepseek-ai/deepseek-v4-flash"); + expect(route.modelId).toBe("deepseek-ai/deepseek-v4-flash"); + }); + + test("vision sidecar plans for text-only NIM models but not vision-capable ones", () => { + const config = nvidiaConfig(); + const openAiSidecar = { + providerName: "openai" as const, + provider: { adapter: "openai-responses", baseUrl: "https://chatgpt.test/v1", authMode: "forward" as const }, + accountMode: "direct" as const, + authContext: { kind: "main" as const, accountId: null }, + headers: new Headers({ authorization: "Bearer chatgpt" }), + }; + const withImage = parseRequest({ + model: "nvidia/deepseek-ai/deepseek-v4-flash", + input: [{ + type: "message", + role: "user", + content: [ + { type: "input_text", text: "What is in this screenshot?" }, + { type: "input_image", image_url: "data:image/png;base64,aGVsbG8=" }, + ], + }], + }); + const noImage = parseRequest({ + model: "nvidia/deepseek-ai/deepseek-v4-flash", + input: [{ type: "message", role: "user", content: [{ type: "input_text", text: "hi" }] }], + }); + + const textRoute = routeModel(config, "nvidia/deepseek-ai/deepseek-v4-flash"); + expect(planVisionSidecar(config, textRoute.provider, textRoute.modelId, withImage, openAiSidecar)) + .toMatchObject({ backend: "openai" }); + expect(planVisionSidecar(config, textRoute.provider, textRoute.modelId, noImage, openAiSidecar)) + .toBeUndefined(); + + const visionRoute = routeModel(config, "nvidia/meta/llama-3.2-11b-vision-instruct"); + expect(planVisionSidecar(config, visionRoute.provider, visionRoute.modelId, withImage, openAiSidecar)) + .toBeUndefined(); + }); + + test("catalog advertises image input for text-only NIM models, not for vision-capable ones", () => { + const config = nvidiaConfig(); + const textRoute = routeModel(config, "nvidia/deepseek-ai/deepseek-v4-flash"); + const hinted = applyProviderConfigHints("nvidia", textRoute.provider, { + id: "deepseek-ai/deepseek-v4-flash", + provider: "nvidia", + }); + expect(hinted.inputModalities).toContain("image"); + + const visionRoute = routeModel(config, "nvidia/meta/llama-3.2-11b-vision-instruct"); + const visionHinted = applyProviderConfigHints("nvidia", visionRoute.provider, { + id: "meta/llama-3.2-11b-vision-instruct", + provider: "nvidia", + }); + expect(visionHinted.inputModalities).toBeUndefined(); + }); }); describe("formatOpenAIChatErrorBody (web-search sidecar detail surfacing)", () => {