From e90b7d5a7231aa6a764c3754d59eb46d4382f421 Mon Sep 17 00:00:00 2001 From: Melissa Barca Date: Thu, 27 Aug 2026 11:36:59 -0400 Subject: [PATCH 1/5] Add model policy helpers Expose pure ai-config helpers for resolving model IDs, checking discovery policy, and filtering discovered model rows through the existing model resolver pipeline. --- packages/ai-config/package.json | 5 + .../src/__tests__/resolve-models.test.ts | 104 ++++++++++++++++++ packages/ai-config/src/index.ts | 8 +- packages/ai-config/src/model-policy.ts | 12 ++ packages/ai-config/src/resolve-models.ts | 102 +++++++++++++++-- 5 files changed, 222 insertions(+), 9 deletions(-) create mode 100644 packages/ai-config/src/model-policy.ts diff --git a/packages/ai-config/package.json b/packages/ai-config/package.json index 93e9758..8c58684 100644 --- a/packages/ai-config/package.json +++ b/packages/ai-config/package.json @@ -19,6 +19,11 @@ "import": "./dist/edit-jsonc.js", "require": "./dist/edit-jsonc.js" }, + "./model-policy": { + "types": "./dist/model-policy.d.ts", + "import": "./dist/model-policy.js", + "require": "./dist/model-policy.js" + }, "./node": { "types": "./dist/node/index.d.ts", "import": "./dist/node/index.js", diff --git a/packages/ai-config/src/__tests__/resolve-models.test.ts b/packages/ai-config/src/__tests__/resolve-models.test.ts index 0419046..89840f3 100644 --- a/packages/ai-config/src/__tests__/resolve-models.test.ts +++ b/packages/ai-config/src/__tests__/resolve-models.test.ts @@ -4,6 +4,12 @@ import { describe, it, expect } from "vitest"; +import { + filterDiscoveredModelsByPolicy, + isModelDiscoveryEnabled, + isModelIdAllowedByPolicy, + resolveModelIds, +} from "../model-policy.js"; import { resolveModels } from "../resolve-models.js"; import type { ModelInfoLike, ModelsBlock, ResolvedConnection } from "../types.js"; @@ -399,3 +405,101 @@ describe("resolveModels", () => { expect(result[1].id).toBe("custom-1"); }); }); + +describe("model policy helpers", () => { + it("isModelDiscoveryEnabled treats discovery off as the only disabled state", () => { + expect([ + isModelDiscoveryEnabled(undefined), + isModelDiscoveryEnabled({}), + isModelDiscoveryEnabled({ discovery: "auto" }), + isModelDiscoveryEnabled({ discovery: "off" }), + ]).toEqual([true, true, true, false]); + }); + + it("resolveModelIds applies the complete resolver pipeline", () => { + const block: ModelsBlock = { + custom: [ + { + id: "custom-1", + name: "Custom 1", + maxContextLength: 50000, + supportsTools: true, + supportsImages: false, + supportsToolResultImages: false, + supportsWebSearch: false, + }, + ], + allow: ["model-a", "model-b", "custom-1"], + deny: ["model-b"], + }; + + expect(resolveModelIds(block, ["model-a", "model-b", "model-c"])).toEqual([ + "model-a", + "custom-1", + ]); + }); + + it("filterDiscoveredModelsByPolicy preserves host model rows while applying allow and deny", () => { + const block: ModelsBlock = { + allow: ["model-a", "model-b"], + deny: ["model-b"], + }; + const models = [ + { id: "model-a", label: "A" }, + { id: "model-b", label: "B" }, + { id: "model-c", label: "C" }, + ]; + + expect(filterDiscoveredModelsByPolicy(block, models, (model) => model.id)).toEqual([ + { id: "model-a", label: "A" }, + ]); + }); + + it("filterDiscoveredModelsByPolicy does not let custom models authorize discovered rows", () => { + const block: ModelsBlock = { + discovery: "off", + custom: [ + { + id: "model-a", + name: "Custom Model A", + maxContextLength: 50000, + supportsTools: true, + supportsImages: false, + supportsToolResultImages: false, + supportsWebSearch: false, + }, + ], + }; + const models = [{ id: "model-a", label: "Discovered A" }]; + + expect({ + discoveredModels: filterDiscoveredModelsByPolicy(block, models, (model) => model.id), + resolvedIds: resolveModelIds(block, ["model-a"]), + modelAAllowed: isModelIdAllowedByPolicy(block, "model-a"), + }).toEqual({ + discoveredModels: [], + resolvedIds: ["model-a"], + modelAAllowed: true, + }); + }); + + it("isModelIdAllowedByPolicy checks the requested id, not unrelated custom declarations", () => { + const block: ModelsBlock = { + custom: [ + { + id: "custom-1", + name: "Custom 1", + maxContextLength: 50000, + supportsTools: true, + supportsImages: false, + supportsToolResultImages: false, + supportsWebSearch: false, + }, + ], + allow: ["custom-1"], + }; + + expect(isModelIdAllowedByPolicy(block, "model-a")).toBe(false); + expect(isModelIdAllowedByPolicy(block, "custom-1")).toBe(true); + }); +}); diff --git a/packages/ai-config/src/index.ts b/packages/ai-config/src/index.ts index be39d6b..de0c560 100644 --- a/packages/ai-config/src/index.ts +++ b/packages/ai-config/src/index.ts @@ -153,7 +153,13 @@ export { } from "./defaults.js"; // --- Resolution helpers (public) ------------------------------------------- -export { resolveModels } from "./resolve-models.js"; +export { + filterDiscoveredModelsByPolicy, + isModelDiscoveryEnabled, + isModelIdAllowedByPolicy, + resolveModelIds, + resolveModels, +} from "./resolve-models.js"; // --- Bare-host base URL correction ------------------------------------------ export { diff --git a/packages/ai-config/src/model-policy.ts b/packages/ai-config/src/model-policy.ts new file mode 100644 index 0000000..1a7c83b --- /dev/null +++ b/packages/ai-config/src/model-policy.ts @@ -0,0 +1,12 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (C) 2026 Posit Software, PBC. All rights reserved. + *--------------------------------------------------------------------------------------------*/ + +export { + filterDiscoveredModelsByPolicy, + isModelDiscoveryEnabled, + isModelIdAllowedByPolicy, + resolveModelIds, + resolveModels, +} from "./resolve-models.js"; +export type { ModelInfoLike, ModelsBlock, ResolvedModelInfo } from "./types.js"; diff --git a/packages/ai-config/src/resolve-models.ts b/packages/ai-config/src/resolve-models.ts index c7cb18b..b14d177 100644 --- a/packages/ai-config/src/resolve-models.ts +++ b/packages/ai-config/src/resolve-models.ts @@ -71,10 +71,13 @@ interface UserRouting { /** No user-configured routing — discovered models start with this. */ const NO_USER_ROUTING: UserRouting = { protocol: undefined, baseUrl: undefined }; -/** A model paired with its user-configured routing (pipeline-internal). */ +type PipelineEntrySource = "discovered" | "custom"; + +/** A model paired with its source and user-configured routing (pipeline-internal). */ interface PipelineEntry { model: ModelInfoLike; userRouting: UserRouting; + source: PipelineEntrySource; } /** @@ -93,17 +96,29 @@ export function resolveModels( discovered: readonly ModelInfoLike[], providerConnection?: ResolvedConnection, ): ResolvedModelInfo[] { + return resolveModelEntries(modelsBlock, discovered).map((e) => + attachRouting(e.model, e.userRouting, providerConnection), + ); +} + +function resolveModelEntries( + modelsBlock: ModelsBlock | undefined, + discovered: readonly ModelInfoLike[], +): PipelineEntry[] { if (!modelsBlock) { // No models block — pass through discovered models with routing resolved. // Discovered protocol is built-in inference only (lowest precedence). - return discovered.map((m) => attachRouting(m, NO_USER_ROUTING, providerConnection)); + return discovered.map((m) => ({ + model: m, + userRouting: NO_USER_ROUTING, + source: "discovered", + })); } // 1. Discovery gate - const base: PipelineEntry[] = - modelsBlock.discovery === "off" - ? [] - : discovered.map((m) => ({ model: m, userRouting: NO_USER_ROUTING })); + const base: PipelineEntry[] = isModelDiscoveryEnabled(modelsBlock) + ? discovered.map((m) => ({ model: m, userRouting: NO_USER_ROUTING, source: "discovered" })) + : []; // 2. Add custom models (protocol/baseUrl are user-configured) const customs = modelsBlock.custom; @@ -112,10 +127,15 @@ export function resolveModels( base.push({ model: customModelToModelInfo(custom), userRouting: { protocol: custom.protocol, baseUrl: custom.baseUrl }, + source: "custom", }); } } + return applyModelPolicy(modelsBlock, base); +} + +function applyModelPolicy(modelsBlock: ModelsBlock, base: PipelineEntry[]): PipelineEntry[] { // 3. Apply overrides (protocol/baseUrl from overrides are user-configured) const overrides = modelsBlock.overrides; if (overrides) { @@ -145,8 +165,61 @@ export function resolveModels( result = result.filter((e) => !denySet.has(e.model.id)); } - // 6. Resolve routing for each surviving model - return result.map((e) => attachRouting(e.model, e.userRouting, providerConnection)); + return result; +} + +/** + * Whether a provider should query runtime discovery for models. + */ +export function isModelDiscoveryEnabled(modelsBlock: ModelsBlock | undefined): boolean { + return modelsBlock?.discovery !== "off"; +} + +/** + * Resolve a provider's final model ids after discovery policy, custom models, + * overrides, allow, and deny are applied. + */ +export function resolveModelIds( + modelsBlock: ModelsBlock | undefined, + discoveredModelIds: readonly string[], +): string[] { + return resolveModels( + modelsBlock, + discoveredModelIds.map((id) => modelIdToModelInfo(id)), + ).map((model) => model.id); +} + +/** + * Filters discovered model rows through the same model resolver that handles + * providers.json allow/deny semantics. Custom declarations do not authorize a + * caller-provided row with the same id; use resolveModels for the complete list. + */ +export function filterDiscoveredModelsByPolicy( + modelsBlock: ModelsBlock | undefined, + models: readonly T[], + getModelId: (model: T) => string, +): T[] { + if (!modelsBlock) { + return [...models]; + } + + const resolvedDiscoveredIds = new Set( + resolveModelEntries( + modelsBlock, + models.map((model) => modelIdToModelInfo(getModelId(model))), + ) + .filter((entry) => entry.source === "discovered") + .map((entry) => entry.model.id), + ); + return models.filter((model) => resolvedDiscoveredIds.has(getModelId(model))); +} + +/** Checks whether a raw provider model id survives complete model policy resolution. */ +export function isModelIdAllowedByPolicy( + modelsBlock: ModelsBlock | undefined, + modelId: string, +): boolean { + return resolveModelIds(modelsBlock, [modelId]).includes(modelId); } /** @@ -190,6 +263,18 @@ function attachRouting( }; } +function modelIdToModelInfo(id: string): ModelInfoLike { + return { + id, + name: id, + maxContextLength: 0, + supportsTools: false, + supportsImages: false, + supportsToolResultImages: false, + supportsWebSearch: false, + }; +} + /** Convert a custom model definition to a ModelInfoLike. */ function customModelToModelInfo(custom: CustomModel): ModelInfoLike { return { @@ -225,6 +310,7 @@ function applyOverrideEntry(entry: PipelineEntry, override: ModelOverride): Pipe return { model, + source: entry.source, userRouting: { // Override routing takes precedence over any prior user routing // (e.g. a custom model that also gets overridden). From f57e5bfaa7c5cd0285345734f5501acb56a6aed3 Mon Sep 17 00:00:00 2001 From: Melissa Barca Date: Thu, 27 Aug 2026 14:39:38 -0400 Subject: [PATCH 2/5] Remove ai-config model policy subpath Keep model policy helpers on the package root export instead of maintaining a duplicate public subpath. --- packages/ai-config/package.json | 5 ----- .../ai-config/src/__tests__/resolve-models.test.ts | 4 ++-- packages/ai-config/src/model-policy.ts | 12 ------------ 3 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 packages/ai-config/src/model-policy.ts diff --git a/packages/ai-config/package.json b/packages/ai-config/package.json index 8c58684..93e9758 100644 --- a/packages/ai-config/package.json +++ b/packages/ai-config/package.json @@ -19,11 +19,6 @@ "import": "./dist/edit-jsonc.js", "require": "./dist/edit-jsonc.js" }, - "./model-policy": { - "types": "./dist/model-policy.d.ts", - "import": "./dist/model-policy.js", - "require": "./dist/model-policy.js" - }, "./node": { "types": "./dist/node/index.d.ts", "import": "./dist/node/index.js", diff --git a/packages/ai-config/src/__tests__/resolve-models.test.ts b/packages/ai-config/src/__tests__/resolve-models.test.ts index 89840f3..c9cce54 100644 --- a/packages/ai-config/src/__tests__/resolve-models.test.ts +++ b/packages/ai-config/src/__tests__/resolve-models.test.ts @@ -9,8 +9,8 @@ import { isModelDiscoveryEnabled, isModelIdAllowedByPolicy, resolveModelIds, -} from "../model-policy.js"; -import { resolveModels } from "../resolve-models.js"; + resolveModels, +} from "../index.js"; import type { ModelInfoLike, ModelsBlock, ResolvedConnection } from "../types.js"; function makeModel(id: string, overrides?: Partial): ModelInfoLike { diff --git a/packages/ai-config/src/model-policy.ts b/packages/ai-config/src/model-policy.ts deleted file mode 100644 index 1a7c83b..0000000 --- a/packages/ai-config/src/model-policy.ts +++ /dev/null @@ -1,12 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (C) 2026 Posit Software, PBC. All rights reserved. - *--------------------------------------------------------------------------------------------*/ - -export { - filterDiscoveredModelsByPolicy, - isModelDiscoveryEnabled, - isModelIdAllowedByPolicy, - resolveModelIds, - resolveModels, -} from "./resolve-models.js"; -export type { ModelInfoLike, ModelsBlock, ResolvedModelInfo } from "./types.js"; From ef31956e982bdb309902c8212acd8299ffc20037 Mon Sep 17 00:00:00 2001 From: Melissa Barca Date: Thu, 27 Aug 2026 14:58:48 -0400 Subject: [PATCH 3/5] Clarify model ID resolution docs Document that discovery policy drops discovered IDs only and custom models still participate in model ID resolution. --- packages/ai-config/src/resolve-models.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/ai-config/src/resolve-models.ts b/packages/ai-config/src/resolve-models.ts index b14d177..4b9fe26 100644 --- a/packages/ai-config/src/resolve-models.ts +++ b/packages/ai-config/src/resolve-models.ts @@ -176,8 +176,9 @@ export function isModelDiscoveryEnabled(modelsBlock: ModelsBlock | undefined): b } /** - * Resolve a provider's final model ids after discovery policy, custom models, - * overrides, allow, and deny are applied. + * Resolve a provider's final model ids from discovered ids plus custom models. + * `discovery: "off"` drops discovered ids only; custom models still participate + * before overrides, allow, and deny. */ export function resolveModelIds( modelsBlock: ModelsBlock | undefined, From 9fa430b9a74423edaeed04e97f0235f92ba8d25c Mon Sep 17 00:00:00 2001 From: Melissa Barca Date: Thu, 27 Aug 2026 15:30:47 -0400 Subject: [PATCH 4/5] Allow custom models to replace discovered metadata Treat same-id custom model declarations as replacements for discovered model entries, preserving discovered optional metadata that the custom declaration does not set. Remove the discovered-row filter helper that encoded the opposite assumption. --- .../src/__tests__/resolve-models.test.ts | 83 +++++++++---------- packages/ai-config/src/index.ts | 1 - packages/ai-config/src/resolve-models.ts | 83 ++++++++++--------- packages/ai-config/src/schema.ts | 8 +- 4 files changed, 85 insertions(+), 90 deletions(-) diff --git a/packages/ai-config/src/__tests__/resolve-models.test.ts b/packages/ai-config/src/__tests__/resolve-models.test.ts index c9cce54..05ba17c 100644 --- a/packages/ai-config/src/__tests__/resolve-models.test.ts +++ b/packages/ai-config/src/__tests__/resolve-models.test.ts @@ -5,7 +5,6 @@ import { describe, it, expect } from "vitest"; import { - filterDiscoveredModelsByPolicy, isModelDiscoveryEnabled, isModelIdAllowedByPolicy, resolveModelIds, @@ -89,6 +88,44 @@ describe("resolveModels", () => { expect(result[3].id).toBe("extra"); }); + it("custom models replace discovered models with the same id", () => { + const block: ModelsBlock = { + custom: [ + { + id: "model-a", + name: "Custom Model A", + maxContextLength: 50000, + supportsTools: false, + supportsImages: true, + supportsToolResultImages: false, + supportsWebSearch: true, + }, + ], + }; + const result = resolveModels(block, [ + makeModel("model-a", { + name: "Discovered Model A", + family: "discovered-family", + maxInputTokens: 25000, + protocol: "openai", + }), + makeModel("model-b"), + ]); + + expect(result.map((m) => m.id)).toEqual(["model-a", "model-b"]); + expect(result[0]).toMatchObject({ + id: "model-a", + name: "Custom Model A", + family: "discovered-family", + maxContextLength: 50000, + maxInputTokens: 25000, + supportsTools: false, + supportsImages: true, + supportsWebSearch: true, + resolvedProtocol: "openai-chat", + }); + }); + // --- Overrides --- it("applies overrides to matching models", () => { @@ -439,50 +476,6 @@ describe("model policy helpers", () => { ]); }); - it("filterDiscoveredModelsByPolicy preserves host model rows while applying allow and deny", () => { - const block: ModelsBlock = { - allow: ["model-a", "model-b"], - deny: ["model-b"], - }; - const models = [ - { id: "model-a", label: "A" }, - { id: "model-b", label: "B" }, - { id: "model-c", label: "C" }, - ]; - - expect(filterDiscoveredModelsByPolicy(block, models, (model) => model.id)).toEqual([ - { id: "model-a", label: "A" }, - ]); - }); - - it("filterDiscoveredModelsByPolicy does not let custom models authorize discovered rows", () => { - const block: ModelsBlock = { - discovery: "off", - custom: [ - { - id: "model-a", - name: "Custom Model A", - maxContextLength: 50000, - supportsTools: true, - supportsImages: false, - supportsToolResultImages: false, - supportsWebSearch: false, - }, - ], - }; - const models = [{ id: "model-a", label: "Discovered A" }]; - - expect({ - discoveredModels: filterDiscoveredModelsByPolicy(block, models, (model) => model.id), - resolvedIds: resolveModelIds(block, ["model-a"]), - modelAAllowed: isModelIdAllowedByPolicy(block, "model-a"), - }).toEqual({ - discoveredModels: [], - resolvedIds: ["model-a"], - modelAAllowed: true, - }); - }); - it("isModelIdAllowedByPolicy checks the requested id, not unrelated custom declarations", () => { const block: ModelsBlock = { custom: [ diff --git a/packages/ai-config/src/index.ts b/packages/ai-config/src/index.ts index de0c560..74aea17 100644 --- a/packages/ai-config/src/index.ts +++ b/packages/ai-config/src/index.ts @@ -154,7 +154,6 @@ export { // --- Resolution helpers (public) ------------------------------------------- export { - filterDiscoveredModelsByPolicy, isModelDiscoveryEnabled, isModelIdAllowedByPolicy, resolveModelIds, diff --git a/packages/ai-config/src/resolve-models.ts b/packages/ai-config/src/resolve-models.ts index 4b9fe26..1a0de52 100644 --- a/packages/ai-config/src/resolve-models.ts +++ b/packages/ai-config/src/resolve-models.ts @@ -10,7 +10,8 @@ * * Pipeline: * 1. If `discovery === "off"`, discovered = []. - * 2. Candidates = discovered + custom models. + * 2. Candidates = discovered + custom models, with custom models replacing + * same-id discovered models. * 3. Apply overrides to matching candidates by id. * 4. If `allow` is non-empty, filter to only allowed ids (exclusive allowlist). * 5. Subtract `deny` (deny always wins). @@ -71,13 +72,10 @@ interface UserRouting { /** No user-configured routing — discovered models start with this. */ const NO_USER_ROUTING: UserRouting = { protocol: undefined, baseUrl: undefined }; -type PipelineEntrySource = "discovered" | "custom"; - -/** A model paired with its source and user-configured routing (pipeline-internal). */ +/** A model paired with its user-configured routing (pipeline-internal). */ interface PipelineEntry { model: ModelInfoLike; userRouting: UserRouting; - source: PipelineEntrySource; } /** @@ -111,24 +109,25 @@ function resolveModelEntries( return discovered.map((m) => ({ model: m, userRouting: NO_USER_ROUTING, - source: "discovered", })); } // 1. Discovery gate const base: PipelineEntry[] = isModelDiscoveryEnabled(modelsBlock) - ? discovered.map((m) => ({ model: m, userRouting: NO_USER_ROUTING, source: "discovered" })) + ? discovered.map((m) => ({ model: m, userRouting: NO_USER_ROUTING })) : []; - // 2. Add custom models (protocol/baseUrl are user-configured) + // 2. Add or replace with custom models (protocol/baseUrl are user-configured) const customs = modelsBlock.custom; if (customs) { for (const custom of customs) { - base.push({ - model: customModelToModelInfo(custom), - userRouting: { protocol: custom.protocol, baseUrl: custom.baseUrl }, - source: "custom", - }); + const index = base.findIndex((entry) => entry.model.id === custom.id); + const customEntry = customModelToEntry(custom, index === -1 ? undefined : base[index]); + if (index === -1) { + base.push(customEntry); + } else { + base[index] = customEntry; + } } } @@ -190,31 +189,6 @@ export function resolveModelIds( ).map((model) => model.id); } -/** - * Filters discovered model rows through the same model resolver that handles - * providers.json allow/deny semantics. Custom declarations do not authorize a - * caller-provided row with the same id; use resolveModels for the complete list. - */ -export function filterDiscoveredModelsByPolicy( - modelsBlock: ModelsBlock | undefined, - models: readonly T[], - getModelId: (model: T) => string, -): T[] { - if (!modelsBlock) { - return [...models]; - } - - const resolvedDiscoveredIds = new Set( - resolveModelEntries( - modelsBlock, - models.map((model) => modelIdToModelInfo(getModelId(model))), - ) - .filter((entry) => entry.source === "discovered") - .map((entry) => entry.model.id), - ); - return models.filter((model) => resolvedDiscoveredIds.has(getModelId(model))); -} - /** Checks whether a raw provider model id survives complete model policy resolution. */ export function isModelIdAllowedByPolicy( modelsBlock: ModelsBlock | undefined, @@ -296,6 +270,37 @@ function customModelToModelInfo(custom: CustomModel): ModelInfoLike { }; } +function customModelToEntry( + custom: CustomModel, + existing: PipelineEntry | undefined, +): PipelineEntry { + return { + model: mergeDefinedModelInfo(existing?.model, customModelToModelInfo(custom)), + userRouting: { + protocol: custom.protocol ?? existing?.userRouting.protocol, + baseUrl: custom.baseUrl ?? existing?.userRouting.baseUrl, + }, + }; +} + +function mergeDefinedModelInfo( + base: ModelInfoLike | undefined, + override: ModelInfoLike, +): ModelInfoLike { + return { + ...override, + family: override.family ?? base?.family, + maxInputTokens: override.maxInputTokens ?? base?.maxInputTokens, + maxOutputTokens: override.maxOutputTokens ?? base?.maxOutputTokens, + protocol: override.protocol ?? base?.protocol, + baseUrl: override.baseUrl ?? base?.baseUrl, + supportedInputMediaTypes: override.supportedInputMediaTypes ?? base?.supportedInputMediaTypes, + thinkingEffortLevels: override.thinkingEffortLevels ?? base?.thinkingEffortLevels, + requiresChatTemplateKwargs: + override.requiresChatTemplateKwargs ?? base?.requiresChatTemplateKwargs, + }; +} + /** * Apply an override to a pipeline entry. Merges metadata onto the model and * promotes any routing fields (protocol, baseUrl) from the override into the @@ -308,10 +313,8 @@ function applyOverrideEntry(entry: PipelineEntry, override: ModelOverride): Pipe (model as Record)[key] = value; } } - return { model, - source: entry.source, userRouting: { // Override routing takes precedence over any prior user routing // (e.g. a custom model that also gets overridden). diff --git a/packages/ai-config/src/schema.ts b/packages/ai-config/src/schema.ts index 4cfd436..a38a2d8 100644 --- a/packages/ai-config/src/schema.ts +++ b/packages/ai-config/src/schema.ts @@ -58,9 +58,9 @@ export const modelOverrideSchema = z .strict(); /** - * Complete model definition declared for a model discovery does NOT return. - * Required fields enforced at schema time (strict) — see `custom` in the - * models block. + * Complete model definition declared for a model discovery does NOT return, or + * to replace metadata for a discovered model with the same id. Required fields + * are enforced at schema time (strict) — see `custom` in the models block. */ export const customModelSchema = z .object({ @@ -88,7 +88,7 @@ export const customModelSchema = z * - `allow` — when non-empty, an EXCLUSIVE allowlist of model ids. * - `deny` — subtracted from candidates; always wins over `allow`. * - `overrides` — partial patches keyed by model id. - * - `custom` — complete model definitions discovery does not return. + * - `custom` — complete model definitions discovery does not return, or same-id replacements. */ export const modelsBlockSchema = z .object({ From 6ac0d196a4d57a274d8aca1c6da4df5a5289b3d4 Mon Sep 17 00:00:00 2001 From: Melissa Barca Date: Thu, 27 Aug 2026 15:49:25 -0400 Subject: [PATCH 5/5] Revert custom-model replace-on-collision behavior Keep resolveModels's existing pipeline unchanged: a custom model declaration and a discovered model sharing an id remain separate candidates, as on main. The new policy helpers wrap that pipeline as-is rather than changing what it does with a same-id collision. --- .../src/__tests__/resolve-models.test.ts | 38 ---------- packages/ai-config/src/resolve-models.ts | 76 ++++--------------- packages/ai-config/src/schema.ts | 8 +- 3 files changed, 18 insertions(+), 104 deletions(-) diff --git a/packages/ai-config/src/__tests__/resolve-models.test.ts b/packages/ai-config/src/__tests__/resolve-models.test.ts index 05ba17c..ee32c8e 100644 --- a/packages/ai-config/src/__tests__/resolve-models.test.ts +++ b/packages/ai-config/src/__tests__/resolve-models.test.ts @@ -88,44 +88,6 @@ describe("resolveModels", () => { expect(result[3].id).toBe("extra"); }); - it("custom models replace discovered models with the same id", () => { - const block: ModelsBlock = { - custom: [ - { - id: "model-a", - name: "Custom Model A", - maxContextLength: 50000, - supportsTools: false, - supportsImages: true, - supportsToolResultImages: false, - supportsWebSearch: true, - }, - ], - }; - const result = resolveModels(block, [ - makeModel("model-a", { - name: "Discovered Model A", - family: "discovered-family", - maxInputTokens: 25000, - protocol: "openai", - }), - makeModel("model-b"), - ]); - - expect(result.map((m) => m.id)).toEqual(["model-a", "model-b"]); - expect(result[0]).toMatchObject({ - id: "model-a", - name: "Custom Model A", - family: "discovered-family", - maxContextLength: 50000, - maxInputTokens: 25000, - supportsTools: false, - supportsImages: true, - supportsWebSearch: true, - resolvedProtocol: "openai-chat", - }); - }); - // --- Overrides --- it("applies overrides to matching models", () => { diff --git a/packages/ai-config/src/resolve-models.ts b/packages/ai-config/src/resolve-models.ts index 1a0de52..1ad1aa3 100644 --- a/packages/ai-config/src/resolve-models.ts +++ b/packages/ai-config/src/resolve-models.ts @@ -10,8 +10,7 @@ * * Pipeline: * 1. If `discovery === "off"`, discovered = []. - * 2. Candidates = discovered + custom models, with custom models replacing - * same-id discovered models. + * 2. Candidates = discovered + custom models. * 3. Apply overrides to matching candidates by id. * 4. If `allow` is non-empty, filter to only allowed ids (exclusive allowlist). * 5. Subtract `deny` (deny always wins). @@ -94,47 +93,29 @@ export function resolveModels( discovered: readonly ModelInfoLike[], providerConnection?: ResolvedConnection, ): ResolvedModelInfo[] { - return resolveModelEntries(modelsBlock, discovered).map((e) => - attachRouting(e.model, e.userRouting, providerConnection), - ); -} - -function resolveModelEntries( - modelsBlock: ModelsBlock | undefined, - discovered: readonly ModelInfoLike[], -): PipelineEntry[] { if (!modelsBlock) { // No models block — pass through discovered models with routing resolved. // Discovered protocol is built-in inference only (lowest precedence). - return discovered.map((m) => ({ - model: m, - userRouting: NO_USER_ROUTING, - })); + return discovered.map((m) => attachRouting(m, NO_USER_ROUTING, providerConnection)); } // 1. Discovery gate - const base: PipelineEntry[] = isModelDiscoveryEnabled(modelsBlock) - ? discovered.map((m) => ({ model: m, userRouting: NO_USER_ROUTING })) - : []; + const base: PipelineEntry[] = + modelsBlock.discovery === "off" + ? [] + : discovered.map((m) => ({ model: m, userRouting: NO_USER_ROUTING })); - // 2. Add or replace with custom models (protocol/baseUrl are user-configured) + // 2. Add custom models (protocol/baseUrl are user-configured) const customs = modelsBlock.custom; if (customs) { for (const custom of customs) { - const index = base.findIndex((entry) => entry.model.id === custom.id); - const customEntry = customModelToEntry(custom, index === -1 ? undefined : base[index]); - if (index === -1) { - base.push(customEntry); - } else { - base[index] = customEntry; - } + base.push({ + model: customModelToModelInfo(custom), + userRouting: { protocol: custom.protocol, baseUrl: custom.baseUrl }, + }); } } - return applyModelPolicy(modelsBlock, base); -} - -function applyModelPolicy(modelsBlock: ModelsBlock, base: PipelineEntry[]): PipelineEntry[] { // 3. Apply overrides (protocol/baseUrl from overrides are user-configured) const overrides = modelsBlock.overrides; if (overrides) { @@ -164,7 +145,8 @@ function applyModelPolicy(modelsBlock: ModelsBlock, base: PipelineEntry[]): Pipe result = result.filter((e) => !denySet.has(e.model.id)); } - return result; + // 6. Resolve routing for each surviving model + return result.map((e) => attachRouting(e.model, e.userRouting, providerConnection)); } /** @@ -270,37 +252,6 @@ function customModelToModelInfo(custom: CustomModel): ModelInfoLike { }; } -function customModelToEntry( - custom: CustomModel, - existing: PipelineEntry | undefined, -): PipelineEntry { - return { - model: mergeDefinedModelInfo(existing?.model, customModelToModelInfo(custom)), - userRouting: { - protocol: custom.protocol ?? existing?.userRouting.protocol, - baseUrl: custom.baseUrl ?? existing?.userRouting.baseUrl, - }, - }; -} - -function mergeDefinedModelInfo( - base: ModelInfoLike | undefined, - override: ModelInfoLike, -): ModelInfoLike { - return { - ...override, - family: override.family ?? base?.family, - maxInputTokens: override.maxInputTokens ?? base?.maxInputTokens, - maxOutputTokens: override.maxOutputTokens ?? base?.maxOutputTokens, - protocol: override.protocol ?? base?.protocol, - baseUrl: override.baseUrl ?? base?.baseUrl, - supportedInputMediaTypes: override.supportedInputMediaTypes ?? base?.supportedInputMediaTypes, - thinkingEffortLevels: override.thinkingEffortLevels ?? base?.thinkingEffortLevels, - requiresChatTemplateKwargs: - override.requiresChatTemplateKwargs ?? base?.requiresChatTemplateKwargs, - }; -} - /** * Apply an override to a pipeline entry. Merges metadata onto the model and * promotes any routing fields (protocol, baseUrl) from the override into the @@ -313,6 +264,7 @@ function applyOverrideEntry(entry: PipelineEntry, override: ModelOverride): Pipe (model as Record)[key] = value; } } + return { model, userRouting: { diff --git a/packages/ai-config/src/schema.ts b/packages/ai-config/src/schema.ts index a38a2d8..4cfd436 100644 --- a/packages/ai-config/src/schema.ts +++ b/packages/ai-config/src/schema.ts @@ -58,9 +58,9 @@ export const modelOverrideSchema = z .strict(); /** - * Complete model definition declared for a model discovery does NOT return, or - * to replace metadata for a discovered model with the same id. Required fields - * are enforced at schema time (strict) — see `custom` in the models block. + * Complete model definition declared for a model discovery does NOT return. + * Required fields enforced at schema time (strict) — see `custom` in the + * models block. */ export const customModelSchema = z .object({ @@ -88,7 +88,7 @@ export const customModelSchema = z * - `allow` — when non-empty, an EXCLUSIVE allowlist of model ids. * - `deny` — subtracted from candidates; always wins over `allow`. * - `overrides` — partial patches keyed by model id. - * - `custom` — complete model definitions discovery does not return, or same-id replacements. + * - `custom` — complete model definitions discovery does not return. */ export const modelsBlockSchema = z .object({