diff --git a/docs-site/src/content/docs/reference/configuration/routing.md b/docs-site/src/content/docs/reference/configuration/routing.md index 1cf9781c0..780f815ab 100644 --- a/docs-site/src/content/docs/reference/configuration/routing.md +++ b/docs-site/src/content/docs/reference/configuration/routing.md @@ -17,16 +17,18 @@ Routing turns the model id sent by a client into one concrete provider and upstr opencodex resolves the requested model in this order: -1. A configured `/` namespace, routed through exactly the +1. An explicit `policy/` or configured routing-profile alias, executing the policy evaluator + and routing the selected candidate. An unknown profile id fails closed. +2. A configured `/` namespace, routed through exactly the mapped stored Codex account. An invalid or unavailable exact target fails closed. -2. A canonical `combo/` or configured combo alias. Canonical ids win before alias matching. -3. An explicit `/` namespace whose prefix names a configured provider. -4. A bare native OpenAI-family id such as `gpt-*`, `o1-*`, `o3-*`, or `o4-*`, routed through the +3. A canonical `combo/` or configured combo alias. Canonical ids win before alias matching. +4. An explicit `/` namespace whose prefix names a configured provider. +5. A bare native OpenAI-family id such as `gpt-*`, `o1-*`, `o3-*`, or `o4-*`, routed through the canonical enabled `openai` provider. -5. An exact match for a provider's `defaultModel`. -6. A known provider-family model prefix. -7. An exact model in a provider's configured `models` list. -8. `defaultProvider`, preserving the requested model id. +6. An exact match for a provider's `defaultModel`. +7. A known provider-family model prefix. +8. An exact model in a provider's configured `models` list. +9. `defaultProvider`, preserving the requested model id. Disabled providers are excluded. An explicit namespace for a disabled provider fails instead of falling through. Provider entries are checked in their JSON insertion order for rules that can match @@ -87,10 +89,10 @@ commands, see [Combos](/guides/combos/). Routing policy profiles are the Router Intelligence selection layer: an explicitly requested `policy/` (or configured alias) selects among a fixed candidate allowlist using hard capability -requirements and deterministic, explainable scoring. In this release profiles are configuration and -dry-run evaluation only: production requests are not yet routed through them (execution wiring -arrives with RI-05), and existing model ids are **never** routed through a profile implicitly. -Policy ids do not participate in the model resolution order above until execution lands. +requirements and deterministic, explainable scoring. An explicit `policy/` request (or a +configured alias) executes the evaluator and routes the selected candidate. Existing model ids are +**never** routed through a profile implicitly: the `policy/` namespace and profile aliases are the +only entry points, and both are validated against the model resolution order above. Each key is an id matching `[A-Za-z0-9][A-Za-z0-9._-]{0,63}`, always addressable as `policy/`, with one optional `alias`. Aliases must be unique and cannot collide with configured providers, @@ -110,9 +112,15 @@ namespace, or reserved bare native families (`gpt-*`, `o1-*`, `o3-*`, `o4-*`, `c `structuredOutput`, `localOnly`, `remoteAllowed`, `encryptedCodexTasks`; plus `reasoningEffort` and `serviceTier` strings. -Request evidence supplied to a dry-run (context window, tools, image input, structured output, -reasoning effort, service tier, encrypted Codex tasks) is evaluated against candidate capabilities -together with the profile `require` block; a candidate must satisfy both to be eligible. +For `unknownEvidence.capability`, `penalize` currently behaves like `allow`: scoring has only a +configured-priority component until a capability score dimension ships (planned with RI-06+), so +`penalize` cannot yet change the selected candidate. + +Request evidence is evaluated against candidate capabilities together with the profile `require` +block; a candidate must satisfy both to be eligible. On the live request path the proxy derives +tools and image-input evidence from the request body; context-window size and the remaining +evidence dimensions stay unknown at routing time. Use the dry-run API/CLI to inspect the full +evidence surface for context-sensitive profiles. The CLI dry-run accepts request-evidence flags but cannot supply candidate capability evidence yet; candidate evidence is provided through the API (`POST /api/routing-profiles/dry-run`). @@ -140,9 +148,9 @@ candidate evidence is provided through the API (`POST /api/routing-profiles/dry- } ``` -CLI: `ocx route policy list`, `ocx route policy show `, and -`ocx route policy dry-run --model-context --tools`. Dry-run evaluates candidates -without sending any upstream request. +CLI: `ocx route policy list [--json]`, `ocx route policy show [--json]`, and +`ocx route policy dry-run [--model-context ] [--tools] [--image] [--structured-output] [--json]`. +Dry-run evaluates candidates without sending any upstream request. ### Combos vs policy profiles @@ -152,8 +160,9 @@ without sending any upstream request. requirements filter first, then deterministic scoring ranks the survivors. Both are virtual namespaces with aliases and collision validation; they differ in *how* a candidate -is chosen. Profile scoring will expand with capability (RI-05), health (RI-06), quota (RI-07), and -cost (RI-08) dimensions; per-request trace recording arrives with execution (RI-05). +is chosen. Profile scoring currently uses the configured-priority component only; health (RI-06), +quota (RI-07), and cost (RI-08) score dimensions are planned. Per-request route-decision traces are +recorded when a policy profile executes. ### Catalog eligibility diff --git a/src/config.ts b/src/config.ts index 2b3a0c8e8..3e0315141 100644 --- a/src/config.ts +++ b/src/config.ts @@ -603,7 +603,18 @@ const providerConfigSchema = z.object({ responsesSnapshotRepair: z.boolean().optional(), }).passthrough(); -const RESERVED_PROVIDER_NAMES = new Set(["__proto__", "prototype", "constructor"]); +const RESERVED_PROVIDER_NAMES = new Set([ + // JavaScript prototype-pollution guards. + "__proto__", + "prototype", + "constructor", + // System-reserved routing namespace (resolved before provider/account + // namespaces in routeModelInternal). "combo" is intentionally NOT reserved: + // a physical provider named `combo` is a supported pattern (combo aliases + // hosted on the combo provider), and the combo selector only wins when an + // actual combo id matches. + "policy", +]); const PROVIDER_NAME_PATTERN = /^[A-Za-z0-9](?:[A-Za-z0-9._-]{0,62}[A-Za-z0-9])?$/; const HEADER_NAME_PATTERN = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/; const SENSITIVE_PROVIDER_HEADERS = new Set([ @@ -1062,7 +1073,7 @@ const configSchema = z.object({ ctx.addIssue({ code: "custom", path: ["providers", name], - message: "provider names must use letters, numbers, dot, underscore, or hyphen and cannot be reserved JavaScript object keys", + message: "provider names must use letters, numbers, dot, underscore, or hyphen and cannot be reserved JavaScript object keys or routing namespaces (policy)", }); } const provider = config.providers[name]; diff --git a/src/router.ts b/src/router.ts index 9b1fa5db1..9ced30e83 100644 --- a/src/router.ts +++ b/src/router.ts @@ -28,6 +28,20 @@ import { type RouteDecisionTraceV1, type TraceCandidateInput, } from "./routing/trace"; +import { getRoutingProfile, resolvePolicyProfileId } from "./routing/profile"; +import { evaluatePolicyProfile, type PolicyRequestEvidence } from "./routing/evaluator"; +import { candidateCapabilityEvidence } from "./routing/capability"; + +export class NoEligiblePolicyCandidateError extends Error { + /** Evaluation trace (with per-candidate exclusions) when nothing qualified. */ + readonly trace?: RouteDecisionTraceV1; + + constructor(readonly profileId: string, trace?: RouteDecisionTraceV1) { + super(`No eligible candidates for policy profile: ${profileId}`); + this.name = "NoEligiblePolicyCandidateError"; + this.trace = trace; + } +} export interface RouteResult { providerName: string; @@ -466,8 +480,39 @@ function comboRouteCandidates( }); } -function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: boolean): RouteResult { +function routeModelInternal( + config: OcxConfig, + modelId: string, + bypassCombos: boolean, + policyEvidence?: PolicyRequestEvidence, +): RouteResult { const slash = modelId.indexOf("/"); + // Policy namespace is system-reserved: an explicit `policy/` or a + // configured profile alias executes the policy evaluator and routes the + // selected candidate. Only explicit requests reach this branch. + const policyId = resolvePolicyProfileId(config, modelId); + if (policyId) { + const profile = getRoutingProfile(config, policyId); + if (!profile) throw new Error(`Unknown routing profile: ${policyId}`); + const candidateEvidence = profile.candidates.map(candidate => ({ + provider: candidate.provider, + model: candidate.model, + capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), + })); + const evaluation = evaluatePolicyProfile(config, policyId, policyEvidence ?? {}, candidateEvidence); + if (evaluation.selectedIndex === null) { + throw new NoEligiblePolicyCandidateError(policyId, evaluation.trace); + } + const selected = evaluation.candidates[evaluation.selectedIndex]!; + const concrete = `${selected.provider}/${selected.model}`; + const routed = routeModelInternal(config, concrete, true); + return { + ...routed, + routeKind: "policy" as const, + routeReason: "policy-selected", + routeDecision: evaluation.trace, + }; + } if (slash > 0) { const namespace = modelId.slice(0, slash); const binding = codexAccountNamespaceEntries(config) @@ -506,7 +551,7 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo const concrete = `${combo.target.provider}/${combo.target.model}`; // The selected target is already a concrete provider/model reference. Resolve it without // consulting combo aliases again, otherwise an alias that shadows the target can recurse. - const routed = routeModelInternal(config, concrete, true); + const routed = routeModelInternal(config, concrete, true, undefined); return { ...routed, combo, routeKind: "combo" as const, routeReason: "combo-pick" }; } } @@ -581,8 +626,14 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo throw new Error(`No provider configured for model: ${modelId}`); } -export function routeModel(config: OcxConfig, modelId: string): RouteResult { - const route = routeModelInternal(config, modelId, false); +export function routeModel( + config: OcxConfig, + modelId: string, + policyEvidence?: PolicyRequestEvidence, +): RouteResult { + const route = routeModelInternal(config, modelId, false, policyEvidence); + // Policy routes carry a full evaluation trace already; never rebuild it. + if (route.routeDecision) return route; const accountRef = route.codexAccountNamespace; const combo = route.combo ? getCombo(config, route.combo.comboId) : undefined; route.routeDecision = buildRouteDecisionTrace({ diff --git a/src/routing/capability.ts b/src/routing/capability.ts new file mode 100644 index 000000000..ae9919f37 --- /dev/null +++ b/src/routing/capability.ts @@ -0,0 +1,179 @@ +/** + * Candidate capability evidence for policy routing (RI-05). + * + * Evidence comes from canonical local sources only - provider config maps, + * the provider registry, the cached Codex catalog file, and the native-model + * metadata helpers. No live network fetch happens at routing time. + * + * "Unknown is not zero": any dimension without canonical evidence stays + * `undefined` (unknown) and the profile's `unknownEvidence` policy decides + * how that affects eligibility. + */ + +import type { OcxConfig } from "../types"; +import { isCanonicalOpenAiForwardProvider } from "../providers/openai-tiers"; +import { PROVIDER_REGISTRY } from "../providers/registry"; +import { + nativeInputModalities, + nativeOpenAiContextWindow, + nativeParallelToolCalls, + nativeReasoningEfforts, +} from "../codex/catalog/metadata"; +import { readCatalog, readCodexCatalogPath } from "../codex/catalog/parsing"; +import { statSync } from "node:fs"; +import type { RouteCapabilityEvidence } from "./trace"; + +type CatalogModelRow = { + provider: string; + id: string; + contextWindow?: number; + inputModalities?: string[]; + reasoningEfforts?: string[]; + capabilities?: string[]; +}; + +/** + * Catalog rows memoized by path + mtime: the cached Codex catalog is stable + * between refreshes, and re-reading/parsing the whole file per candidate on + * the request path would multiply a synchronous disk + JSON cost by the + * profile candidate count for every policy-routed request. + */ +let catalogCache: { path: string; mtimeMs: number; rows: CatalogModelRow[] } | null = null; + +function cachedCatalogModels(): CatalogModelRow[] { + try { + const path = readCodexCatalogPath(); + const mtimeMs = statSync(path).mtimeMs; + if (catalogCache && catalogCache.path === path && catalogCache.mtimeMs === mtimeMs) { + return catalogCache.rows; + } + const catalog = readCatalog(path); + const models = catalog?.models; + if (!Array.isArray(models)) return []; + const rows = models + .filter((model): model is Record & { id: string; provider: string } => + typeof model === "object" && model !== null && typeof model.id === "string" && typeof model.provider === "string") + .map(model => ({ + provider: model.provider, + id: model.id, + ...(typeof model.contextWindow === "number" ? { contextWindow: model.contextWindow } : {}), + ...(Array.isArray(model.inputModalities) + ? { inputModalities: model.inputModalities.filter((value): value is string => typeof value === "string") } + : {}), + ...(Array.isArray(model.reasoningEfforts) + ? { reasoningEfforts: model.reasoningEfforts.filter((value): value is string => typeof value === "string") } + : {}), + ...(Array.isArray(model.capabilities) + ? { capabilities: model.capabilities.filter((value): value is string => typeof value === "string") } + : {}), + })); + catalogCache = { path, mtimeMs, rows }; + return rows; + } catch { + return []; + } +} + +/** + * Classify a hostname for locality evidence. `URL.hostname` keeps IPv6 + * literals bracketed (`[::1]`), so strip the brackets before matching. + * Anything not positively local or private stays unknown: "unknown is not + * zero", so an unrecognized host must never assert `remoteAllowed`. + */ +function classifyHostname(hostname: string): "local" | "private" | null { + const host = hostname.trim().toLowerCase().replace(/\.$/, "").replace(/^\[|\]$/g, ""); + if (host === "localhost" || host.endsWith(".localhost") || host === "0.0.0.0") return "local"; + if (host === "::1" || /^127\./.test(host)) return "local"; + if (/^10\./.test(host) + || /^192\.168\./.test(host) + || /^169\.254\./.test(host) + || /^172\.(1[6-9]|2\d|3[01])\./.test(host) + || /^f[cd][0-9a-f]{2}:/.test(host) + || /^fe80:/.test(host) + || /^::ffff:(?:10\.|127\.|192\.168\.|169\.254\.|172\.(?:1[6-9]|2\d|3[01])\.)/.test(host)) { + return "private"; + } + return null; +} + +function localRemoteEvidence(baseUrl: string | undefined): Pick { + if (typeof baseUrl !== "string" || baseUrl.length === 0) return {}; + try { + const hostname = new URL(baseUrl).hostname; + if (!hostname) return {}; + const kind = classifyHostname(hostname); + if (kind === null) return {}; + // Both booleans are emitted once classified: definitive negative evidence, + // so a local host cannot satisfy `require.remoteAllowed` (or vice versa) + // under `unknownEvidence.capability: "allow"`/`"penalize"`. + return kind === "local" || kind === "private" + ? { localOnly: true, remoteAllowed: false } + : { remoteAllowed: true, localOnly: false }; + } catch { + return {}; + } +} + +/** + * Assemble canonical capability evidence for one `provider/model` candidate. + * Sources (in priority order): provider config maps, provider registry hints, + * cached Codex catalog row, native-model metadata. + */ +export function candidateCapabilityEvidence( + config: OcxConfig, + providerName: string, + modelId: string, +): RouteCapabilityEvidence { + const provider = config.providers[providerName]; + const registryEntry = PROVIDER_REGISTRY.find(entry => entry.id === providerName); + const catalogRow = cachedCatalogModels().find(model => model.provider === providerName && model.id === modelId); + const isNative = providerName === "openai" && !modelId.includes("/"); + + const contextWindow = provider?.modelContextWindows?.[modelId] + ?? provider?.contextWindow + ?? registryEntry?.modelContextWindows?.[modelId] + ?? catalogRow?.contextWindow + ?? (isNative ? nativeOpenAiContextWindow(modelId) : undefined); + + const modalities = provider?.modelInputModalities?.[modelId] + ?? registryEntry?.modelInputModalities?.[modelId] + ?? catalogRow?.inputModalities + ?? (isNative ? nativeInputModalities(modelId) : undefined); + const image = Array.isArray(modalities) + ? modalities.includes("image") + : undefined; + + const capabilities = catalogRow?.capabilities ?? []; + // `parallelToolCalls` is provider-level evidence that the provider accepts + // parallel tool calls (registry-set per provider); the catalog `capabilities` + // list is the per-model signal. Both are positive local evidence only. + const tools = capabilities.includes("tools") + || (isNative ? true : provider?.parallelToolCalls === true) + || undefined; + + const reasoningEfforts = provider?.modelReasoningEfforts?.[modelId] + ?? registryEntry?.modelReasoningEfforts?.[modelId] + ?? catalogRow?.reasoningEfforts + ?? (isNative ? nativeReasoningEfforts(modelId) : undefined); + + const tierSupport = provider?.supportsServiceTier + ?? registryEntry?.supportsServiceTier; + const serviceTier = tierSupport === true + ? "supported" + : tierSupport === false ? "unsupported" : "unknown"; + + const localRemote = localRemoteEvidence(provider?.baseUrl); + const encryptedCodexTasks = isCanonicalOpenAiForwardProvider( + provider ?? { adapter: "", authMode: undefined, baseUrl: undefined }, + ); + + return { + ...(typeof contextWindow === "number" ? { contextWindow } : {}), + ...(typeof image === "boolean" ? { image } : {}), + ...(typeof tools === "boolean" ? { tools } : {}), + ...(reasoningEfforts !== undefined && reasoningEfforts.length > 0 ? { reasoningEfforts } : {}), + ...(serviceTier !== "unknown" ? { serviceTier } : {}), + ...localRemote, + encryptedCodexTasks, + }; +} diff --git a/src/routing/evaluator.ts b/src/routing/evaluator.ts index 880403e2e..6ecf4fc61 100644 --- a/src/routing/evaluator.ts +++ b/src/routing/evaluator.ts @@ -9,6 +9,7 @@ import type { OcxConfig } from "../types"; import { buildRouteDecisionTrace, + MAX_REQUIREMENTS, type RouteCapabilityEvidence, type RouteCostEvidence, type RouteDecisionTraceV1, @@ -265,8 +266,9 @@ export function evaluatePolicyProfile( const unsatisfied = bad.some(requirement => requirement.outcome === "unsatisfied"); const unknown = bad.some(requirement => requirement.outcome === "unknown"); // Unknown capability handling per profile: exclude (default), penalize, - // or allow. "penalize" currently cannot move the score because RI-04 has - // no capability component yet - the capability score arrives with RI-05. + // or allow. "penalize" cannot move the score while configuredPriorityScore + // is the only component; the capability score dimension is still future + // (RI-06+), so "penalize" behaves identically to "allow" in this release. const excludedByUnknown = unknown && profile.unknownEvidence.capability === "exclude"; const costLimit = profile.limits.maxEstimatedCostUsd; const estimatedCost = evidence.cost?.estimatedUsd; @@ -311,7 +313,7 @@ export function evaluatePolicyProfile( profile: { id: profile.id, revision: profile.revision }, // Flat, capped summary (truncation is flagged by the builder); per-candidate // attribution lives in each candidate's `requirements`/`exclusions`. - requirements: candidates.flatMap(candidate => candidate.requirements).slice(0, 16), + requirements: candidates.flatMap(candidate => candidate.requirements).slice(0, MAX_REQUIREMENTS), candidates: candidates.map(candidate => ({ provider: candidate.provider, model: candidate.model, diff --git a/src/routing/request-evidence.ts b/src/routing/request-evidence.ts new file mode 100644 index 000000000..ccca228c2 --- /dev/null +++ b/src/routing/request-evidence.ts @@ -0,0 +1,45 @@ +/** + * Cheap request-side evidence extraction for policy routing (RI-05). + * + * Extracts only what the request body can prove: whether the caller asked for + * tools and whether the input contains image parts. Context-window size is + * left unknown at routing time (documented limitation) - the dry-run API/CLI + * remains the evidence-inspection surface for context-sensitive profiles. + */ + +import type { PolicyRequestEvidence } from "./evaluator"; + +/** + * Walk a body fragment for image parts. Real request shapes nest image blocks: + * Responses puts them under `input[].content[]` (type `input_image`), Chat + * Completions under `messages[].content[]` (type `image_url`), and Claude + * Messages under `messages[].content[]` (type `image`), so the scan recurses + * into arrays and `content` fields instead of only checking the top level. + */ +function containsImagePart(value: unknown): boolean { + if (typeof value === "string") return false; + if (!value || typeof value !== "object") return false; + if (Array.isArray(value)) return value.some(containsImagePart); + const record = value as Record; + if (record.type === "image" || record.type === "input_image") return true; + if (record.image_url !== undefined || record.image !== undefined) return true; + if (record.content !== undefined && containsImagePart(record.content)) return true; + return false; +} + +function inputContainsImage(input: unknown): boolean { + if (typeof input === "string") return false; + if (!Array.isArray(input)) return false; + return input.some(containsImagePart); +} + +export function evidenceFromBody(body: unknown): PolicyRequestEvidence { + if (!body || typeof body !== "object" || Array.isArray(body)) return {}; + const record = body as Record; + const tools = Array.isArray(record.tools) && record.tools.length > 0; + const image = inputContainsImage(record.input) || inputContainsImage(record.messages); + return { + ...(tools ? { toolsRequired: true } : {}), + ...(image ? { imageInputRequired: true } : {}), + }; +} diff --git a/src/server/chat-completions.ts b/src/server/chat-completions.ts index 0f352a249..3ea74e972 100644 --- a/src/server/chat-completions.ts +++ b/src/server/chat-completions.ts @@ -18,7 +18,8 @@ import { classifyError, CYBER_POLICY_ERROR_CODE, isCyberPolicyCode } from "../li import { redactSecretString } from "../lib/redact"; import { resolveClientRetryAfter } from "../lib/retry-after"; import { estimateTokens } from "../lib/token-estimate"; -import { routeModel } from "../router"; +import { NoEligiblePolicyCandidateError, routeModel } from "../router"; +import { evidenceFromBody } from "../routing/request-evidence"; import { resolveWireProtocolOverride } from "./adapter-resolve"; import type { OcxConfig } from "../types"; import { readJsonRequestBody } from "./request-decompress"; @@ -110,7 +111,7 @@ async function handleChatCompletionsWithBudget( let nativeRoute = false; let directRoute = false; try { - const route = routeModel(config, internalBody.model as string); + const route = routeModel(config, internalBody.model as string, evidenceFromBody(internalBody)); // Settle the wire once so every branch below reads the adapter this model will // actually use, not the provider-wide default (#404). route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, "chat"); @@ -148,7 +149,12 @@ async function handleChatCompletionsWithBudget( const ladder = supportedLadderFor({ provider: route.provider, modelId: route.modelId }); if (ladder !== undefined && ladder.length === 0) delete internalBody.reasoning; } - } catch { + } catch (err) { + if (err instanceof NoEligiblePolicyCandidateError) { + logCtx.routeDecision = err.trace; + if (logIds) addFinalRequestLog(logIds.requestId, logIds.start, logCtx, 404, { closeReason: "non_stream" }); + return chatCompletionsErrorResponse(404, err.message, "invalid_request_error"); + } /* unknown model: let handleResponses shape the 404 */ } void nativeRoute; diff --git a/src/server/claude-messages.ts b/src/server/claude-messages.ts index d0b5a045b..875368ccc 100644 --- a/src/server/claude-messages.ts +++ b/src/server/claude-messages.ts @@ -25,7 +25,8 @@ import { } from "../claude/outbound"; import { clearableDeadline, idleDeadline } from "../lib/abort"; import { estimateTokens } from "../lib/token-estimate"; -import { routeModel } from "../router"; +import { NoEligiblePolicyCandidateError, routeModel } from "../router"; +import { evidenceFromBody } from "../routing/request-evidence"; import { resolveWireProtocolOverride } from "./adapter-resolve"; import type { OcxConfig } from "../types"; import { readJsonRequestBody } from "./request-decompress"; @@ -628,7 +629,7 @@ async function handleClaudeMessagesWithBudget( // verified live 2026-07-11). Strip them for that route; routed providers keep them. let nativeRoute = false; try { - const route = routeModel(config, internalBody.model as string); + const route = routeModel(config, internalBody.model as string, evidenceFromBody(internalBody)); // Settle the wire once so the sampling decision below reads the effective // adapter rather than the provider-wide default (#404). route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, "anthropic"); @@ -663,7 +664,14 @@ async function handleClaudeMessagesWithBudget( const ladder = supportedLadderFor({ provider: route.provider, modelId: route.modelId }); if (ladder !== undefined && ladder.length === 0) delete internalBody.reasoning; } - } catch { /* unknown model: let handleResponses shape the 404 */ } + } catch (err) { + if (err instanceof NoEligiblePolicyCandidateError) { + logCtx.routeDecision = err.trace; + if (logIds) addFinalRequestLog(logIds.requestId, logIds.start, logCtx, 404, { closeReason: "non_stream" }); + return anthropicErrorResponse(404, err.message, "invalid_request_error"); + } + /* unknown model: let handleResponses shape the 404 */ + } const headers = new Headers({ "content-type": "application/json" }); for (const name of FORWARD_HEADERS) { diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index 883f076fb..59f2fd273 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -9,7 +9,8 @@ import { parseRequest } from "../../responses/parser"; import { buildCompactV1Output, COMPACT_PROMPT, decodeCompactionSummary, extractCompactUserMessages } from "../../responses/compaction"; import { FORWARD_HEADERS, sanitizeReasoningInputContent } from "../../adapters/openai-responses"; import { expandPreviousResponseInput, previousResponseProviderState, rememberResponseState } from "../../responses/state"; -import { routeModel } from "../../router"; +import { NoEligiblePolicyCandidateError, routeModel } from "../../router"; +import { evidenceFromBody } from "../../routing/request-evidence"; import { advanceComboAfterFailure, comboDefaultEffort, @@ -269,8 +270,17 @@ export async function handleResponsesCompact( let route; try { - route = routeModel(config, raw.model); + // Compact requests route through the same policy evaluation as normal + // turns, so body-derived evidence (tools/image) must reach the first + // evaluation too - not only the later handleResponses dispatch. + route = routeModel(config, raw.model, evidenceFromBody(raw)); } catch (err) { + if (err instanceof NoEligiblePolicyCandidateError) { + // Persist the evaluation trace (per-candidate exclusions + the + // no-eligible reason) so a failed compact policy request stays + // auditable, matching the other request handlers. + logCtx.routeDecision = err.trace; + } return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); } const selectedModelId = route.modelId; diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index f1e20a6ba..dc9e902c5 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -16,7 +16,8 @@ import { previousResponseReplayFailure, rememberResponseState, } from "../../responses/state"; -import { comboRouteDecisionTrace, routeModel, type RouteResult } from "../../router"; +import { comboRouteDecisionTrace, NoEligiblePolicyCandidateError, routeModel, type RouteResult } from "../../router"; +import { evidenceFromBody } from "../../routing/request-evidence"; import { advanceComboAfterFailure, comboDefaultEffort, @@ -1400,12 +1401,17 @@ async function handleResponsesInner( let route: RouteResult; try { - route = routeModel(config, parsed.modelId); + route = routeModel(config, parsed.modelId, evidenceFromBody(parsed._rawBody)); logCtx.routeDecision = route.routeDecision; } catch (err) { if (err instanceof NoAvailableComboTargetsError) { return comboUnavailableResponse(err.message); } + if (err instanceof NoEligiblePolicyCandidateError) { + // Persist the evaluation trace (per-candidate exclusions + the + // no-eligible reason) so failed policy requests stay auditable. + logCtx.routeDecision = err.trace; + } return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); } @@ -1473,12 +1479,15 @@ async function handleResponsesInner( if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { try { - route = routeModel(config, fallback.to); + route = routeModel(config, fallback.to, evidenceFromBody(parsed._rawBody)); logCtx.routeDecision = route.routeDecision; } catch (err) { if (err instanceof NoAvailableComboTargetsError) { return comboUnavailableResponse(err.message); } + if (err instanceof NoEligiblePolicyCandidateError) { + logCtx.routeDecision = err.trace; + } return formatErrorResponse(404, "invalid_request_error", err instanceof Error ? err.message : String(err)); } } diff --git a/tests/policy-execution.test.ts b/tests/policy-execution.test.ts new file mode 100644 index 000000000..ef6e242f0 --- /dev/null +++ b/tests/policy-execution.test.ts @@ -0,0 +1,274 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { NoEligiblePolicyCandidateError, routeModel } from "../src/router"; +import { isValidProviderName } from "../src/config"; +import { getRoutingProfile } from "../src/routing/profile"; +import { evidenceFromBody } from "../src/routing/request-evidence"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-policy-exec-")); + process.env.OPENCODEX_HOME = testDir; +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function baseConfig(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { + adapter: "openai-chat", + baseUrl: "https://a.example/v1", + apiKey: "ka", + models: ["m1"], + modelContextWindows: { m1: 200_000 }, + modelInputModalities: { m1: ["text", "image"] }, + parallelToolCalls: true, + }, + b: { + adapter: "openai-chat", + baseUrl: "https://b.example/v1", + apiKey: "kb", + models: ["m2"], + modelContextWindows: { m2: 64_000 }, + modelInputModalities: { m2: ["text"] }, + }, + openai: { adapter: "openai-responses", authMode: "forward", baseUrl: "https://chatgpt.com/backend-api/codex" }, + }, + combos: { + free: { + strategy: "failover", + targets: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + }, + }, + routingProfiles: { + fast: { + alias: "ocx/fast", + candidates: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + require: { tools: true, minContextWindow: 128000 }, + unknownEvidence: { capability: "exclude", health: "penalize", quota: "penalize", cost: "penalize" }, + }, + }, + ...overrides, + }; +} + +describe("policy execution (RI-05)", () => { + test("explicit policy/ request executes the evaluator and routes the winner", () => { + const config = baseConfig(); + const route = routeModel(config, "policy/fast"); + expect(route.routeKind).toBe("policy"); + expect(route.providerName).toBe("a"); + expect(route.modelId).toBe("m1"); + const trace = route.routeDecision!; + expect(trace.routeKind).toBe("policy"); + expect(trace.profile).toEqual({ id: "fast", revision: getRoutingProfile(config, "fast")!.revision }); + expect(trace.candidates.length).toBe(2); + expect(trace.candidates[0]).toMatchObject({ provider: "a", model: "m1", eligible: true }); + expect(trace.candidates[1]).toMatchObject({ provider: "b", model: "m2", eligible: false }); + expect(trace.candidates[1]!.exclusions[0]!.code).toBe("capability-unsatisfied"); + expect(trace.selected.provider).toBe("a"); + expect(trace.selected.model).toBe("m1"); + expect(trace.candidates[0]!.score).toEqual({ total: 1, components: { configuredPriority: 1 } }); + }); + + test("profile alias executes the same policy", () => { + const route = routeModel(baseConfig(), "ocx/fast"); + expect(route.routeKind).toBe("policy"); + expect(route.providerName).toBe("a"); + expect(route.modelId).toBe("m1"); + }); + + test("existing explicit, combo, native and default routes are unchanged", () => { + const config = baseConfig(); + expect(routeModel(config, "a/m1")).toMatchObject({ routeKind: "explicit-provider", providerName: "a", modelId: "m1" }); + expect(routeModel(config, "combo/free")).toMatchObject({ routeKind: "combo", combo: { comboId: "free" } }); + expect(routeModel(config, "gpt-5.6")).toMatchObject({ routeKind: "native", providerName: "openai", modelId: "gpt-5.6" }); + expect(routeModel(config, "totally-unknown")).toMatchObject({ routeKind: "default-provider", providerName: "a" }); + }); + + test("all candidates excluded throws NoEligiblePolicyCandidateError", () => { + const config = baseConfig({ + routingProfiles: { + strict: { + candidates: [{ provider: "b", model: "m2" }], + require: { minContextWindow: 128000 }, + }, + }, + }); + expect(() => routeModel(config, "policy/strict")).toThrow(NoEligiblePolicyCandidateError); + }); + + test("unknown capability follows the profile unknownEvidence (exclude default)", () => { + // Provider "b" has no parallelToolCalls and no catalog row: tools unknown. + const config = baseConfig({ + routingProfiles: { + toolsOnly: { + candidates: [{ provider: "b", model: "m2" }], + require: { tools: true }, + }, + }, + }); + expect(() => routeModel(config, "policy/toolsOnly")).toThrow(NoEligiblePolicyCandidateError); + + const permissive = baseConfig({ + routingProfiles: { + toolsOnly: { + candidates: [{ provider: "b", model: "m2" }], + require: { tools: true }, + unknownEvidence: { capability: "allow", health: "penalize", quota: "penalize", cost: "penalize" }, + }, + }, + }); + const route = routeModel(permissive, "policy/toolsOnly"); + expect(route.providerName).toBe("b"); + expect(route.modelId).toBe("m2"); + }); + + test("request evidence constrains candidates: image input excludes non-image models", () => { + const config = baseConfig({ + routingProfiles: { + image: { candidates: [{ provider: "b", model: "m2" }] }, + }, + }); + // No image in the request: the request requirement is absent; b is eligible. + const plain = routeModel(config, "policy/image"); + expect(plain.providerName).toBe("b"); + // Image request: b's modalities are text-only -> excluded. + expect(() => routeModel(config, "policy/image", { imageInputRequired: true })).toThrow(NoEligiblePolicyCandidateError); + }); + + test("request tools requirement is enforced when provably needed", () => { + const config = baseConfig({ + routingProfiles: { + tools: { candidates: [{ provider: "b", model: "m2" }] }, + }, + }); + expect(routeModel(config, "policy/tools")).toMatchObject({ providerName: "b", modelId: "m2" }); + // b's tools support is unknown -> request requiring tools excludes it. + expect(() => routeModel(config, "policy/tools", { toolsRequired: true })).toThrow(NoEligiblePolicyCandidateError); + }); + + test("policy selection is deterministic across calls", () => { + const config = baseConfig(); + const first = routeModel(config, "policy/fast"); + const second = routeModel(config, "policy/fast"); + expect(first.providerName).toBe(second.providerName); + expect(first.modelId).toBe(second.modelId); + expect(first.routeDecision!.selected).toEqual(second.routeDecision!.selected); + }); + + test("evidenceFromBody detects nested image parts in real request shapes", () => { + // Responses-shaped body: image block nested under input[].content[]. + const responses = { + model: "policy/image", + input: [ + { + role: "user", + type: "message", + content: [ + { type: "input_text", text: "look" }, + { type: "input_image", image_url: "https://example.test/x.png" }, + ], + }, + ], + }; + expect(evidenceFromBody(responses)).toEqual({ imageInputRequired: true }); + + // Chat-shaped body: image block nested under messages[].content[]. + const chat = { + model: "policy/image", + messages: [ + { + role: "user", + content: [ + { type: "text", text: "look" }, + { type: "image_url", image_url: { url: "https://example.test/x.png" } }, + ], + }, + ], + }; + expect(evidenceFromBody(chat)).toEqual({ imageInputRequired: true }); + + // Tools remain a top-level signal. + expect(evidenceFromBody({ model: "policy/t", tools: [{ type: "function", function: { name: "f" } }] })) + .toEqual({ toolsRequired: true }); + + // Plain text-only bodies produce no evidence. + expect(evidenceFromBody({ + model: "policy/x", + input: [{ role: "user", type: "message", content: [{ type: "input_text", text: "hi" }] }], + })).toEqual({}); + }); + + test("compact-style policy dispatch applies request evidence", () => { + const config = baseConfig({ + routingProfiles: { + image: { candidates: [{ provider: "b", model: "m2" }] }, + }, + }); + // Mirrors src/server/responses/compact.ts: routeModel(config, raw.model, evidenceFromBody(raw)). + const compactBody = { + model: "policy/image", + input: [{ + role: "user", + type: "message", + content: [{ type: "input_image", image_url: "https://example.test/x.png" }], + }], + }; + // b/m2 is text-only, so a provably-image request must be excluded - the + // evidence has to reach the first policy evaluation. + expect(() => routeModel(config, compactBody.model as string, evidenceFromBody(compactBody))) + .toThrow(NoEligiblePolicyCandidateError); + }); + + test("no-eligible policy error carries the evaluation trace", () => { + const config = baseConfig({ + routingProfiles: { + strict: { + candidates: [{ provider: "b", model: "m2" }], + require: { minContextWindow: 128000 }, + }, + }, + }); + let caught: NoEligiblePolicyCandidateError | undefined; + try { + routeModel(config, "policy/strict"); + } catch (err) { + caught = err as NoEligiblePolicyCandidateError; + } + expect(caught).toBeDefined(); + expect(caught!.profileId).toBe("strict"); + expect(caught!.trace).toBeDefined(); + expect(caught!.trace!.selected.reason).toBe("no-eligible-candidate"); + expect(caught!.trace!.candidates).toHaveLength(1); + expect(caught!.trace!.candidates![0]!.exclusions[0]!.code).toBe("capability-unsatisfied"); + }); + + test("policy provider name is a reserved routing namespace (combo stays usable)", () => { + expect(isValidProviderName("policy")).toBe(false); + // A physical provider named `combo` is a supported pattern (combo aliases + // hosted on the combo provider); only the policy namespace is reserved. + expect(isValidProviderName("combo")).toBe(true); + expect(isValidProviderName("openai")).toBe(true); + expect(isValidProviderName("my-provider_2")).toBe(true); + }); +}); diff --git a/tests/responses-compaction-routing.test.ts b/tests/responses-compaction-routing.test.ts index d6cd644b4..4aedbe110 100644 --- a/tests/responses-compaction-routing.test.ts +++ b/tests/responses-compaction-routing.test.ts @@ -930,3 +930,28 @@ describe("compact alternate-account attempt (#913)", () => { }); }); }); + +test("a no-eligible policy compact request persists the evaluation trace", async () => { + const config = { + ...keyProviderConfig(), + routingProfiles: { + strict: { + candidates: [{ provider: "gw", model: "gpt-5.5" }], + require: { minContextWindow: 128000 }, + }, + }, + } as unknown as OcxConfig; + globalThis.fetch = (async () => { + throw new Error("compact must not send upstream when policy evaluation has no eligible candidate"); + }) as typeof fetch; + const logCtx: RequestLogContext = { model: "", provider: "" }; + const response = await handleResponsesCompact( + compactionRequest(baseCompactionBody({ model: "policy/strict" })), + config, + logCtx, + ); + expect(response.status).toBe(404); + expect(logCtx.routeDecision).toBeDefined(); + expect(logCtx.routeDecision!.selected.reason).toBe("no-eligible-candidate"); + expect(logCtx.routeDecision!.candidates).toHaveLength(1); +});