From ae2c3297da26447001dc8626987df27982954db3 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 4 Aug 2026 19:11:14 +0200 Subject: [PATCH] feat(control-plane): expose route decision explanations (RI-09) --- .../001_pr_stack_status.md | 30 +- src/cli/observe.ts | 15 +- src/cli/route-policy.ts | 10 +- src/routing/capability.ts | 11 +- .../management/request-history-routes.ts | 58 ++++ .../management/routing-profile-routes.ts | 77 +++-- tests/route-explainability.test.ts | 269 ++++++++++++++++++ 7 files changed, 427 insertions(+), 43 deletions(-) create mode 100644 tests/route-explainability.test.ts diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index bf50bcb4d5..f75b7d26d6 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -46,9 +46,9 @@ other; closing one is a maintainer decision and neither is stale. | RI-04 | `feat/ri-04-policy-profile-core` | `dev` (post-#1005 merge) | `31c9f0b28` | #1011 | https://github.com/lidge-jun/opencodex/pull/1011 | MERGED | | RI-05 | `feat/ri-05-capability-aware-routing` | `dev` (post-#1011 merge) | `088194a3a` | #1012 | https://github.com/lidge-jun/opencodex/pull/1012 | MERGED | | RI-06 | `feat/ri-06-health-aware-routing` | `dev` (post-#1012 merge) | `af692bb7a` | #1013 | https://github.com/lidge-jun/opencodex/pull/1013 | MERGED | -| RI-07 | `feat/ri-07-quota-aware-routing` | `dev` (post-#1013 merge) | `0c5100271` (pre-restack) | #1014 | https://github.com/lidge-jun/opencodex/pull/1014 | in progress | -| RI-08 | `feat/ri-08-cost-aware-routing` | `feat/ri-07` head | pending | pending | pending | queued | -| RI-09 | `feat/ri-09-route-explainability-api` | `feat/ri-08` head | pending | pending | pending | queued | +| RI-07 | `feat/ri-07-quota-aware-routing` | `dev` (post-#1013 merge) | `1f07c00b8` | #1014 | https://github.com/lidge-jun/opencodex/pull/1014 | MERGED | +| RI-08 | `feat/ri-08-cost-aware-routing` | `dev` (post-#1014 merge) | `410db97e4` | #1015 | https://github.com/lidge-jun/opencodex/pull/1015 | MERGED | +| RI-09 | `feat/ri-09-route-explainability-api` | `dev` (post-#1015 merge) | `d887c1202` | #1016 | https://github.com/lidge-jun/opencodex/pull/1016 | rebased / review in progress | | RI-10 | `feat/ri-10-routing-intelligence-ui` | `feat/ri-09` head | pending | pending | pending | queued | ## Per-PR acceptance log @@ -258,6 +258,30 @@ other; closing one is a maintainer decision and neither is stale. - Base sync deferred: waiting for RI-05 (#1012) to merge before updating these branches from `dev`. +### RI-09 - feat/ri-09-route-explainability-api + +- Base SHA: `410db97e4cd9e9b4f8aba60682d20946e211d6dd` (`dev` after #1015 merge) +- Reviewed commit: `d887c120282c8d381f92cd11c1eebe2373a27281` +- Findings (self-review / full-review): fixed on this head - + (1) dry-run preserves `parseCandidateEvidence(...) === null` as + `400 invalid_candidates`; (2) combo explanations report the physical last + attempt; (3) absent providers leave `encryptedCodexTasks` unknown; + (4) CLI USAGE documents `evaluate` and rejects option-like profile ids; + (5) assembleCandidateEvidence typed as `OcxConfig` after the RI-08 health/ + quota/cost merge. +- Final commit: `d887c120282c8d381f92cd11c1eebe2373a27281` +- PR: #1016 https://github.com/lidge-jun/opencodex/pull/1016 +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run test tests/route-explainability.test.ts`: 10/10 pass - + trace+attempts+outcome merge, 404 unknown ids, pre-trace rows, combo + physical final attempt, dry-run auto-evidence, malformed candidates 400, + absent-provider encryptedCodexTasks unknown, CLI logs explain encode/json, + CLI logs explain missing-id, CLI route policy evaluate dry-run + id guard + - Focused regression suites: cost/quota/routing-profile + explainability green + - `bun run privacy:scan`: passed +- Remaining Low findings: none + ## Baseline note The full-suite baseline on this Windows machine did not complete within the diff --git a/src/cli/observe.ts b/src/cli/observe.ts index 1e2c47df30..a1e852cdd3 100644 --- a/src/cli/observe.ts +++ b/src/cli/observe.ts @@ -14,6 +14,7 @@ import { const USAGE = `Usage: ocx observe logs [--provider ] [--model ] [--status ] [--limit ] [--follow] [--json|--jsonl] + ocx logs explain [--json] ocx logs rebuild-index ocx logs index-status ocx observe usage [--range <7d|30d|all>] [--surface ] [--json] @@ -81,6 +82,17 @@ async function logs(argv: string[], deps: RuntimeApiDeps): Promise { } while (true); } +async function explain(argv: string[], deps: RuntimeApiDeps): Promise { + const args = [...argv]; + const requestId = args.shift(); + const wantsJson = takeFlag(args, "--json"); + if (!requestId) throw new CliUsageError("request id is required", USAGE); + rejectArgs(args, USAGE); + const encoded = encodeURIComponent(requestId); + const result = await runtimeRequest(`/api/request-history/${encoded}/route-decision`, {}, deps); + printData(result, wantsJson, wantsJson ? undefined : [JSON.stringify(result, null, 2)]); +} + async function rebuildIndex(argv: string[], deps: RuntimeApiDeps): Promise { const args = [...argv]; const wantsJson = takeFlag(args, "--json"); @@ -140,7 +152,8 @@ export async function handleObserveCommand(argv: string[], deps: RuntimeApiDeps const [sub = "logs", ...rest] = argv; if (sub === "logs") { const action = rest[0]; - if (action === "rebuild-index") await rebuildIndex(rest.slice(1), deps); + if (action === "explain") await explain(rest.slice(1), deps); + else if (action === "rebuild-index") await rebuildIndex(rest.slice(1), deps); else if (action === "index-status") await indexStatus(rest.slice(1), deps); else await logs(rest, deps); } diff --git a/src/cli/route-policy.ts b/src/cli/route-policy.ts index eb58552e74..1d5e7809d7 100644 --- a/src/cli/route-policy.ts +++ b/src/cli/route-policy.ts @@ -13,6 +13,8 @@ const USAGE = `Usage: ocx route policy list [--json] ocx route policy show [--json] ocx route policy dry-run [--model-context ] [--tools] + [--image] [--structured-output] [--json] + ocx route policy evaluate [--model-context ] [--tools] [--image] [--structured-output] [--json]`; interface ProfileRow { @@ -40,7 +42,7 @@ async function show(argv: string[], deps: RuntimeApiDeps): Promise { const args = [...argv]; const id = args.shift(); const wantsJson = takeFlag(args, "--json"); - if (!id) throw new CliUsageError("profile id is required", USAGE); + if (!id || id.startsWith("-")) throw new CliUsageError("profile id is required", USAGE); rejectArgs(args, USAGE); const result = await runtimeRequest<{ profiles?: ProfileRow[] }>("/api/routing-profiles", {}, deps); const profile = (result.profiles ?? []).find(candidate => candidate.id === id); @@ -52,7 +54,7 @@ async function dryRun(argv: string[], deps: RuntimeApiDeps): Promise { const args = [...argv]; const id = args.shift(); const wantsJson = takeFlag(args, "--json"); - if (!id) throw new CliUsageError("profile id is required", USAGE); + if (!id || id.startsWith("-")) throw new CliUsageError("profile id is required", USAGE); const modelContext = takeIntegerOption(args, "--model-context", { min: 1 }); const tools = takeFlag(args, "--tools"); const image = takeFlag(args, "--image"); @@ -81,10 +83,10 @@ async function dryRun(argv: string[], deps: RuntimeApiDeps): Promise { export async function handleRoutePolicyCommand(argv: string[], deps: RuntimeApiDeps = {}): Promise { return runCliAction(async () => { const [sub, ...rest] = argv; - if (!sub) throw new CliUsageError("route policy requires a subcommand (list, show, dry-run)", USAGE); + if (!sub) throw new CliUsageError("route policy requires a subcommand (list, show, dry-run, evaluate)", USAGE); if (sub === "list") await list(rest, deps); else if (sub === "show") await show(rest, deps); - else if (sub === "dry-run") await dryRun(rest, deps); + else if (sub === "dry-run" || sub === "evaluate") await dryRun(rest, deps); else throw new CliUsageError(`unknown route policy command: ${sub}`, USAGE); }); } diff --git a/src/routing/capability.ts b/src/routing/capability.ts index b628e55df8..c3a27cfebc 100644 --- a/src/routing/capability.ts +++ b/src/routing/capability.ts @@ -185,9 +185,12 @@ export function candidateCapabilityEvidence( : tierSupport === false ? "unsupported" : "unknown"; const localRemote = localRemoteEvidence(provider?.baseUrl); - const encryptedCodexTasks = isCanonicalOpenAiForwardProvider( - provider ?? { adapter: "", authMode: undefined, baseUrl: undefined }, - ); + // Only emit a definitive encryptedCodexTasks value when the provider is + // present. An absent/unconfigured provider must stay unknown so + // require.encryptedCodexTasks does not fail closed on missing config. + const encryptedCodexTasks = provider === undefined + ? undefined + : isCanonicalOpenAiForwardProvider(provider); return { ...(typeof contextWindow === "number" ? { contextWindow } : {}), @@ -196,6 +199,6 @@ export function candidateCapabilityEvidence( ...(reasoningEfforts !== undefined && reasoningEfforts.length > 0 ? { reasoningEfforts } : {}), ...(serviceTier !== "unknown" ? { serviceTier } : {}), ...localRemote, - encryptedCodexTasks, + ...(typeof encryptedCodexTasks === "boolean" ? { encryptedCodexTasks } : {}), }; } diff --git a/src/server/management/request-history-routes.ts b/src/server/management/request-history-routes.ts index db318031cb..0af7e860c9 100644 --- a/src/server/management/request-history-routes.ts +++ b/src/server/management/request-history-routes.ts @@ -3,6 +3,8 @@ * * - `GET /api/request-history` - keyset-paginated rows with filters * - `GET /api/request-history/:requestId` - one canonical row + * - `GET /api/request-history/:requestId/route-decision` - why-this-route + * explanation (RI-09): durable trace + attempts + outcome * * The index is a derived projection of `usage.jsonl`; every response carries * an `index` status block so callers can see schema version, indexed rows and @@ -19,6 +21,7 @@ import { requestLogEntryFromPersistedUsage } from "../request-log"; import { requestLogDto } from "./shared"; import { jsonResponse } from "../auth-cors"; import type { ManagementContext } from "./context"; +import type { PersistedUsageEntry } from "../../usage/log"; function parseQueryInt(raw: string | null): number | undefined | "invalid" { if (raw === null) return undefined; @@ -28,6 +31,17 @@ function parseQueryInt(raw: string | null): number | undefined | "invalid" { return Number.isInteger(value) ? value : "invalid"; } +function finalAttemptTarget(entry: PersistedUsageEntry): { provider: string; model: string } { + const attempts = entry.attempts; + if (Array.isArray(attempts) && attempts.length > 0) { + const last = attempts[attempts.length - 1]; + if (last && typeof last.provider === "string" && typeof last.model === "string") { + return { provider: last.provider, model: last.model }; + } + } + return { provider: entry.provider, model: entry.model }; +} + export async function handleRequestHistoryRoutes(ctx: ManagementContext): Promise { const { url, req, config } = ctx; if (!url.pathname.startsWith("/api/request-history")) return null; @@ -113,6 +127,50 @@ export async function handleRequestHistoryRoutes(ctx: ManagementContext): Promis } if (url.pathname.startsWith("/api/request-history/") && req.method === "GET") { + // Why-this-route explanation (RI-09): trace + attempts + outcome. + if (url.pathname.endsWith("/route-decision")) { + let requestId: string; + try { + requestId = decodeURIComponent( + url.pathname.slice("/api/request-history/".length, -"/route-decision".length), + ); + } catch { + return jsonResponse({ error: { code: "not_found", message: "unknown request" } }, 404, req, config); + } + if (!requestId || requestId.includes("/")) { + return jsonResponse({ error: { code: "not_found", message: "unknown request" } }, 404, req, config); + } + const entry = await requestHistoryRowById(requestId); + if (!entry) { + return jsonResponse({ error: { code: "not_found", message: "unknown request" } }, 404, req, config); + } + const trace = entry.routeDecision ?? null; + const final = finalAttemptTarget(entry); + return jsonResponse({ + requestId, + routeDecision: trace, + attemptSequence: entry.attempts ?? [], + outcome: { + status: entry.status, + ...(entry.terminalStatus ? { terminalStatus: entry.terminalStatus } : {}), + ...(entry.closeReason ? { closeReason: entry.closeReason } : {}), + ...(entry.errorCode ? { errorCode: entry.errorCode } : {}), + ...(entry.durationMs !== undefined ? { durationMs: entry.durationMs } : {}), + ...(entry.usageStatus ? { usageStatus: entry.usageStatus } : {}), + }, + summary: { + requestedModel: entry.requestedModel ?? entry.model, + routeKind: trace?.routeKind ?? null, + ...(trace?.profile ? { profileId: trace.profile.id, revision: trace.profile.revision } : {}), + selected: trace?.selected + ? { provider: trace.selected.provider, model: trace.selected.model } + : null, + finalProvider: final.provider, + finalModel: final.model, + }, + }, 200, req, config); + } + let requestId: string; try { requestId = decodeURIComponent(url.pathname.slice("/api/request-history/".length)); diff --git a/src/server/management/routing-profile-routes.ts b/src/server/management/routing-profile-routes.ts index ebfa491d9f..9144f9eb4d 100644 --- a/src/server/management/routing-profile-routes.ts +++ b/src/server/management/routing-profile-routes.ts @@ -11,6 +11,7 @@ import { evaluatePolicyProfile, type PolicyCandidateEvidence, type PolicyRequest import { candidateCapabilityEvidence } from "../../routing/capability"; import { policyCandidateHealthEvidence } from "../../routing/health"; import { quotaEvidenceForCandidate } from "../../routing/quota"; +import { costEvidenceForCandidate } from "../../routing/cost"; import { providerCodexAccountMode } from "../../providers/registry"; import { getEffectiveActiveCodexAccountId } from "../../codex/routing"; import { getAccountSet } from "../../oauth/store"; @@ -19,6 +20,7 @@ import { isPlainRecord } from "./shared"; import { readManagementJsonBody, rethrowManagementBodyTooLarge } from "./body"; import { jsonResponse } from "../auth-cors"; import type { ManagementContext } from "./context"; +import type { OcxConfig } from "../../types"; function profileDto(config: Parameters[0], id: string): Record | null { const profile = getRoutingProfile(config, id); @@ -86,6 +88,49 @@ function parseCandidateEvidence(raw: unknown): PolicyCandidateEvidence[] | null return out; } +function assembleCandidateEvidence( + config: OcxConfig, + profile: NonNullable>, +): PolicyCandidateEvidence[] { + // Match execution: fill the same candidate evidence the router would + // assemble, so dry-run/evaluate reports the same eligibility as real routing + // instead of treating every capability as unknown. Cost is always present so + // evaluate mode can surface the profile limit even without a usage estimate. + return profile.candidates.map(candidate => ({ + provider: candidate.provider, + model: candidate.model, + capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), + health: policyCandidateHealthEvidence(config, candidate), + quota: quotaEvidenceForCandidate({ + provider: candidate.provider, + model: candidate.model, + ...(candidate.provider === OPENAI_CODEX_PROVIDER_ID + && providerCodexAccountMode( + OPENAI_CODEX_PROVIDER_ID, + config.providers[OPENAI_CODEX_PROVIDER_ID], + ) === "pool" + ? (() => { + const codexAccountId = getEffectiveActiveCodexAccountId(config); + return { + codexAccountId, + codexAccountPlan: codexAccountId + ? config.codexAccounts?.find(account => account.id === codexAccountId)?.plan + : undefined, + }; + })() + : {}), + accountRef: candidate.provider === "anthropic" + ? getAccountSet("anthropic")?.activeAccountId + : undefined, + }), + cost: costEvidenceForCandidate({ + provider: candidate.provider, + model: candidate.model, + limitUsd: profile.limits.maxEstimatedCostUsd, + }), + })); +} + export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promise { const { req, url, config } = ctx; @@ -119,37 +164,7 @@ export async function handleRoutingProfileRoutes(ctx: ManagementContext): Promis return jsonResponse({ error: { code: "invalid_evidence", message: "evidence must be an object" } }, 400, req, config); } const candidateEvidence = body.candidates === undefined - // Match execution: fill the same candidate evidence the router would - // assemble, so dry-run reports the same eligibility as real routing - // instead of treating every capability as unknown. - ? resolvedProfile.candidates.map(candidate => ({ - provider: candidate.provider, - model: candidate.model, - capability: candidateCapabilityEvidence(config, candidate.provider, candidate.model), - health: policyCandidateHealthEvidence(config, candidate), - quota: quotaEvidenceForCandidate({ - provider: candidate.provider, - model: candidate.model, - ...(candidate.provider === OPENAI_CODEX_PROVIDER_ID - && providerCodexAccountMode( - OPENAI_CODEX_PROVIDER_ID, - config.providers[OPENAI_CODEX_PROVIDER_ID], - ) === "pool" - ? (() => { - const codexAccountId = getEffectiveActiveCodexAccountId(config); - return { - codexAccountId, - codexAccountPlan: codexAccountId - ? config.codexAccounts?.find(account => account.id === codexAccountId)?.plan - : undefined, - }; - })() - : {}), - accountRef: candidate.provider === "anthropic" - ? getAccountSet("anthropic")?.activeAccountId - : undefined, - }), - })) + ? assembleCandidateEvidence(config, resolvedProfile) : parseCandidateEvidence(body.candidates); if (candidateEvidence === null) { return jsonResponse({ error: { code: "invalid_candidates", message: "candidates must be an array of evidence objects" } }, 400, req, config); diff --git a/tests/route-explainability.test.ts b/tests/route-explainability.test.ts new file mode 100644 index 0000000000..db84463294 --- /dev/null +++ b/tests/route-explainability.test.ts @@ -0,0 +1,269 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleManagementAPI } from "../src/server/management-api"; +import { ManagementRequest } from "./helpers/management-auth"; +import { appendUsageEntry, resetUsageReadCacheForTests, type PersistedUsageEntry } from "../src/usage/log"; +import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; +import { candidateCapabilityEvidence } from "../src/routing/capability"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-explain-")); + process.env.OPENCODEX_HOME = testDir; + resetUsageReadCacheForTests(); + closeRequestHistoryIndex(); +}); + +afterEach(() => { + closeRequestHistoryIndex(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function config(): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { + adapter: "openai-chat", + baseUrl: "https://a.example/v1", + apiKey: "ka", + models: ["m1"], + modelContextWindows: { m1: 200_000 }, + parallelToolCalls: true, + }, + }, + routingProfiles: { + fast: { candidates: [{ provider: "a", model: "m1" }] }, + }, + }; +} + +function tracedEntry(requestId: string): PersistedUsageEntry { + return { + requestId, + timestamp: 1_700_000_000_000, + provider: "a", + model: "m1", + requestedModel: "policy/fast", + status: 200, + durationMs: 1234, + usageStatus: "reported", + routeDecision: { + version: 1, + decisionId: "abc123def456", + createdAt: 1_700_000_000_000, + requestedModel: "policy/fast", + routeKind: "policy", + profile: { id: "fast", revision: "0123456789abcdef" }, + requirements: [], + candidates: [{ + provider: "a", + model: "m1", + eligible: true, + exclusions: [], + score: { total: 1, components: { configuredPriority: 1 } }, + }], + selected: { candidateIndex: 0, provider: "a", model: "m1", reason: "policy-selected" }, + }, + attempts: [ + { ordinal: 1, provider: "a", model: "m1", adapter: "openai-chat", status: 200, durationMs: 1200, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, + ], + }; +} + +async function apiGet(path: string, cfg: OcxConfig): Promise { + const req = new ManagementRequest(`http://localhost${path}`, { method: "GET" }); + const response = await handleManagementAPI(req, new URL(req.url), cfg, { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + return response!; +} + +describe("route explainability (RI-09)", () => { + test("route-decision endpoint merges trace, attempts, and outcome", async () => { + appendUsageEntry(tracedEntry("explain-me")); + const response = await apiGet("/api/request-history/explain-me/route-decision", config()); + expect(response.status).toBe(200); + const body = await response.json() as { + requestId?: string; + routeDecision?: { routeKind?: string; profile?: { id?: string; revision?: string } }; + attemptSequence?: Array<{ ordinal?: number }>; + outcome?: { status?: number; durationMs?: number }; + summary?: { requestedModel?: string; routeKind?: string | null; profileId?: string; finalProvider?: string; finalModel?: string }; + }; + expect(body.requestId).toBe("explain-me"); + expect(body.routeDecision?.routeKind).toBe("policy"); + expect(body.routeDecision?.profile).toEqual({ id: "fast", revision: "0123456789abcdef" }); + expect(body.attemptSequence).toHaveLength(1); + expect(body.attemptSequence![0]!.ordinal).toBe(1); + expect(body.outcome).toMatchObject({ status: 200, durationMs: 1234 }); + expect(body.summary).toMatchObject({ + requestedModel: "policy/fast", + routeKind: "policy", + profileId: "fast", + finalProvider: "a", + finalModel: "m1", + }); + }); + + test("unknown request ids return 404", async () => { + const response = await apiGet("/api/request-history/missing/route-decision", config()); + expect(response.status).toBe(404); + }); + + test("pre-trace rows explain with null routeDecision and their attempts", async () => { + appendUsageEntry({ + requestId: "legacy-row", + timestamp: 1_700_000_000_000, + provider: "a", + model: "m1", + status: 503, + durationMs: 50, + usageStatus: "unreported", + attempts: [ + { ordinal: 1, provider: "a", model: "m1", adapter: "openai-chat", status: 503, durationMs: 50, sendCount: 1, recoveryKinds: ["transient-5xx"], usageStatus: "unreported" }, + ], + }); + const response = await apiGet("/api/request-history/legacy-row/route-decision", config()); + expect(response.status).toBe(200); + const body = await response.json() as { routeDecision?: unknown; summary?: { routeKind?: string | null } }; + expect(body.routeDecision).toBeNull(); + expect(body.summary?.routeKind).toBeNull(); + }); + + test("combo rows report the physical final attempt, not the virtual combo model", async () => { + appendUsageEntry({ + requestId: "combo-row", + timestamp: 1_700_000_000_000, + provider: "combo", + model: "fast-fallback", + requestedModel: "combo/fast-fallback", + status: 200, + durationMs: 900, + usageStatus: "reported", + attempts: [ + { ordinal: 1, provider: "a", model: "m1", adapter: "openai-chat", status: 503, durationMs: 100, sendCount: 1, recoveryKinds: ["transient-5xx"], usageStatus: "unreported" }, + { ordinal: 2, provider: "b", model: "m2", adapter: "openai-chat", status: 200, durationMs: 800, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, + ], + }); + const response = await apiGet("/api/request-history/combo-row/route-decision", config()); + expect(response.status).toBe(200); + const body = await response.json() as { + summary?: { finalProvider?: string; finalModel?: string }; + attemptSequence?: Array<{ provider?: string; model?: string }>; + }; + expect(body.attemptSequence).toHaveLength(2); + expect(body.summary).toMatchObject({ finalProvider: "b", finalModel: "m2" }); + }); + + test("dry-run without candidate evidence assembles canonical evidence", async () => { + const cfg = config(); + const req = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "fast", evidence: {} }), + }); + const response = await handleManagementAPI(req, new URL(req.url), cfg, { refreshCodexCatalog: async () => {} }); + expect(response!.status).toBe(200); + const body = await response!.json() as { + candidates?: Array<{ capability?: { contextWindow?: number; encryptedCodexTasks?: boolean }; health?: object; quota?: object; cost?: object }>; + }; + expect(body.candidates?.[0]?.capability?.contextWindow).toBe(200_000); + expect(body.candidates?.[0]?.health).toBeDefined(); + expect(body.candidates?.[0]?.quota).toBeDefined(); + expect(body.candidates?.[0]?.cost).toBeDefined(); + }); + + test("malformed candidates remain invalid_candidates 400", async () => { + const cfg = config(); + const req = new ManagementRequest("http://localhost/api/routing-profiles/dry-run", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ profile: "fast", candidates: [{ model: "m1" }] }), + }); + const response = await handleManagementAPI(req, new URL(req.url), cfg, { refreshCodexCatalog: async () => {} }); + expect(response!.status).toBe(400); + const body = await response!.json() as { error?: { code?: string } }; + expect(body.error?.code).toBe("invalid_candidates"); + }); + + test("absent provider leaves encryptedCodexTasks unknown", () => { + const evidence = candidateCapabilityEvidence(config(), "missing-provider", "m1"); + expect(Object.prototype.hasOwnProperty.call(evidence, "encryptedCodexTasks")).toBe(false); + }); + + test("CLI logs explain encodes request ids and supports --json", async () => { + const { handleObserveCommand } = await import("../src/cli/observe"); + const calls: Array<{ path: string; init?: RequestInit }> = []; + const payload = { + requestId: "id with spaces", + summary: { finalProvider: "a", finalModel: "m1" }, + }; + const code = await handleObserveCommand(["logs", "explain", "id with spaces", "--json"], { + baseUrl: "http://cli.test", + fetchImpl: async (input, init) => { + const path = String(input).replace("http://cli.test", ""); + calls.push({ path, init }); + return new Response(JSON.stringify(payload), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }, + }); + expect(code).toBe(0); + expect(calls).toHaveLength(1); + expect(calls[0]!.path).toBe("/api/request-history/id%20with%20spaces/route-decision"); + }); + + test("CLI logs explain rejects missing request ids", async () => { + const { handleObserveCommand } = await import("../src/cli/observe"); + const code = await handleObserveCommand(["logs", "explain"], { + baseUrl: "http://cli.test", + fetchImpl: async () => { + throw new Error("should not request"); + }, + }); + expect(code).toBe(2); + }); + + test("CLI route policy evaluate posts dry-run evidence and rejects option-like ids", async () => { + const { handleRoutePolicyCommand } = await import("../src/cli/route-policy"); + const calls: Array<{ path: string; init?: RequestInit }> = []; + const ok = await handleRoutePolicyCommand(["evaluate", "fast", "--tools", "--json"], { + baseUrl: "http://cli.test", + fetchImpl: async (input, init) => { + const path = String(input).replace("http://cli.test", ""); + calls.push({ path, init }); + return new Response(JSON.stringify({ selectedIndex: 0, candidates: [] }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }, + }); + expect(ok).toBe(0); + expect(calls).toHaveLength(1); + expect(calls[0]!.path).toBe("/api/routing-profiles/dry-run"); + expect(calls[0]!.init?.method).toBe("POST"); + const body = JSON.parse(String(calls[0]!.init?.body ?? "{}")) as { + profile?: string; + evidence?: { toolsRequired?: boolean }; + }; + expect(body).toEqual({ profile: "fast", evidence: { toolsRequired: true } }); + + const bad = await handleRoutePolicyCommand(["evaluate", "--json"], { + baseUrl: "http://cli.test", + fetchImpl: async () => { + throw new Error("should not request"); + }, + }); + expect(bad).toBe(2); + }); +});