diff --git a/devlog/_plan/260804_router_intelligence/000_master_plan.md b/devlog/_plan/260804_router_intelligence/000_master_plan.md new file mode 100644 index 0000000000..b0461191de --- /dev/null +++ b/devlog/_plan/260804_router_intelligence/000_master_plan.md @@ -0,0 +1,490 @@ +# 000 - Master plan: Router Intelligence / Routing Control Plane + +Status: ACTIVE +Created: 2026-08-04 +Owner: Codex agent (this programme) +Target repository: `lidge-jun/opencodex` (`upstream`), target branch `dev` +Stack base: `upstream/dev` `e44d234f08e03dd4dbf0c4aa13af43046d86b0a6` + +This unit implements the missing routing-intelligence vertical: durable "why +this route?" decision traces, a rebuildable full-history query index, routing +analytics, and explicit user-configured policy profiles with capability, +health, quota and cost scoring. It is **inference routing only**: no Agent +Fabric tasks, harness handoffs, ACP/A2A, worktree orchestration, agent teams, +or portable coding-task state. + +## 1. Verified current state (2026-08-04, live repository) + +Environment: + +- Bun: `1.3.14` (`bun --version` in the dedicated worktree) +- Package version: `2.10.0` (`package.json`) +- Remotes: `origin` = `Wibias/opencodex` (user fork), `upstream` = `lidge-jun/opencodex` +- `origin/dev` = `be177ea501e5007f4a56d19d069ef5cd76ea24b9` (2026-07-30, merge of #761) +- `upstream/dev` = `e44d234f08e03dd4dbf0c4aa13af43046d86b0a6` (2026-08-04, merge of #949) +- `origin/dev` is a strict ancestor of `upstream/dev` (`git merge-base` = `be177ea`) +- Stack base decision: **`upstream/dev` head** is the base for every branch + because PRs target `lidge-jun/opencodex:dev` and the routing/combos/quota + infrastructure this programme extends only exists on `upstream/dev` + (landed 2026-08-01..04). `origin/dev` is recorded above for provenance. +- Worktree: `D:\codex-worktrees\ocx-router-intelligence` (one task = one + folder = one branch; branches switch inside it, worktree stays clean) +- Test baseline: full `bun run test` was started in the background on a clean + `upstream/dev` checkout; result recorded in `001_pr_stack_status.md`. The + suite is large (~8k tests; a 15-minute foreground run did not finish on + this machine, so the baseline is collected out-of-band). + +Relevant existing modules (all verified by direct read): + +| Module | Role for this programme | +|---|---| +| `src/router.ts` | Central `routeModel(config, modelId)` resolver. Precedence: codex account namespace -> combo -> explicit `/` -> bare OpenAI family -> `defaultModel` -> pattern -> provider model lists -> `config.defaultProvider`. Returns `RouteResult { providerName, provider, modelId, codexAccountMode?, codexAccountId?, codexAccountNamespace?, combo? }`. **RI-01 capture point.** | +| `src/combos/types.ts` | Combo schema validation (`comboConfigIssues`, alias rules, `NATIVE_OPENAI_FAMILY_PATTERN`), normalization (`normalizeComboConfig`), `COMBO_NAMESPACE = "combo"`, `resolveComboId`, `getCombo`, `isValidComboId`. **RI-04 mirrors this pattern for profiles.** | +| `src/combos/resolve.ts` | `pickComboTarget` (failover / smooth weighted round-robin), `advanceComboAfterFailure`, `noteComboSuccess/Failure`, `tryPickComboModel`. Candidate/attempt semantics RI-01 traces. | +| `src/combos/failover.ts` | Per-target cooldown (`coolComboTarget`, `isComboTargetInCooldown`). | +| `src/combos/request.ts` | `comboIdFromRawBody`, `concreteComboRequestBody` (target model rewrite). | +| `src/codex/routing.ts` (1677) | Codex account-pool routing: per-account upstream health maps, cooldown (retry-after / reset-derived / default), quota recovery probe leases, thread affinity, pool strategies (quota / round-robin / fill-first), `recordCodexUpstreamOutcome`, `getCodexAccountHealthSnapshot`, `isCodexAccountInCooldown`, soft-avoid, `computeCodexUsageScore`. **RI-06/07 evidence source.** | +| `src/codex/auth-context.ts` | `resolveCodexAuthContext`, account selection + credential fencing. Boundary: RI-07 must not change exact-account fail-closed semantics. | +| `src/usage/log.ts` | `PersistedUsageEntry`/`PersistedUsageAttempt`, `appendUsageEntry` -> `normalizeUsageEntry` whitelist serializer, revision-keyed cooperative reads, 64 MiB / 200k-entry management truncation, `readRecentUsageEntries` for hydration. **RI-01 extends this schema additively.** | +| `src/server/request-log.ts` | `RequestLogContext` -> `RequestLogEntry` -> `addRequestLog` (retains ring + `appendUsageEntry` with failure diagnostics) and `addFinalRequestLog`; `requestLogEntryFromPersistedUsage`; `hydrateRequestLogsFromDisk`. **RI-01 hydration + DTO extension points.** | +| `src/server/management/logs-usage-routes.ts` | `GET /api/logs` (in-memory ring, offset/limit/tail), `/api/usage` summary. SQLite mentions are Codex `state.sqlite` busy-error strings only - no usage index exists. | +| `src/server/management/shared.ts` | `requestLogDto` with display-only cost/TPS metrics; `costResult`, `unavailableCostReason`. | +| `src/usage/summary.ts` | Daily rollups (`summarizeUsage`), ranges/surfaces. No percentiles, no breakdowns. | +| `src/usage/cost.ts`, `src/usage/expected-prices.ts` | `estimateRequestCost`, `estimateComboCost`, `normalizeCostTokens`, `tokensPerSecond`, `serviceTierContext`; expected-price overlay with `source: "expected"`. **RI-08 evidence source.** | +| `src/providers/quota.ts` (1276) | Provider quota reports + cache (`fetchProviderQuotaReports`, `clearProviderQuotaCache`). **RI-07 evidence source.** | +| `src/providers/context-cap.ts` | Global/provider context caps. **RI-05 requirement input.** | +| `src/codex/catalog/` | Catalog aggregation (`CatalogModel`: contextWindow, inputModalities, reasoningEfforts, ...), native windows (`nativeOpenAiContextWindow`), provider-fetch/sync. **RI-05 capability evidence.** | +| `src/config.ts` | `validateConfigCandidate`; combos validated via `comboConfigIssues` (~line 1233); `saveConfigPreservingClaudeCode`. **RI-04 wiring point.** | +| `src/types.ts` | `OcxConfig` (line 542), `combos` (line 775), `OcxComboConfig` (794). **RI-04 adds `routingProfiles` here.** | +| `src/cli/index.ts`, `src/cli/combo.ts` | Hand-rolled dispatch; `ocx route combo ` already exists. **RI-04/09 CLI wiring points.** | +| `gui/src/pages/Logs.tsx` (1059), `Combos.tsx`, `gui/src/i18n/{en,de,ja,ko,ru,zh}.ts` | Existing dashboard grammar and strict locale-key enforcement (`lint:i18n`). **RI-10 surfaces.** | +| `docs-site/src/content/docs/reference/configuration/routing.md` (96), `guides/combos.md` (286) | Docs anchors for configuration and combos. **RI-04/10 docs.** | + +Mandatory searches (run on `upstream/dev` tree; authoritative): + +- `route decision`, `decision trace`, `routing profile`, `policy profile`, + `candidate exclusion`, `health score`, `quota headroom`, + `latency percentile`, `cursor pagination`: **zero matches** in `src/`, + `tests/`, `gui/src/`, `docs-site/src/`. The vertical is genuinely missing. +- `bun:sqlite`: used by `src/oauth/kiro-credentials.ts`, + `src/codex/history-provider.ts`, `src/codex/model-cache.ts`, + `src/codex/native-profile-store.ts`, `src/storage/*`. Bun's built-in SQLite + is therefore available and in active use; no third-party DB dependency is + needed for RI-02. +- `SQLite`: `src/server/management/logs-usage-routes.ts` mentions + `state.sqlite` only in Codex-busy error strings. **No usage-history index + exists.** + +Related in-flight work (checked 2026-08-04; no blocking overlap): + +| In-flight item | Relationship | Boundary this programme keeps | +|---|---|---| +| PR #922 `fix/914-account-neutral-network` (luvs01, DRAFT, 19 commits, updated 2026-08-04) | Implements host/account transport-health separation for #914 | RI-06 consumes its outcome classification (`connect-neutral` / host ledger) as health evidence. We do **not** re-implement failure classification, redirect policy, or host circuits. | +| PR #966 `codex/260804-issue914-transport-attribution` (Yuxin-Qiao, DRAFT, updated 2026-08-04) | Alternative #914 implementation (pre-connection classifier + host ledger) | Same: consumed as evidence input, never duplicated. The two PRs overlap each other; that is a maintainer decision, not ours to close - neither is stale. | +| PR #715 `feat/priority-levels` (DRAFT) | Codex pool selection order | Documented boundary: pool strategies remain authoritative inside their scope; policy profiles own candidate selection only when explicitly invoked. | +| PR #988 `codex/providers-copy-doctor` (GUI) | Providers/combos layout work | RI-10 keeps its GUI surface to new pages + Logs/route-detail additions; conflict-check `Providers.tsx`/`Combos.tsx` ownership at RI-10 time. | +| PR #998 `codex/260803-integration-switches` | Write-substrate changes | Watch for conflict with `request-log.ts`; rebase boundary noted in ledger. | +| `devlog/_plan/260803_transport_attribution` | #914/#919 policy unit | Now embodied by #922/#966; we consume, not implement. | +| `devlog/_plan/260803_cooldown_recovery_probe` | #915 recovery probe (Not started) | RI-06 reads cooldown state only; never touches probe leases/generations. | +| `devlog/_plan/260730_kiro_usage_cumulative_cache` | Usage persist layer (`usage/log.ts`) | Our JSONL extension is additive (`routeDecision` field); we do not touch the `normalizeUsageValue` whitelist or `contextTotalTokens`. | +| `devlog/_plan/260804_stacked_pr_ci` | Empty placeholder created 2026-08-04 | Referenced; no concrete overlap. | + +## 2. Reuse-versus-new-work table + +| Work item | Reuse | New work | +|---|---|---| +| Trace types + bounds | `PersistedUsageEntry` shape, `capMetadataString`/whitelist discipline in `usage/log.ts`, `RequestLogContext` plumbing | `src/routing/trace.ts` (types, builder, normalizer, redaction, truncation) | +| Trace capture | `routeModel()` call sites in `src/server/responses/core.ts`, `compact.ts`, `chat-completions.ts`, `claude-messages.ts`, `fetch-helpers.ts` | One-line `routeDecision` attachment per call site; candidate derivation for combo routes from `getCombo` + `isComboTargetInCooldown` | +| Full-history index | `bun:sqlite` (already used repo-wide); `usageLogRevision` file-identity discipline | `src/routing/history/*` (schema, indexer, cursor, queries, rebuild) + `ocx logs rebuild-index` | +| Cursor API | `jsonResponse`/auth-cors helpers, management route registration | `GET /api/request-history`, `GET /api/request-history/:id` | +| Analytics | `usage/summary.ts` rollup style | Percentiles, rates, breakdowns, confidence, truncation flags over the SQLite index | +| Profile schema | Combos validation/normalization pattern (`comboConfigIssues`) | `src/routing/profile.ts`, config wiring in `src/config.ts` + `src/types.ts` | +| Policy execution | `routeModel` integration point, catalog capability data | `src/routing/evaluator.ts` + `policy-execution.ts` | +| Health scoring | `src/codex/routing.ts` health/cooldown APIs, `recordCodexUpstreamOutcome` fields | Deterministic formula + trace components (`src/routing/health.ts`) | +| Quota scoring | `src/providers/quota.ts`, codex quota caches | Unknown-safe evidence adapter + scoring (`src/routing/quota.ts`) | +| Cost scoring | `src/usage/cost.ts`, `expected-prices.ts` | Cost limits, provenance, incomplete-estimate flags (`src/routing/cost.ts`) | +| Explainability | `requestLogDto`/management API patterns, `handleLogsUsageRoutes` | `GET .../route-decision`, profile endpoints, `ocx logs explain`, `ocx route policy evaluate` | +| GUI | `Logs.tsx` detail modal grammar, `Combos.tsx` card grammar, i18n key discipline | Profiles list/detail/dry-run view, why-this-route view, analytics chips | +| Docs | `routing.md`, `combos.md` structure, five locales (en/ja/ko/ru/zh-cn) | Policy profiles guide, history API, explainability, migration notes | + +## 3. Architecture decision records + +- **ADR-1 - Canonical ledger stays authoritative.** `usage.jsonl` remains the + canonical append-only request evidence. `routing-history.sqlite` is a + disposable, rebuildable derived query index. No index state is ever written + back into the JSONL, and the index never owns data the JSONL does not carry. +- **ADR-2 - Trace rides the usage entry.** `RouteDecisionTraceV1` is persisted + as an optional, additive field (`routeDecision`) on `PersistedUsageEntry`. + Rejected alternative: a separate trace store (two-phase commit hazards, + ordering complexity, duplicated privacy surface). Old rows parse; new rows + are forward-compatible by whitelist normalization. +- **ADR-3 - Selection and execution stay separate.** The trace records the + selection decision before dispatch. Fallback execution attempts remain the + existing `attempts[]` array on the usage entry. The trace's + `selected`/`candidates` never mutate after dispatch; the explain API merges + trace + attempts + final outcome at read time. +- **ADR-4 - Explicit routing keeps precedence.** Policy routing activates + only for an explicitly requested `policy/` or configured alias. Existing + selectors (account namespace, provider/model, `combo/`, native model + ids, default-provider) are byte-for-byte unchanged. +- **ADR-5 - Unknown is not zero.** Unknown capability/health/quota/price stays + unknown in evidence. Each profile's `unknownEvidence` map (allow / + penalize / exclude) decides scoring; safe defaults: capability `exclude`, + health `penalize`, quota `penalize`, cost `penalize`. +- **ADR-6 - Deterministic scoring.** Weights normalize by exact division with + fixed-point truncation; candidates score component-wise with documented + constants; ties break by candidate declaration order, then provider/model + lexicographic order. No randomness, no wall-clock jitter. +- **ADR-7 - Revision digest.** Profile revision = first 16 hex chars of + SHA-256 over canonical normalized JSON (sorted keys, trimmed strings). Every + decision trace for a policy carries `profile.id` + `profile.revision`. +- **ADR-8 - Index rebuild contract.** `schema_meta` stores schema version, + source file identity (dev/ino/birthtime/size/mtime), and indexed byte + offset. Missing/corrupt/stale/incompatible index or a changed source + identity => full rebuild from JSONL (transactional batches, `requestId` + primary key dedupe). Partial final JSONL lines are skipped and re-read on + the next append. Crash during indexing rolls back to the last committed + batch (WAL + batch transaction). +- **ADR-9 - Keyset cursor pagination.** `ORDER BY timestamp DESC, + request_id DESC`, cursor = base64url(`{t, i}`) of the last returned row. + No offset for unbounded history; max page size 100; invalid cursors return + `400 invalid_cursor`. +- **ADR-10 - No automatic self-tuning.** Analytics are read-only. The system + never rewrites profile weights, budgets, or candidate sets from observed + outcomes. Explainability is the only feedback loop. +- **ADR-11 - Transparent formulas.** Health/quota/cost scores use explicit + deterministic formulas with named constants (documented in `src/routing/` + and the docs). No learned model, no hidden priors. + +## 4. Durable data model + +### 4.1 `usage.jsonl` row extension (RI-01) + +```ts +interface RouteDecisionTraceV1 { + version: 1; + decisionId: string; // 12-hex, random per decision + createdAt: number; // epoch ms + requestedModel: string; // capped 128 (MAX_TRACE_STRING) + routeKind: "explicit-account" | "explicit-provider" | "native" + | "combo" | "policy" | "default-provider"; + profile?: { id: string; revision: string }; // policy routes only + requirements: RouteRequirementEvidence[]; // max 16 + candidates: RouteCandidateTrace[]; // max 8 + selected: { + candidateIndex: number; // index into candidates + provider: string; // capped 128 + model: string; // capped 128 + accountRef?: string; // opaque id or privacy-safe handle, capped 128 + reason: string; // stable code, capped 128 + tieBreak?: string; + }; + truncated?: { candidates?: true; exclusions?: true; strings?: true }; +} +``` + +`RouteCandidateTrace`: `{ provider, model, accountRef?, eligible, exclusions +(max 16 of `{code, detail?}`), capability?, health?, quota?, cost?, score? }`. +Evidence shapes: + +- `capability`: `{ contextWindow?, tools?, image?, structuredOutput?, + reasoningEfforts?, serviceTier?, localOnly?, remoteAllowed?, + encryptedCodexTasks? }` - every field `number | boolean | "unknown"`. +- `health`: `{ cooldownUntilMs?, softAvoidUntilMs?, successRate?, failures?, + incompleteStreamRate?, recentLatencyMs?, sampleCount?, recencyWeight? }`. +- `quota`: `{ known, headroomTokens?, exhausted?, resetAtMs?, + reauthOrCooling?, reservedHeadroomTokens?, source }`. +- `cost`: `{ estimatedUsd?, priceSource?, incomplete?, limitUsd?, + excludedByLimit? }`. +- `score`: `{ total, components: { capability?, health?, quota?, cost?, + latency?, configuredPriority? } }`. + +Bounds: `MAX_CANDIDATES = 8`, `MAX_EXCLUSIONS_PER_CANDIDATE = 16`, +`MAX_REQUIREMENTS = 16`, `MAX_TRACE_STRING = 128`, `MAX_TRACE_BYTES approx 16 KiB` +(enforced by builder; oversized inputs truncated with `truncated` flags). +Stable wire values only; no localized strings in persisted data. + +### 4.2 `routing-history.sqlite` (RI-02, derived index) + +```sql +CREATE TABLE schema_meta ( + key TEXT PRIMARY KEY, value TEXT NOT NULL +); -- schema_version, source_path/dev/ino/birthtime_ms/size/mtime_ms, + -- indexed_offset, indexed_rows, built_at_ms, last_error + +CREATE TABLE requests ( + request_id TEXT PRIMARY KEY, + timestamp INTEGER NOT NULL, + provider TEXT NOT NULL, + model TEXT NOT NULL, + requested_model TEXT, + status INTEGER NOT NULL, + surface TEXT, + inbound_protocol TEXT, + api_key_id TEXT, + conversation_id TEXT, + route_kind TEXT, + profile_id TEXT, + profile_revision TEXT, + fallback INTEGER NOT NULL DEFAULT 0, -- attempts.length > 1 + duration_ms INTEGER NOT NULL, + first_output_ms INTEGER, + usage_status TEXT, + usage_json TEXT, + total_tokens INTEGER, + error_code TEXT, + terminal_status TEXT, + close_reason TEXT, + attempt_count INTEGER NOT NULL DEFAULT 1, + decision_json TEXT, -- RouteDecisionTraceV1 when present + row_json TEXT NOT NULL -- full normalized PersistedUsageEntry +); +CREATE INDEX idx_requests_ts ON requests(timestamp DESC, request_id DESC); +CREATE INDEX idx_requests_provider ON requests(provider); +CREATE INDEX idx_requests_model ON requests(model); +CREATE INDEX idx_requests_requested_model ON requests(requested_model); +CREATE INDEX idx_requests_status ON requests(status); +CREATE INDEX idx_requests_conversation ON requests(conversation_id); +CREATE INDEX idx_requests_api_key ON requests(api_key_id); +CREATE INDEX idx_requests_profile ON requests(profile_id); +``` + +Indexer appends by byte offset, skips a trailing partial line, and commits +batches of 500 rows in one transaction (WAL mode). Duplicate `request_id` +replay is ignored (`INSERT OR IGNORE`). File identity change, size regression +(truncation), or `PRAGMA integrity_check` failure => automatic full rebuild +plus a `last_error`/`rebuilt_at` audit row. + +### 4.3 Routing profile config (RI-04) + +```jsonc +{ + "routingProfiles": { + "fast": { + "alias": "ocx/fast", // optional; canonical id is policy/fast + "candidates": ["anthropic/claude-sonnet-5", "openai/gpt-5.6-sol", "google/gemini-3.6-pro"], + "require": { "tools": true, "minContextWindow": 128000 }, + "optimize": { "latency": 0.55, "health": 0.25, "cost": 0.10, "quota": 0.10 }, + "limits": { "maxEstimatedCostUsd": 0.50 }, + "unknownEvidence": { "capability": "exclude", "health": "penalize", "quota": "penalize", "cost": "penalize" } + } + } +} +``` + +Normalized profile: strategy-free v1 (explicit candidate allowlist only; no +implicit expansion), `alias` optional (one `/` segment at most, bare aliases +reject the native OpenAI family per the combos rule), hard `require` +requirements evaluated before scoring, weights normalized to sum 1 with +deterministic truncation, revision digest per ADR-7. + +## 5. Privacy and security threat model + +- Never persisted anywhere in this vertical: prompt bodies, message bodies, + tool arguments/results, API keys, OAuth tokens, raw account emails, raw + provider quota responses, authorization headers, hidden reasoning / CoT, + raw upstream response bodies. +- Account references: existing opaque ids (`codexAccountId`, `apiKeyId`) or + privacy-safe handles only. Provider quota evidence is reduced to + `{known, headroomTokens?, exhausted?, resetAtMs?, reauthOrCooling?}` - + never the raw response. +- Diagnostic strings: capped at `MAX_TRACE_STRING` and passed through the + existing `redactSecretString` discipline where they originate from upstream + text. Trace builder rejects `provider.apiKey` and URL credentials by + construction (it only receives names/ids, never config objects with + secrets). +- The SQLite index lives in the config dir with the same `0o600` file mode as + `usage.jsonl` and carries only data already present in the canonical ledger. +- `privacy:scan` gate must stay green; any new log/CLI output uses bounded, + redacted values. +- Trust boundary: management APIs already require a dashboard session / admin + token (`auth-cors`); new endpoints register through the same path. +- Threat model summary: local attacker with filesystem access already owns + `usage.jsonl`; the index adds no new secret surface. Network attacker sees + no new data (index is local). Upstream attacker cannot inject into the + index beyond what already lands in the canonical ledger; row parsing is + defensive (malformed JSONL rows are skipped, oversized traces truncated). + +## 6. Compatibility strategy + +- Old `usage.jsonl` rows (no `routeDecision`) parse unchanged. +- New rows remain valid for every existing reader (`requestLogEntryFromPersistedUsage`, + `/api/logs`, `/api/usage`, per-key rollups) because the field is additive + and the normalizer whitelists it. +- Existing config files need no migration: `routingProfiles` is optional and + absent-by-default. +- `/api/logs` contract unchanged by RI-02; the new `/api/request-history` is + additive. +- Combo and Codex pool behavior is untouched by all ten PRs; policy routing + is inert until a `policy/`/alias model id is requested. +- All new API/CLI output uses stable wire codes plus display-ready summaries; + no locale strings in persisted data. + +## 7. Migration and recovery strategy + +- RI-01: additive field; no migration. Hydration of old rows is automatic. +- RI-02: first open creates the schema; subsequent opens append by offset. + `ocx logs rebuild-index` forces a full rebuild. Corruption => automatic + rebuild (documented in `last_error` + `rebuilt_at`). +- RI-04+: config validation rejects invalid profiles at load; a broken config + fails closed exactly like a broken combo (existing `validateConfigCandidate` + behavior) and never falls back to scoring existing routes. +- Rollback: each PR is independently revertable. Removing RI-01's field write + is safe (readers ignore it); removing RI-02 deletes only the disposable + index; removing RI-04..10 leaves canonical routing byte-identical. + +## 8. PR dependency graph + +```text +dev (upstream e44d234f) +`- RI-01 feat/ri-01-route-decision-traces + `- RI-02 feat/ri-02-request-history-index + `- RI-03 feat/ri-03-routing-analytics + `- RI-04 feat/ri-04-policy-profile-core + `- RI-05 feat/ri-05-capability-aware-routing + `- RI-06 feat/ri-06-health-aware-routing + `- RI-07 feat/ri-07-quota-aware-routing + `- RI-08 feat/ri-08-cost-aware-routing + `- RI-09 feat/ri-09-route-explainability-api + `- RI-10 feat/ri-10-routing-intelligence-ui +``` + +Each PR targets its predecessor's head while open; after merge it can be +retargeted to `dev`. Branches are pushed to `origin` (Wibias fork), draft PRs +target `lidge-jun/opencodex:dev`. **Never merged by this programme.** + +## 9. Acceptance criteria per PR + +- **RI-01**: traces exist for all five existing route kinds; same + provider/model before/after; combo/pool tests unchanged; old JSONL rows + parse; oversized traces truncate deterministically; no secrets/prompts in + traces; trace round-trips through usage.jsonl -> `/api/logs`. +- **RI-02**: index rebuilds from empty/missing/corrupt/truncated/replaced + JSONL; partial final line skipped; duplicate replay ignored; cursor stable + under append; invalid cursor 400; page bounds enforced; `/api/logs` and + usage.jsonl untouched; `ocx logs rebuild-index` works. +- **RI-03**: success/failure/fallback rates, attempt count, p50/p95/p99 + duration + TTFT, incomplete-stream rate, cooldown-failure count, + provider/model/account breakdown, profile breakdown, estimated cost per + success, coverage + confidence + truncation indicators - all sourced from + the index; no routing change. +- **RI-04**: schema validation, collision rules, revision digest, normalized + weights, dry-run evaluator, management API + CLI read/dry-run; no production + routing change. +- **RI-05**: `policy/`/alias requests execute with hard capability + requirements (context window, text/image input, tools, structured output, + reasoning effort, service tier, local/remote, encrypted-Codex-task + readability) using catalog/registry evidence; unknown handling per profile; + deterministic priority scoring; all other routes unchanged. +- **RI-06**: health score from cooldown, recent success rate, consecutive + failures, incomplete-stream rate, latency, sample count, recency, failure + class; network-neutral and client-cancel never damage health; low sample = + reduced confidence; cooldown stays authoritative; components in trace. +- **RI-07**: quota evidence from existing sources with unknown-safe handling; + headroom preference; reset/exhausted/reauth state; reserved headroom; + profile min-headroom; exact account selectors fail closed; pool strategies + authoritative; selection/account-pool boundary documented. +- **RI-08**: cost estimate + price source + incomplete flag + limit + exclusion + cost component in trace; no billing/budgets. +- **RI-09**: `/api/request-history/:id/route-decision`, + `/api/routing-profiles`, `/api/routing-profiles/dry-run`, `ocx logs explain + `, `ocx route policy evaluate `; stable codes + display + summaries; attempt sequence + outcome included. +- **RI-10**: GUI surfaces (profiles list/detail/candidates/requirements/ + weights/unknown-evidence/dry-run; why-this-route with exclusions, scores, + fallback timeline; analytics) with existing grammar, no invented green + health for unknown evidence, accessible + keyboard-operable, all locale + keys present in all six GUI locales (en/de/ja/ko/ru/zh), responsive; docs + (config, CLI, API, migration, combo-vs-profile distinction) in all five + docs-site locales (en/ja/ko/ru/zh-cn - the docs site intentionally ships + no German edition; the GUI does). + +## 10. Test strategy + +Per-PR gates: `bun x tsc --noEmit`, focused `bun run test `, and +`bun run privacy:scan`; full suite once per PR where feasible (recorded in +ledger). GUI PRs add `bun run lint:gui`, GUI unit tests, production build, +locale parity, and a screenshot in the PR body (required by `enforce-target` +when the description mentions `gui`). Docs PRs run the docs build and link +validation where available. + +Mandatory matrix coverage (per the programme brief): + +- Trace: explicit routes unchanged; combo routes unchanged; old JSONL parse; + oversized truncation; no secrets; corruption; trace matches decision. +- Index: empty, large, corrupt, missing DB, old schema, partial final line, + replacement, truncation, duplicate replay, concurrent query during append, + cursor stability, invalid cursor, page bounds, rebuild equivalence. +- Policy: explicit routes unchanged; combo unchanged; alias collision; all + candidates excluded; unknown capability/health/quota/price; deterministic + tie; hard cost limit; cooldown; client cancellation; account-neutral + network error; fallback after selected-target failure; trace-exactly- + matches-decision. + +## 11. Rollback strategy + +Per-PR revert order is reverse of the stack. RI-01 removal is lossless for +new features (field simply stops being written); RI-02 removal deletes a +disposable index; RI-04..10 removal restores canonical routing with zero +behavioral delta. Config files written with `routingProfiles` remain valid in +older versions (unknown top-level keys are tolerated; to be verified against +`validateConfigCandidate` behavior in RI-04 and documented). + +## 12. Rejected alternatives + +- Separate trace store (ADR-2): ordering/commit hazards, duplicated privacy + surface. +- Full-file JSONL scan for analytics instead of SQLite index: O(file) per + query, already bounded at 64 MiB/200k entries for management reads; fails + the full-history query requirement. +- Third-party DB (better-sqlite3 etc.): `bun:sqlite` is available and used; + a new native dependency is unjustified. +- Implicit candidate expansion ("all models on the internet"): rejected for + determinism, privacy, and explainability; v1 is explicit allowlist only. +- Automatic weight self-tuning from analytics: rejected (ADR-10); explainable + routing is the product. +- ML-based health prediction: rejected (ADR-11); transparent constants. +- Persisting raw quota responses / upstream bodies for richer traces: + rejected by the privacy threat model. + +## 13. Final user-facing behavior + +- Every request records a bounded, privacy-safe route-decision trace that + answers: which candidates were considered, why each was rejected/penalized, + which evidence was used, which profile/revision decided, what was selected, + and what happened during fallback. +- `ocx logs explain ` and the GUI "why this route?" view render + the full explanation from the durable trace + attempts + outcome. +- `/api/request-history` gives cursor-paginated full-history querying with + filters; `/api/routing-analytics` gives reliability/latency/cost + breakdowns with confidence and truncation indicators. +- `routingProfiles` in config.json let a user declare policy profiles + (`policy/fast` or a public alias) with hard requirements, optimization + weights, cost limits, and unknown-evidence policy; dry-run evaluation is + available via CLI and management API before any traffic is routed. +- Explicitly requested policy profiles execute with capability-aware scoring; + health, quota and cost scoring are additive, deterministic, and auditable. +- Nothing auto-tunes; routing stays explainable and user-configured. + +## 14. Exact non-goals + +- No Agent Fabric tasks, harness handoffs, ACP, A2A, worktree orchestration, + agent teams, or portable coding-task state. +- No automatic self-tuning or silent profile mutation. +- No monthly billing, invoicing, or hidden automatic budgets. +- No implicit candidate discovery beyond the configured allowlist. +- No changes to explicit account/provider/native/combo/default routing + behavior. +- No re-implementation of #914/#919 transport attribution (owned by #922/ + #966; consumed as evidence). +- No changes to Codex pool selection order (#715) or cooldown recovery probes + (#915); RI-06/07 only read their state. +- No prompt/message/tool content persistence anywhere. +- No generic enterprise dashboard redesign in RI-10. + +## 15. Related docs + +- Stack ledger: `devlog/_plan/260804_router_intelligence/001_pr_stack_status.md` +- Combos vs policy profiles: `docs-site/src/content/docs/guides/combos.md` + (extended in RI-10); configuration reference `routing.md` (extended in + RI-04/RI-10). diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md new file mode 100644 index 0000000000..11c8d3fb57 --- /dev/null +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -0,0 +1,93 @@ +# 001 - PR stack status ledger + +Continuously updated during the programme. Every branch records: base SHA, +head SHA, PR number/URL, verification result, and review state. + +## Programme facts + +- Stack base (dev): `e44d234f08e03dd4dbf0c4aa13af43046d86b0a6` (`upstream/dev`) +- `origin/dev` (fork, stale ancestor): `be177ea501e5007f4a56d19d069ef5cd76ea24b9` +- Bun: `1.3.14`; package version: `2.10.0` +- Worktree: `D:\codex-worktrees\ocx-router-intelligence` +- Push remote: `origin` (Wibias/opencodex); PR target: `lidge-jun/opencodex:dev` +- All PRs opened as DRAFT; nothing merged by this programme. + +## Related in-flight PRs (not superseded by this stack) + +| PR | Branch | Note | +|---|---|---| +| #922 | `fix/914-account-neutral-network` | #914 alternative; consumed as health evidence input by RI-06 | +| #966 | `codex/260804-issue914-transport-attribution` | #914 alternative; consumed as health evidence input by RI-06 | +| #715 | `feat/priority-levels` | Pool selection order; out of scope | +| #988 | `codex/providers-copy-doctor` | GUI providers/combos; conflict-checked at RI-10 | +| #998 | `codex/260803-integration-switches` | Write substrate; rebase watch on request-log.ts | + +No open PR found that implements the same vertical as any PR in this stack, +so no stale PR is closed by this programme. Both #914 drafts overlap each +other; closing one is a maintainer decision and neither is stale. + +## Baseline + +- Full-suite baseline on clean `upstream/dev` (worktree + `D:\codex-worktrees\ocx-typecheck-base`, head `e44d234f0`): running in + background; exact pass/fail counts appended here when done. +- `bun x tsc --noEmit` on clean `upstream/dev`: **PASSED** (0 errors, verified + in the pristine base worktree). +- `bun run privacy:scan`: passed per-PR (see RI-01 below). + +## Stack status + +| RI | Branch | Base | Head SHA | PR | URL | Status | +|---|---|---|---|---|---|---| +| RI-01 | `feat/ri-01-route-decision-traces` | `e44d234f0` | pending | pending | pending | in progress | +| RI-02 | `feat/ri-02-request-history-index` | `feat/ri-01` head | pending | pending | pending | queued | +| RI-03 | `feat/ri-03-routing-analytics` | `feat/ri-02` head | pending | pending | pending | queued | +| RI-04 | `feat/ri-04-policy-profile-core` | `feat/ri-03` head | pending | pending | pending | queued | +| RI-05 | `feat/ri-05-capability-aware-routing` | `feat/ri-04` head | pending | pending | pending | queued | +| RI-06 | `feat/ri-06-health-aware-routing` | `feat/ri-05` head | pending | pending | pending | queued | +| RI-07 | `feat/ri-07-quota-aware-routing` | `feat/ri-06` head | pending | pending | pending | queued | +| RI-08 | `feat/ri-08-cost-aware-routing` | `feat/ri-07` head | pending | pending | pending | queued | +| RI-09 | `feat/ri-09-route-explainability-api` | `feat/ri-08` head | pending | pending | pending | queued | +| RI-10 | `feat/ri-10-routing-intelligence-ui` | `feat/ri-09` head | pending | pending | pending | queued | + +## Per-PR acceptance log + +### RI-01 - feat/ri-01-route-decision-traces + +- Base SHA: `e44d234f08e03dd4dbf0c4aa13af43046d86b0a6` +- Reviewed commits: + - `b5a8e7c4c` (implementation; author self-review + CodeRabbit review) + - `2e0522b2` (privacy-scan fix after CI `gates` failure) + - `pending` (CodeRabbit findings round; recorded after commit) +- Findings (self-review): 3 test failures caught pre-push - (1) missing value + import for `normalizeRouteDecisionTrace` in request-log hydration, + (2) selected combo target marked ineligible because `ComboPick.attempted` + includes the winner, (3) account-namespace fixture missing the canonical + ChatGPT forward `baseUrl` (test-fixture bug, not product code). +- Fixes: import fixed; `comboRouteCandidates` now excludes the selected target + from `already-attempted`; fixture uses `https://chatgpt.com/backend-api/codex`. +- Regression tests: all three cases are covered by the final + `tests/route-decision-trace.test.ts` (14 tests, 75 assertions). +- Findings (CodeRabbit, verified against code): 12 comments - 9 accepted + (locale/plan docs, requestedModel bound doc, ledger SHA, combo tieBreak + + duplicate getCombo, `truncated.requirements` flag, byte-accurate budget, + parse-once evidence, hydration guard drops invalid traces, 2 regression + tests, credential-test assertion hardening); 2 design-judgment comments + (persist trace on every row - kept: bounded ~200 B single-candidate traces, + plan mandates one trace per decision; docstring coverage - docstrings + added to trace helpers); the privacy-scan finding was already fixed in + `2e0522b2`. +- Final commit: recorded after commit (round applies CodeRabbit + simplify + fixes; new head pushes to #1003) +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run test tests/route-decision-trace.test.ts`: 14/14 pass + - Focused regression suites: 253/253 pass across combos, codex-routing, + usage-log, request-log, combo-management-api, codex-account-namespaces + - `tests/server-combo-failover-e2e.test.ts`: 44/44 pass + - `bun run privacy:scan`: passed +- Remaining Low findings: none + +### RI-02..RI-10 + +Appended as each PR is implemented. diff --git a/src/router.ts b/src/router.ts index 25aba0902c..fd8af30f15 100644 --- a/src/router.ts +++ b/src/router.ts @@ -1,5 +1,13 @@ import type { CodexAccountMode, OcxConfig, OcxProviderConfig } from "./types"; -import { preservesPhysicalComboProvider, tryPickComboModel, type ComboPick } from "./combos"; +import { + getCombo, + isComboTargetInCooldown, + preservesPhysicalComboProvider, + targetKey, + tryPickComboModel, + type ComboPick, +} from "./combos"; +import type { NormalizedComboConfig } from "./combos/types"; import { hasOwnProvider, resolveEnvValue } from "./config"; import { assertProviderDestinationAllowed } from "./lib/destination-policy"; import { redactSecretString, redactUrlForLog } from "./lib/redact"; @@ -14,17 +22,29 @@ import { import { decodeRoutedModelId, encodeRoutedModelId } from "./providers/slug-codec"; import { getStaleCached } from "./codex/model-cache"; import { codexAccountNamespaceEntries } from "./codex/account-namespaces"; +import { + buildRouteDecisionTrace, + type RouteDecisionKind, + type RouteDecisionTraceV1, + type TraceCandidateInput, +} from "./routing/trace"; export interface RouteResult { providerName: string; provider: OcxProviderConfig; modelId: string; + /** Which deterministic routing path produced this route (RI-01). */ + routeKind: RouteDecisionKind; + /** Stable wire reason code for the selected route (RI-01). */ + routeReason: string; codexAccountMode?: CodexAccountMode; /** Exact account selected by an account-qualified native model. */ codexAccountId?: string; /** Public namespace used by the account-qualified selector. */ codexAccountNamespace?: string; combo?: ComboPick; + /** Bounded route-decision trace (RI-01); never contains secrets. */ + routeDecision?: RouteDecisionTraceV1; } const MODEL_PROVIDER_PATTERNS: Array<{ providerNames: string[]; prefixes: string[] }> = [ @@ -335,6 +355,34 @@ export class NoEnabledOpenAiProviderError extends Error { } } +/** + * One immutable selection trace for a combo request: built once from the + * initial pick, before any child dispatch. Fallback execution stays in the + * usage entry's `attempts[]`; the trace never changes after selection. + */ +export function comboRouteDecisionTrace( + config: OcxConfig, + comboId: string, + pick: ComboPick, + requestedModel: string, +): RouteDecisionTraceV1 { + const combo = getCombo(config, comboId); + return buildRouteDecisionTrace({ + requestedModel, + routeKind: "combo", + selected: { + provider: pick.target.provider, + model: pick.target.model, + reason: "combo-pick", + candidateIndex: pick.targetIndex, + ...(combo + ? { tieBreak: combo.strategy === "round-robin" ? "round-robin" : "failover" } + : {}), + }, + candidates: combo ? comboRouteCandidates(config, pick, combo) : undefined, + }); +} + // Codex uses a small number of control-plane model ids that are not part of the public GPT/o // naming families. Keep this exact: a broad `codex-*` rule could capture a third-party model. const CODEX_INTERNAL_OPENAI_MODELS = new Set(["codex-auto-review"]); @@ -344,16 +392,63 @@ function isBareOpenAiFamilyModel(modelId: string): boolean { && (/^(?:gpt-|o1-|o3-|o4-)/.test(modelId) || CODEX_INTERNAL_OPENAI_MODELS.has(modelId)); } -function routeResult(providerName: string, provider: OcxProviderConfig, modelId: string): RouteResult { +function routeResult( + providerName: string, + provider: OcxProviderConfig, + modelId: string, + routeKind: RouteDecisionKind, + routeReason: string, +): RouteResult { const codexAccountMode = providerCodexAccountMode(providerName, provider); return { providerName, provider: routedProviderConfig(providerName, provider), modelId, + routeKind, + routeReason, ...(codexAccountMode ? { codexAccountMode } : {}), }; } +/** + * Candidate evidence for a combo route: every configured target with its + * selection-time eligibility and exclusion reasons. Purely observational; the + * pick already happened and this never re-selects. + */ +function comboRouteCandidates( + config: OcxConfig, + pick: NonNullable, + combo: NormalizedComboConfig, +): TraceCandidateInput[] { + const now = Date.now(); + return combo.targets.map((target, index) => { + const key = targetKey(target); + const provider = config.providers[target.provider]; + const configured = provider !== undefined; + const enabled = configured && provider.disabled !== true; + const inCooldown = isComboTargetInCooldown(pick.comboId, target, now); + const isSelected = index === pick.targetIndex; + // The pick's `attempted` list includes the winner itself; only non-selected + // targets can be "already-attempted" (fallback picks exclude earlier tries). + const alreadyAttempted = !isSelected && pick.attempted.includes(key); + const exclusions: TraceCandidateInput["exclusions"] = []; + if (!configured) exclusions.push({ code: "unconfigured" }); + if (configured && !enabled) exclusions.push({ code: "disabled" }); + if (inCooldown) exclusions.push({ code: "cooldown" }); + if (isSelected && inCooldown) exclusions.push({ code: "selected-despite-cooldown" }); + if (!isSelected && alreadyAttempted && exclusions.length === 0) { + exclusions.push({ code: "already-attempted" }); + } + if (!isSelected && exclusions.length === 0) exclusions.push({ code: "not-selected" }); + return { + provider: target.provider, + model: target.model, + eligible: enabled && !inCooldown && !alreadyAttempted, + exclusions, + }; + }); +} + function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: boolean): RouteResult { const slash = modelId.indexOf("/"); if (slash > 0) { @@ -378,7 +473,7 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo throw new NoEnabledOpenAiProviderError(nativeModelId); } return { - ...routeResult(OPENAI_CODEX_PROVIDER_ID, provider, nativeModelId), + ...routeResult(OPENAI_CODEX_PROVIDER_ID, provider, nativeModelId, "explicit-account", "account-namespace"), // Exact account injection uses the pool credential machinery even when the canonical // provider is globally Direct. The fixed id bypasses pool selection entirely. codexAccountMode: "pool", @@ -395,7 +490,7 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo // The selected target is already a concrete provider/model reference. Resolve it without // consulting combo aliases again, otherwise an alias that shadows the target can recurse. const routed = routeModelInternal(config, concrete, true); - return { ...routed, combo }; + return { ...routed, combo, routeKind: "combo" as const, routeReason: "combo-pick" }; } } @@ -415,23 +510,33 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo // Self-namespaced native id — the vendor segment equals the provider id, so the FULL ref is // itself a known model (e.g. orcarouter/auto). Route it whole instead of stripping to the // remainder, which would send a bare `auto` the upstream cannot resolve. - if (known.includes(modelId)) return routeResult(provName, prov, modelId); + if (known.includes(modelId)) { + return routeResult(provName, prov, modelId, "explicit-provider", "explicit-provider-namespace"); + } // Codex-facing alias ids (`provider/vendor-model`) decode back to the native // slash id via an exact known-id lookup; raw full-slash selectors keep working. - return routeResult(provName, prov, decodeRoutedModelId(modelId.slice(slash + 1), known)); + return routeResult( + provName, + prov, + decodeRoutedModelId(modelId.slice(slash + 1), known), + "explicit-provider", + "explicit-provider-namespace", + ); } } if (isBareOpenAiFamilyModel(modelId)) { const provider = config.providers[OPENAI_CODEX_PROVIDER_ID]; - if (provider && provider.disabled !== true) return routeResult(OPENAI_CODEX_PROVIDER_ID, provider, modelId); + if (provider && provider.disabled !== true) { + return routeResult(OPENAI_CODEX_PROVIDER_ID, provider, modelId, "native", "native-family"); + } throw new NoEnabledOpenAiProviderError(modelId); } for (const [provName, prov] of activeProviderEntries(config)) { if (prov.defaultModel === modelId || (typeof prov.defaultModel === "string" && encodeRoutedModelId(prov.defaultModel) === modelId)) { - return routeResult(provName, prov, prov.defaultModel as string); + return routeResult(provName, prov, prov.defaultModel as string, "explicit-provider", "configured-default-model"); } } @@ -441,7 +546,9 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo for (const [provName, prov] of activeProviderEntries(config)) { if (prov.models && Array.isArray(prov.models)) { const hit = (prov.models as string[]).find(id => id === modelId || encodeRoutedModelId(id) === modelId); - if (hit !== undefined) return routeResult(provName, prov, hit); + if (hit !== undefined) { + return routeResult(provName, prov, hit, "explicit-provider", "configured-model-list"); + } } } @@ -451,14 +558,34 @@ function routeModelInternal(config: OcxConfig, modelId: string, bypassCombos: bo if (hasOwnProvider(config.providers, config.defaultProvider)) { const defaultProv = config.providers[config.defaultProvider]; if (defaultProv.disabled === true) throw new Error(`Default provider is disabled: ${config.defaultProvider}`); - return routeResult(config.defaultProvider, defaultProv, modelId); + return routeResult(config.defaultProvider, defaultProv, modelId, "default-provider", "default-provider"); } throw new Error(`No provider configured for model: ${modelId}`); } export function routeModel(config: OcxConfig, modelId: string): RouteResult { - return routeModelInternal(config, modelId, false); + const route = routeModelInternal(config, modelId, false); + const accountRef = route.codexAccountNamespace; + const combo = route.combo ? getCombo(config, route.combo.comboId) : undefined; + route.routeDecision = buildRouteDecisionTrace({ + requestedModel: modelId, + routeKind: route.routeKind, + selected: { + provider: route.providerName, + model: route.modelId, + ...(accountRef ? { accountRef } : {}), + reason: route.routeReason, + ...(route.combo ? { candidateIndex: route.combo.targetIndex } : {}), + ...(combo + ? { tieBreak: combo.strategy === "round-robin" ? "round-robin" : "failover" } + : {}), + }, + candidates: route.routeKind === "combo" && route.combo && combo + ? comboRouteCandidates(config, route.combo, combo) + : undefined, + }); + return route; } function routeByKnownModelPattern(config: OcxConfig, modelId: string): RouteResult | undefined { @@ -469,7 +596,7 @@ function routeByKnownModelPattern(config: OcxConfig, modelId: string): RouteResu ); if (matchingProvider) { const [provName, prov] = matchingProvider; - return routeResult(provName, prov, modelId); + return routeResult(provName, prov, modelId, "explicit-provider", "model-pattern"); } } } diff --git a/src/routing/trace.ts b/src/routing/trace.ts new file mode 100644 index 0000000000..2215e3655a --- /dev/null +++ b/src/routing/trace.ts @@ -0,0 +1,662 @@ +/** + * Route decision trace: bounded, versioned, privacy-safe evidence of WHY a + * provider/model/account was selected for a request (RI-01). + * + * Contract rules (devlog/_plan/260804_router_intelligence/000_master_plan.md): + * - One trace per routing decision; fallback EXECUTION attempts stay in the + * usage entry's existing `attempts[]` array, never in this trace. + * - Never persists prompts, message bodies, tool payloads, credentials, + * authorization headers, raw quota responses, or hidden reasoning. + * - Stable wire values only; no localized strings. + * - Deterministic bounds: candidates <= 8, exclusions per candidate <= 16, + * requirements <= 16, strings <= 128 chars, serialized trace <= 16 KiB. + * - Truncation is explicit: `truncated` flags are set whenever a limit was + * enforced. + */ + +import { randomBytes } from "node:crypto"; + +export type RouteDecisionKind = + | "explicit-account" + | "explicit-provider" + | "native" + | "combo" + | "policy" + | "default-provider"; + +export type Unknownable = number | boolean | "unknown"; + +export interface RouteRequirementEvidence { + /** Stable wire code, e.g. "min-context-window" | "tools" | "image-input". */ + id: string; + expected?: string | number | boolean; + actual?: Unknownable | string; + outcome: "satisfied" | "unsatisfied" | "unknown"; +} + +export interface RouteExclusionReason { + /** Stable wire code, e.g. "cooldown" | "disabled" | "cost-limit". */ + code: string; + detail?: string; +} + +export interface RouteCapabilityEvidence { + contextWindow?: number; + tools?: Unknownable; + image?: Unknownable; + structuredOutput?: Unknownable; + reasoningEfforts?: string[]; + serviceTier?: Unknownable; + localOnly?: Unknownable; + remoteAllowed?: Unknownable; + encryptedCodexTasks?: Unknownable; +} + +export interface RouteHealthEvidence { + cooldownUntilMs?: number; + softAvoidUntilMs?: number; + successRate?: number; + failures?: number; + incompleteStreamRate?: number; + recentLatencyMs?: number; + sampleCount?: number; + recencyWeight?: number; +} + +export interface RouteQuotaEvidence { + known: boolean; + headroomTokens?: number; + exhausted?: boolean; + resetAtMs?: number; + reauthOrCooling?: boolean; + reservedHeadroomTokens?: number; + /** Stable wire code for the evidence source (e.g. "provider-report"). */ + source?: string; +} + +export interface RouteCostEvidence { + estimatedUsd?: number; + /** Stable wire code for the price source (e.g. "registry" | "expected"). */ + priceSource?: string; + incomplete?: boolean; + limitUsd?: number; + excludedByLimit?: boolean; +} + +export interface RouteScoreEvidence { + total: number; + components: { + capability?: number; + health?: number; + quota?: number; + cost?: number; + latency?: number; + configuredPriority?: number; + }; +} + +export interface RouteCandidateTrace { + provider: string; + model: string; + accountRef?: string; + eligible: boolean; + exclusions: RouteExclusionReason[]; + capability?: RouteCapabilityEvidence; + health?: RouteHealthEvidence; + quota?: RouteQuotaEvidence; + cost?: RouteCostEvidence; + score?: RouteScoreEvidence; +} + +export interface RouteDecisionTraceV1 { + version: 1; + decisionId: string; + createdAt: number; + requestedModel: string; + routeKind: RouteDecisionKind; + profile?: { id: string; revision: string }; + requirements: RouteRequirementEvidence[]; + candidates: RouteCandidateTrace[]; + selected: { + candidateIndex: number; + provider: string; + model: string; + accountRef?: string; + reason: string; + tieBreak?: string; + }; + truncated?: { + candidates?: true; + exclusions?: true; + requirements?: true; + strings?: true; + }; +} + +export const MAX_TRACE_CANDIDATES = 8; +export const MAX_EXCLUSIONS_PER_CANDIDATE = 16; +export const MAX_REQUIREMENTS = 16; +export const MAX_TRACE_STRING = 128; +export const MAX_TRACE_BYTES = 16 * 1024; + +const ROUTE_KINDS = new Set([ + "explicit-account", + "explicit-provider", + "native", + "combo", + "policy", + "default-provider", +]); + +const REQUIREMENT_OUTCOMES = new Set(["satisfied", "unsatisfied", "unknown"]); + +/** Cap a string at MAX_TRACE_STRING and record the truncation flag. */ +function capString(value: string, budget: { strings?: true }): string { + if (value.length <= MAX_TRACE_STRING) return value; + budget.strings = true; + return value.slice(0, MAX_TRACE_STRING); +} + +function isPlainRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function finiteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function unknownable(value: unknown): Unknownable | undefined { + if (typeof value === "boolean") return value; + if (finiteNumber(value)) return value; + if (value === "unknown") return "unknown"; + return undefined; +} + +export interface TraceCandidateInput { + provider: string; + model: string; + accountRef?: string; + eligible: boolean; + exclusions: RouteExclusionReason[]; +} + +export interface TraceBuildInput { + requestedModel: string; + routeKind: RouteDecisionKind; + selected: { + provider: string; + model: string; + accountRef?: string; + reason: string; + tieBreak?: string; + /** Index into `candidates`; defaults to 0. */ + candidateIndex?: number; + }; + profile?: { id: string; revision: string }; + requirements?: RouteRequirementEvidence[]; + candidates?: TraceCandidateInput[]; + now?: number; +} + +/** Bounded candidate copy: strings capped, exclusions sliced, score/evidence kept. */ +function buildCandidate(input: TraceCandidateInput, budget: { strings?: true; exclusions?: true }): RouteCandidateTrace { + const exclusions = input.exclusions.slice(0, MAX_EXCLUSIONS_PER_CANDIDATE); + if (exclusions.length < input.exclusions.length) budget.exclusions = true; + return { + provider: capString(input.provider, budget), + model: capString(input.model, budget), + ...(input.accountRef !== undefined + ? { accountRef: capString(input.accountRef, budget) } + : {}), + eligible: input.eligible, + exclusions: exclusions.map(exclusion => ({ + code: capString(exclusion.code, budget), + ...(exclusion.detail !== undefined + ? { detail: capString(exclusion.detail, budget) } + : {}), + })), + }; +} + +/** Bounded requirement copy with capped strings. */ +function buildRequirement(requirement: RouteRequirementEvidence, budget: { strings?: true }): RouteRequirementEvidence { + return { + id: capString(requirement.id, budget), + ...(requirement.expected !== undefined + ? { expected: typeof requirement.expected === "string" + ? capString(requirement.expected, budget) + : requirement.expected } + : {}), + ...(requirement.actual !== undefined + ? { actual: typeof requirement.actual === "string" + ? capString(requirement.actual, budget) + : requirement.actual } + : {}), + outcome: requirement.outcome, + }; +} + +/** + * Build a bounded decision trace. The builder never receives credentials: callers + * pass provider/model NAME strings and opaque account references only. + */ +export function buildRouteDecisionTrace(input: TraceBuildInput): RouteDecisionTraceV1 { + const budget: { strings?: true; exclusions?: true; candidates?: true } = {}; + const now = input.now ?? Date.now(); + const truncated: RouteDecisionTraceV1["truncated"] = {}; + let selectedIndex = Number.isInteger(input.selected.candidateIndex ?? 0) + ? (input.selected.candidateIndex ?? 0) + : 0; + + let candidates = (input.candidates ?? []).map(candidate => buildCandidate(candidate, budget)); + if (candidates.length > MAX_TRACE_CANDIDATES) { + // Keep the selected candidate even when it sits beyond the slice: a trace + // whose selected candidate vanished would contradict the decision. + candidates = selectedIndex < MAX_TRACE_CANDIDATES + ? candidates.slice(0, MAX_TRACE_CANDIDATES) + : [...candidates.slice(0, MAX_TRACE_CANDIDATES - 1), candidates[selectedIndex]!]; + selectedIndex = Math.min(selectedIndex, candidates.length - 1); + truncated.candidates = true; + } + if (candidates.length === 0) { + // Invariant: every decision names at least the selected route as a candidate. + candidates = [{ + provider: capString(input.selected.provider, budget), + model: capString(input.selected.model, budget), + ...(input.selected.accountRef !== undefined + ? { accountRef: capString(input.selected.accountRef, budget) } + : {}), + eligible: true, + exclusions: [], + }]; + } + + let requirements = (input.requirements ?? []).map(requirement => buildRequirement(requirement, budget)); + if (requirements.length > MAX_REQUIREMENTS) { + requirements = requirements.slice(0, MAX_REQUIREMENTS); + truncated.requirements = true; + } + + if (selectedIndex < 0 || selectedIndex >= candidates.length) selectedIndex = 0; + + const trace: RouteDecisionTraceV1 = { + version: 1, + decisionId: randomBytes(6).toString("hex"), + createdAt: now, + requestedModel: capString(input.requestedModel, budget), + routeKind: input.routeKind, + ...(input.profile + ? { + profile: { + id: capString(input.profile.id, budget), + revision: capString(input.profile.revision, budget), + }, + } + : {}), + requirements, + candidates, + selected: { + candidateIndex: selectedIndex, + provider: capString(input.selected.provider, budget), + model: capString(input.selected.model, budget), + ...(input.selected.accountRef !== undefined + ? { accountRef: capString(input.selected.accountRef, budget) } + : {}), + reason: capString(input.selected.reason, budget), + ...(input.selected.tieBreak !== undefined + ? { tieBreak: capString(input.selected.tieBreak, budget) } + : {}), + }, + }; + + if (budget.strings) truncated.strings = true; + if (budget.exclusions) truncated.exclusions = true; + if (budget.candidates) truncated.candidates = true; + if (Object.keys(truncated).length > 0) trace.truncated = truncated; + + return enforceByteBudget(trace); +} + +/** Serialized UTF-8 length of a value (JSON is measured in bytes, not code units). */ +function serializedByteLength(value: unknown): number { + return Buffer.byteLength(JSON.stringify(value), "utf8"); +} + +/** Deterministic byte-budget enforcement: drop details, then shrink candidates. */ +function enforceByteBudget(trace: RouteDecisionTraceV1): RouteDecisionTraceV1 { + if (serializedByteLength(trace) <= MAX_TRACE_BYTES) return trace; + const truncated = { ...trace.truncated, strings: true as const }; + const candidates = trace.candidates.map(candidate => ({ + ...candidate, + exclusions: candidate.exclusions.map(exclusion => ({ code: exclusion.code })), + })); + const slimmed: RouteDecisionTraceV1 = { ...trace, truncated, candidates }; + if (serializedByteLength(slimmed) <= MAX_TRACE_BYTES) return slimmed; + // Second stage: shrink candidates. The selected candidate must survive and + // `selected.candidateIndex` must keep pointing at it (same invariant as the + // candidate-cap branch above). + const half = Math.max(1, Math.floor(MAX_TRACE_CANDIDATES / 2)); + const selectedIndex = trace.selected.candidateIndex; + const kept = selectedIndex < half + ? slimmed.candidates.slice(0, half) + : [...slimmed.candidates.slice(0, half - 1), slimmed.candidates[selectedIndex]!]; + let result: RouteDecisionTraceV1 = { + ...slimmed, + truncated: { ...truncated, candidates: true as const }, + candidates: kept, + selected: { ...slimmed.selected, candidateIndex: Math.min(selectedIndex, kept.length - 1) }, + }; + // Last resort: keep shrinking deterministically until the byte budget holds. + // Each stage reduces a bounded dimension, and a single candidate with no + // exclusions is provably below MAX_TRACE_BYTES given the string cap, so the + // loop terminates. + for (let stage = 0; stage < 4 && serializedByteLength(result) > MAX_TRACE_BYTES; stage++) { + if (stage === 0) { + result = { + ...result, + truncated: { ...result.truncated, exclusions: true as const }, + candidates: result.candidates.map(candidate => ({ + ...candidate, + exclusions: candidate.exclusions.slice(0, 8), + })), + }; + } else if (stage === 1) { + result = { + ...result, + truncated: { ...result.truncated, exclusions: true as const }, + candidates: result.candidates.map(candidate => ({ ...candidate, exclusions: [] })), + }; + } else { + const half = Math.max(1, Math.ceil(result.candidates.length / 2)); + const index = result.selected.candidateIndex; + const shrinkKept = index < half + ? result.candidates.slice(0, half) + : [...result.candidates.slice(0, half - 1), result.candidates[index]!]; + result = { + ...result, + truncated: { ...result.truncated, candidates: true as const }, + candidates: shrinkKept, + selected: { ...result.selected, candidateIndex: Math.min(index, shrinkKept.length - 1) }, + }; + } + } + return result; +} + +// ---- defensive parsing of persisted rows -------------------------------------- + +/** Caps applied by the normalizer; unioned into `truncated` on the result. */ +interface ParseCaps { + candidates?: true; + exclusions?: true; + requirements?: true; + strings?: true; +} + +/** Defensive parse of one persisted exclusion reason. */ +function parseExclusion(raw: unknown, caps: ParseCaps): RouteExclusionReason | null { + if (!isPlainRecord(raw)) return null; + const code = raw.code; + if (typeof code !== "string" || code.length === 0) return null; + if (code.length > MAX_TRACE_STRING) caps.strings = true; + const out: RouteExclusionReason = { code: code.slice(0, MAX_TRACE_STRING) }; + if (typeof raw.detail === "string") { + if (raw.detail.length > MAX_TRACE_STRING) caps.strings = true; + out.detail = raw.detail.slice(0, MAX_TRACE_STRING); + } + return out; +} + +/** Defensive parse of one persisted requirement; rejects unknown outcomes. */ +function parseRequirement(raw: unknown, caps: ParseCaps): RouteRequirementEvidence | null { + if (!isPlainRecord(raw)) return null; + const id = raw.id; + const outcome = raw.outcome; + if (typeof id !== "string" || id.length === 0) return null; + if (typeof outcome !== "string" || !REQUIREMENT_OUTCOMES.has(outcome)) return null; + if (id.length > MAX_TRACE_STRING) caps.strings = true; + const out: RouteRequirementEvidence = { + id: id.slice(0, MAX_TRACE_STRING), + outcome: outcome as RouteRequirementEvidence["outcome"], + }; + if (typeof raw.expected === "string") { + if (raw.expected.length > MAX_TRACE_STRING) caps.strings = true; + out.expected = raw.expected.slice(0, MAX_TRACE_STRING); + } else if (typeof raw.expected === "number" || typeof raw.expected === "boolean") { + out.expected = raw.expected; + } + if (typeof raw.actual === "string") { + if (raw.actual.length > MAX_TRACE_STRING) caps.strings = true; + out.actual = raw.actual.slice(0, MAX_TRACE_STRING); + } else if (unknownable(raw.actual) !== undefined) { + out.actual = unknownable(raw.actual); + } + return out; +} + +/** Whitelisted capability-evidence parse; unknown fields are dropped. */ +function parseCapability(raw: unknown, caps: ParseCaps): RouteCapabilityEvidence | undefined { + if (!isPlainRecord(raw)) return undefined; + const out: RouteCapabilityEvidence = {}; + if (finiteNumber(raw.contextWindow)) out.contextWindow = raw.contextWindow; + const tools = unknownable(raw.tools); + if (tools !== undefined) out.tools = tools; + const image = unknownable(raw.image); + if (image !== undefined) out.image = image; + const structuredOutput = unknownable(raw.structuredOutput); + if (structuredOutput !== undefined) out.structuredOutput = structuredOutput; + if (Array.isArray(raw.reasoningEfforts) + && raw.reasoningEfforts.slice(0, 8).every((value): value is string => typeof value === "string")) { + if (raw.reasoningEfforts.some((value: unknown) => typeof value === "string" + && value.length > MAX_TRACE_STRING)) caps.strings = true; + out.reasoningEfforts = raw.reasoningEfforts + .slice(0, 8) + .map(value => value.slice(0, MAX_TRACE_STRING)); + } + const serviceTier = unknownable(raw.serviceTier); + if (serviceTier !== undefined) out.serviceTier = serviceTier; + const localOnly = unknownable(raw.localOnly); + if (localOnly !== undefined) out.localOnly = localOnly; + const remoteAllowed = unknownable(raw.remoteAllowed); + if (remoteAllowed !== undefined) out.remoteAllowed = remoteAllowed; + const encryptedCodexTasks = unknownable(raw.encryptedCodexTasks); + if (encryptedCodexTasks !== undefined) out.encryptedCodexTasks = encryptedCodexTasks; + return Object.keys(out).length > 0 ? out : undefined; +} + +/** Whitelisted health-evidence parse; non-numeric fields are dropped. */ +function parseHealth(raw: unknown): RouteHealthEvidence | undefined { + if (!isPlainRecord(raw)) return undefined; + const out: RouteHealthEvidence = {}; + if (finiteNumber(raw.cooldownUntilMs)) out.cooldownUntilMs = raw.cooldownUntilMs; + if (finiteNumber(raw.softAvoidUntilMs)) out.softAvoidUntilMs = raw.softAvoidUntilMs; + if (finiteNumber(raw.successRate)) out.successRate = raw.successRate; + if (finiteNumber(raw.failures)) out.failures = raw.failures; + if (finiteNumber(raw.incompleteStreamRate)) out.incompleteStreamRate = raw.incompleteStreamRate; + if (finiteNumber(raw.recentLatencyMs)) out.recentLatencyMs = raw.recentLatencyMs; + if (finiteNumber(raw.sampleCount)) out.sampleCount = raw.sampleCount; + if (finiteNumber(raw.recencyWeight)) out.recencyWeight = raw.recencyWeight; + return Object.keys(out).length > 0 ? out : undefined; +} + +/** Whitelisted quota-evidence parse; requires a boolean `known`. */ +function parseQuota(raw: unknown, caps: ParseCaps): RouteQuotaEvidence | undefined { + if (!isPlainRecord(raw)) return undefined; + if (typeof raw.known !== "boolean") return undefined; + const out: RouteQuotaEvidence = { known: raw.known }; + if (finiteNumber(raw.headroomTokens)) out.headroomTokens = raw.headroomTokens; + if (typeof raw.exhausted === "boolean") out.exhausted = raw.exhausted; + if (finiteNumber(raw.resetAtMs)) out.resetAtMs = raw.resetAtMs; + if (typeof raw.reauthOrCooling === "boolean") out.reauthOrCooling = raw.reauthOrCooling; + if (finiteNumber(raw.reservedHeadroomTokens)) out.reservedHeadroomTokens = raw.reservedHeadroomTokens; + if (typeof raw.source === "string" && raw.source) { + if (raw.source.length > MAX_TRACE_STRING) caps.strings = true; + out.source = raw.source.slice(0, MAX_TRACE_STRING); + } + return out; +} + +/** Whitelisted cost-evidence parse. */ +function parseCost(raw: unknown, caps: ParseCaps): RouteCostEvidence | undefined { + if (!isPlainRecord(raw)) return undefined; + const out: RouteCostEvidence = {}; + if (finiteNumber(raw.estimatedUsd)) out.estimatedUsd = raw.estimatedUsd; + if (typeof raw.priceSource === "string" && raw.priceSource) { + if (raw.priceSource.length > MAX_TRACE_STRING) caps.strings = true; + out.priceSource = raw.priceSource.slice(0, MAX_TRACE_STRING); + } + if (typeof raw.incomplete === "boolean") out.incomplete = raw.incomplete; + if (finiteNumber(raw.limitUsd)) out.limitUsd = raw.limitUsd; + if (typeof raw.excludedByLimit === "boolean") out.excludedByLimit = raw.excludedByLimit; + return Object.keys(out).length > 0 ? out : undefined; +} + +/** Whitelisted score parse; requires a finite `total` and bounded components. */ +function parseScore(raw: unknown): RouteScoreEvidence | undefined { + if (!isPlainRecord(raw)) return undefined; + if (!finiteNumber(raw.total)) return undefined; + const components = isPlainRecord(raw.components) ? raw.components : {}; + const parsedComponents: RouteScoreEvidence["components"] = {}; + for (const key of ["capability", "health", "quota", "cost", "latency", "configuredPriority"] as const) { + if (finiteNumber(components[key])) parsedComponents[key] = components[key]; + } + return { total: raw.total, components: parsedComponents }; +} + +/** Defensive parse of one persisted candidate with bounded evidence blocks. */ +function parseCandidate(raw: unknown, caps: ParseCaps): RouteCandidateTrace | null { + if (!isPlainRecord(raw)) return null; + const provider = raw.provider; + const model = raw.model; + if (typeof provider !== "string" || provider.length === 0) return null; + if (typeof model !== "string" || model.length === 0) return null; + if (provider.length > MAX_TRACE_STRING) caps.strings = true; + if (model.length > MAX_TRACE_STRING) caps.strings = true; + if (typeof raw.accountRef === "string" && raw.accountRef.length > MAX_TRACE_STRING) caps.strings = true; + if (typeof raw.eligible !== "boolean") return null; + if (Array.isArray(raw.exclusions) && raw.exclusions.length > MAX_EXCLUSIONS_PER_CANDIDATE) caps.exclusions = true; + const exclusions = Array.isArray(raw.exclusions) + ? raw.exclusions.slice(0, MAX_EXCLUSIONS_PER_CANDIDATE) + .map(value => parseExclusion(value, caps)) + .filter((value): value is RouteExclusionReason => value !== null) + : []; + const capability = parseCapability(raw.capability, caps); + const health = parseHealth(raw.health); + const quota = parseQuota(raw.quota, caps); + const cost = parseCost(raw.cost, caps); + const score = parseScore(raw.score); + return { + provider: provider.slice(0, MAX_TRACE_STRING), + model: model.slice(0, MAX_TRACE_STRING), + ...(typeof raw.accountRef === "string" + ? { accountRef: raw.accountRef.slice(0, MAX_TRACE_STRING) } + : {}), + eligible: raw.eligible, + exclusions: exclusions.slice(0, MAX_EXCLUSIONS_PER_CANDIDATE), + ...(capability ? { capability } : {}), + ...(health ? { health } : {}), + ...(quota ? { quota } : {}), + ...(cost ? { cost } : {}), + ...(score ? { score } : {}), + }; +} + +/** + * Defensive parse of a persisted trace. Returns null when the row is not a + * version-1 trace; otherwise returns a bounded, whitelisted copy. Invalid + * evidence objects are dropped rather than poisoning the DTO. + */ +export function normalizeRouteDecisionTrace(raw: unknown): RouteDecisionTraceV1 | null { + if (!isPlainRecord(raw)) return null; + if (raw.version !== 1) return null; + const caps: ParseCaps = {}; + const decisionId = raw.decisionId; + const createdAt = raw.createdAt; + const requestedModel = raw.requestedModel; + const routeKind = raw.routeKind; + if (typeof decisionId !== "string" || decisionId.length === 0) return null; + if (!/^[0-9a-f]{12}$/.test(decisionId)) return null; + if (decisionId.length > MAX_TRACE_STRING) caps.strings = true; + if (!finiteNumber(createdAt)) return null; + if (typeof requestedModel !== "string" || requestedModel.length === 0) return null; + if (requestedModel.length > MAX_TRACE_STRING) caps.strings = true; + if (typeof routeKind !== "string" || !ROUTE_KINDS.has(routeKind as RouteDecisionKind)) return null; + if (!Array.isArray(raw.candidates) || raw.candidates.length === 0) return null; + if (raw.candidates.length > MAX_TRACE_CANDIDATES) caps.candidates = true; + + const candidates = raw.candidates + .slice(0, MAX_TRACE_CANDIDATES) + .map(value => parseCandidate(value, caps)) + .filter((value): value is RouteCandidateTrace => value !== null) + .slice(0, MAX_TRACE_CANDIDATES); + if (candidates.length === 0) return null; + + const selected = isPlainRecord(raw.selected) ? raw.selected : null; + if (!selected) return null; + const selectedProvider = selected.provider; + const selectedModel = selected.model; + const selectedReason = selected.reason; + if (typeof selectedProvider !== "string" || typeof selectedModel !== "string") return null; + if (typeof selectedReason !== "string") return null; + if (selectedProvider.length > MAX_TRACE_STRING) caps.strings = true; + if (selectedModel.length > MAX_TRACE_STRING) caps.strings = true; + if (selectedReason.length > MAX_TRACE_STRING) caps.strings = true; + if (typeof selected.accountRef === "string" && selected.accountRef.length > MAX_TRACE_STRING) caps.strings = true; + if (typeof selected.tieBreak === "string" && selected.tieBreak.length > MAX_TRACE_STRING) caps.strings = true; + const candidateIndex = Number.isInteger(selected.candidateIndex) + ? selected.candidateIndex as number + : 0; + if (candidateIndex < 0 || candidateIndex >= candidates.length) return null; + + const rawRequirements = Array.isArray(raw.requirements) ? raw.requirements : []; + if (rawRequirements.length > MAX_REQUIREMENTS) caps.requirements = true; + const requirements = rawRequirements + .slice(0, MAX_REQUIREMENTS) + .map(value => parseRequirement(value, caps)) + .filter((value): value is RouteRequirementEvidence => value !== null) + .slice(0, MAX_REQUIREMENTS); + + const profile = isPlainRecord(raw.profile) + && typeof raw.profile.id === "string" + && typeof raw.profile.revision === "string" + ? (() => { + if (raw.profile.id.length > MAX_TRACE_STRING) caps.strings = true; + if (raw.profile.revision.length > MAX_TRACE_STRING) caps.strings = true; + return { + id: raw.profile.id.slice(0, MAX_TRACE_STRING), + revision: raw.profile.revision.slice(0, MAX_TRACE_STRING), + }; + })() + : undefined; + + const incoming = isPlainRecord(raw.truncated) ? raw.truncated : {}; + const truncated: RouteDecisionTraceV1["truncated"] = {}; + if (incoming.candidates === true || caps.candidates) truncated.candidates = true; + if (incoming.exclusions === true || caps.exclusions) truncated.exclusions = true; + if (incoming.requirements === true || caps.requirements) truncated.requirements = true; + if (incoming.strings === true || caps.strings) truncated.strings = true; + + return { + version: 1, + decisionId: decisionId.slice(0, MAX_TRACE_STRING), + createdAt, + requestedModel: requestedModel.slice(0, MAX_TRACE_STRING), + routeKind: routeKind as RouteDecisionKind, + ...(profile ? { profile } : {}), + requirements, + candidates, + selected: { + candidateIndex, + provider: selectedProvider.slice(0, MAX_TRACE_STRING), + model: selectedModel.slice(0, MAX_TRACE_STRING), + ...(typeof selected.accountRef === "string" + ? { accountRef: selected.accountRef.slice(0, MAX_TRACE_STRING) } + : {}), + reason: selectedReason.slice(0, MAX_TRACE_STRING), + ...(typeof selected.tieBreak === "string" + ? { tieBreak: selected.tieBreak.slice(0, MAX_TRACE_STRING) } + : {}), + }, + ...(truncated && Object.keys(truncated).length > 0 ? { truncated } : {}), + }; +} diff --git a/src/server/chat-completions.ts b/src/server/chat-completions.ts index a4489d591a..0f352a249f 100644 --- a/src/server/chat-completions.ts +++ b/src/server/chat-completions.ts @@ -118,6 +118,7 @@ async function handleChatCompletionsWithBudget( logCtx.providerAdapter = route.provider.adapter; logCtx.requestedModel = requestedModel; logCtx.provider = route.providerName; + logCtx.routeDecision = route.routeDecision; if (route.provider.adapter === "openai-responses") { nativeRoute = true; directRoute = route.codexAccountMode === "direct"; diff --git a/src/server/claude-messages.ts b/src/server/claude-messages.ts index 0d6711e00c..d0b5a045b6 100644 --- a/src/server/claude-messages.ts +++ b/src/server/claude-messages.ts @@ -632,6 +632,7 @@ async function handleClaudeMessagesWithBudget( // Settle the wire once so the sampling decision below reads the effective // adapter rather than the provider-wide default (#404). route.provider = resolveWireProtocolOverride(route.providerName, route.modelId, route.provider, "anthropic"); + logCtx.routeDecision = route.routeDecision; if (route.provider.adapter === "openai-responses") { nativeRoute = true; delete internalBody.max_output_tokens; diff --git a/src/server/request-log.ts b/src/server/request-log.ts index b9982e64b4..4e317967b6 100644 --- a/src/server/request-log.ts +++ b/src/server/request-log.ts @@ -8,6 +8,7 @@ import { import { CODEX_CONFIG_PATH, readRootTomlString } from "../codex/paths"; import { readCodexCatalogPath } from "../codex/catalog"; import type { OcxUsage } from "../types"; +import { normalizeRouteDecisionTrace, type RouteDecisionTraceV1 } from "../routing/trace"; import type { AdapterRequest } from "../adapters/base"; import { redactSecretString } from "../lib/redact"; import { @@ -96,6 +97,8 @@ export interface RequestLogContext { affinity?: "reused" | "new_bind" | "rebound" | "cleared"; transportPhase?: "pre_headers" | "mid_stream" | "terminal_sse"; terminalSource?: "upstream" | "synthetic"; + /** Bounded route-decision trace (RI-01); never contains secrets. */ + routeDecision?: RouteDecisionTraceV1; } export interface RequestLogEntry { @@ -147,6 +150,8 @@ export interface RequestLogEntry { transportPhase?: "pre_headers" | "mid_stream" | "terminal_sse"; /** Whether the terminal came from a real upstream SSE event or a proxy synthetic tail. */ terminalSource?: "upstream" | "synthetic"; + /** Bounded route-decision trace (RI-01); never contains secrets. */ + routeDecision?: RouteDecisionTraceV1; } const requestLog: RequestLogEntry[] = []; @@ -215,6 +220,7 @@ function asCloseReason(value: string | undefined): RequestLogEntry["closeReason" export function requestLogEntryFromPersistedUsage(entry: PersistedUsageEntry): RequestLogEntry { const terminalStatus = asTerminalStatus(entry.terminalStatus); const closeReason = asCloseReason(entry.closeReason); + const routeDecision = normalizeRouteDecisionTraceForLog(entry.routeDecision); return { requestId: entry.requestId, timestamp: entry.timestamp, @@ -247,9 +253,21 @@ export function requestLogEntryFromPersistedUsage(entry: PersistedUsageEntry): R ...(entry.usage ? { usage: entry.usage } : {}), ...(entry.totalTokens !== undefined ? { totalTokens: entry.totalTokens } : {}), ...(entry.attempts?.length ? { attempts: entry.attempts } : {}), + ...(routeDecision ? { routeDecision } : {}), }; } +/** + * Hydration guard: persisted traces are re-normalized before they enter the + * in-memory ring buffer so a hand-edited or corrupt row cannot poison the DTO. + * A row that fails validation is dropped, never forwarded unvalidated. + */ +function normalizeRouteDecisionTraceForLog( + entry: RouteDecisionTraceV1 | undefined, +): RouteDecisionTraceV1 | null { + return entry ? normalizeRouteDecisionTrace(entry) : null; +} + /** * Seed the in-memory Logs ring buffer from usage.jsonl so GUI /api/logs survives * `ocx stop` / `ocx start` (process restart). Idempotent per process; no-ops when @@ -330,6 +348,7 @@ export function addRequestLog(entry: RequestLogEntry) { ...(entry.totalTokens !== undefined ? { totalTokens: entry.totalTokens } : {}), ...(entry.attempts?.length ? { attempts: entry.attempts } : {}), ...failureDiagnostics, + ...(entry.routeDecision ? { routeDecision: entry.routeDecision } : {}), }); } catch { /* request logging must never fail a user request */ @@ -817,6 +836,7 @@ export function addFinalRequestLog( ...(logCtx.affinity ? { affinity: logCtx.affinity } : {}), ...(logCtx.transportPhase ? { transportPhase: logCtx.transportPhase } : {}), ...(logCtx.terminalSource ? { terminalSource: logCtx.terminalSource } : {}), + ...(logCtx.routeDecision ? { routeDecision: logCtx.routeDecision } : {}), }); if (isUsageDebugEnabled()) { appendUsageDebug({ diff --git a/src/server/responses/compact.ts b/src/server/responses/compact.ts index fc963ad623..74ba6171b9 100644 --- a/src/server/responses/compact.ts +++ b/src/server/responses/compact.ts @@ -272,6 +272,7 @@ export async function handleResponsesCompact( const selectedModelId = route.modelId; logCtx.requestedModel = raw.model; logCtx.model = selectedModelId; + logCtx.routeDecision = route.routeDecision; logCtx.provider = route.codexAccountNamespace ? `${route.providerName}-${route.codexAccountNamespace}` : route.providerName; diff --git a/src/server/responses/core.ts b/src/server/responses/core.ts index 619a2e7f08..08cc244b4d 100644 --- a/src/server/responses/core.ts +++ b/src/server/responses/core.ts @@ -16,7 +16,7 @@ import { previousResponseReplayFailure, rememberResponseState, } from "../../responses/state"; -import { routeModel, type RouteResult } from "../../router"; +import { comboRouteDecisionTrace, routeModel, type RouteResult } from "../../router"; import { advanceComboAfterFailure, comboDefaultEffort, @@ -853,6 +853,7 @@ async function applyFinalRouteRequestNormalization(args: { logCtx.model = route.modelId; logCtx.provider = route.providerName; logCtx.providerAdapter = route.provider.adapter; + logCtx.routeDecision = route.routeDecision; if (websocketUpstreamStreaming === false) { parsed.stream = false; @@ -965,6 +966,7 @@ export async function handleComboResponses( model: requestedModel, provider: "combo", comboId, + routeDecision: logCtx.routeDecision, attempts: logCtx.attempts, activeAttempt: undefined, activeAttemptStartedAt: undefined, @@ -999,6 +1001,9 @@ export async function handleComboResponses( if (!pick) { return comboUnavailableResponse(`No available targets for combo: ${comboId}`); } + // One immutable combo selection trace, before any child dispatch; child + // adoption below must never replace it with a concrete child route trace. + logCtx.routeDecision = comboRouteDecisionTrace(config, comboId, pick, requestedModel); let lastFailure: Response | null = null; while (pick) { @@ -1102,6 +1107,7 @@ export async function handleComboResponses( model: requestedModel, provider: "combo", comboId, + routeDecision: logCtx.routeDecision, attempts: logCtx.attempts, activeAttempt: attempt, activeAttemptStartedAt: started, @@ -1372,6 +1378,7 @@ async function handleResponsesInner( let route: RouteResult; try { route = routeModel(config, parsed.modelId); + logCtx.routeDecision = route.routeDecision; } catch (err) { if (err instanceof NoAvailableComboTargetsError) { return comboUnavailableResponse(err.message); @@ -1444,6 +1451,7 @@ async function handleResponsesInner( if (fallback?.to && !slugsEquivalent(fallback.to, route.modelId)) { try { route = routeModel(config, fallback.to); + logCtx.routeDecision = route.routeDecision; } catch (err) { if (err instanceof NoAvailableComboTargetsError) { return comboUnavailableResponse(err.message); diff --git a/src/server/search.ts b/src/server/search.ts index 27ce98cabf..76ace97cdd 100644 --- a/src/server/search.ts +++ b/src/server/search.ts @@ -79,6 +79,7 @@ export async function handleSearch( } exactAccount = { accountId: route.codexAccountId, modelId: route.modelId }; logCtx.provider = `${route.providerName}-${accountNamespace}`; + logCtx.routeDecision = route.routeDecision; // The ChatGPT search endpoint only understands the native model slug. The // account namespace is proxy routing syntax and must not cross the wire. relayBody = { ...(body as Record), model: route.modelId }; diff --git a/src/usage/log.ts b/src/usage/log.ts index 9a66422b8f..b86aa05d3b 100644 --- a/src/usage/log.ts +++ b/src/usage/log.ts @@ -4,6 +4,7 @@ import { getConfigDir } from "../config"; import { recordOwnedConfigPath } from "../lib/config-ownership"; import { usageDisplayTotalTokens } from "./totals"; import type { OcxUsage } from "../types"; +import { normalizeRouteDecisionTrace, type RouteDecisionTraceV1 } from "../routing/trace"; export type UsageStatus = "reported" | "unreported" | "unsupported" | "estimated"; @@ -86,6 +87,12 @@ export interface PersistedUsageEntry { closeReason?: "terminal" | "client_cancel" | "non_stream" | "body_stall" | "body_overflow"; /** Already redacted + capped at capture (request-log.ts redactSecretString().slice(0,500)). */ upstreamError?: string; + /** + * Bounded route-decision trace (RI-01): why this provider/model/account was + * selected. Additive field; old rows without it parse unchanged. Never + * contains prompts, credentials, or hidden reasoning. + */ + routeDecision?: RouteDecisionTraceV1; } const KNOWN_USAGE_SURFACES = new Set>([ @@ -315,6 +322,9 @@ export function normalizeUsageEntryForTest(entry: PersistedUsageEntry): Persiste function normalizeUsageEntry(entry: PersistedUsageEntry): PersistedUsageEntry { const attempts = normalizedAttempts(entry.attempts); + const routeDecision = entry.routeDecision + ? normalizeRouteDecisionTrace(entry.routeDecision) + : undefined; return { requestId: entry.requestId, timestamp: entry.timestamp, @@ -380,6 +390,7 @@ function normalizeUsageEntry(entry: PersistedUsageEntry): PersistedUsageEntry { ...(entry.terminalStatus ? { terminalStatus: entry.terminalStatus } : {}), ...(entry.closeReason ? { closeReason: entry.closeReason } : {}), ...(entry.upstreamError ? { upstreamError: entry.upstreamError } : {}), + ...(routeDecision ? { routeDecision } : {}), }; } @@ -644,9 +655,11 @@ export function readRecentUsageEntries(limit: number): PersistedUsageEntry[] { fd = openSync(path, "r"); const size = fstatSync(fd).size; if (size <= 0) return []; - // ~4 KiB/row budget with a floor; expand once if the window yields too few lines. - let windowBytes = Math.min(size, Math.max(64 * 1024, Math.ceil(limit) * 4 * 1024)); - for (let attempt = 0; attempt < 2; attempt++) { + // Trace-sized rows (up to MAX_TRACE_BYTES, RI-01) need a larger per-row + // budget than the pre-trace ledger; keep expanding until the window covers + // the file start or the whole file so a restart never hydrates nothing. + let windowBytes = Math.min(size, Math.max(64 * 1024, Math.ceil(limit) * 20 * 1024)); + while (true) { const start = Math.max(0, size - windowBytes); const buf = Buffer.alloc(size - start); readSync(fd, buf, 0, buf.length, start); @@ -666,6 +679,7 @@ export function readRecentUsageEntries(limit: number): PersistedUsageEntry[] { // most recent N valid rows (not N physical lines minus corrupt ones). const entries = parseUsageLines(lines); if (entries.length >= limit || start === 0 || windowBytes >= size) return entries.slice(-limit); + if (windowBytes >= size) break; windowBytes = Math.min(size, windowBytes * 4); } return []; diff --git a/tests/route-decision-trace.test.ts b/tests/route-decision-trace.test.ts new file mode 100644 index 0000000000..b7f4aa8516 --- /dev/null +++ b/tests/route-decision-trace.test.ts @@ -0,0 +1,414 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { routeModel } from "../src/router"; +import { requestLogEntryFromPersistedUsage } from "../src/server/request-log"; +import { + appendUsageEntry, + normalizeUsageEntryForTest, + readRecentUsageEntries, + readUsageEntries, + resetUsageReadCacheForTests, + type PersistedUsageEntry, +} from "../src/usage/log"; +import { + MAX_EXCLUSIONS_PER_CANDIDATE, + MAX_TRACE_CANDIDATES, + MAX_TRACE_BYTES, + MAX_TRACE_STRING, + MAX_REQUIREMENTS, + buildRouteDecisionTrace, + normalizeRouteDecisionTrace, + type RouteDecisionTraceV1, +} from "../src/routing/trace"; +import type { OcxConfig } from "../src/types"; + +/** Near-budget trace: 8 candidates x 16 exclusions with max-length strings. */ +function oversizedTrace(): RouteDecisionTraceV1 { + const longCode = "x".repeat(MAX_TRACE_STRING); + const candidates = Array.from({ length: MAX_TRACE_CANDIDATES }, (_, index) => ({ + provider: "p".repeat(MAX_TRACE_STRING), + model: `m${index}`.padEnd(MAX_TRACE_STRING, "y"), + eligible: index === MAX_TRACE_CANDIDATES - 1, + exclusions: Array.from({ length: 16 }, () => ({ code: longCode, detail: "z".repeat(MAX_TRACE_STRING) })), + })); + return buildRouteDecisionTrace({ + requestedModel: "combo/big", + routeKind: "combo", + selected: { + provider: candidates[MAX_TRACE_CANDIDATES - 1]!.provider, + model: candidates[MAX_TRACE_CANDIDATES - 1]!.model, + reason: "combo-pick", + candidateIndex: MAX_TRACE_CANDIDATES - 1, + }, + candidates, + }); +} + +let testDir = ""; +let previousHome: string | undefined; + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-route-trace-")); + process.env.OPENCODEX_HOME = testDir; + resetUsageReadCacheForTests(); +}); + +afterEach(() => { + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function baseConfig(overrides: Partial = {}): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { + a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1"] }, + b: { adapter: "openai-chat", baseUrl: "https://b.example/v1", apiKey: "kb", models: ["m2"] }, + openai: { + adapter: "openai-responses", + authMode: "forward", + baseUrl: "https://chatgpt.com/backend-api/codex", + }, + }, + combos: { + free: { + strategy: "failover", + targets: [ + { provider: "a", model: "m1" }, + { provider: "b", model: "m2" }, + ], + }, + }, + codexAccountNamespaces: { work: "acct-1" }, + ...overrides, + }; +} + +describe("route decision traces (RI-01)", () => { + test("explicit provider namespace records an explicit-provider trace", () => { + const route = routeModel(baseConfig(), "a/m1"); + expect(route.providerName).toBe("a"); + expect(route.modelId).toBe("m1"); + const trace = route.routeDecision!; + expect(trace.version).toBe(1); + expect(trace.routeKind).toBe("explicit-provider"); + expect(trace.requestedModel).toBe("a/m1"); + expect(trace.selected.provider).toBe("a"); + expect(trace.selected.model).toBe("m1"); + expect(trace.selected.reason).toBe("explicit-provider-namespace"); + expect(trace.candidates.length).toBe(1); + expect(trace.candidates[0]).toMatchObject({ provider: "a", model: "m1", eligible: true }); + expect(trace.selected.candidateIndex).toBe(0); + }); + + test("bare OpenAI family model records a native trace", () => { + const route = routeModel(baseConfig(), "gpt-5.6"); + expect(route.providerName).toBe("openai"); + expect(route.routeDecision!.routeKind).toBe("native"); + expect(route.routeDecision!.selected.reason).toBe("native-family"); + }); + + test("account namespace records an explicit-account trace with privacy-safe handle", () => { + const route = routeModel(baseConfig(), "work/gpt-5.6"); + expect(route.routeDecision!.routeKind).toBe("explicit-account"); + expect(route.routeDecision!.selected.accountRef).toBe("work"); + expect(route.routeDecision!.selected.provider).toBe("openai"); + expect(route.codexAccountId).toBe("acct-1"); + // The opaque account id is never needed in the trace; the namespace handle is enough. + expect(JSON.stringify(route.routeDecision)).not.toContain("acct-1"); + }); + + test("combo route records every target with eligibility and exclusion reasons", () => { + const config = baseConfig(); + const route = routeModel(config, "combo/free"); + const trace = route.routeDecision!; + expect(trace.routeKind).toBe("combo"); + expect(trace.requestedModel).toBe("combo/free"); + expect(trace.candidates.length).toBe(2); + const selected = trace.candidates[trace.selected.candidateIndex]!; + expect(selected.eligible).toBe(true); + for (const candidate of trace.candidates) { + if (candidate === selected) continue; + // Failover: the not-selected target is eligible but skipped. + expect(candidate.eligible).toBe(true); + expect(candidate.exclusions.map(exclusion => exclusion.code)).toContain("not-selected"); + } + }); + + test("default-provider fallback records a default-provider trace", () => { + const route = routeModel(baseConfig(), "totally-unknown-model"); + expect(route.providerName).toBe("a"); + expect(route.routeDecision!.routeKind).toBe("default-provider"); + expect(route.routeDecision!.selected.reason).toBe("default-provider"); + }); + + test("combo candidates are capped and the selected candidate survives truncation", () => { + const targets = Array.from({ length: 12 }, (_, index) => ({ + provider: index % 2 === 0 ? "a" : "b", + model: `m${index}`, + })); + const config = baseConfig({ combos: { big: { strategy: "failover", targets } } }); + const route = routeModel(config, "combo/big"); + const trace = route.routeDecision!; + expect(trace.candidates.length).toBeLessThanOrEqual(MAX_TRACE_CANDIDATES); + expect(trace.truncated?.candidates).toBe(true); + expect(trace.candidates[trace.selected.candidateIndex]).toMatchObject({ + provider: "a", + model: "m0", + }); + }); + + test("a selected candidate beyond the cap survives truncation at the last slot", () => { + const candidates = Array.from({ length: 12 }, (_, index) => ({ + provider: "a", + model: `m${index}`, + eligible: index === 11, + exclusions: index === 11 ? [] : [{ code: "not-selected" }], + })); + const trace = buildRouteDecisionTrace({ + requestedModel: "combo/big", + routeKind: "combo", + selected: { provider: "a", model: "m11", reason: "combo-pick", candidateIndex: 11 }, + candidates, + }); + expect(trace.candidates.length).toBe(MAX_TRACE_CANDIDATES); + expect(trace.truncated?.candidates).toBe(true); + expect(trace.selected.candidateIndex).toBe(MAX_TRACE_CANDIDATES - 1); + expect(trace.candidates[trace.selected.candidateIndex]).toMatchObject({ model: "m11" }); + }); + + test("the byte-budget fallback keeps the selected candidate and re-points the index", () => { + const trace = oversizedTrace(); + expect(trace.truncated?.candidates).toBe(true); + expect(trace.selected.candidateIndex).toBe(MAX_TRACE_CANDIDATES / 2 - 1); + expect(trace.candidates[trace.selected.candidateIndex]?.model) + .toBe(`m${MAX_TRACE_CANDIDATES - 1}`.padEnd(MAX_TRACE_STRING, "y")); + // The serialized trace respects the byte budget. + expect(Buffer.byteLength(JSON.stringify(trace), "utf8")).toBeLessThanOrEqual(MAX_TRACE_BYTES); + }); + + test("normalization marks truncation applied to oversized persisted rows", () => { + const raw = { + version: 1, + decisionId: "abcdef012345", + createdAt: 1, + requestedModel: "y".repeat(300), + routeKind: "combo", + requirements: Array.from({ length: 20 }, (_, index) => ({ + id: `req-${index}`, + outcome: "satisfied", + })), + candidates: Array.from({ length: 12 }, (_, index) => ({ + provider: "a", + model: `m${index}`, + eligible: true, + exclusions: Array.from({ length: 20 }, () => ({ code: "x" })), + })), + selected: { candidateIndex: 0, provider: "a", model: "m0", reason: "r" }, + }; + const trace = normalizeRouteDecisionTrace(raw)!; + expect(trace.truncated).toMatchObject({ + candidates: true, + exclusions: true, + requirements: true, + strings: true, + }); + expect(trace.candidates).toHaveLength(MAX_TRACE_CANDIDATES); + expect(trace.requirements).toHaveLength(MAX_REQUIREMENTS); + expect(trace.candidates.every(candidate => candidate.exclusions.length === MAX_EXCLUSIONS_PER_CANDIDATE)).toBe(true); + }); + + test("startup hydration reads trace-sized usage rows", () => { + const trace = oversizedTrace(); + for (let index = 0; index < 20; index++) { + appendUsageEntry({ + requestId: `big-${index}`, + timestamp: 1_700_000_000_000 + index, + provider: "a", + model: "m1", + status: 200, + durationMs: 1, + usageStatus: "reported", + routeDecision: trace, + }); + } + const entries = readRecentUsageEntries(20); + expect(entries.length).toBe(20); + expect(entries.every(entry => entry.routeDecision?.routeKind === "combo")).toBe(true); + }); + + test("oversized candidate exclusions are capped with a truncation flag", () => { + const exclusions = Array.from({ length: 30 }, (_, index) => ({ + code: `reason-${index}`, + detail: "x".repeat(200), + })); + const trace = buildRouteDecisionTrace({ + requestedModel: "a/m1", + routeKind: "policy", + selected: { provider: "a", model: "m1", reason: "policy" }, + candidates: [{ provider: "a", model: "m1", eligible: false, exclusions }], + }); + expect(trace.candidates[0]!.exclusions.length).toBe(MAX_EXCLUSIONS_PER_CANDIDATE); + expect(trace.truncated?.exclusions).toBe(true); + expect(trace.truncated?.strings).toBe(true); + for (const exclusion of trace.candidates[0]!.exclusions) { + expect(exclusion.detail!.length).toBeLessThanOrEqual(MAX_TRACE_STRING); + } + }); + + test("long strings are capped deterministically", () => { + const trace = buildRouteDecisionTrace({ + requestedModel: "x".repeat(500), + routeKind: "policy", + selected: { provider: "p".repeat(300), model: "m".repeat(300), reason: "r".repeat(300) }, + }); + expect(trace.requestedModel.length).toBe(MAX_TRACE_STRING); + expect(trace.selected.provider.length).toBe(MAX_TRACE_STRING); + expect(trace.selected.model.length).toBe(MAX_TRACE_STRING); + expect(trace.selected.reason.length).toBe(MAX_TRACE_STRING); + expect(trace.truncated?.strings).toBe(true); + }); + + test("trace never contains credentials or prompt content", () => { + const config = baseConfig(); + // Built at runtime so the privacy scanner's key-pattern grep does not + // treat the fixture itself as a leaked credential. + const secretKey = ["sk", "super-secret-token-12345"].join("-"); + config.providers.a = { ...config.providers.a!, apiKey: secretKey }; + const route = routeModel(config, "a/m1"); + const serialized = JSON.stringify(route.routeDecision); + expect(serialized).not.toContain(secretKey); + // The trace carries only provider/model identifiers and stable wire codes: + // the provider config's credential-carrying fields never reach it. + expect(serialized).not.toContain("apiKey"); + expect(serialized).not.toContain("baseUrl"); + expect(serialized).not.toContain("https://a.example/v1"); + }); + + test("trace round-trips through usage.jsonl and request-log hydration", () => { + const route = routeModel(baseConfig(), "combo/free"); + const entry: PersistedUsageEntry = { + requestId: "ocx-trace-roundtrip", + timestamp: 1700000000000, + provider: "combo", + model: "combo/free", + status: 200, + durationMs: 42, + usageStatus: "reported", + routeDecision: route.routeDecision, + }; + appendUsageEntry(entry); + const persisted = readUsageEntries(); + expect(persisted.length).toBe(1); + expect(persisted[0]!.routeDecision).toEqual(route.routeDecision); + + const hydrated = requestLogEntryFromPersistedUsage(persisted[0]!); + expect(hydrated.routeDecision).toEqual(route.routeDecision); + }); + + test("old JSONL rows without a trace parse unchanged", () => { + const entry: PersistedUsageEntry = { + requestId: "ocx-legacy-row", + timestamp: 1700000000000, + provider: "a", + model: "m1", + status: 200, + durationMs: 7, + usageStatus: "reported", + }; + const normalized = normalizeUsageEntryForTest(entry); + expect(normalized.routeDecision).toBeUndefined(); + expect(normalized.requestId).toBe("ocx-legacy-row"); + }); + + test("hand-edited corrupt trace rows are dropped or normalized, never poisoned", () => { + const entry: PersistedUsageEntry = { + requestId: "ocx-corrupt-trace", + timestamp: 1700000000000, + provider: "a", + model: "m1", + status: 200, + durationMs: 7, + usageStatus: "reported", + routeDecision: { version: 1, decisionId: "d", createdAt: 1, requestedModel: "x", + routeKind: "native", requirements: [], candidates: [], selected: { candidateIndex: 0, provider: "a", model: "m1", reason: "r" } } as RouteDecisionTraceV1, + }; + const normalized = normalizeUsageEntryForTest(entry); + expect(normalized.routeDecision).toBeUndefined(); + expect(normalizeRouteDecisionTrace({ version: 99 })).toBeNull(); + expect(normalizeRouteDecisionTrace("junk")).toBeNull(); + expect(normalizeRouteDecisionTrace(null)).toBeNull(); + }); + + test("request-log hydration drops a corrupt persisted trace", () => { + const entry: PersistedUsageEntry = { + requestId: "ocx-corrupt-hydration", + timestamp: 1700000000000, + provider: "a", + model: "m1", + status: 200, + durationMs: 7, + usageStatus: "reported", + // Empty candidates: normalizeRouteDecisionTrace rejects this row. + routeDecision: { + version: 1, decisionId: "d", createdAt: 1, requestedModel: "x", + routeKind: "native", requirements: [], candidates: [], + selected: { candidateIndex: 0, provider: "a", model: "m1", reason: "r" }, + } as RouteDecisionTraceV1, + }; + const hydrated = requestLogEntryFromPersistedUsage(entry); + expect(hydrated.routeDecision).toBeUndefined(); + }); + + test("normalizer bounds hand-edited oversized rows", () => { + const raw = { + version: 1, + decisionId: "0123456789ab", + createdAt: 1, + requestedModel: "y".repeat(400), + routeKind: "native", + requirements: [], + candidates: [{ + provider: "a", + model: "m1", + eligible: true, + exclusions: [], + capability: { contextWindow: 100, tools: true, junk: "drop-me" }, + health: { sampleCount: 3, junkField: true }, + score: { total: 1, components: { health: 1 } }, + }], + selected: { candidateIndex: 0, provider: "a", model: "m1", reason: "r" }, + }; + const trace = normalizeRouteDecisionTrace(raw)!; + expect(trace.requestedModel.length).toBe(MAX_TRACE_STRING); + expect(trace.candidates[0]!.capability).toEqual({ contextWindow: 100, tools: true }); + expect(trace.candidates[0]!.health).toEqual({ sampleCount: 3 }); + expect(trace.candidates[0]!.score).toEqual({ total: 1, components: { health: 1 } }); + expect(JSON.stringify(trace)).not.toContain("drop-me"); + }); + + test("deterministic: same input produces identical traces except decisionId/createdAt", () => { + const now = 1700000000000; + const first = buildRouteDecisionTrace({ + requestedModel: "a/m1", + routeKind: "explicit-provider", + selected: { provider: "a", model: "m1", reason: "explicit-provider-namespace" }, + now, + }); + const second = buildRouteDecisionTrace({ + requestedModel: "a/m1", + routeKind: "explicit-provider", + selected: { provider: "a", model: "m1", reason: "explicit-provider-namespace" }, + now, + }); + expect(first.candidates).toEqual(second.candidates); + expect(first.selected).toEqual(second.selected); + expect(first.decisionId).not.toBe(second.decisionId); + expect(first.decisionId).toMatch(/^[0-9a-f]{12}$/); + }); +}); diff --git a/tests/server-combo-failover-e2e.test.ts b/tests/server-combo-failover-e2e.test.ts index 718b1773cf..17970dbba2 100644 --- a/tests/server-combo-failover-e2e.test.ts +++ b/tests/server-combo-failover-e2e.test.ts @@ -424,6 +424,57 @@ describe("server combo failover 030 activation matrix", () => { } }); + test("persists one immutable combo route trace, not the child route trace", async () => { + const a = serve(() => chatSuccess("winner", "m1")); + const b = serve(() => chatSuccess("backup", "m2")); + const config = comboConfig({ + a: provider("openai-chat", baseUrl(a), "key-a"), + b: provider("openai-chat", baseUrl(b), "key-b"), + }); + const response = await postLogged(config); + expect(response.status).toBe(200); + await response.text(); + const { log, usage } = await latestAttemptReceipts(config); + for (const receipt of [log, usage]) { + expect(receipt.routeDecision).toBeDefined(); + expect(receipt.routeDecision.routeKind).toBe("combo"); + expect(receipt.routeDecision.requestedModel).toBe("combo/free"); + expect(receipt.routeDecision.candidates).toHaveLength(2); + expect(receipt.routeDecision.selected).toMatchObject({ + provider: "a", + model: "m1", + reason: "combo-pick", + }); + // Selection trace stays immutable: exactly one physical attempt happened + // and the trace still describes the combo decision, not the child route. + expect(receipt.attempts).toHaveLength(1); + expect(receipt.attempts![0]).toMatchObject({ provider: "a", model: "m1" }); + } + }); + + test("terminal combo failure keeps the combo trace through child adoption", async () => { + const a = serve(() => Response.json({ error: { message: "overloaded" } }, { status: 503 })); + const b = serve(() => Response.json({ error: { message: "overloaded" } }, { status: 503 })); + const config = comboConfig({ + a: provider("openai-chat", baseUrl(a), "key-a"), + b: provider("openai-chat", baseUrl(b), "key-b"), + }); + const response = await postLogged(config); + expect(response.status).toBeGreaterThanOrEqual(500); + await response.text(); + const { log, usage } = await latestAttemptReceipts(config); + for (const receipt of [log, usage]) { + expect(receipt.routeDecision).toBeDefined(); + expect(receipt.routeDecision.routeKind).toBe("combo"); + expect(receipt.routeDecision.selected).toMatchObject({ + provider: "a", + model: "m1", + reason: "combo-pick", + }); + expect(receipt.attempts).toHaveLength(2); + } + }); + test("preserves distinct failed and winning reasoning wires through restart hydration", async () => { const bodies: Array<{ provider: string; effort?: unknown }> = []; const a = serve(async request => {