From a91550c823e4fa9614e85317e72f9bc4d7da978b Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:12:27 +0200 Subject: [PATCH 1/4] feat(analytics): add route reliability and latency analytics (RI-03) --- .../001_pr_stack_status.md | 28 +- src/routing/analytics.ts | 372 ++++++++++++++++++ src/routing/history/indexer.ts | 10 + src/server/management-api.ts | 2 + .../management/routing-analytics-routes.ts | 37 ++ tests/routing-analytics.test.ts | 201 ++++++++++ 6 files changed, 647 insertions(+), 3 deletions(-) create mode 100644 src/routing/analytics.ts create mode 100644 src/server/management/routing-analytics-routes.ts create mode 100644 tests/routing-analytics.test.ts diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index f687d020d1..3e80edf990 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -40,8 +40,8 @@ other; closing one is a maintainer decision and neither is stale. | RI | Branch | Base | Head SHA | PR | URL | Status | |---|---|---|---|---|---|---| | RI-01 | `feat/ri-01-route-decision-traces` | `e44d234f0` | `b5a8e7c4c` | #1003 | https://github.com/lidge-jun/opencodex/pull/1003 | MERGED | -| RI-02 | `feat/ri-02-request-history-index` | `dev` (post-#1003 merge) | `03b0eafa7` | #1004 | https://github.com/lidge-jun/opencodex/pull/1004 | OPEN | -| RI-03 | `feat/ri-03-routing-analytics` | `feat/ri-02` head | pending | pending | pending | queued | +| RI-02 | `feat/ri-02-request-history-index` | `dev` (post-#1003 merge) | `2a72aa4a9` | #1004 | https://github.com/lidge-jun/opencodex/pull/1004 | MERGED | +| RI-03 | `feat/ri-03-routing-analytics` | `dev` (post-#1004 merge) | pending | #1005 | https://github.com/lidge-jun/opencodex/pull/1005 | OPEN (resync) | | RI-04 | `feat/ri-04-policy-profile-core` | `feat/ri-03` head | pending | pending | pending | queued | | RI-05 | `feat/ri-05-capability-aware-routing` | `feat/ri-04` head | pending | pending | pending | queued | | RI-06 | `feat/ri-06-health-aware-routing` | `feat/ri-05` head | pending | pending | pending | queued | @@ -107,7 +107,7 @@ other; closing one is a maintainer decision and neither is stale. 4. duplicate-replay accounting counted ignored rows in `indexedRows` - now counts real `INSERT` changes. - Fixes: all four above; tests cover every one. -- PR: #1004 (OPEN) https://github.com/lidge-jun/opencodex/pull/1004 +- PR: #1004 (MERGED) https://github.com/lidge-jun/opencodex/pull/1004 - Final commit: recorded after review round (rebase + CodeRabbit/simplify fixes; new head pushes to #1004) - Verification: @@ -121,3 +121,25 @@ other; closing one is a maintainer decision and neither is stale. codex-routing, codex-account-namespaces) - `bun run privacy:scan`: passed - Remaining Low findings: none + +### RI-03 - feat/ri-03-routing-analytics + +- Base SHA: `2a72aa4a9b0870c629adf842da659a5c521c6bfa` (`dev` after #1004 merge; + rebased off RI-02 head `7efb6e842` / `2069e724e`) +- Reviewed commit: same as final (author self-review before push) +- Findings (self-review): 3 fixed pre-push - (1) `requestHistoryDb` accessor + missing from the indexer (analytics needs the handle after open); + (2) SQL column names are snake_case - analytics SELECT now aliases to + camelCase; (3) cost field is `estimate.cost.total` (CostBreakdown), not + `costUsd`; plus the row-cap is injectable for truncation tests. +- Final commit: pending (recorded after commit) +- PR: #1005 (OPEN) https://github.com/lidge-jun/opencodex/pull/1005 +- Verification: + - `bun x tsc --noEmit`: PASSED (0 errors) + - `bun run test tests/routing-analytics.test.ts`: 8/8 pass (32 assertions): + classification (success/failure/cancel/incomplete), percentiles + + coverage, fallback rate, provider/model/account + profile breakdown, + unknown-price honesty, filters, truncation flag, API payload + - Focused regression suites: 144/144 pass across 6 files + - `bun run privacy:scan`: passed +- Remaining Low findings: none diff --git a/src/routing/analytics.ts b/src/routing/analytics.ts new file mode 100644 index 0000000000..7ddfed9ef8 --- /dev/null +++ b/src/routing/analytics.ts @@ -0,0 +1,372 @@ +/** + * Source-backed routing analytics (RI-03). + * + * All metrics derive from the rebuildable request-history index + * (`routing-history.sqlite`), never from repeated full JSONL scans. The + * analysis is read-only: no routing decision, profile, or weight changes here + * (ADR-10 - no automatic self-tuning). + * + * Bounds: at most ANALYTICS_MAX_ROWS matching rows are analyzed per call; a + * larger population sets `historyTruncated: true` so readers never mistake a + * sample for the full history. + */ + +import type { PersistedUsageEntry, PersistedUsageAttempt } from "../usage/log"; +import { estimateRequestCost, serviceTierContext } from "../usage/cost"; +import { openRequestHistoryIndex, requestHistoryDb } from "./history/indexer"; + +export const ANALYTICS_MAX_ROWS = 50_000; + +export interface RoutingAnalyticsFilters { + provider?: string; + model?: string; + profileId?: string; + surface?: string; + from?: number; + to?: number; +} + +export type AnalyticsConfidence = "high" | "medium" | "low"; + +export interface AnalyticsBreakdownRow { + provider: string; + model: string; + accountRef?: string; + profileId?: string; + requests: number; + successes: number; + failures: number; + cancelled: number; + successRate: number | null; + p50DurationMs?: number; + estimatedCostUsdPerSuccessfulRequest?: number | null; +} + +export interface AnalyticsProfileRow { + profileId: string; + profileRevision?: string; + requests: number; + successes: number; + failures: number; + fallbacks: number; + successRate: number | null; +} + +export interface RoutingAnalyticsResult { + generatedAt: number; + totalRequests: number; + scannedRows: number; + historyTruncated: boolean; + confidence: AnalyticsConfidence | null; + successRate: number | null; + failureRate: number | null; + cancelledRate: number | null; + fallbackRate: number | null; + totalAttempts: number; + averageAttemptsPerRequest: number | null; + incompleteStreamRate: number | null; + cooldownTriggeringFailures: number; + durationMs: { + p50?: number; + p95?: number; + p99?: number; + sampleCount: number; + }; + firstOutputMs: { + p50?: number; + p95?: number; + p99?: number; + sampleCount: number; + /** Share of scanned requests with a TTFT measurement (0..1). */ + coverage: number | null; + }; + estimatedCostUsdPerSuccessfulRequest: number | null; + estimatedCostUsdTotalSuccessful: number | null; + usageCoverage: number | null; + priceCoverage: number | null; + breakdown: AnalyticsBreakdownRow[]; + profileBreakdown: AnalyticsProfileRow[]; +} + +interface ScannedRow { + provider: string; + model: string; + apiKeyId?: string | null; + profileId?: string | null; + profileRevision?: string | null; + status: number; + durationMs: number; + firstOutputMs?: number | null; + closeReason?: string | null; + terminalStatus?: string | null; + usageStatus: string; + usageJson?: string | null; + attemptCount: number; + fallback: number; + rowJson: string; +} + +interface Bucket extends AnalyticsBreakdownRow { + durations: number[]; + costUsdSum: number; + costRows: number; +} + +const COOLDOWN_RECOVERY_KINDS = new Set([ + "rate-limit-429", + "key-429", + "oauth-401", + "anthropic-oauth-429", +]); + +function percentile(sorted: number[], p: number): number | undefined { + if (sorted.length === 0) return undefined; + const index = Math.max(0, Math.ceil((p / 100) * sorted.length) - 1); + return sorted[Math.min(index, sorted.length - 1)]; +} + +function classifyRow(row: ScannedRow): "success" | "failure" | "cancelled" { + if (row.closeReason === "client_cancel" || row.status === 499) return "cancelled"; + if (row.terminalStatus === "incomplete") return "failure"; + if (row.terminalStatus && row.terminalStatus !== "completed") return "failure"; + if (row.status >= 400) return "failure"; + return "success"; +} + +function parseEntry(rowJson: string): PersistedUsageEntry | null { + try { + const parsed = JSON.parse(rowJson) as PersistedUsageEntry; + return parsed && typeof parsed === "object" && typeof parsed.requestId === "string" ? parsed : null; + } catch { + return null; + } +} + +function attemptsOf(entry: PersistedUsageEntry | null): PersistedUsageAttempt[] | undefined { + return entry?.attempts; +} + +function cooldownTriggering(entry: PersistedUsageEntry | null, status: number): boolean { + if (status === 429) return true; + const attempts = attemptsOf(entry) ?? []; + return attempts.some(attempt => attempt.recoveryKinds.some(kind => COOLDOWN_RECOVERY_KINDS.has(kind))); +} + +function isSuccessStatus(status: number): boolean { + return status >= 200 && status < 400; +} + +export async function computeRoutingAnalytics( + filters: RoutingAnalyticsFilters, + options: { maxRows?: number } = {}, +): Promise { + await openRequestHistoryIndex(); + const handle = requestHistoryDb(); + const maxRows = Math.min( + Math.max(1, Math.trunc(options.maxRows ?? ANALYTICS_MAX_ROWS)), + ANALYTICS_MAX_ROWS, + ); + + const where: string[] = []; + const values: Array = []; + const add = (clause: string, value: string | number) => { + where.push(clause); + values.push(value); + }; + if (filters.provider !== undefined) add("provider = ?", filters.provider); + if (filters.model !== undefined) add("model = ?", filters.model); + if (filters.profileId !== undefined) add("profile_id = ?", filters.profileId); + if (filters.surface !== undefined) add("surface = ?", filters.surface); + if (filters.from !== undefined) add("timestamp >= ?", filters.from); + if (filters.to !== undefined) add("timestamp <= ?", filters.to); + const whereSql = where.length > 0 ? ` WHERE ${where.join(" AND ")}` : ""; + + const rows = handle.query( + `SELECT provider, model, api_key_id AS apiKeyId, profile_id AS profileId, + profile_revision AS profileRevision, status, + duration_ms AS durationMs, first_output_ms AS firstOutputMs, + close_reason AS closeReason, terminal_status AS terminalStatus, + usage_status AS usageStatus, usage_json AS usageJson, + attempt_count AS attemptCount, fallback, row_json AS rowJson + FROM requests${whereSql} ORDER BY timestamp DESC LIMIT ?`, + ).all(...values, maxRows + 1) as ScannedRow[]; + + const scanned = rows.slice(0, maxRows); + const historyTruncated = rows.length > maxRows; + + let successes = 0; + let failures = 0; + let cancelled = 0; + let fallbacks = 0; + let totalAttempts = 0; + let incompleteStreams = 0; + let cooldownFailures = 0; + let usageReported = 0; + const durations: number[] = []; + const firstOutputs: number[] = []; + let costTotalUsd = 0; + let costCount = 0; + + const byKey = new Map(); + const byProfile = new Map(); + + for (const row of scanned) { + const kind = classifyRow(row); + if (kind === "success") successes += 1; + else if (kind === "failure") failures += 1; + else cancelled += 1; + if (row.fallback === 1) fallbacks += 1; + totalAttempts += row.attemptCount; + if (row.terminalStatus === "incomplete") incompleteStreams += 1; + durations.push(row.durationMs); + if (row.firstOutputMs !== null && row.firstOutputMs !== undefined && row.firstOutputMs >= 0) { + firstOutputs.push(row.firstOutputMs); + } + if (row.usageStatus !== "unreported") usageReported += 1; + + const entry = kind === "success" || row.status >= 400 ? parseEntry(row.rowJson) : null; + if (cooldownTriggering(entry, row.status)) cooldownFailures += 1; + + if (kind === "success" && entry?.usage) { + const estimate = estimateRequestCost({ + provider: row.provider, + model: row.model, + usage: entry.usage, + usageStatus: entry.usageStatus, + serviceTier: serviceTierContext(entry), + }); + if (estimate) { + costTotalUsd += estimate.cost.total; + costCount += 1; + } + } + + const key = `${row.provider}\0${row.model}\0${row.apiKeyId ?? ""}\0${row.profileId ?? ""}`; + let bucket: Bucket | undefined = byKey.get(key); + if (!bucket) { + bucket = { + provider: row.provider, + model: row.model, + ...(row.apiKeyId ? { accountRef: row.apiKeyId } : {}), + ...(row.profileId ? { profileId: row.profileId } : {}), + requests: 0, + successes: 0, + failures: 0, + cancelled: 0, + successRate: null, + durations: [], + costUsdSum: 0, + costRows: 0, + }; + byKey.set(key, bucket); + } + bucket.requests += 1; + if (kind === "success") bucket.successes += 1; + else if (kind === "failure") bucket.failures += 1; + else bucket.cancelled += 1; + bucket.durations.push(row.durationMs); + if (kind === "success" && entry?.usage) { + const estimate = estimateRequestCost({ + provider: row.provider, + model: row.model, + usage: entry.usage, + usageStatus: entry.usageStatus, + serviceTier: serviceTierContext(entry), + }); + if (estimate) { + bucket.costUsdSum += estimate.cost.total; + bucket.costRows += 1; + } + } + + if (row.profileId) { + let profile = byProfile.get(row.profileId); + if (!profile) { + profile = { + profileId: row.profileId, + ...(row.profileRevision ? { profileRevision: row.profileRevision } : {}), + requests: 0, + successes: 0, + failures: 0, + fallbacks: 0, + successRate: null, + }; + byProfile.set(row.profileId, profile); + } + profile.requests += 1; + if (kind === "success") profile.successes += 1; + else if (kind === "failure") profile.failures += 1; + if (row.fallback === 1) profile.fallbacks += 1; + } + } + + durations.sort((a, b) => a - b); + firstOutputs.sort((a, b) => a - b); + const total = scanned.length; + const rate = (count: number): number | null => (total > 0 ? count / total : null); + + const breakdown: AnalyticsBreakdownRow[] = [...byKey.values()].map(bucket => { + const sorted = bucket.durations.sort((a, b) => a - b); + return { + provider: bucket.provider, + model: bucket.model, + ...(bucket.accountRef ? { accountRef: bucket.accountRef } : {}), + ...(bucket.profileId ? { profileId: bucket.profileId } : {}), + requests: bucket.requests, + successes: bucket.successes, + failures: bucket.failures, + cancelled: bucket.cancelled, + successRate: bucket.requests > 0 ? bucket.successes / bucket.requests : null, + ...(percentile(sorted, 50) !== undefined ? { p50DurationMs: percentile(sorted, 50) } : {}), + ...(bucket.requests > 0 + ? { estimatedCostUsdPerSuccessfulRequest: bucket.costRows > 0 + ? bucket.costUsdSum / bucket.costRows + : null } + : {}), + }; + }).sort((a, b) => b.requests - a.requests); + + const profileBreakdown: AnalyticsProfileRow[] = [...byProfile.values()].map(profile => ({ + ...profile, + successRate: profile.requests > 0 ? profile.successes / profile.requests : null, + })).sort((a, b) => b.requests - a.requests); + + const confidence: AnalyticsConfidence | null = total === 0 + ? null + : total >= 100 ? "high" : total >= 20 ? "medium" : "low"; + + return { + generatedAt: Date.now(), + totalRequests: total, + scannedRows: scanned.length, + historyTruncated, + confidence, + successRate: rate(successes), + failureRate: rate(failures), + cancelledRate: rate(cancelled), + fallbackRate: rate(fallbacks), + totalAttempts, + averageAttemptsPerRequest: total > 0 ? totalAttempts / total : null, + incompleteStreamRate: rate(incompleteStreams), + cooldownTriggeringFailures: cooldownFailures, + durationMs: { + ...(percentile(durations, 50) !== undefined ? { p50: percentile(durations, 50) } : {}), + ...(percentile(durations, 95) !== undefined ? { p95: percentile(durations, 95) } : {}), + ...(percentile(durations, 99) !== undefined ? { p99: percentile(durations, 99) } : {}), + sampleCount: durations.length, + }, + firstOutputMs: { + ...(percentile(firstOutputs, 50) !== undefined ? { p50: percentile(firstOutputs, 50) } : {}), + ...(percentile(firstOutputs, 95) !== undefined ? { p95: percentile(firstOutputs, 95) } : {}), + ...(percentile(firstOutputs, 99) !== undefined ? { p99: percentile(firstOutputs, 99) } : {}), + sampleCount: firstOutputs.length, + coverage: total > 0 ? firstOutputs.length / total : null, + }, + estimatedCostUsdPerSuccessfulRequest: costCount > 0 ? costTotalUsd / costCount : null, + estimatedCostUsdTotalSuccessful: costCount > 0 ? costTotalUsd : null, + usageCoverage: total > 0 ? usageReported / total : null, + priceCoverage: successes > 0 ? costCount / successes : null, + breakdown, + profileBreakdown, + }; +} diff --git a/src/routing/history/indexer.ts b/src/routing/history/indexer.ts index 6ccb7547f9..b8c915de7f 100644 --- a/src/routing/history/indexer.ts +++ b/src/routing/history/indexer.ts @@ -567,3 +567,13 @@ export async function requestHistoryRowById(requestId: string): Promise { return openRequestHistoryIndex(); } + +/** + * Raw handle for analytics-style queries. Callers must await + * `openRequestHistoryIndex()` first; the handle is valid until + * `closeRequestHistoryIndex()`. + */ +export function requestHistoryDb(): Database { + if (!db) throw new Error("request-history index is not open"); + return db; +} diff --git a/src/server/management-api.ts b/src/server/management-api.ts index 63129d5dfe..af8fd499db 100644 --- a/src/server/management-api.ts +++ b/src/server/management-api.ts @@ -60,6 +60,7 @@ import type { ManagementApiDeps } from "./management/context"; import { handleConfigRoutes } from "./management/config-routes"; import { handleLogsUsageRoutes } from "./management/logs-usage-routes"; import { handleRequestHistoryRoutes } from "./management/request-history-routes"; +import { handleRoutingAnalyticsRoutes } from "./management/routing-analytics-routes"; import { handleProviderRoutes } from "./management/provider-routes"; import { handleModelRoutes } from "./management/model-routes"; import { handleAgentSettingsRoutes } from "./management/agent-settings-routes"; @@ -140,6 +141,7 @@ export async function handleManagementAPI( routed = (await handleConfigRoutes(ctx)) ?? (await handleLogsUsageRoutes(ctx)) ?? (await handleRequestHistoryRoutes(ctx)) + ?? (await handleRoutingAnalyticsRoutes(ctx)) ?? (await handleProviderRoutes(ctx)) ?? (await handleModelRoutes(ctx)) ?? (await handleIntegrationRoutes(ctx)) diff --git a/src/server/management/routing-analytics-routes.ts b/src/server/management/routing-analytics-routes.ts new file mode 100644 index 0000000000..2e06199eb3 --- /dev/null +++ b/src/server/management/routing-analytics-routes.ts @@ -0,0 +1,37 @@ +/** + * Routing analytics API (RI-03): `GET /api/routing-analytics`. + * + * Returns source-backed reliability/latency/cost metrics over the + * request-history index. Read-only; never changes routing behavior. + */ + +import { computeRoutingAnalytics } from "../../routing/analytics"; +import { jsonResponse } from "../auth-cors"; +import type { ManagementContext } from "./context"; + +function parseOptionalInt(raw: string | null): number | undefined { + if (raw === null) return undefined; + const value = Number(raw.trim()); + return Number.isInteger(value) ? value : undefined; +} + +export async function handleRoutingAnalyticsRoutes(ctx: ManagementContext): Promise { + const { url, req, config } = ctx; + if (url.pathname !== "/api/routing-analytics" || req.method !== "GET") return null; + + const from = parseOptionalInt(url.searchParams.get("from")); + const to = parseOptionalInt(url.searchParams.get("to")); + if (from !== undefined && to !== undefined && from > to) { + return jsonResponse({ error: { code: "invalid_range", message: "from must not be after to" } }, 400, req, config); + } + + const result = await computeRoutingAnalytics({ + provider: url.searchParams.get("provider")?.trim() || undefined, + model: url.searchParams.get("model")?.trim() || undefined, + profileId: url.searchParams.get("profileId")?.trim() || undefined, + surface: url.searchParams.get("surface")?.trim() || undefined, + from, + to, + }); + return jsonResponse(result, 200, req, config); +} diff --git a/tests/routing-analytics.test.ts b/tests/routing-analytics.test.ts new file mode 100644 index 0000000000..7c3657b044 --- /dev/null +++ b/tests/routing-analytics.test.ts @@ -0,0 +1,201 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { handleManagementAPI } from "../src/server/management-api"; +import { ManagementRequest } from "./helpers/management-auth"; +import { + appendUsageEntry, + resetUsageReadCacheForTests, + type PersistedUsageEntry, +} from "../src/usage/log"; +import { closeRequestHistoryIndex } from "../src/routing/history/indexer"; +import { computeRoutingAnalytics } from "../src/routing/analytics"; +import type { OcxConfig } from "../src/types"; + +let testDir = ""; +let previousHome: string | undefined; + +function entry( + requestId: string, + overrides: Partial & { timestamp: number; status: number; durationMs: number }, +): PersistedUsageEntry { + return { + requestId, + provider: "a", + model: "m1", + usageStatus: "reported", + ...overrides, + }; +} + +beforeEach(() => { + previousHome = process.env.OPENCODEX_HOME; + testDir = mkdtempSync(join(tmpdir(), "ocx-analytics-")); + process.env.OPENCODEX_HOME = testDir; + resetUsageReadCacheForTests(); + closeRequestHistoryIndex(); +}); + +afterEach(() => { + closeRequestHistoryIndex(); + if (previousHome === undefined) delete process.env.OPENCODEX_HOME; + else process.env.OPENCODEX_HOME = previousHome; + if (testDir) rmSync(testDir, { recursive: true, force: true }); +}); + +function config(): OcxConfig { + return { + port: 10100, + defaultProvider: "a", + providers: { a: { adapter: "openai-chat", baseUrl: "https://a.example/v1", apiKey: "ka", models: ["m1"] } }, + }; +} + +describe("routing analytics (RI-03)", () => { + test("classifies success, failure, cancellation and incomplete streams", async () => { + appendUsageEntry(entry("r1", { timestamp: 1000, status: 200, durationMs: 100, firstOutputMs: 10 })); + appendUsageEntry(entry("r2", { timestamp: 2000, status: 200, durationMs: 200, firstOutputMs: 30 })); + appendUsageEntry(entry("r3", { timestamp: 3000, status: 429, durationMs: 300 })); + appendUsageEntry(entry("r4", { timestamp: 4000, status: 499, durationMs: 50, closeReason: "client_cancel" })); + appendUsageEntry(entry("r5", { timestamp: 5000, status: 200, durationMs: 400, terminalStatus: "incomplete" })); + + const result = await computeRoutingAnalytics({}); + expect(result.totalRequests).toBe(5); + expect(result.successRate).toBe(0.4); + expect(result.failureRate).toBe(0.4); + expect(result.cancelledRate).toBe(0.2); + expect(result.incompleteStreamRate).toBe(0.2); + expect(result.cooldownTriggeringFailures).toBe(1); + expect(result.confidence).toBe("low"); + expect(result.historyTruncated).toBe(false); + }); + + test("computes duration and TTFT percentiles with coverage", async () => { + appendUsageEntry(entry("r1", { timestamp: 1, status: 200, durationMs: 100, firstOutputMs: 10 })); + appendUsageEntry(entry("r2", { timestamp: 2, status: 200, durationMs: 200, firstOutputMs: 20 })); + appendUsageEntry(entry("r3", { timestamp: 3, status: 200, durationMs: 300, firstOutputMs: 30 })); + appendUsageEntry(entry("r4", { timestamp: 4, status: 200, durationMs: 400 })); + + const result = await computeRoutingAnalytics({}); + // Nearest-rank percentiles over [100,200,300,400]: + expect(result.durationMs.p50).toBe(200); + expect(result.durationMs.p95).toBe(400); + expect(result.durationMs.p99).toBe(400); + expect(result.durationMs.sampleCount).toBe(4); + expect(result.firstOutputMs.p50).toBe(20); + expect(result.firstOutputMs.sampleCount).toBe(3); + expect(result.firstOutputMs.coverage).toBe(0.75); + }); + + test("fallback rate counts multi-attempt requests", async () => { + appendUsageEntry(entry("r1", { + timestamp: 1, + status: 200, + durationMs: 100, + attempts: [ + { ordinal: 1, provider: "a", model: "m1", adapter: "openai-chat", status: 503, durationMs: 50, sendCount: 1, recoveryKinds: ["transient-5xx"], usageStatus: "unreported" }, + { ordinal: 2, provider: "a", model: "m1", adapter: "openai-chat", status: 200, durationMs: 50, sendCount: 1, recoveryKinds: [], usageStatus: "reported" }, + ], + })); + appendUsageEntry(entry("r2", { timestamp: 2, status: 200, durationMs: 100 })); + + const result = await computeRoutingAnalytics({}); + expect(result.fallbackRate).toBe(0.5); + expect(result.totalAttempts).toBe(3); + expect(result.averageAttemptsPerRequest).toBe(1.5); + }); + + test("breakdown groups by provider/model/account and profile", async () => { + appendUsageEntry(entry("r1", { + timestamp: 1, + status: 200, + durationMs: 100, + apiKeyId: "key-a", + routeDecision: { + version: 1, + decisionId: "d1", + createdAt: 1, + requestedModel: "policy/fast", + routeKind: "policy", + profile: { id: "fast", revision: "abc123" }, + requirements: [], + candidates: [{ provider: "a", model: "m1", eligible: true, exclusions: [] }], + selected: { candidateIndex: 0, provider: "a", model: "m1", reason: "policy" }, + }, + })); + appendUsageEntry(entry("r2", { + timestamp: 2, + status: 500, + durationMs: 200, + apiKeyId: "key-a", + routeDecision: { + version: 1, + decisionId: "d2", + createdAt: 2, + requestedModel: "policy/fast", + routeKind: "policy", + profile: { id: "fast", revision: "abc123" }, + requirements: [], + candidates: [{ provider: "a", model: "m1", eligible: true, exclusions: [] }], + selected: { candidateIndex: 0, provider: "a", model: "m1", reason: "policy" }, + }, + })); + + const result = await computeRoutingAnalytics({}); + expect(result.breakdown.length).toBe(1); + expect(result.breakdown[0]).toMatchObject({ + provider: "a", + model: "m1", + accountRef: "key-a", + profileId: "fast", + requests: 2, + successes: 1, + failures: 1, + successRate: 0.5, + }); + expect(result.profileBreakdown).toEqual([ + { profileId: "fast", profileRevision: "abc123", requests: 2, successes: 1, failures: 1, fallbacks: 0, successRate: 0.5 }, + ]); + }); + + test("usage and price coverage are honest about unknown data", async () => { + appendUsageEntry(entry("r1", { timestamp: 1, status: 200, durationMs: 100, usageStatus: "reported", usage: { inputTokens: 1000, outputTokens: 100 } })); + appendUsageEntry(entry("r2", { timestamp: 2, status: 200, durationMs: 100, usageStatus: "unreported" })); + + const result = await computeRoutingAnalytics({}); + expect(result.usageCoverage).toBe(0.5); + // Unknown price for provider "a": the estimate stays null, never zero. + expect(result.estimatedCostUsdPerSuccessfulRequest).toBeNull(); + expect(result.priceCoverage).toBe(0); + }); + + test("filters scope the analysis", async () => { + appendUsageEntry(entry("r1", { timestamp: 1, status: 200, durationMs: 100, provider: "a" })); + appendUsageEntry(entry("r2", { timestamp: 2, status: 200, durationMs: 100, provider: "b", model: "m2" })); + + const result = await computeRoutingAnalytics({ provider: "b" }); + expect(result.totalRequests).toBe(1); + expect(result.breakdown[0]).toMatchObject({ provider: "b", model: "m2" }); + }); + + test("explicit truncated-history indicator when the cap is hit", async () => { + for (let index = 0; index < 12; index++) { + appendUsageEntry(entry(`r${index}`, { timestamp: index, status: 200, durationMs: 10 })); + } + const result = await computeRoutingAnalytics({}, { maxRows: 10 }); + expect(result.scannedRows).toBe(10); + expect(result.historyTruncated).toBe(true); + }); + + test("API endpoint returns the analytics payload", async () => { + appendUsageEntry(entry("r1", { timestamp: 1, status: 200, durationMs: 100 })); + const req = new ManagementRequest("http://localhost/api/routing-analytics", { method: "GET" }); + const response = await handleManagementAPI(req, new URL(req.url), config(), { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + expect(response!.status).toBe(200); + const body = await response!.json() as { totalRequests?: number; successRate?: number | null }; + expect(body.totalRequests).toBe(1); + expect(body.successRate).toBe(1); + }); +}); From e732d02ef41a111327dc4649e61c6b933c4a7434 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:08:58 +0200 Subject: [PATCH 2/4] fix(analytics): valid RI-01 decisionIds in tests; simplify helpers and query validation (RI-03) --- src/routing/analytics.ts | 48 +++++++++---------- .../management/routing-analytics-routes.ts | 20 ++++++-- tests/routing-analytics.test.ts | 4 +- 3 files changed, 40 insertions(+), 32 deletions(-) diff --git a/src/routing/analytics.ts b/src/routing/analytics.ts index 7ddfed9ef8..0f745cac61 100644 --- a/src/routing/analytics.ts +++ b/src/routing/analytics.ts @@ -152,8 +152,19 @@ function cooldownTriggering(entry: PersistedUsageEntry | null, status: number): return attempts.some(attempt => attempt.recoveryKinds.some(kind => COOLDOWN_RECOVERY_KINDS.has(kind))); } -function isSuccessStatus(status: number): boolean { - return status >= 200 && status < 400; +function successCostUsd( + row: Pick, + entry: PersistedUsageEntry, +): number | null { + if (!entry.usage) return null; + const estimate = estimateRequestCost({ + provider: row.provider, + model: row.model, + usage: entry.usage, + usageStatus: entry.usageStatus, + serviceTier: serviceTierContext(entry), + }); + return estimate ? estimate.cost.total : null; } export async function computeRoutingAnalytics( @@ -227,16 +238,11 @@ export async function computeRoutingAnalytics( const entry = kind === "success" || row.status >= 400 ? parseEntry(row.rowJson) : null; if (cooldownTriggering(entry, row.status)) cooldownFailures += 1; - if (kind === "success" && entry?.usage) { - const estimate = estimateRequestCost({ - provider: row.provider, - model: row.model, - usage: entry.usage, - usageStatus: entry.usageStatus, - serviceTier: serviceTierContext(entry), - }); - if (estimate) { - costTotalUsd += estimate.cost.total; + let rowCostUsd: number | null = null; + if (kind === "success" && entry) { + rowCostUsd = successCostUsd(row, entry); + if (rowCostUsd !== null) { + costTotalUsd += rowCostUsd; costCount += 1; } } @@ -265,18 +271,9 @@ export async function computeRoutingAnalytics( else if (kind === "failure") bucket.failures += 1; else bucket.cancelled += 1; bucket.durations.push(row.durationMs); - if (kind === "success" && entry?.usage) { - const estimate = estimateRequestCost({ - provider: row.provider, - model: row.model, - usage: entry.usage, - usageStatus: entry.usageStatus, - serviceTier: serviceTierContext(entry), - }); - if (estimate) { - bucket.costUsdSum += estimate.cost.total; - bucket.costRows += 1; - } + if (rowCostUsd !== null) { + bucket.costUsdSum += rowCostUsd; + bucket.costRows += 1; } if (row.profileId) { @@ -307,6 +304,7 @@ export async function computeRoutingAnalytics( const breakdown: AnalyticsBreakdownRow[] = [...byKey.values()].map(bucket => { const sorted = bucket.durations.sort((a, b) => a - b); + const p50DurationMs = percentile(sorted, 50); return { provider: bucket.provider, model: bucket.model, @@ -317,7 +315,7 @@ export async function computeRoutingAnalytics( failures: bucket.failures, cancelled: bucket.cancelled, successRate: bucket.requests > 0 ? bucket.successes / bucket.requests : null, - ...(percentile(sorted, 50) !== undefined ? { p50DurationMs: percentile(sorted, 50) } : {}), + ...(p50DurationMs !== undefined ? { p50DurationMs } : {}), ...(bucket.requests > 0 ? { estimatedCostUsdPerSuccessfulRequest: bucket.costRows > 0 ? bucket.costUsdSum / bucket.costRows diff --git a/src/server/management/routing-analytics-routes.ts b/src/server/management/routing-analytics-routes.ts index 2e06199eb3..71ce69ddb4 100644 --- a/src/server/management/routing-analytics-routes.ts +++ b/src/server/management/routing-analytics-routes.ts @@ -9,18 +9,28 @@ import { computeRoutingAnalytics } from "../../routing/analytics"; import { jsonResponse } from "../auth-cors"; import type { ManagementContext } from "./context"; -function parseOptionalInt(raw: string | null): number | undefined { +function parseQueryInt(raw: string | null): number | undefined | "invalid" { if (raw === null) return undefined; - const value = Number(raw.trim()); - return Number.isInteger(value) ? value : undefined; + const trimmed = raw.trim(); + if (trimmed.length === 0) return "invalid"; + const value = Number(trimmed); + return Number.isInteger(value) ? value : "invalid"; } export async function handleRoutingAnalyticsRoutes(ctx: ManagementContext): Promise { const { url, req, config } = ctx; if (url.pathname !== "/api/routing-analytics" || req.method !== "GET") return null; - const from = parseOptionalInt(url.searchParams.get("from")); - const to = parseOptionalInt(url.searchParams.get("to")); + const fromParsed = parseQueryInt(url.searchParams.get("from")); + if (fromParsed === "invalid") { + return jsonResponse({ error: { code: "invalid_from", message: "from must be an integer timestamp" } }, 400, req, config); + } + const toParsed = parseQueryInt(url.searchParams.get("to")); + if (toParsed === "invalid") { + return jsonResponse({ error: { code: "invalid_to", message: "to must be an integer timestamp" } }, 400, req, config); + } + const from = fromParsed; + const to = toParsed; if (from !== undefined && to !== undefined && from > to) { return jsonResponse({ error: { code: "invalid_range", message: "from must not be after to" } }, 400, req, config); } diff --git a/tests/routing-analytics.test.ts b/tests/routing-analytics.test.ts index 7c3657b044..7ddc5568a9 100644 --- a/tests/routing-analytics.test.ts +++ b/tests/routing-analytics.test.ts @@ -114,7 +114,7 @@ describe("routing analytics (RI-03)", () => { apiKeyId: "key-a", routeDecision: { version: 1, - decisionId: "d1", + decisionId: "a00000000001", createdAt: 1, requestedModel: "policy/fast", routeKind: "policy", @@ -131,7 +131,7 @@ describe("routing analytics (RI-03)", () => { apiKeyId: "key-a", routeDecision: { version: 1, - decisionId: "d2", + decisionId: "a00000000002", createdAt: 2, requestedModel: "policy/fast", routeKind: "policy", From 5d9386bf21eb9c42429a84620f84031955b833a1 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:21:18 +0200 Subject: [PATCH 3/4] fix(routing): address CodeRabbit RI-03 review (cooldown, limit, tests, devlog) --- .../001_pr_stack_status.md | 4 +- src/routing/analytics.ts | 23 ++++++---- .../management/routing-analytics-routes.ts | 31 +++++++++++++- tests/routing-analytics.test.ts | 42 +++++++++++++++++++ 4 files changed, 88 insertions(+), 12 deletions(-) diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 3e80edf990..72392e629d 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -10,7 +10,7 @@ head SHA, PR number/URL, verification result, and review state. - Bun: `1.3.14`; package version: `2.10.0` - Worktree: `D:\codex-worktrees\ocx-router-intelligence` - Push remote: `origin` (Wibias/opencodex); PR target: `lidge-jun/opencodex:dev` -- All PRs opened as DRAFT; nothing merged by this programme. +- Programme stack: #1003 (RI-01) and #1004 (RI-02) merged to `dev`; #1005 (RI-03) open. ## Related in-flight PRs (not superseded by this stack) @@ -132,7 +132,7 @@ other; closing one is a maintainer decision and neither is stale. (2) SQL column names are snake_case - analytics SELECT now aliases to camelCase; (3) cost field is `estimate.cost.total` (CostBreakdown), not `costUsd`; plus the row-cap is injectable for truncation tests. -- Final commit: pending (recorded after commit) +- Final commit: `e732d02e` (post-review fixes on cooldown parsing, API row cap, tests) - PR: #1005 (OPEN) https://github.com/lidge-jun/opencodex/pull/1005 - Verification: - `bun x tsc --noEmit`: PASSED (0 errors) diff --git a/src/routing/analytics.ts b/src/routing/analytics.ts index 0f745cac61..bcf8f75ed1 100644 --- a/src/routing/analytics.ts +++ b/src/routing/analytics.ts @@ -16,6 +16,8 @@ import { estimateRequestCost, serviceTierContext } from "../usage/cost"; import { openRequestHistoryIndex, requestHistoryDb } from "./history/indexer"; export const ANALYTICS_MAX_ROWS = 50_000; +/** Default row cap for the management API (full cap remains available via `limit`). */ +export const ANALYTICS_API_DEFAULT_ROWS = 5_000; export interface RoutingAnalyticsFilters { provider?: string; @@ -235,18 +237,23 @@ export async function computeRoutingAnalytics( } if (row.usageStatus !== "unreported") usageReported += 1; - const entry = kind === "success" || row.status >= 400 ? parseEntry(row.rowJson) : null; - if (cooldownTriggering(entry, row.status)) cooldownFailures += 1; - let rowCostUsd: number | null = null; - if (kind === "success" && entry) { - rowCostUsd = successCostUsd(row, entry); - if (rowCostUsd !== null) { - costTotalUsd += rowCostUsd; - costCount += 1; + if (kind === "success") { + const entry = parseEntry(row.rowJson); + if (entry) { + rowCostUsd = successCostUsd(row, entry); + if (rowCostUsd !== null) { + costTotalUsd += rowCostUsd; + costCount += 1; + } } } + if (kind === "failure") { + const failureEntry = parseEntry(row.rowJson); + if (cooldownTriggering(failureEntry, row.status)) cooldownFailures += 1; + } + const key = `${row.provider}\0${row.model}\0${row.apiKeyId ?? ""}\0${row.profileId ?? ""}`; let bucket: Bucket | undefined = byKey.get(key); if (!bucket) { diff --git a/src/server/management/routing-analytics-routes.ts b/src/server/management/routing-analytics-routes.ts index 71ce69ddb4..e0e0d0ea88 100644 --- a/src/server/management/routing-analytics-routes.ts +++ b/src/server/management/routing-analytics-routes.ts @@ -5,7 +5,11 @@ * request-history index. Read-only; never changes routing behavior. */ -import { computeRoutingAnalytics } from "../../routing/analytics"; +import { + ANALYTICS_API_DEFAULT_ROWS, + ANALYTICS_MAX_ROWS, + computeRoutingAnalytics, +} from "../../routing/analytics"; import { jsonResponse } from "../auth-cors"; import type { ManagementContext } from "./context"; @@ -34,6 +38,29 @@ export async function handleRoutingAnalyticsRoutes(ctx: ManagementContext): Prom if (from !== undefined && to !== undefined && from > to) { return jsonResponse({ error: { code: "invalid_range", message: "from must not be after to" } }, 400, req, config); } + const limitParsed = parseQueryInt(url.searchParams.get("limit")); + if (limitParsed === "invalid") { + return jsonResponse( + { error: { code: "invalid_limit", message: "limit must be an integer" } }, + 400, + req, + config, + ); + } + const maxRows = limitParsed ?? ANALYTICS_API_DEFAULT_ROWS; + if (maxRows < 1 || maxRows > ANALYTICS_MAX_ROWS) { + return jsonResponse( + { + error: { + code: "invalid_limit", + message: `limit must be between 1 and ${ANALYTICS_MAX_ROWS}`, + }, + }, + 400, + req, + config, + ); + } const result = await computeRoutingAnalytics({ provider: url.searchParams.get("provider")?.trim() || undefined, @@ -42,6 +69,6 @@ export async function handleRoutingAnalyticsRoutes(ctx: ManagementContext): Prom surface: url.searchParams.get("surface")?.trim() || undefined, from, to, - }); + }, { maxRows }); return jsonResponse(result, 200, req, config); } diff --git a/tests/routing-analytics.test.ts b/tests/routing-analytics.test.ts index 7ddc5568a9..bc824bbe4b 100644 --- a/tests/routing-analytics.test.ts +++ b/tests/routing-analytics.test.ts @@ -198,4 +198,46 @@ describe("routing analytics (RI-03)", () => { expect(body.totalRequests).toBe(1); expect(body.successRate).toBe(1); }); + + test("counts cooldownTriggeringFailures for non-4xx failures with recovery attempts", async () => { + appendUsageEntry( + entry("r1", { + timestamp: 1, + status: 503, + durationMs: 100, + attempts: [ + { + ordinal: 1, + provider: "a", + model: "m1", + adapter: "openai-chat", + status: 503, + durationMs: 100, + sendCount: 1, + recoveryKinds: ["rate-limit-429"], + usageStatus: "unreported", + }, + ], + }), + ); + const result = await computeRoutingAnalytics({}); + expect(result.cooldownTriggeringFailures).toBe(1); + }); + + test("routing analytics API returns 400 for invalid from/to/limit", async () => { + const cases = [ + { query: "from=abc", code: "invalid_from" }, + { query: "to=xyz", code: "invalid_to" }, + { query: "from=10&to=5", code: "invalid_range" }, + { query: "limit=0", code: "invalid_limit" }, + ] as const; + for (const { query, code } of cases) { + const req = new ManagementRequest(`http://localhost/api/routing-analytics?${query}`, { method: "GET" }); + const response = await handleManagementAPI(req, new URL(req.url), config(), { refreshCodexCatalog: async () => {} }); + expect(response).not.toBeNull(); + expect(response!.status).toBe(400); + const body = await response!.json() as { error?: { code?: string } }; + expect(body.error?.code).toBe(code); + } + }); }); From b25ffa4bd6ed55536178361ca6188d715452be95 Mon Sep 17 00:00:00 2001 From: Wibias <37517432+Wibias@users.noreply.github.com> Date: Wed, 5 Aug 2026 00:24:08 +0200 Subject: [PATCH 4/4] docs(devlog): sync RI-03 stack status after CodeRabbit round 2 --- .../001_pr_stack_status.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md index 72392e629d..3e32843b57 100644 --- a/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md +++ b/devlog/_plan/260804_router_intelligence/001_pr_stack_status.md @@ -124,22 +124,22 @@ other; closing one is a maintainer decision and neither is stale. ### RI-03 - feat/ri-03-routing-analytics -- Base SHA: `2a72aa4a9b0870c629adf842da659a5c521c6bfa` (`dev` after #1004 merge; - rebased off RI-02 head `7efb6e842` / `2069e724e`) -- Reviewed commit: same as final (author self-review before push) +- Base SHA: `2a72aa4a9b0870c629adf842da659a5c521c6bfa` (`dev` after #1004 squash-merge) +- Reviewed commit: `e732d02e` (pre–CodeRabbit review round) - Findings (self-review): 3 fixed pre-push - (1) `requestHistoryDb` accessor missing from the indexer (analytics needs the handle after open); (2) SQL column names are snake_case - analytics SELECT now aliases to camelCase; (3) cost field is `estimate.cost.total` (CostBreakdown), not `costUsd`; plus the row-cap is injectable for truncation tests. -- Final commit: `e732d02e` (post-review fixes on cooldown parsing, API row cap, tests) +- Final commit: `5f464c730` (CodeRabbit: cooldown parse gate, API `limit` default 5k, devlog + tests) - PR: #1005 (OPEN) https://github.com/lidge-jun/opencodex/pull/1005 - Verification: - `bun x tsc --noEmit`: PASSED (0 errors) - - `bun run test tests/routing-analytics.test.ts`: 8/8 pass (32 assertions): + - `bun run test tests/routing-analytics.test.ts`: 10/10 pass: classification (success/failure/cancel/incomplete), percentiles + coverage, fallback rate, provider/model/account + profile breakdown, - unknown-price honesty, filters, truncation flag, API payload + unknown-price honesty, filters, truncation flag, API payload, + cooldown on failure+attempts, API validation (`invalid_from`/`invalid_to`/`invalid_range`) - Focused regression suites: 144/144 pass across 6 files - `bun run privacy:scan`: passed - Remaining Low findings: none