diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx new file mode 100644 index 0000000000..cf2a56776c --- /dev/null +++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx @@ -0,0 +1,79 @@ +--- +title: Backtest & calibration +description: How LoopOver measures whether its own review rules are right — and backtests threshold and logic changes against real history before they ship. +--- + +## What this is + +Every time a configured gate blocker fires, LoopOver records it. Every time a human later +overrides that call (or confirms it), LoopOver records that too. Paired together, the two +histories become a labeled corpus: _this rule fired against this PR, and a person later said it +was right ("confirmed") or wrong ("reversed")_. + +That corpus makes a rule change **testable before it merges**. A PR that tunes a confidence +threshold or rewrites detection logic gets scored against the real recorded history — the same +targets, the same raw inputs — instead of being eyeballed. + + + Everything on this page is advisory-only today. A backtest verdict — even a regression — never + blocks a merge. Whether a REGRESSED verdict should ever gate merges is a tracked, deliberately + separate decision that waits for real production track-record data. + + +## The corpus + +- **Fired events** — recorded when a configured blocker actually carries gate authority, with + bounded raw context: the issue text, PR title/body, and diff the rule evaluated, and (for the + AI-judgment rule `linked_issue_scope_mismatch`) the model's own raw response text. +- **Override events** — recorded when a human reverses an automated call, or confirms one (for + example, an owner closing a PR that was held for low AI confidence confirms the hold). +- **`secret_leak` is permanently excluded from raw-context capture.** Storing the diff that + triggered a leaked-credential finding would store the credential itself in the audit trail. + This rule can therefore never be logic-backtested — by design, not omission. + +## Two backtests, two mechanisms + +**Threshold changes** run inside LoopOver's own review pass. When a PR's diff changes a known +confidence-threshold constant, the old and new values are replayed as classifiers over the +rule's recorded history, and the comparison is rendered directly into the unified review +comment. No code from the PR is ever executed — a threshold is just a number. + +**Logic/regex changes** run in a dedicated CI workflow instead, because honestly verifying a +rewritten detection function requires _executing the PR's own code_ against history — something +the production review service must never do. The workflow checks out both the PR's head and its +base, replays each recorded case's captured raw context through both versions of the detection +function, and posts its own clearly-labeled **"Logic backtest"** PR comment, separate from the +unified review comment. It triggers only for PRs touching the watched detection-logic paths, +skips drafts and forks (fork runs get no secrets), and fails open: an infrastructure problem +produces a skip notice, never a red check. + +## Reading the comparison + +Both backtests score the same way: + +- **"Reversed" is the positive class.** A classifier predicting "reversed" is saying the rule's + original firing was wrong. Precision and recall are computed against the human labels, and + stay `N/A` when there is not enough decided data — unknown is never coerced to zero. +- **The Pareto floor decides the verdict.** A candidate that regresses on _any_ axis is + **REGRESSED**, even if the other axis improved. Trading precision for recall is a regression, + not a net win. Otherwise the verdict is _improved_ (some axis moved up) or _unchanged_. +- Cases recorded before raw-context capture existed are skipped and counted in the comment — + replaying them against empty inputs would bias both sides, so they are never scored. + +## The track record + +Every backtest run persists its comparison as structured audit data. The accumulated record — +how often runs regress, per rule — is the evidence base for the pending merge-gating decision, +readable at any time with the maintainer CLI: + +```bash +npx tsx scripts/backtest-track-record.ts --db loopover --remote +``` + +The corpus itself can be exported as a versioned, checksummed snapshot: + +```bash +npx tsx scripts/backtest-corpus-export.ts --rule-id --output corpus.json --remote +``` + +Both CLIs are strictly read-only against the database. diff --git a/apps/loopover-ui/src/components/site/command-palette.tsx b/apps/loopover-ui/src/components/site/command-palette.tsx index 06c67af17c..527f38a611 100644 --- a/apps/loopover-ui/src/components/site/command-palette.tsx +++ b/apps/loopover-ui/src/components/site/command-palette.tsx @@ -61,6 +61,7 @@ const DEFAULT_ITEMS: PaletteItem[] = [ { label: "Scoreability", to: "/docs/scoreability", group: "Docs" }, { label: "Upstream drift", to: "/docs/upstream-drift", group: "Docs" }, { label: "AI summaries policy", to: "/docs/ai-summaries", group: "Docs" }, + { label: "Backtest & calibration", to: "/docs/backtest-calibration", group: "Docs" }, { label: "Privacy & security", to: "/docs/privacy-security", group: "Docs" }, { label: "Troubleshooting", to: "/docs/troubleshooting", group: "Docs" }, { label: "API reference", to: "/api", group: "Reference" }, diff --git a/apps/loopover-ui/src/components/site/docs-nav.tsx b/apps/loopover-ui/src/components/site/docs-nav.tsx index 5f6876f001..bab4b2a82a 100644 --- a/apps/loopover-ui/src/components/site/docs-nav.tsx +++ b/apps/loopover-ui/src/components/site/docs-nav.tsx @@ -99,6 +99,7 @@ export const docsNav: DocsGroup[] = [ { to: "/docs/branch-analysis", label: "Branch analysis" }, { to: "/docs/scoreability", label: "Scoreability" }, { to: "/docs/upstream-drift", label: "Upstream drift" }, + { to: "/docs/backtest-calibration", label: "Backtest & calibration" }, ], }, { diff --git a/apps/loopover-ui/src/lib/docs-source-server-isolation.test.ts b/apps/loopover-ui/src/lib/docs-source-server-isolation.test.ts index d45b00e2e0..4cbbc746a8 100644 --- a/apps/loopover-ui/src/lib/docs-source-server-isolation.test.ts +++ b/apps/loopover-ui/src/lib/docs-source-server-isolation.test.ts @@ -29,7 +29,7 @@ describe("docs.*.tsx routes never import docs-source(.server) directly", () => { const inScope = docsRouteFiles.filter((name) => !outOfScope.has(name)); it("found the expected set of in-scope docs route files", () => { - expect(inScope.length).toBe(48); + expect(inScope.length).toBe(49); }); it.each(inScope)( diff --git a/apps/loopover-ui/src/routeTree.gen.ts b/apps/loopover-ui/src/routeTree.gen.ts index 0b0fdf902f..ddf4b1534e 100644 --- a/apps/loopover-ui/src/routeTree.gen.ts +++ b/apps/loopover-ui/src/routeTree.gen.ts @@ -63,6 +63,7 @@ import { Route as DocsFederatedFleetIntelligenceRouteImport } from './routes/doc import { Route as DocsCapacityRouteImport } from './routes/docs.capacity' import { Route as DocsBranchAnalysisRouteImport } from './routes/docs.branch-analysis' import { Route as DocsBetaOnboardingRouteImport } from './routes/docs.beta-onboarding' +import { Route as DocsBacktestCalibrationRouteImport } from './routes/docs.backtest-calibration' import { Route as DocsAmsUnattendedSchedulingRouteImport } from './routes/docs.ams-unattended-scheduling' import { Route as DocsAmsSizingRouteImport } from './routes/docs.ams-sizing' import { Route as DocsAmsOperationsRunbookRouteImport } from './routes/docs.ams-operations-runbook' @@ -376,6 +377,11 @@ const DocsBetaOnboardingRoute = DocsBetaOnboardingRouteImport.update({ path: '/beta-onboarding', getParentRoute: () => DocsRoute, } as any) +const DocsBacktestCalibrationRoute = DocsBacktestCalibrationRouteImport.update({ + id: '/backtest-calibration', + path: '/backtest-calibration', + getParentRoute: () => DocsRoute, +} as any) const DocsAmsUnattendedSchedulingRoute = DocsAmsUnattendedSchedulingRouteImport.update({ id: '/ams-unattended-scheduling', @@ -554,6 +560,7 @@ export interface FileRoutesByFullPath { '/docs/ams-operations-runbook': typeof DocsAmsOperationsRunbookRoute '/docs/ams-sizing': typeof DocsAmsSizingRoute '/docs/ams-unattended-scheduling': typeof DocsAmsUnattendedSchedulingRoute + '/docs/backtest-calibration': typeof DocsBacktestCalibrationRoute '/docs/beta-onboarding': typeof DocsBetaOnboardingRoute '/docs/branch-analysis': typeof DocsBranchAnalysisRoute '/docs/capacity': typeof DocsCapacityRoute @@ -633,6 +640,7 @@ export interface FileRoutesByTo { '/docs/ams-operations-runbook': typeof DocsAmsOperationsRunbookRoute '/docs/ams-sizing': typeof DocsAmsSizingRoute '/docs/ams-unattended-scheduling': typeof DocsAmsUnattendedSchedulingRoute + '/docs/backtest-calibration': typeof DocsBacktestCalibrationRoute '/docs/beta-onboarding': typeof DocsBetaOnboardingRoute '/docs/branch-analysis': typeof DocsBranchAnalysisRoute '/docs/capacity': typeof DocsCapacityRoute @@ -717,6 +725,7 @@ export interface FileRoutesById { '/docs/ams-operations-runbook': typeof DocsAmsOperationsRunbookRoute '/docs/ams-sizing': typeof DocsAmsSizingRoute '/docs/ams-unattended-scheduling': typeof DocsAmsUnattendedSchedulingRoute + '/docs/backtest-calibration': typeof DocsBacktestCalibrationRoute '/docs/beta-onboarding': typeof DocsBetaOnboardingRoute '/docs/branch-analysis': typeof DocsBranchAnalysisRoute '/docs/capacity': typeof DocsCapacityRoute @@ -802,6 +811,7 @@ export interface FileRouteTypes { | '/docs/ams-operations-runbook' | '/docs/ams-sizing' | '/docs/ams-unattended-scheduling' + | '/docs/backtest-calibration' | '/docs/beta-onboarding' | '/docs/branch-analysis' | '/docs/capacity' @@ -881,6 +891,7 @@ export interface FileRouteTypes { | '/docs/ams-operations-runbook' | '/docs/ams-sizing' | '/docs/ams-unattended-scheduling' + | '/docs/backtest-calibration' | '/docs/beta-onboarding' | '/docs/branch-analysis' | '/docs/capacity' @@ -964,6 +975,7 @@ export interface FileRouteTypes { | '/docs/ams-operations-runbook' | '/docs/ams-sizing' | '/docs/ams-unattended-scheduling' + | '/docs/backtest-calibration' | '/docs/beta-onboarding' | '/docs/branch-analysis' | '/docs/capacity' @@ -1405,6 +1417,13 @@ declare module '@tanstack/react-router' { preLoaderRoute: typeof DocsBetaOnboardingRouteImport parentRoute: typeof DocsRoute } + '/docs/backtest-calibration': { + id: '/docs/backtest-calibration' + path: '/backtest-calibration' + fullPath: '/docs/backtest-calibration' + preLoaderRoute: typeof DocsBacktestCalibrationRouteImport + parentRoute: typeof DocsRoute + } '/docs/ams-unattended-scheduling': { id: '/docs/ams-unattended-scheduling' path: '/ams-unattended-scheduling' @@ -1658,6 +1677,7 @@ interface DocsRouteChildren { DocsAmsOperationsRunbookRoute: typeof DocsAmsOperationsRunbookRoute DocsAmsSizingRoute: typeof DocsAmsSizingRoute DocsAmsUnattendedSchedulingRoute: typeof DocsAmsUnattendedSchedulingRoute + DocsBacktestCalibrationRoute: typeof DocsBacktestCalibrationRoute DocsBetaOnboardingRoute: typeof DocsBetaOnboardingRoute DocsBranchAnalysisRoute: typeof DocsBranchAnalysisRoute DocsCapacityRoute: typeof DocsCapacityRoute @@ -1711,6 +1731,7 @@ const DocsRouteChildren: DocsRouteChildren = { DocsAmsOperationsRunbookRoute: DocsAmsOperationsRunbookRoute, DocsAmsSizingRoute: DocsAmsSizingRoute, DocsAmsUnattendedSchedulingRoute: DocsAmsUnattendedSchedulingRoute, + DocsBacktestCalibrationRoute: DocsBacktestCalibrationRoute, DocsBetaOnboardingRoute: DocsBetaOnboardingRoute, DocsBranchAnalysisRoute: DocsBranchAnalysisRoute, DocsCapacityRoute: DocsCapacityRoute, diff --git a/apps/loopover-ui/src/routes/docs-routes-loading-state.test.tsx b/apps/loopover-ui/src/routes/docs-routes-loading-state.test.tsx index bc1a8d3d93..1d0c07bcda 100644 --- a/apps/loopover-ui/src/routes/docs-routes-loading-state.test.tsx +++ b/apps/loopover-ui/src/routes/docs-routes-loading-state.test.tsx @@ -45,7 +45,7 @@ describe("docs route Suspense fallback (#6982)", () => { "docs.tsx", ]); const inScope = docsRouteFiles.filter((name) => !outOfScope.has(name)); - expect(inScope.length).toBe(48); + expect(inScope.length).toBe(49); for (const file of inScope) { const source = readFileSync(join(routesDir, file), "utf8"); diff --git a/apps/loopover-ui/src/routes/docs.backtest-calibration.tsx b/apps/loopover-ui/src/routes/docs.backtest-calibration.tsx new file mode 100644 index 0000000000..5092800d44 --- /dev/null +++ b/apps/loopover-ui/src/routes/docs.backtest-calibration.tsx @@ -0,0 +1,50 @@ +import { createFileRoute, notFound } from "@tanstack/react-router"; +import { Suspense } from "react"; + +import { DocsPage } from "@/components/site/docs-page"; +import { LoadingState } from "@/components/site/state-views"; +import { docsClientLoader } from "@/lib/docs-client-loader"; +import { getDocPage } from "@/lib/docs-source.functions"; + +// Rendered from content/docs/backtest-calibration.mdx via fumadocs-mdx's browser entry +// (docsClientLoader), through the existing DocsPage/Callout/CodeBlock primitives -- not fumadocs-ui's +// bundled components. See docs-source.server.ts's comment for why the loader below resolves only a +// plain, serializable path string. +export const Route = createFileRoute("/docs/backtest-calibration")({ + loader: async () => { + const page = await getDocPage({ data: { slugs: ["backtest-calibration"] } }); + if (!page) throw notFound(); + return page; + }, + head: () => ({ + meta: [ + { title: "Backtest & calibration — LoopOver docs" }, + { + name: "description", + content: + "How LoopOver measures whether its own review rules are right, and backtests threshold and logic changes against real recorded history before they ship.", + }, + { property: "og:title", content: "Backtest & calibration — LoopOver docs" }, + { + property: "og:description", + content: + "How LoopOver measures whether its own review rules are right, and backtests threshold and logic changes against real recorded history before they ship.", + }, + { property: "og:url", content: "/docs/backtest-calibration" }, + ], + links: [{ rel: "canonical", href: "/docs/backtest-calibration" }], + }), + component: BacktestCalibrationDoc, +}); + +function BacktestCalibrationDoc() { + const { path, title, description } = Route.useLoaderData(); + const Content = docsClientLoader.getComponent(path); + return ( + + }> + + + + ); +} diff --git a/apps/loopover-ui/src/routes/docs.index.tsx b/apps/loopover-ui/src/routes/docs.index.tsx index 2e99a8e5ef..26b8985dbd 100644 --- a/apps/loopover-ui/src/routes/docs.index.tsx +++ b/apps/loopover-ui/src/routes/docs.index.tsx @@ -94,6 +94,7 @@ const AUDIENCES: Audience[] = [ { to: "/docs/self-hosting-rees-analyzers", label: "REES analyzers" }, { to: "/docs/upstream-drift", label: "Upstream drift" }, { to: "/docs/ai-summaries", label: "AI summaries policy" }, + { to: "/docs/backtest-calibration", label: "Backtest & calibration" }, ], }, {