diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx
new file mode 100644
index 0000000000..cf2a56776c
--- /dev/null
+++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx
@@ -0,0 +1,79 @@
+---
+title: Backtest & calibration
+description: How LoopOver measures whether its own review rules are right — and backtests threshold and logic changes against real history before they ship.
+---
+
+## What this is
+
+Every time a configured gate blocker fires, LoopOver records it. Every time a human later
+overrides that call (or confirms it), LoopOver records that too. Paired together, the two
+histories become a labeled corpus: _this rule fired against this PR, and a person later said it
+was right ("confirmed") or wrong ("reversed")_.
+
+That corpus makes a rule change **testable before it merges**. A PR that tunes a confidence
+threshold or rewrites detection logic gets scored against the real recorded history — the same
+targets, the same raw inputs — instead of being eyeballed.
+
+
+ Everything on this page is advisory-only today. A backtest verdict — even a regression — never
+ blocks a merge. Whether a REGRESSED verdict should ever gate merges is a tracked, deliberately
+ separate decision that waits for real production track-record data.
+
+
+## The corpus
+
+- **Fired events** — recorded when a configured blocker actually carries gate authority, with
+ bounded raw context: the issue text, PR title/body, and diff the rule evaluated, and (for the
+ AI-judgment rule `linked_issue_scope_mismatch`) the model's own raw response text.
+- **Override events** — recorded when a human reverses an automated call, or confirms one (for
+ example, an owner closing a PR that was held for low AI confidence confirms the hold).
+- **`secret_leak` is permanently excluded from raw-context capture.** Storing the diff that
+ triggered a leaked-credential finding would store the credential itself in the audit trail.
+ This rule can therefore never be logic-backtested — by design, not omission.
+
+## Two backtests, two mechanisms
+
+**Threshold changes** run inside LoopOver's own review pass. When a PR's diff changes a known
+confidence-threshold constant, the old and new values are replayed as classifiers over the
+rule's recorded history, and the comparison is rendered directly into the unified review
+comment. No code from the PR is ever executed — a threshold is just a number.
+
+**Logic/regex changes** run in a dedicated CI workflow instead, because honestly verifying a
+rewritten detection function requires _executing the PR's own code_ against history — something
+the production review service must never do. The workflow checks out both the PR's head and its
+base, replays each recorded case's captured raw context through both versions of the detection
+function, and posts its own clearly-labeled **"Logic backtest"** PR comment, separate from the
+unified review comment. It triggers only for PRs touching the watched detection-logic paths,
+skips drafts and forks (fork runs get no secrets), and fails open: an infrastructure problem
+produces a skip notice, never a red check.
+
+## Reading the comparison
+
+Both backtests score the same way:
+
+- **"Reversed" is the positive class.** A classifier predicting "reversed" is saying the rule's
+ original firing was wrong. Precision and recall are computed against the human labels, and
+ stay `N/A` when there is not enough decided data — unknown is never coerced to zero.
+- **The Pareto floor decides the verdict.** A candidate that regresses on _any_ axis is
+ **REGRESSED**, even if the other axis improved. Trading precision for recall is a regression,
+ not a net win. Otherwise the verdict is _improved_ (some axis moved up) or _unchanged_.
+- Cases recorded before raw-context capture existed are skipped and counted in the comment —
+ replaying them against empty inputs would bias both sides, so they are never scored.
+
+## The track record
+
+Every backtest run persists its comparison as structured audit data. The accumulated record —
+how often runs regress, per rule — is the evidence base for the pending merge-gating decision,
+readable at any time with the maintainer CLI:
+
+```bash
+npx tsx scripts/backtest-track-record.ts --db loopover --remote
+```
+
+The corpus itself can be exported as a versioned, checksummed snapshot:
+
+```bash
+npx tsx scripts/backtest-corpus-export.ts --rule-id --output corpus.json --remote
+```
+
+Both CLIs are strictly read-only against the database.
diff --git a/apps/loopover-ui/src/components/site/command-palette.tsx b/apps/loopover-ui/src/components/site/command-palette.tsx
index 06c67af17c..527f38a611 100644
--- a/apps/loopover-ui/src/components/site/command-palette.tsx
+++ b/apps/loopover-ui/src/components/site/command-palette.tsx
@@ -61,6 +61,7 @@ const DEFAULT_ITEMS: PaletteItem[] = [
{ label: "Scoreability", to: "/docs/scoreability", group: "Docs" },
{ label: "Upstream drift", to: "/docs/upstream-drift", group: "Docs" },
{ label: "AI summaries policy", to: "/docs/ai-summaries", group: "Docs" },
+ { label: "Backtest & calibration", to: "/docs/backtest-calibration", group: "Docs" },
{ label: "Privacy & security", to: "/docs/privacy-security", group: "Docs" },
{ label: "Troubleshooting", to: "/docs/troubleshooting", group: "Docs" },
{ label: "API reference", to: "/api", group: "Reference" },
diff --git a/apps/loopover-ui/src/components/site/docs-nav.tsx b/apps/loopover-ui/src/components/site/docs-nav.tsx
index 5f6876f001..bab4b2a82a 100644
--- a/apps/loopover-ui/src/components/site/docs-nav.tsx
+++ b/apps/loopover-ui/src/components/site/docs-nav.tsx
@@ -99,6 +99,7 @@ export const docsNav: DocsGroup[] = [
{ to: "/docs/branch-analysis", label: "Branch analysis" },
{ to: "/docs/scoreability", label: "Scoreability" },
{ to: "/docs/upstream-drift", label: "Upstream drift" },
+ { to: "/docs/backtest-calibration", label: "Backtest & calibration" },
],
},
{
diff --git a/apps/loopover-ui/src/lib/docs-source-server-isolation.test.ts b/apps/loopover-ui/src/lib/docs-source-server-isolation.test.ts
index d45b00e2e0..4cbbc746a8 100644
--- a/apps/loopover-ui/src/lib/docs-source-server-isolation.test.ts
+++ b/apps/loopover-ui/src/lib/docs-source-server-isolation.test.ts
@@ -29,7 +29,7 @@ describe("docs.*.tsx routes never import docs-source(.server) directly", () => {
const inScope = docsRouteFiles.filter((name) => !outOfScope.has(name));
it("found the expected set of in-scope docs route files", () => {
- expect(inScope.length).toBe(48);
+ expect(inScope.length).toBe(49);
});
it.each(inScope)(
diff --git a/apps/loopover-ui/src/routeTree.gen.ts b/apps/loopover-ui/src/routeTree.gen.ts
index 0b0fdf902f..ddf4b1534e 100644
--- a/apps/loopover-ui/src/routeTree.gen.ts
+++ b/apps/loopover-ui/src/routeTree.gen.ts
@@ -63,6 +63,7 @@ import { Route as DocsFederatedFleetIntelligenceRouteImport } from './routes/doc
import { Route as DocsCapacityRouteImport } from './routes/docs.capacity'
import { Route as DocsBranchAnalysisRouteImport } from './routes/docs.branch-analysis'
import { Route as DocsBetaOnboardingRouteImport } from './routes/docs.beta-onboarding'
+import { Route as DocsBacktestCalibrationRouteImport } from './routes/docs.backtest-calibration'
import { Route as DocsAmsUnattendedSchedulingRouteImport } from './routes/docs.ams-unattended-scheduling'
import { Route as DocsAmsSizingRouteImport } from './routes/docs.ams-sizing'
import { Route as DocsAmsOperationsRunbookRouteImport } from './routes/docs.ams-operations-runbook'
@@ -376,6 +377,11 @@ const DocsBetaOnboardingRoute = DocsBetaOnboardingRouteImport.update({
path: '/beta-onboarding',
getParentRoute: () => DocsRoute,
} as any)
+const DocsBacktestCalibrationRoute = DocsBacktestCalibrationRouteImport.update({
+ id: '/backtest-calibration',
+ path: '/backtest-calibration',
+ getParentRoute: () => DocsRoute,
+} as any)
const DocsAmsUnattendedSchedulingRoute =
DocsAmsUnattendedSchedulingRouteImport.update({
id: '/ams-unattended-scheduling',
@@ -554,6 +560,7 @@ export interface FileRoutesByFullPath {
'/docs/ams-operations-runbook': typeof DocsAmsOperationsRunbookRoute
'/docs/ams-sizing': typeof DocsAmsSizingRoute
'/docs/ams-unattended-scheduling': typeof DocsAmsUnattendedSchedulingRoute
+ '/docs/backtest-calibration': typeof DocsBacktestCalibrationRoute
'/docs/beta-onboarding': typeof DocsBetaOnboardingRoute
'/docs/branch-analysis': typeof DocsBranchAnalysisRoute
'/docs/capacity': typeof DocsCapacityRoute
@@ -633,6 +640,7 @@ export interface FileRoutesByTo {
'/docs/ams-operations-runbook': typeof DocsAmsOperationsRunbookRoute
'/docs/ams-sizing': typeof DocsAmsSizingRoute
'/docs/ams-unattended-scheduling': typeof DocsAmsUnattendedSchedulingRoute
+ '/docs/backtest-calibration': typeof DocsBacktestCalibrationRoute
'/docs/beta-onboarding': typeof DocsBetaOnboardingRoute
'/docs/branch-analysis': typeof DocsBranchAnalysisRoute
'/docs/capacity': typeof DocsCapacityRoute
@@ -717,6 +725,7 @@ export interface FileRoutesById {
'/docs/ams-operations-runbook': typeof DocsAmsOperationsRunbookRoute
'/docs/ams-sizing': typeof DocsAmsSizingRoute
'/docs/ams-unattended-scheduling': typeof DocsAmsUnattendedSchedulingRoute
+ '/docs/backtest-calibration': typeof DocsBacktestCalibrationRoute
'/docs/beta-onboarding': typeof DocsBetaOnboardingRoute
'/docs/branch-analysis': typeof DocsBranchAnalysisRoute
'/docs/capacity': typeof DocsCapacityRoute
@@ -802,6 +811,7 @@ export interface FileRouteTypes {
| '/docs/ams-operations-runbook'
| '/docs/ams-sizing'
| '/docs/ams-unattended-scheduling'
+ | '/docs/backtest-calibration'
| '/docs/beta-onboarding'
| '/docs/branch-analysis'
| '/docs/capacity'
@@ -881,6 +891,7 @@ export interface FileRouteTypes {
| '/docs/ams-operations-runbook'
| '/docs/ams-sizing'
| '/docs/ams-unattended-scheduling'
+ | '/docs/backtest-calibration'
| '/docs/beta-onboarding'
| '/docs/branch-analysis'
| '/docs/capacity'
@@ -964,6 +975,7 @@ export interface FileRouteTypes {
| '/docs/ams-operations-runbook'
| '/docs/ams-sizing'
| '/docs/ams-unattended-scheduling'
+ | '/docs/backtest-calibration'
| '/docs/beta-onboarding'
| '/docs/branch-analysis'
| '/docs/capacity'
@@ -1405,6 +1417,13 @@ declare module '@tanstack/react-router' {
preLoaderRoute: typeof DocsBetaOnboardingRouteImport
parentRoute: typeof DocsRoute
}
+ '/docs/backtest-calibration': {
+ id: '/docs/backtest-calibration'
+ path: '/backtest-calibration'
+ fullPath: '/docs/backtest-calibration'
+ preLoaderRoute: typeof DocsBacktestCalibrationRouteImport
+ parentRoute: typeof DocsRoute
+ }
'/docs/ams-unattended-scheduling': {
id: '/docs/ams-unattended-scheduling'
path: '/ams-unattended-scheduling'
@@ -1658,6 +1677,7 @@ interface DocsRouteChildren {
DocsAmsOperationsRunbookRoute: typeof DocsAmsOperationsRunbookRoute
DocsAmsSizingRoute: typeof DocsAmsSizingRoute
DocsAmsUnattendedSchedulingRoute: typeof DocsAmsUnattendedSchedulingRoute
+ DocsBacktestCalibrationRoute: typeof DocsBacktestCalibrationRoute
DocsBetaOnboardingRoute: typeof DocsBetaOnboardingRoute
DocsBranchAnalysisRoute: typeof DocsBranchAnalysisRoute
DocsCapacityRoute: typeof DocsCapacityRoute
@@ -1711,6 +1731,7 @@ const DocsRouteChildren: DocsRouteChildren = {
DocsAmsOperationsRunbookRoute: DocsAmsOperationsRunbookRoute,
DocsAmsSizingRoute: DocsAmsSizingRoute,
DocsAmsUnattendedSchedulingRoute: DocsAmsUnattendedSchedulingRoute,
+ DocsBacktestCalibrationRoute: DocsBacktestCalibrationRoute,
DocsBetaOnboardingRoute: DocsBetaOnboardingRoute,
DocsBranchAnalysisRoute: DocsBranchAnalysisRoute,
DocsCapacityRoute: DocsCapacityRoute,
diff --git a/apps/loopover-ui/src/routes/docs-routes-loading-state.test.tsx b/apps/loopover-ui/src/routes/docs-routes-loading-state.test.tsx
index bc1a8d3d93..1d0c07bcda 100644
--- a/apps/loopover-ui/src/routes/docs-routes-loading-state.test.tsx
+++ b/apps/loopover-ui/src/routes/docs-routes-loading-state.test.tsx
@@ -45,7 +45,7 @@ describe("docs route Suspense fallback (#6982)", () => {
"docs.tsx",
]);
const inScope = docsRouteFiles.filter((name) => !outOfScope.has(name));
- expect(inScope.length).toBe(48);
+ expect(inScope.length).toBe(49);
for (const file of inScope) {
const source = readFileSync(join(routesDir, file), "utf8");
diff --git a/apps/loopover-ui/src/routes/docs.backtest-calibration.tsx b/apps/loopover-ui/src/routes/docs.backtest-calibration.tsx
new file mode 100644
index 0000000000..5092800d44
--- /dev/null
+++ b/apps/loopover-ui/src/routes/docs.backtest-calibration.tsx
@@ -0,0 +1,50 @@
+import { createFileRoute, notFound } from "@tanstack/react-router";
+import { Suspense } from "react";
+
+import { DocsPage } from "@/components/site/docs-page";
+import { LoadingState } from "@/components/site/state-views";
+import { docsClientLoader } from "@/lib/docs-client-loader";
+import { getDocPage } from "@/lib/docs-source.functions";
+
+// Rendered from content/docs/backtest-calibration.mdx via fumadocs-mdx's browser entry
+// (docsClientLoader), through the existing DocsPage/Callout/CodeBlock primitives -- not fumadocs-ui's
+// bundled components. See docs-source.server.ts's comment for why the loader below resolves only a
+// plain, serializable path string.
+export const Route = createFileRoute("/docs/backtest-calibration")({
+ loader: async () => {
+ const page = await getDocPage({ data: { slugs: ["backtest-calibration"] } });
+ if (!page) throw notFound();
+ return page;
+ },
+ head: () => ({
+ meta: [
+ { title: "Backtest & calibration — LoopOver docs" },
+ {
+ name: "description",
+ content:
+ "How LoopOver measures whether its own review rules are right, and backtests threshold and logic changes against real recorded history before they ship.",
+ },
+ { property: "og:title", content: "Backtest & calibration — LoopOver docs" },
+ {
+ property: "og:description",
+ content:
+ "How LoopOver measures whether its own review rules are right, and backtests threshold and logic changes against real recorded history before they ship.",
+ },
+ { property: "og:url", content: "/docs/backtest-calibration" },
+ ],
+ links: [{ rel: "canonical", href: "/docs/backtest-calibration" }],
+ }),
+ component: BacktestCalibrationDoc,
+});
+
+function BacktestCalibrationDoc() {
+ const { path, title, description } = Route.useLoaderData();
+ const Content = docsClientLoader.getComponent(path);
+ return (
+
+ }>
+
+
+
+ );
+}
diff --git a/apps/loopover-ui/src/routes/docs.index.tsx b/apps/loopover-ui/src/routes/docs.index.tsx
index 2e99a8e5ef..26b8985dbd 100644
--- a/apps/loopover-ui/src/routes/docs.index.tsx
+++ b/apps/loopover-ui/src/routes/docs.index.tsx
@@ -94,6 +94,7 @@ const AUDIENCES: Audience[] = [
{ to: "/docs/self-hosting-rees-analyzers", label: "REES analyzers" },
{ to: "/docs/upstream-drift", label: "Upstream drift" },
{ to: "/docs/ai-summaries", label: "AI summaries policy" },
+ { to: "/docs/backtest-calibration", label: "Backtest & calibration" },
],
},
{