diff --git a/scripts/backfill-calibration-corpus-core.ts b/scripts/backfill-calibration-corpus-core.ts new file mode 100644 index 000000000..deef093df --- /dev/null +++ b/scripts/backfill-calibration-corpus-core.ts @@ -0,0 +1,191 @@ +// Pure core for the calibration-corpus backfill (#8157 phase 1, epic #8082). Transforms historical +// review_targets decisions (decision-level AI verdict + confidence, terminal outcome) into the synthesized +// signal.rule_fired / signal.human_override audit rows the live capture writers (#8101) would have produced +// had they existed then — so the shipped threshold backtest (#8138) and trend view (#8113) start from the +// ledger's real history instead of an empty corpus. No IO here — the CLI (backfill-calibration-corpus.ts) +// does the D1 reads/writes — mirrors backtest-corpus-export-core.ts's identical pure-core / thin-IO split. +// +// Integrity rules (#8157's own Requirements, plus the mapping decision ratified on the issue): +// • Mapping (a): decision-level CLOSE verdicts synthesize firings for `ai_consensus_defect` — ONE rule id +// (the close-authority consensus code KNOWN_THRESHOLDS maps to DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE), +// never duplicated across sibling ids, and every synthesized row carries `backfilled: true` + +// `provenance` so consumers can include/exclude the era explicitly. +// • Never fabricate: a row without a close verdict, a numeric confidence, or a terminal outcome yields +// NOTHING (counted, not guessed). modelResponseText is never synthesized. +// • Idempotent by construction: deterministic ids + INSERT OR IGNORE, so re-runs are no-ops. +export const BACKFILL_PROVENANCE = "review_targets_decision_level"; +/** Mapping (a) — see #8157. The single rule id historical close decisions are synthesized under. */ +export const BACKFILL_RULE_ID = "ai_consensus_defect"; + +const FIRED_EVENT_TYPE = `signal.rule_fired:${BACKFILL_RULE_ID}`; +const OVERRIDE_EVENT_TYPE = `signal.human_override:${BACKFILL_RULE_ID}`; + +/** The projection of a review_targets row this transform reads. `confidence` is the decision-level value + * json-extracted by the CLI's query; null when absent from decision_json. */ +export type ReviewTargetDecisionRow = { + repo: string; + number: number; + verdict: string | null; + status: string | null; + confidence: number | null; + terminalAt: string | null; +}; + +export type SynthesizedAuditRow = { + id: string; + eventType: string; + actor: string; + targetKey: string; + outcome: string; + detail: string; + metadataJson: string; + createdAt: string; +}; + +export type BackfillReport = { + eligible: number; + reversed: number; + confirmed: number; + skippedWrongVerdict: number; + skippedNoConfidence: number; + skippedNotTerminal: number; + skippedDuplicateTarget: number; + rows: SynthesizedAuditRow[]; +}; + +/** SQLite "YYYY-MM-DD HH:MM:SS" (no zone) normalized to ISO-8601 UTC; already-ISO strings pass through. + * Returns null for a blank value so eligibility can fail closed on it. */ +function normalizeLedgerTimestamp(value: string | null): string | null { + if (!value || !value.trim()) return null; + const t = value.includes("T") ? value : value.replace(" ", "T"); + const hasZone = t.endsWith("Z") || /[+-]\d\d:?\d\d$/.test(t); + const ms = Date.parse(hasZone ? t : `${t}Z`); + if (!Number.isFinite(ms)) return null; + return new Date(ms).toISOString(); +} + +/** + * Synthesize the backfill's fired + override audit rows from historical decisions. Eligibility (all + * required, each miss counted separately, priority in the listed order): verdict `close`, a numeric + * decision-level confidence, a parseable terminal timestamp, and a terminal `closed`/`merged` status. + * Label: `closed` (the close stood) ⇒ `confirmed`; `merged` (a closed-verdict PR that ended MERGED — the + * decision was wrong) ⇒ `reversed`. One synthesized pair per targetKey (a re-reviewed target keeps its + * LATEST terminal decision; earlier ones count as duplicates). The override's createdAt sits 1s after the + * firing's so buildBacktestCorpus's strictly-after pairing always matches. Deterministic output for + * deterministic input — ids derive from the targetKey alone. + */ +export function synthesizeBackfillRows(rows: readonly ReviewTargetDecisionRow[]): BackfillReport { + const report: BackfillReport = { + eligible: 0, + reversed: 0, + confirmed: 0, + skippedWrongVerdict: 0, + skippedNoConfidence: 0, + skippedNotTerminal: 0, + skippedDuplicateTarget: 0, + rows: [], + }; + + // Latest terminal decision wins per target — sort desc by normalized terminal time, first seen kept. + const eligible: Array<{ row: ReviewTargetDecisionRow; terminalIso: string }> = []; + for (const row of rows) { + if (row.verdict !== "close") { + report.skippedWrongVerdict += 1; + continue; + } + if (typeof row.confidence !== "number" || !Number.isFinite(row.confidence)) { + report.skippedNoConfidence += 1; + continue; + } + const terminalIso = normalizeLedgerTimestamp(row.terminalAt); + if (!terminalIso || (row.status !== "closed" && row.status !== "merged")) { + report.skippedNotTerminal += 1; + continue; + } + eligible.push({ row, terminalIso }); + } + eligible.sort((a, b) => (a.terminalIso < b.terminalIso ? 1 : a.terminalIso > b.terminalIso ? -1 : 0)); + + const seen = new Set(); + for (const { row, terminalIso } of eligible) { + const targetKey = `${row.repo}#${row.number}`; + if (seen.has(targetKey)) { + report.skippedDuplicateTarget += 1; + continue; + } + seen.add(targetKey); + const label = row.status === "merged" ? "reversed" : "confirmed"; + report.eligible += 1; + if (label === "reversed") report.reversed += 1; + else report.confirmed += 1; + + const overrideIso = new Date(Date.parse(terminalIso) + 1000).toISOString(); + report.rows.push( + { + id: `backfill:${BACKFILL_RULE_ID}:${targetKey}:fired`, + eventType: FIRED_EVENT_TYPE, + actor: "loopover", + targetKey, + outcome: "completed", + detail: `rule ${BACKFILL_RULE_ID} fired (close) against ${targetKey} [backfilled]`, + metadataJson: JSON.stringify({ outcome: "close", confidence: row.confidence, backfilled: true, provenance: BACKFILL_PROVENANCE }), + createdAt: terminalIso, + }, + { + id: `backfill:${BACKFILL_RULE_ID}:${targetKey}:override`, + eventType: OVERRIDE_EVENT_TYPE, + actor: "human", + targetKey, + outcome: "completed", + detail: `human ${label} rule ${BACKFILL_RULE_ID} against ${targetKey} [backfilled]`, + metadataJson: JSON.stringify({ verdict: label, backfilled: true, provenance: BACKFILL_PROVENANCE }), + createdAt: overrideIso, + }, + ); + } + return report; +} + +/** Single-quoted SQL string literal — mirrors backtest-corpus-export.ts's sqlStringLiteral exactly. */ +export function sqlStringLiteral(value: string): string { + return `'${value.replace(/'/g, "''")}'`; +} + +/** + * Render the synthesized rows as chunked UPSERT statements. Idempotency comes from the deterministic + * targetKey-derived ids; `ON CONFLICT(id) DO UPDATE` (rather than OR IGNORE) makes the semantics "latest + * decision wins" ACROSS runs too, matching the transform's own latest-terminal-wins dedupe: a target whose + * terminal decision changed between two backfill runs gets its pair UPDATED, never silently dropped + * (ORB-review finding on the original OR IGNORE shape), while a re-run over identical data remains an + * effective no-op. Chunked so a statement never grows past what `wrangler d1 execute --command` sanely + * carries. Returns [] for an empty report. + */ +export function buildBackfillInsertStatements(rows: readonly SynthesizedAuditRow[], chunkSize = 50): string[] { + const statements: string[] = []; + for (let start = 0; start < rows.length; start += Math.max(1, chunkSize)) { + const chunk = rows.slice(start, start + Math.max(1, chunkSize)); + const values = chunk + .map( + (row) => + `(${[row.id, row.eventType, row.actor, row.targetKey, row.outcome, row.detail, row.metadataJson, row.createdAt] + .map(sqlStringLiteral) + .join(", ")})`, + ) + .join(", "); + statements.push( + `INSERT INTO audit_events (id, event_type, actor, target_key, outcome, detail, metadata_json, created_at) VALUES ${values} ` + + `ON CONFLICT(id) DO UPDATE SET detail = excluded.detail, metadata_json = excluded.metadata_json, created_at = excluded.created_at`, + ); + } + return statements; +} + +/** The human-readable dry-run/apply summary the CLI prints and #8157's report requires. Pure string build. */ +export function renderBackfillReport(report: BackfillReport, mode: "dry-run" | "apply"): string { + return [ + `Calibration corpus backfill (${mode}) — mapping (a), rule ${BACKFILL_RULE_ID}, provenance ${BACKFILL_PROVENANCE}`, + ` eligible decisions: ${report.eligible} (confirmed ${report.confirmed}, reversed ${report.reversed})`, + ` synthesized audit rows: ${report.rows.length} (fired + override pairs)`, + ` skipped: wrong-verdict ${report.skippedWrongVerdict}, no-confidence ${report.skippedNoConfidence}, not-terminal ${report.skippedNotTerminal}, duplicate-target ${report.skippedDuplicateTarget}`, + ].join("\n"); +} diff --git a/scripts/backfill-calibration-corpus.ts b/scripts/backfill-calibration-corpus.ts new file mode 100644 index 000000000..ede9073b2 --- /dev/null +++ b/scripts/backfill-calibration-corpus.ts @@ -0,0 +1,84 @@ +#!/usr/bin/env node +// Calibration-corpus backfill CLI (#8157 phase 1, epic #8082). Reads historical review_targets decisions +// out of D1 via `wrangler d1 execute --json`, synthesizes the fired/override pairs the live capture +// writers (#8101) would have produced, and — ONLY with --apply — writes them back as idempotent +// `INSERT OR IGNORE` rows. Dry-run is the default and prints the report #8157 requires before any apply. +// All transform logic lives in backfill-calibration-corpus-core.ts (unit-tested); this file is the thin IO +// wrapper — mirrors backtest-corpus-export.ts's identical split. +// +// tsx scripts/backfill-calibration-corpus.ts --db loopover [--remote] [--apply] +// +// Deployment note (#8157): source AND destination are the same D1 — the ledger of record for this +// deployment. Self-host operators' corpora live in their own Postgres; a pg-driver variant is an explicit +// non-goal here. +import { spawnSync } from "node:child_process"; +import { + buildBackfillInsertStatements, + renderBackfillReport, + synthesizeBackfillRows, + type ReviewTargetDecisionRow, +} from "./backfill-calibration-corpus-core.js"; + +type Args = { db: string; remote: boolean; apply: boolean }; + +function parseArgs(argv: string[]): Args { + const args: Args = { db: "loopover", remote: false, apply: false }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + if (flag === "--remote") args.remote = true; + else if (flag === "--apply") args.apply = true; + else if (flag === "--db") args.db = argv[++i]!; + } + return args; +} + +// Mirrors export-d1-data.ts's d1Query: fail-loud so a partial read/write never passes silently. +function d1Execute(db: string, remote: boolean, sql: string): Array> { + const result = spawnSync("npx", ["wrangler", "d1", "execute", db, remote ? "--remote" : "--local", "--json", "--command", sql], { + encoding: "utf8", + maxBuffer: 256 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`wrangler d1 execute failed (${result.status}): ${(result.stderr || result.stdout || "").slice(0, 500)}`); + } + const parsed = JSON.parse(result.stdout); + const first = Array.isArray(parsed) ? parsed[0] : parsed; + return first?.results ?? []; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + + const rows = d1Execute( + args.db, + args.remote, + `SELECT repo, number, verdict, status, json_extract(decision_json, '$.confidence') AS confidence, terminal_at + FROM review_targets WHERE kind = 'pull_request'`, + ); + const projected: ReviewTargetDecisionRow[] = rows.map((row) => ({ + repo: typeof row.repo === "string" ? row.repo : "", + number: typeof row.number === "number" ? row.number : Number(row.number ?? 0), + verdict: typeof row.verdict === "string" ? row.verdict : null, + status: typeof row.status === "string" ? row.status : null, + confidence: typeof row.confidence === "number" ? row.confidence : null, + terminalAt: typeof row.terminal_at === "string" ? row.terminal_at : null, + })); + + const report = synthesizeBackfillRows(projected); + console.log(renderBackfillReport(report, args.apply ? "apply" : "dry-run")); + + if (!args.apply) { + console.error("dry-run only — re-run with --apply to write. Rows are INSERT OR IGNORE with deterministic ids (idempotent)."); + return; + } + const statements = buildBackfillInsertStatements(report.rows); + let written = 0; + for (const statement of statements) { + d1Execute(args.db, args.remote, statement); + written += 1; + console.error(`applied statement ${written}/${statements.length}`); + } + console.error(`backfill applied: ${report.rows.length} row(s) across ${statements.length} statement(s) (re-runs are no-ops).`); +} + +main(); diff --git a/test/unit/backfill-calibration-corpus-core.test.ts b/test/unit/backfill-calibration-corpus-core.test.ts new file mode 100644 index 000000000..b28c4335d --- /dev/null +++ b/test/unit/backfill-calibration-corpus-core.test.ts @@ -0,0 +1,153 @@ +import { describe, expect, it } from "vitest"; +import { buildBacktestCorpus, type HumanOverrideEvent, type RuleFiredEvent } from "@loopover/engine"; +import { + BACKFILL_PROVENANCE, + BACKFILL_RULE_ID, + buildBackfillInsertStatements, + renderBackfillReport, + sqlStringLiteral, + synthesizeBackfillRows, + type ReviewTargetDecisionRow, +} from "../../scripts/backfill-calibration-corpus-core.js"; + +function decisionRow(overrides: Partial = {}): ReviewTargetDecisionRow { + return { + repo: "acme/widgets", + number: 7, + verdict: "close", + status: "closed", + confidence: 0.82, + terminalAt: "2026-06-15 10:00:00", + ...overrides, + }; +} + +describe("synthesizeBackfillRows (#8157)", () => { + it("synthesizes an idempotent fired+override pair for a confirmed close decision, tagged with provenance", () => { + const report = synthesizeBackfillRows([decisionRow()]); + expect(report.eligible).toBe(1); + expect(report.confirmed).toBe(1); + expect(report.reversed).toBe(0); + expect(report.rows).toHaveLength(2); + + const [fired, override] = report.rows; + expect(fired!.id).toBe(`backfill:${BACKFILL_RULE_ID}:acme/widgets#7:fired`); + expect(fired!.eventType).toBe(`signal.rule_fired:${BACKFILL_RULE_ID}`); + expect(fired!.actor).toBe("loopover"); + expect(fired!.createdAt).toBe("2026-06-15T10:00:00.000Z"); // SQLite timestamp normalized to UTC ISO + expect(JSON.parse(fired!.metadataJson)).toEqual({ outcome: "close", confidence: 0.82, backfilled: true, provenance: BACKFILL_PROVENANCE }); + + expect(override!.id).toBe(`backfill:${BACKFILL_RULE_ID}:acme/widgets#7:override`); + expect(override!.eventType).toBe(`signal.human_override:${BACKFILL_RULE_ID}`); + expect(override!.actor).toBe("human"); + expect(override!.createdAt).toBe("2026-06-15T10:00:01.000Z"); // strictly after the firing, for pairing + expect(JSON.parse(override!.metadataJson)).toEqual({ verdict: "confirmed", backfilled: true, provenance: BACKFILL_PROVENANCE }); + }); + + it("labels a close-verdict decision whose PR ended MERGED as reversed — the decision was wrong", () => { + const report = synthesizeBackfillRows([decisionRow({ status: "merged" })]); + expect(report.reversed).toBe(1); + expect(report.confirmed).toBe(0); + expect(JSON.parse(report.rows[1]!.metadataJson).verdict).toBe("reversed"); + }); + + it("never fabricates: wrong verdict, missing/NaN confidence, missing/garbled terminal time, and non-terminal status are skipped and counted", () => { + const report = synthesizeBackfillRows([ + decisionRow({ verdict: "merge" }), + decisionRow({ verdict: null }), + decisionRow({ confidence: null }), + decisionRow({ confidence: Number.NaN }), + decisionRow({ terminalAt: null }), + decisionRow({ terminalAt: "not a time" }), + decisionRow({ status: "manual" }), + decisionRow({ status: null }), + ]); + expect(report.eligible).toBe(0); + expect(report.rows).toEqual([]); + expect(report.skippedWrongVerdict).toBe(2); + expect(report.skippedNoConfidence).toBe(2); + expect(report.skippedNotTerminal).toBe(4); + }); + + it("keeps only the LATEST terminal decision per target and counts earlier ones as duplicates", () => { + const report = synthesizeBackfillRows([ + decisionRow({ terminalAt: "2026-06-10 09:00:00", confidence: 0.5 }), + decisionRow({ terminalAt: "2026-06-20T09:00:00.000Z", confidence: 0.9 }), + ]); + expect(report.eligible).toBe(1); + expect(report.skippedDuplicateTarget).toBe(1); + expect(JSON.parse(report.rows[0]!.metadataJson).confidence).toBe(0.9); + }); + + it("orders deterministically when two targets share an identical terminal timestamp (comparator's equal arm)", () => { + const report = synthesizeBackfillRows([ + decisionRow({ number: 21, terminalAt: "2026-06-15 10:00:00" }), + decisionRow({ number: 22, terminalAt: "2026-06-15 10:00:00" }), + // Shuffled distinct timestamps around the pair force BOTH direction arms of the sort comparator. + decisionRow({ number: 23, terminalAt: "2026-06-15 09:00:00" }), + decisionRow({ number: 24, terminalAt: "2026-06-15 11:00:00" }), + ]); + expect(report.eligible).toBe(4); + expect(report.skippedDuplicateTarget).toBe(0); + }); + + it("passes an already-ISO terminal timestamp through unchanged (both timestamp eras)", () => { + const report = synthesizeBackfillRows([decisionRow({ terminalAt: "2026-06-15T10:00:00.000Z" })]); + expect(report.rows[0]!.createdAt).toBe("2026-06-15T10:00:00.000Z"); + }); + + it("round-trips into buildBacktestCorpus: synthesized pairs become labeled BacktestCases with the confidence metadata", () => { + const report = synthesizeBackfillRows([decisionRow(), decisionRow({ number: 8, status: "merged", confidence: 0.4 })]); + const fired: RuleFiredEvent[] = []; + const overrides: HumanOverrideEvent[] = []; + for (const row of report.rows) { + const metadata = JSON.parse(row.metadataJson) as Record; + if (row.eventType.startsWith("signal.rule_fired:")) { + const { outcome, ...extra } = metadata; + fired.push({ ruleId: BACKFILL_RULE_ID, targetKey: row.targetKey, outcome: String(outcome), occurredAt: row.createdAt, metadata: extra }); + } else { + const { verdict } = metadata; + overrides.push({ ruleId: BACKFILL_RULE_ID, targetKey: row.targetKey, verdict: verdict as "confirmed" | "reversed", occurredAt: row.createdAt }); + } + } + const cases = buildBacktestCorpus(BACKFILL_RULE_ID, fired, overrides); + expect(cases).toHaveLength(2); + expect(cases.find((c) => c.targetKey === "acme/widgets#7")!.label).toBe("confirmed"); + const wrongCall = cases.find((c) => c.targetKey === "acme/widgets#8")!; + expect(wrongCall.label).toBe("reversed"); + expect(wrongCall.metadata).toMatchObject({ confidence: 0.4, backfilled: true }); + }); +}); + +describe("buildBackfillInsertStatements (#8157)", () => { + const report = synthesizeBackfillRows([decisionRow(), decisionRow({ number: 8 }), decisionRow({ number: 9 })]); + + it("renders latest-decision-wins UPSERTs with the full audit_events column list and SQL-escaped values", () => { + const escaped = synthesizeBackfillRows([decisionRow({ repo: "o'brien/repo" })]); + const [statement] = buildBackfillInsertStatements(escaped.rows); + expect(statement).toMatch(/^INSERT INTO audit_events \(id, event_type, actor, target_key, outcome, detail, metadata_json, created_at\) VALUES /); + // ORB-review finding: a target whose terminal decision changed between runs must be UPDATED, never + // silently dropped -- the conflict clause updates exactly the decision-bearing columns. + expect(statement).toContain("ON CONFLICT(id) DO UPDATE SET detail = excluded.detail, metadata_json = excluded.metadata_json, created_at = excluded.created_at"); + expect(statement).toContain("o''brien/repo#7"); + expect(sqlStringLiteral("it's")).toBe("'it''s'"); + }); + + it("chunks rows across statements and clamps a degenerate chunk size to 1", () => { + expect(buildBackfillInsertStatements(report.rows, 4)).toHaveLength(2); // 6 rows -> 4 + 2 + expect(buildBackfillInsertStatements(report.rows, 0)).toHaveLength(6); + expect(buildBackfillInsertStatements([])).toEqual([]); + }); +}); + +describe("renderBackfillReport (#8157)", () => { + it("summarizes both modes with every counter", () => { + const report = synthesizeBackfillRows([decisionRow(), decisionRow({ verdict: "merge" })]); + const text = renderBackfillReport(report, "dry-run"); + expect(text).toContain("dry-run"); + expect(text).toContain(`rule ${BACKFILL_RULE_ID}`); + expect(text).toContain("eligible decisions: 1 (confirmed 1, reversed 0)"); + expect(text).toContain("wrong-verdict 1"); + expect(renderBackfillReport(report, "apply")).toContain("(apply)"); + }); +});