diff --git a/.loopover.yml.example b/.loopover.yml.example index d2e3077b13..5358bb6861 100644 --- a/.loopover.yml.example +++ b/.loopover.yml.example @@ -329,6 +329,16 @@ gate: # value. contentLaneDeliverable: off + # Backtest-regression gate (#8105). Governs what a REGRESSED verdict from the + # pre-merge backtest (threshold #8138 / logic #8139) does: advisory (default) + # renders the comparison in the review comment but never blocks; block + # escalates it into a backtest_regression hard blocker; off silences the + # backtest advisory entirely. Flip to block only once the persisted track + # record (scripts/backtest-track-record.ts, #8140) supports it. + # off | advisory | block. Default: advisory. DB-backed (dashboard-settable + # too); this overrides the stored value. + backtestRegression: advisory + # Gate-check dry-run. When true, the posted check conclusion remains the real # non-enforcing verdict while comments/check text may also show the would-be # stricter verdict for AI-review blocker mode. It does not disable downstream diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx index cf2a56776c..8d0d5e1ec9 100644 --- a/apps/loopover-ui/content/docs/backtest-calibration.mdx +++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx @@ -15,9 +15,10 @@ threshold or rewrites detection logic gets scored against the real recorded hist targets, the same raw inputs — instead of being eyeballed. - Everything on this page is advisory-only today. A backtest verdict — even a regression — never - blocks a merge. Whether a REGRESSED verdict should ever gate merges is a tracked, deliberately - separate decision that waits for real production track-record data. + Everything on this page is advisory-only by default. A REGRESSED verdict blocks a merge only if a + repo explicitly opts in via `backtestRegressionGateMode: block` (`.loopover.yml + gate.backtestRegression`) — the shipped default is `advisory`, and the flip is meant to wait for + real production track-record data. ## The corpus diff --git a/apps/loopover-ui/content/docs/tuning.mdx b/apps/loopover-ui/content/docs/tuning.mdx index 2a1e0486e1..6bd9f6b058 100644 --- a/apps/loopover-ui/content/docs/tuning.mdx +++ b/apps/loopover-ui/content/docs/tuning.mdx @@ -259,6 +259,12 @@ forward and wins whenever both are set for the same field. `advisory` renders the assessment in the review comment without blocking; `block` additionally lets a confidence-floor-passing "unaddressed" verdict become a blocker. +- `gate.backtestRegression` — governs what a REGRESSED verdict from the + [pre-merge backtest](/docs/backtest-calibration) does. Default `advisory`: + the comparison renders in the review comment but never blocks. `block` + escalates a REGRESSED verdict into a hard blocker — flip it only once the + persisted track record supports gating. `off` silences the backtest advisory + entirely. - `gate.contentLaneDeliverable` — only meaningful for a repo with a registry content-lane spec configured (`contentLane`); a no-op otherwise. Fully deterministic (a text/path match, no AI call): when the PR's primary linked @@ -381,36 +387,42 @@ testExpectations: linkedIssuePolicy: preferred # Gate policy — checkMode is set explicitly (not the legacy, ambiguous "enabled" alias) + gate: - checkMode: visible - pack: gittensor - duplicates: block - linkedIssue: advisory - readiness: - mode: advisory - minScore: 70 - slop: - mode: block - minScore: 60 - aiAdvisory: true - mergeReadiness: advisory - manifestPolicy: block - aiReview: - mode: advisory - byok: true - provider: anthropic - model: claude-3-5-sonnet-latest +checkMode: visible +pack: gittensor +duplicates: block +linkedIssue: advisory +readiness: +mode: advisory +minScore: 70 +slop: +mode: block +minScore: 60 +aiAdvisory: true +mergeReadiness: advisory +manifestPolicy: block +aiReview: +mode: advisory +byok: true +provider: anthropic +model: claude-3-5-sonnet-latest # Generic settings overrides -- commentMode/checkRunMode/checkRunDetailLevel/badgeEnabled are + # config-as-code only (no DB column or dashboard toggle); this file is their sole source. + settings: - commentMode: detected_contributors_only - checkRunMode: enabled - checkRunDetailLevel: standard - badgeEnabled: true - # Optional path holds. Omitted or [] means no path guardrails. - # hardGuardrailGlobs: - # - "src/selfhost/**"`} +commentMode: detected_contributors_only +checkRunMode: enabled +checkRunDetailLevel: standard +badgeEnabled: true + +# Optional path holds. Omitted or [] means no path guardrails. + +# hardGuardrailGlobs: + +# - "src/selfhost/\*\*"`} /> diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index 06f0ddd1f9..4ef08d1c6a 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -9761,6 +9761,14 @@ "advisory", "block" ] + }, + "backtestRegressionGateMode": { + "type": "string", + "enum": [ + "off", + "advisory", + "block" + ] } }, "required": [ @@ -9782,6 +9790,7 @@ "selfAuthoredLinkedIssueGateMode", "linkedIssueSatisfactionGateMode", "contentLaneDeliverableGateMode", + "backtestRegressionGateMode", "slopAiAdvisory", "aiReviewMode", "aiReviewByok", @@ -10492,6 +10501,14 @@ "advisory", "block" ] + }, + "backtestRegressionGateMode": { + "type": "string", + "enum": [ + "off", + "advisory", + "block" + ] } }, "required": [ @@ -10513,6 +10530,7 @@ "selfAuthoredLinkedIssueGateMode", "linkedIssueSatisfactionGateMode", "contentLaneDeliverableGateMode", + "backtestRegressionGateMode", "autoLabelEnabled", "typeLabelsEnabled", "gittensorLabel", diff --git a/config/examples/loopover.full.yml b/config/examples/loopover.full.yml index 1ee4ca0690..a7cfe2ea61 100644 --- a/config/examples/loopover.full.yml +++ b/config/examples/loopover.full.yml @@ -343,6 +343,16 @@ gate: # value. contentLaneDeliverable: off + # Backtest-regression gate (#8105). Governs what a REGRESSED verdict from the + # pre-merge backtest (threshold #8138 / logic #8139) does: advisory (default) + # renders the comparison in the review comment but never blocks; block + # escalates it into a backtest_regression hard blocker; off silences the + # backtest advisory entirely. Flip to block only once the persisted track + # record (scripts/backtest-track-record.ts, #8140) supports it. + # off | advisory | block. Default: advisory. DB-backed (dashboard-settable + # too); this overrides the stored value. + backtestRegression: advisory + # Gate-check dry-run. When true, the posted check conclusion remains the real # non-enforcing verdict while comments/check text may also show the would-be # stricter verdict for AI-review blocker mode. It does not disable downstream diff --git a/migrations/0174_gate_backtest_regression.sql b/migrations/0174_gate_backtest_regression.sql new file mode 100644 index 0000000000..45a4b4290e --- /dev/null +++ b/migrations/0174_gate_backtest_regression.sql @@ -0,0 +1,6 @@ +-- Backtest-regression gate (#8105, epic #8082): advisory by default -- byte-identical behavior for every +-- existing row (the shipped #8138/#8142 comment-only advisory). block escalates a REGRESSED pre-merge +-- backtest verdict into a backtest_regression hard blocker; off silences the backtest advisory entirely. +-- The flip to block is deliberately a config change made only once #8140's persisted track record supports +-- it -- see #8105's own do-not-gate-before-data boundary. +ALTER TABLE repository_settings ADD COLUMN backtest_regression_gate_mode TEXT NOT NULL DEFAULT 'advisory'; diff --git a/packages/loopover-engine/src/advisory/gate-advisory.ts b/packages/loopover-engine/src/advisory/gate-advisory.ts index 95116fbcd9..f346fb21d6 100644 --- a/packages/loopover-engine/src/advisory/gate-advisory.ts +++ b/packages/loopover-engine/src/advisory/gate-advisory.ts @@ -99,6 +99,10 @@ export type GateCheckPolicy = { * on this mode; see runContentLaneDeliverableCheckForAdvisory, src/queue/processors.ts on the host side), * so this branch only matters once a repo has explicitly opted into `advisory`/`block`. */ contentLaneDeliverableGateMode?: GateRuleMode | undefined; + /** Backtest-regression gate (#8105): `block` turns a `backtest_regression` finding into a hard blocker. + * Default `advisory` — the finding only exists in block mode (the host resolver pushes it there), so + * this branch is defense-in-depth, mirroring content_lane_deliverable_missing's above. */ + backtestRegressionGateMode?: GateRuleMode | undefined; /** CLA / license-compatibility gate (#2564). When `block`, a `cla_consent_missing` finding — raised when * neither configured detection method (a consent phrase in the PR body, or a named CLA-bot check-run * conclusion) confirms consent — becomes a hard blocker. `off` (default) = no finding at all; `advisory` = @@ -636,6 +640,9 @@ function isConfiguredGateBlocker(finding: AdvisoryFinding, policy: GateCheckPoli // opts in with `block`. Fully deterministic (no AI judgment involved), so it is exempt from the // close-precision circuit breaker on the host side. if (code === "content_lane_deliverable_missing") return gatePolicyBlocks(policy.contentLaneDeliverableGateMode, "off"); + // Backtest-regression gate (#8105): blocks only under an explicit opt-in; default advisory (the shipped + // pre-#8105 behavior). The finding itself only exists in block mode -- see the host-side resolver. + if (code === "backtest_regression") return gatePolicyBlocks(policy.backtestRegressionGateMode, "advisory"); // Lockfile-tamper-risk gate (#2563): blocks only when the maintainer opts in with `block`. Defaults to `off` // (the finding is never even produced — see maybeAddLockfileTamperFinding's mode gate in queue/processors.ts), // so this branch only matters once a repo has explicitly turned the scan on. diff --git a/packages/loopover-engine/src/focus-manifest.ts b/packages/loopover-engine/src/focus-manifest.ts index b7288d5072..676ae101c8 100644 --- a/packages/loopover-engine/src/focus-manifest.ts +++ b/packages/loopover-engine/src/focus-manifest.ts @@ -165,6 +165,12 @@ export type FocusManifestGateConfig = { * `linkedIssueSatisfaction` immediately above in shape, but is a purely structural check, not an AI * opinion, so it carries none of that feature's AI-budget/confidence-floor machinery. */ contentLaneDeliverable: GateRuleMode | null; + /** `gate.backtestRegression` (#8105, epic #8082): off|advisory|block, advisory by default — governs what a + * REGRESSED pre-merge backtest verdict (#8138 threshold / #8139 logic) does. `advisory` renders the + * comparison but never blocks (the shipped pre-#8105 behavior); `block` escalates it into a + * `backtest_regression` hard blocker; `off` silences the backtest advisory entirely. DB-backed + * (dashboard-settable too); this overrides the stored value like every other `gate:` field. */ + backtestRegression: GateRuleMode | null; dryRun: boolean | null; /** `gate.premergeContentRecheck` (#2550): for a PR touching `migrations/**`, re-verify against a live, * freshly-fetched tip of the base branch — unioned with this PR's own new migration filenames — for a @@ -1303,6 +1309,7 @@ const EMPTY_GATE_CONFIG: FocusManifestGateConfig = { selfAuthoredLinkedIssue: null, linkedIssueSatisfaction: null, contentLaneDeliverable: null, + backtestRegression: null, dryRun: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, @@ -1788,6 +1795,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu selfAuthoredLinkedIssue: normalizeOptionalGateMode(record.selfAuthoredLinkedIssue, "gate.selfAuthoredLinkedIssue", warnings), linkedIssueSatisfaction: normalizeOptionalGateMode(record.linkedIssueSatisfaction, "gate.linkedIssueSatisfaction", warnings), contentLaneDeliverable: normalizeOptionalGateMode(record.contentLaneDeliverable, "gate.contentLaneDeliverable", warnings), + backtestRegression: normalizeOptionalGateMode(record.backtestRegression, "gate.backtestRegression", warnings), dryRun: normalizeOptionalBoolean(record.dryRun, "gate.dryRun", warnings), premergeContentRecheck: normalizeOptionalBoolean(record.premergeContentRecheck, "gate.premergeContentRecheck", warnings), requireFreshRebaseWindowMinutes: normalizeOptionalPositiveInteger(record.requireFreshRebaseWindow, "gate.requireFreshRebaseWindow", warnings), @@ -1845,6 +1853,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu gate.selfAuthoredLinkedIssue !== null || gate.linkedIssueSatisfaction !== null || gate.contentLaneDeliverable !== null || + gate.backtestRegression !== null || gate.dryRun !== null || gate.premergeContentRecheck !== null || gate.requireFreshRebaseWindowMinutes !== null || diff --git a/scripts/check-docs-drift.ts b/scripts/check-docs-drift.ts index be2d5eb908..7f2d3bf321 100644 --- a/scripts/check-docs-drift.ts +++ b/scripts/check-docs-drift.ts @@ -240,6 +240,7 @@ export const GATE_MODE_MANIFEST: GateModeManifestRow[] = [ { field: "selfAuthoredLinkedIssueGateMode", aliases: ["selfAuthoredLinkedIssueGateMode", "gate.selfAuthoredLinkedIssue"], pages: ["how-reviews-work.mdx", "tuning.mdx", "github-app.mdx"] }, { field: "linkedIssueSatisfactionGateMode", aliases: ["linkedIssueSatisfactionGateMode", "gate.linkedIssueSatisfaction"], pages: ["how-reviews-work.mdx", "tuning.mdx", "github-app.mdx"] }, { field: "contentLaneDeliverableGateMode", aliases: ["contentLaneDeliverableGateMode", "gate.contentLaneDeliverable"], pages: ["how-reviews-work.mdx", "tuning.mdx", "github-app.mdx"] }, + { field: "backtestRegressionGateMode", aliases: ["backtestRegressionGateMode", "gate.backtestRegression"], pages: ["backtest-calibration.mdx", "tuning.mdx"] }, { field: "moderationGateMode", aliases: ["moderationGateMode", "settings.moderationGateMode"], pages: ["how-reviews-work.mdx", "tuning.mdx", "github-app.mdx"] }, ]; diff --git a/src/api/routes.ts b/src/api/routes.ts index 74caa6eef3..858d66860c 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -922,6 +922,7 @@ const maintainerSettingsSchema = z manifestPolicyGateMode: z.enum(["off", "advisory", "block"]), linkedIssueSatisfactionGateMode: z.enum(["off", "advisory", "block"]), contentLaneDeliverableGateMode: z.enum(["off", "advisory", "block"]), + backtestRegressionGateMode: z.enum(["off", "advisory", "block"]), // #6443: mergeTrainMode/gittensorLabel/blacklistLabel/createMissingLabel removed -- no longer DB-backed, // config-as-code only via .loopover.yml's settings: block now. // #6446: firstTimeContributorGrace removed -- a dead, never-wired RESERVED/INERT field (#2266); deleted diff --git a/src/db/repositories.ts b/src/db/repositories.ts index 5c2a7ee91c..b547c3ec9d 100644 --- a/src/db/repositories.ts +++ b/src/db/repositories.ts @@ -624,6 +624,7 @@ export async function getRepositorySettings(env: Env, fullName: string): Promise selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopGateMinScore: null, slopAiAdvisory: false, aiReviewMode: "off", @@ -720,6 +721,9 @@ export async function getRepositorySettings(env: Env, fullName: string): Promise selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: parseGateRuleMode(row.linkedIssueSatisfactionGateMode), contentLaneDeliverableGateMode: parseGateRuleMode(row.contentLaneDeliverableGateMode), + // parseGateRuleMode's fallback IS this field's default ("advisory"), so an unrecognized stored value + // degrades to the safe shipped behavior rather than silently blocking (#8105). + backtestRegressionGateMode: parseGateRuleMode(row.backtestRegressionGateMode), slopGateMinScore: normalizeQualityGateMinScore(row.slopGateMinScore), slopAiAdvisory: row.slopAiAdvisory, aiReviewMode: "off", @@ -865,6 +869,7 @@ export async function upsertRepositorySettings(env: Env, settings: Partial>, + // #8105: what a REGRESSED verdict does. Optional + defaulted so every existing caller/test keeps the + // shipped advisory-only behavior; the live gate call site threads settings.backtestRegressionGateMode. + options: { mode?: GateRuleMode; advisory?: { findings: AdvisoryFinding[] } } = {}, ): Promise { + const mode = options.mode ?? "advisory"; + // `off` silences the whole advisory (no D1 read, no persist, no comment section) -- the same short-circuit + // shape contentLaneDeliverableGateMode's "off" uses. advisory/block both still render the section. + if (mode === "off") return ""; if (!files.some((file) => THRESHOLD_BACKTEST_WATCHED_PATHS.has(file.path))) return ""; try { const { changed, comparisons } = await runThresholdBacktestAdvisory(env, buildSecretScanDiff(files)); if (comparisons.length === 0) return ""; await persistThresholdBacktestRuns(env, repoFullName, pr.number, changed, comparisons); + // #8105 block mode: a REGRESSED verdict becomes a configured gate blocker. The finding only exists in + // block mode (advisory keeps today's comment-only behavior byte-identically), and + // isConfiguredGateBlocker's own `backtest_regression` branch is the defense-in-depth mirror of this + // push -- same split as linked_issue_scope_mismatch's block-mode wiring above. + const regressed = comparisons.filter((comparison) => comparison.verdict === "regressed"); + if (mode === "block" && regressed.length > 0 && options.advisory) { + const ruleIds = regressed.map((comparison) => comparison.ruleId).join(", "); + options.advisory.findings.push({ + code: "backtest_regression", + severity: "warning", + title: "Backtest regression against recorded history", + detail: `Backtesting this PR's threshold change against real recorded history REGRESSED ${ruleIds} on at least one axis (see the Threshold backtest section).`, + action: "Revisit the changed threshold so no backtest axis regresses, or ask a maintainer to override.", + publicText: `Backtest: this change scores worse than the current value against recorded review history (${ruleIds}). See the Threshold backtest section for the comparison.`, + }); + } return thresholdBacktestBlock(comparisons); } catch (error) { console.error(JSON.stringify({ level: "error", event: "threshold_backtest_failed", repoFullName, pullNumber: pr.number, error: errorMessage(error) })); @@ -9485,8 +9508,13 @@ async function maybePublishPrPublicSurface( } } // Threshold-only backtest advisory (#8138, epic #8082): independent of linkedIssueSatisfactionGateMode - // above -- see resolveThresholdBacktestAdvisory's own doc comment for why. - thresholdBacktest = await resolveThresholdBacktestAdvisory(env, repoFullName, pr, await getReviewFiles()); + // above -- see resolveThresholdBacktestAdvisory's own doc comment for why. #8105 threads the repo's + // backtestRegressionGateMode: advisory (default) keeps the shipped comment-only behavior; block lets a + // REGRESSED verdict push a configured gate blocker; off silences the section. + thresholdBacktest = await resolveThresholdBacktestAdvisory(env, repoFullName, pr, await getReviewFiles(), { + mode: settings.backtestRegressionGateMode, + advisory, + }); // Content-lane linked-issue deliverable check (#content-lane-deliverable, opt-in via // contentLaneDeliverableGateMode). Independent gate mode from the AI-based satisfaction assessment above -- // `off` (default) short-circuits before any fetch, so this is byte-identical to before this feature existed diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts index e1fae729c8..6f933e6017 100644 --- a/src/rules/advisory.ts +++ b/src/rules/advisory.ts @@ -100,6 +100,11 @@ export type GateCheckPolicy = { * on this mode; see runContentLaneDeliverableCheckForAdvisory, src/queue/processors.ts), so this branch * only matters once a repo has explicitly opted into `advisory`/`block`. */ contentLaneDeliverableGateMode?: GateRuleMode | undefined; + /** Backtest-regression gate (#8105): `block` turns a `backtest_regression` finding (pushed by + * resolveThresholdBacktestAdvisory when a REGRESSED verdict exists) into a hard blocker. Default + * `advisory` — the finding never even exists below `block` (the resolver only pushes it in block mode), + * so this branch is defense-in-depth like content_lane_deliverable_missing's. */ + backtestRegressionGateMode?: GateRuleMode | undefined; /** CLA / license-compatibility gate (#2564). When `block`, a `cla_consent_missing` finding — raised when * neither configured detection method (a consent phrase in the PR body, or a named CLA-bot check-run * conclusion) confirms consent — becomes a hard blocker. `off` (default) = no finding at all; `advisory` = @@ -1043,6 +1048,9 @@ function isConfiguredGateBlocker(finding: AdvisoryFinding, policy: GateCheckPoli // "off" mode (see runContentLaneDeliverableCheckForAdvisory), so this is a defense-in-depth mirror of that // gate, not the primary enforcement point -- mirrors linked_issue_scope_mismatch immediately above. if (code === "content_lane_deliverable_missing") return gateMode(policy.contentLaneDeliverableGateMode ?? "off") === "block"; + // Backtest-regression gate (#8105): blocks only under an explicit opt-in, mirroring the content-lane gate + // above -- the finding itself only exists in block mode (see resolveThresholdBacktestAdvisory). + if (code === "backtest_regression") return gateMode(policy.backtestRegressionGateMode ?? "advisory") === "block"; // Lockfile-tamper-risk gate (#2563): blocks only when the maintainer opts in with `block`. Defaults to `off` // (the finding is never even produced — see maybeAddLockfileTamperFinding's mode gate in queue/processors.ts), // so this branch only matters once a repo has explicitly turned the scan on. diff --git a/src/signals/focus-manifest.ts b/src/signals/focus-manifest.ts index 2fb63f6ea9..0306bdd53e 100644 --- a/src/signals/focus-manifest.ts +++ b/src/signals/focus-manifest.ts @@ -525,6 +525,7 @@ function applyGateConfigOverrides(effective: RepositorySettings, gate: FocusMani if (gate.selfAuthoredLinkedIssue !== null) effective.selfAuthoredLinkedIssueGateMode = gate.selfAuthoredLinkedIssue; if (gate.linkedIssueSatisfaction !== null) effective.linkedIssueSatisfactionGateMode = gate.linkedIssueSatisfaction; if (gate.contentLaneDeliverable !== null) effective.contentLaneDeliverableGateMode = gate.contentLaneDeliverable; + if (gate.backtestRegression !== null) effective.backtestRegressionGateMode = gate.backtestRegression; if (gate.dryRun !== null) effective.gateDryRun = gate.dryRun; if (gate.premergeContentRecheck !== null) effective.premergeContentRecheck = gate.premergeContentRecheck; if (gate.requireFreshRebaseWindowMinutes !== null) effective.requireFreshRebaseWindowMinutes = gate.requireFreshRebaseWindowMinutes; diff --git a/src/signals/settings-preview.ts b/src/signals/settings-preview.ts index dbae539a89..9e9c2292a9 100644 --- a/src/signals/settings-preview.ts +++ b/src/signals/settings-preview.ts @@ -247,6 +247,7 @@ export type RepoSettingsPreview = { selfAuthoredLinkedIssueGateMode: RepositorySettings["selfAuthoredLinkedIssueGateMode"]; linkedIssueSatisfactionGateMode: RepositorySettings["linkedIssueSatisfactionGateMode"]; contentLaneDeliverableGateMode: RepositorySettings["contentLaneDeliverableGateMode"]; + backtestRegressionGateMode: RepositorySettings["backtestRegressionGateMode"]; slopGateMinScore?: number | null | undefined; autoLabelEnabled: boolean; typeLabelsEnabled: boolean; @@ -384,6 +385,7 @@ export function buildRepoSettingsPreview(args: { selfAuthoredLinkedIssueGateMode: settings.selfAuthoredLinkedIssueGateMode, linkedIssueSatisfactionGateMode: settings.linkedIssueSatisfactionGateMode, contentLaneDeliverableGateMode: settings.contentLaneDeliverableGateMode, + backtestRegressionGateMode: settings.backtestRegressionGateMode, slopGateMinScore: settings.slopGateMinScore ?? null, autoLabelEnabled: settings.autoLabelEnabled, typeLabelsEnabled: settings.typeLabelsEnabled ?? true, diff --git a/src/types.ts b/src/types.ts index bb5d88271e..36a5b32025 100644 --- a/src/types.ts +++ b/src/types.ts @@ -976,6 +976,14 @@ export type RepositorySettings = { * exactly like every other `gate:` field overrides its `RepositorySettings` counterpart. Default `off` — * opt-in. */ contentLaneDeliverableGateMode: GateRuleMode; + /** Backtest-regression gate (#8105, epic #8082). Governs what a REGRESSED verdict from the pre-merge + * backtest (#8138 threshold / #8139 logic) does: `advisory` (default — today's shipped behavior) renders + * the comparison but never blocks; `block` additionally pushes a `backtest_regression` finding that + * isConfiguredGateBlocker treats as a hard blocker; `off` skips the backtest advisory entirely. Default + * deliberately stays `advisory` until the persisted track record (#8140) supports flipping — see #8105's + * own do-not-gate-before-data boundary. `.loopover.yml gate.backtestRegression` overrides it exactly like + * every other `gate:` field. */ + backtestRegressionGateMode: GateRuleMode; /** Slop-risk threshold (0-100) at/above which `slopGateMode: block` blocks. Default 60 (the `high` band). */ slopGateMinScore?: number | null | undefined; /** AI-assisted slop advisory (the `slopAiAdvisory` capability). When true AND `slopGateMode != off`, a diff --git a/test/unit/backtest-regression-gate-setting.test.ts b/test/unit/backtest-regression-gate-setting.test.ts new file mode 100644 index 0000000000..e0d69960d7 --- /dev/null +++ b/test/unit/backtest-regression-gate-setting.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; +import { getRepositorySettings, upsertRepositoryFromGitHub, upsertRepositorySettings } from "../../src/db/repositories"; +import { createTestEnv } from "../helpers/d1"; + +// backtestRegressionGateMode persistence wiring (#8105): defaults, round-trip, and the unrecognized-value +// fallback. The behavioral (blocker/enforcement) sides live in gate-check-policy.test.ts and +// threshold-backtest-advisory-wiring.test.ts — this file pins ONLY the storage layer's three paths. + +async function seedRepo(env: Env, fullName: string) { + await upsertRepositoryFromGitHub(env, { + id: 1, + full_name: fullName, + name: fullName.split("/")[1]!, + owner: { login: fullName.split("/")[0]! }, + private: false, + default_branch: "main", + } as never); +} + +describe("backtestRegressionGateMode setting (#8105)", () => { + it("defaults to advisory — the shipped pre-#8105 behavior — for an unconfigured repo", async () => { + const env = createTestEnv(); + const settings = await getRepositorySettings(env, "acme/unconfigured"); + expect(settings.backtestRegressionGateMode).toBe("advisory"); + }); + + it("round-trips an explicit block, and an upsert omitting the field keeps the advisory default", async () => { + const env = createTestEnv(); + await seedRepo(env, "acme/widgets"); + await upsertRepositorySettings(env, { repoFullName: "acme/widgets", backtestRegressionGateMode: "block" }); + expect((await getRepositorySettings(env, "acme/widgets")).backtestRegressionGateMode).toBe("block"); + + await seedRepo(env, "acme/other"); + await upsertRepositorySettings(env, { repoFullName: "acme/other" }); + expect((await getRepositorySettings(env, "acme/other")).backtestRegressionGateMode).toBe("advisory"); + }); + + it("degrades an unrecognized stored value to advisory (parseGateRuleMode's fallback), never silently to block", async () => { + const env = createTestEnv(); + await seedRepo(env, "acme/widgets"); + await upsertRepositorySettings(env, { repoFullName: "acme/widgets", backtestRegressionGateMode: "advisory" }); + await env.DB.prepare("UPDATE repository_settings SET backtest_regression_gate_mode = 'bogus' WHERE repo_full_name = ?") + .bind("acme/widgets") + .run(); + expect((await getRepositorySettings(env, "acme/widgets")).backtestRegressionGateMode).toBe("advisory"); + }); +}); diff --git a/test/unit/check-docs-drift-script.test.ts b/test/unit/check-docs-drift-script.test.ts index d0f10cbe71..d28a889209 100644 --- a/test/unit/check-docs-drift-script.test.ts +++ b/test/unit/check-docs-drift-script.test.ts @@ -310,6 +310,8 @@ describe("check-docs-drift script", () => { 'import { PUBLIC_COMMAND_ENTRIES, MAINTAINER_COMMAND_ENTRIES, ACTION_COMMAND_ENTRIES } from "@/lib/command-reference";', "apps/loopover-ui/content/docs/how-reviews-work.mdx": buildGateModePageText(), "apps/loopover-ui/content/docs/github-app.mdx": buildGateModePageText(), + // #8105: backtestRegressionGateMode's manifest row names this page too. + "apps/loopover-ui/content/docs/backtest-calibration.mdx": buildGateModePageText(), }; return files; } @@ -327,10 +329,10 @@ describe("check-docs-drift script", () => { const result = checkDocsDrift({ root: "/fake", readFile: makeReadFile(files) }); expect(result.failures).toEqual([]); - // gateModes bumped 13 -> 14 for contentLaneDeliverableGateMode (#content-lane-deliverable). - // settingsFields = 14 GATE_MODE_MANIFEST fields + 20 synthetic extras; focusManifestFields = 18 + // gateModes bumped 14 -> 15 for backtestRegressionGateMode (#8105). + // settingsFields = 15 GATE_MODE_MANIFEST fields + 20 synthetic extras; focusManifestFields = 18 // synthetic review fields + the nested review.visual.productionUrl leaf (#4617). - expect(result.counts).toEqual({ flags: 10, commands: 19, gateModes: 14, settingsFields: 34, focusManifestFields: 19 }); + expect(result.counts).toEqual({ flags: 10, commands: 19, gateModes: 15, settingsFields: 35, focusManifestFields: 19 }); }); it("catches an unmapped *GateMode field missing from GATE_MODE_MANIFEST", () => { diff --git a/test/unit/focus-manifest.test.ts b/test/unit/focus-manifest.test.ts index 5b2de4d0fe..6ef8c3fd71 100644 --- a/test/unit/focus-manifest.test.ts +++ b/test/unit/focus-manifest.test.ts @@ -287,6 +287,7 @@ describe(".loopover.yml.example field-exhaustiveness (#1670)", () => { selfAuthoredLinkedIssue: "selfAuthoredLinkedIssue:", linkedIssueSatisfaction: "linkedIssueSatisfaction:", contentLaneDeliverable: "contentLaneDeliverable:", + backtestRegression: "backtestRegression:", dryRun: "dryRun:", premergeContentRecheck: "premergeContentRecheck:", requireFreshRebaseWindowMinutes: "requireFreshRebaseWindow:", @@ -943,7 +944,7 @@ describe("compileFocusManifestPolicy", () => { issueDiscoveryPolicy: "neutral", maintainerNotes: [], publicNotes: ["Keep PRs focused.", "Maximize your reward payout"], - gate: { present: false, enabled: null, checkMode: null, pack: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, sizeMaxFiles: null, sizeMaxLines: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewLowConfidenceDisposition: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, linkedIssueSatisfaction: null, contentLaneDeliverable: null, manifestPolicy: null, dryRun: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, staleBaseAheadByThreshold: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, expectedCiContexts: null, advisoryCheckRuns: null, aiJudgmentBlockersMode: null, copycatMode: null, copycatMinScore: null }, + gate: { present: false, enabled: null, checkMode: null, pack: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, sizeMaxFiles: null, sizeMaxLines: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewLowConfidenceDisposition: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, linkedIssueSatisfaction: null, contentLaneDeliverable: null, backtestRegression: null, manifestPolicy: null, dryRun: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, staleBaseAheadByThreshold: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, expectedCiContexts: null, advisoryCheckRuns: null, aiJudgmentBlockersMode: null, copycatMode: null, copycatMinScore: null }, settings: {}, review: { present: false, footerText: null, note: null, fields: {}, enrichmentAnalyzers: {}, profile: null, tone: null, securityFocus: null, inlineComments: null, fixHandoff: null, autoMergeSummary: null, suggestions: null, changedFilesSummary: null, effortScore: null, impactMap: null, cultureProfile: null, selftune: null, sweepWatchdog: null, prReconciliation: null, activeReviewReconciliation: null, reviewMemory: null, findingCategories: null, inlineCommentsPerCategory: null, minFindingSeverity: null, maxFindings: { blockers: null, nits: null }, commentVerbosity: null, e2eTestDelivery: null, e2eTestAutoTrigger: null, pathInstructions: [], instructions: null, excludePaths: [], pathFilters: [], preMergeChecks: [], autoReview: { ...EMPTY_AUTO_REVIEW_CONFIG }, aiModel: { ...EMPTY_SELF_HOST_AI_MODEL_CONFIG }, visual: { ...EMPTY_VISUAL_CONFIG }, linkedIssueSatisfaction: null, sharedConfigSource: null }, features: { present: false, rag: null, reputation: null, safety: null, grounding: null, e2eTests: null, screenshots: null, improvementSignal: null, amsReputationBridge: null }, @@ -1110,7 +1111,7 @@ describe("parseFocusManifest gate config", () => { // the block→advisory deprecation-downgrade behavior itself is covered separately below. const m = parseFocusManifest({ gate: { linkedIssue: "block", duplicates: "advisory", readiness: { mode: "advisory", minScore: 70 } } }); expect(m.present).toBe(true); - expect(m.gate).toEqual({ present: true, enabled: null, checkMode: null, pack: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "advisory", readinessMinScore: 70, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, sizeMaxFiles: null, sizeMaxLines: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewLowConfidenceDisposition: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, linkedIssueSatisfaction: null, contentLaneDeliverable: null, manifestPolicy: null, dryRun: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, staleBaseAheadByThreshold: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, expectedCiContexts: null, advisoryCheckRuns: null, aiJudgmentBlockersMode: null, copycatMode: null, copycatMinScore: null }); + expect(m.gate).toEqual({ present: true, enabled: null, checkMode: null, pack: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "advisory", readinessMinScore: 70, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, sizeMaxFiles: null, sizeMaxLines: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewLowConfidenceDisposition: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, linkedIssueSatisfaction: null, contentLaneDeliverable: null, backtestRegression: null, manifestPolicy: null, dryRun: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, staleBaseAheadByThreshold: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, expectedCiContexts: null, advisoryCheckRuns: null, aiJudgmentBlockersMode: null, copycatMode: null, copycatMinScore: null }); }); it("parses gate.mergeReadiness, round-trips it, and warns on a bad value (#822)", () => { diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts index fddc61d92a..77f0eb59e8 100644 --- a/test/unit/gate-check-policy.test.ts +++ b/test/unit/gate-check-policy.test.ts @@ -989,6 +989,33 @@ describe("linked-issue satisfaction gate blocker (#1961/#3906)", () => { }); }); +describe("backtest-regression gate blocker (#8105)", () => { + const backtestAdvisory = (): Advisory => ({ + ...missingIssueAdvisory(), + findings: [{ code: "backtest_regression", title: "Backtest regression against recorded history", severity: "warning", detail: "Backtesting this PR's threshold change REGRESSED linked_issue_scope_mismatch on at least one axis.", action: "Revisit the changed threshold so no backtest axis regresses, or ask a maintainer to override." }], + }); + + it("blocks (failure) under backtestRegressionGateMode: block", () => { + const result = evaluateGateCheck(backtestAdvisory(), { backtestRegressionGateMode: "block", confirmedContributor: true }); + expect(result.conclusion).toBe("failure"); + expect(result.blockers.map((b) => b.code)).toContain("backtest_regression"); + }); + + it("stays advisory (never blocks) under unset (default advisory), explicit advisory, and off — the shipped pre-#8105 behavior", () => { + expect(evaluateGateCheck(backtestAdvisory(), {}).conclusion).toBe("success"); // unset ⇒ defaults to advisory + expect(evaluateGateCheck(backtestAdvisory(), { backtestRegressionGateMode: "advisory" }).conclusion).toBe("success"); + expect(evaluateGateCheck(backtestAdvisory(), { backtestRegressionGateMode: "off" }).conclusion).toBe("success"); + const advisoryResult = evaluateGateCheck(backtestAdvisory(), { backtestRegressionGateMode: "advisory" }); + expect(advisoryResult.warnings.map((w) => w.code)).toContain("backtest_regression"); + }); + + it("resolveEffectiveSettings maps gate.backtestRegression → backtestRegressionGateMode, and gateCheckPolicy threads it", () => { + const eff = resolveEffectiveSettings(settings({}), parseFocusManifest({ gate: { backtestRegression: "block" } })); + expect(eff.backtestRegressionGateMode).toBe("block"); + expect(gateCheckPolicy(settings({ backtestRegressionGateMode: "block" }), null, true).backtestRegressionGateMode).toBe("block"); + }); +}); + describe("content-lane linked-issue deliverable gate blocker (#content-lane-deliverable)", () => { const deliverableAdvisory = (): Advisory => ({ ...missingIssueAdvisory(), diff --git a/test/unit/maintainer-activation.test.ts b/test/unit/maintainer-activation.test.ts index ae8335ba5e..d334b0ad93 100644 --- a/test/unit/maintainer-activation.test.ts +++ b/test/unit/maintainer-activation.test.ts @@ -39,6 +39,7 @@ function settings(overrides: Partial = {}): RepositorySettin selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, diff --git a/test/unit/policy-sanitizer.test.ts b/test/unit/policy-sanitizer.test.ts index baefe4bc77..4e3c8103b7 100644 --- a/test/unit/policy-sanitizer.test.ts +++ b/test/unit/policy-sanitizer.test.ts @@ -72,6 +72,7 @@ function settingsFor(repoFullName: string, overrides: Partial { expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("duplicate_pr_risk"), block)).toBe(true); expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("ai_consensus_defect"), advisory)).toBe(false); expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("ai_consensus_defect"), block)).toBe(true); + // Backtest-regression gate (#8105): absent (defaults to advisory), explicit advisory, and block. + expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("backtest_regression"), advisory)).toBe(false); + expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("backtest_regression"), { ...advisory, backtestRegressionGateMode: "advisory" as const })).toBe(false); + expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("backtest_regression"), { ...block, backtestRegressionGateMode: "block" as const })).toBe(true); expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("ai_review_split"), advisory)).toBe(false); // #4603: aiReviewLowConfidenceDisposition branches — the default ("hold_for_review", exercised by the // bare `block` policy above with no confidence set) and "one_shot" both ignore confidence entirely and diff --git a/test/unit/registration-readiness.test.ts b/test/unit/registration-readiness.test.ts index b5a47a8a44..edef6dd721 100644 --- a/test/unit/registration-readiness.test.ts +++ b/test/unit/registration-readiness.test.ts @@ -53,6 +53,7 @@ function settingsFor(repoFullName: string, overrides: Partial = {}): RepositorySettin selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, diff --git a/test/unit/repository-settings-enforcement.test.ts b/test/unit/repository-settings-enforcement.test.ts index 8dc00807a5..c306da409b 100644 --- a/test/unit/repository-settings-enforcement.test.ts +++ b/test/unit/repository-settings-enforcement.test.ts @@ -27,6 +27,7 @@ function settings(over: Partial = {}): RepositorySettings { selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, diff --git a/test/unit/self-dogfood-registration-pack.test.ts b/test/unit/self-dogfood-registration-pack.test.ts index a22ba76e9b..aaca6de720 100644 --- a/test/unit/self-dogfood-registration-pack.test.ts +++ b/test/unit/self-dogfood-registration-pack.test.ts @@ -66,6 +66,7 @@ function settingsFor(repoFullName: string, overrides: Partial = {}): RepositorySettin selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, diff --git a/test/unit/signals-coverage.test.ts b/test/unit/signals-coverage.test.ts index 778af041ee..cc1e6b67f6 100644 --- a/test/unit/signals-coverage.test.ts +++ b/test/unit/signals-coverage.test.ts @@ -2389,6 +2389,7 @@ function repoSettings(repoFullName: string): RepositorySettings { selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, diff --git a/test/unit/signals-v2.test.ts b/test/unit/signals-v2.test.ts index 3d981461af..51e61bb91f 100644 --- a/test/unit/signals-v2.test.ts +++ b/test/unit/signals-v2.test.ts @@ -2116,6 +2116,7 @@ describe("v2 signal builders", () => { selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, diff --git a/test/unit/signals.test.ts b/test/unit/signals.test.ts index cab3c6edb0..e5b40757fb 100644 --- a/test/unit/signals.test.ts +++ b/test/unit/signals.test.ts @@ -543,6 +543,7 @@ describe("world-class backend signals", () => { selfAuthoredLinkedIssueGateMode: "advisory" as const, linkedIssueSatisfactionGateMode: "off" as const, contentLaneDeliverableGateMode: "off" as const, + backtestRegressionGateMode: "advisory" as const, slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, @@ -604,6 +605,7 @@ describe("world-class backend signals", () => { selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, @@ -687,6 +689,7 @@ describe("world-class backend signals", () => { selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, @@ -803,6 +806,7 @@ describe("world-class backend signals", () => { selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true, diff --git a/test/unit/threshold-backtest-advisory-wiring.test.ts b/test/unit/threshold-backtest-advisory-wiring.test.ts index cb58b478ee..bb7f5f5f75 100644 --- a/test/unit/threshold-backtest-advisory-wiring.test.ts +++ b/test/unit/threshold-backtest-advisory-wiring.test.ts @@ -4,7 +4,7 @@ import { createSignalStore } from "../../src/review/signal-tracking-wire"; import { listAuditEventsByType } from "../../src/db/repositories"; import * as thresholdBacktestRun from "../../src/services/threshold-backtest-run"; import { THRESHOLD_BACKTEST_EVENT_TYPE } from "../../src/services/threshold-backtest-run"; -import type { PullRequestFileRecord } from "../../src/types"; +import type { AdvisoryFinding, PullRequestFileRecord } from "../../src/types"; import { createTestEnv } from "../helpers/d1"; afterEach(() => { @@ -72,3 +72,77 @@ describe("resolveThresholdBacktestAdvisory (processor wiring, #8138)", () => { expect(result).toBe(""); }); }); + +describe("resolveThresholdBacktestAdvisory backtestRegressionGateMode (#8105)", () => { + // The shipped fixture shape: lowering the floor 0.5 -> 0.2 against a reversed firing recorded at + // confidence 0.35 REGRESSES recall (the old floor caught the bad call, the new one would not). + async function seedReversedHistory(env: ReturnType) { + const now = Date.now(); + await createSignalStore(env).recordRuleFired({ + ruleId: "linked_issue_scope_mismatch", + targetKey: "acme/widgets#1", + outcome: "unaddressed", + occurredAt: new Date(now - 1000).toISOString(), + metadata: { confidence: 0.35 }, + }); + await createSignalStore(env).recordHumanOverride({ + ruleId: "linked_issue_scope_mismatch", + targetKey: "acme/widgets#1", + verdict: "reversed", + occurredAt: new Date(now).toISOString(), + }); + return now; + } + const regressingFiles = () => [ + fileRecord("src/services/linked-issue-satisfaction.ts", thresholdPatch("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.2")), + ]; + + it("off silences the whole advisory: no section, no persisted run, even for a real threshold change", async () => { + const env = createTestEnv(); + const now = await seedReversedHistory(env); + const result = await resolveThresholdBacktestAdvisory(env, "acme/widgets", { number: 7 }, regressingFiles(), { mode: "off" }); + expect(result).toBe(""); + expect(await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString())).toHaveLength(0); + }); + + it("advisory (explicit AND defaulted) renders the section but never pushes a finding, even on a REGRESSED verdict", async () => { + for (const options of [{ mode: "advisory" as const, advisory: { findings: [] } }, {}]) { + const env = createTestEnv(); + await seedReversedHistory(env); + const result = await resolveThresholdBacktestAdvisory(env, "acme/widgets", { number: 7 }, regressingFiles(), options); + expect(result).toContain("REGRESSED"); + if ("advisory" in options) expect(options.advisory.findings).toEqual([]); + } + }); + + it("block escalates a REGRESSED verdict into a backtest_regression finding and still renders the section", async () => { + const env = createTestEnv(); + await seedReversedHistory(env); + const advisory: { findings: AdvisoryFinding[] } = { findings: [] }; + const result = await resolveThresholdBacktestAdvisory(env, "acme/widgets", { number: 7 }, regressingFiles(), { mode: "block", advisory }); + expect(result).toContain("REGRESSED"); + expect(advisory.findings).toHaveLength(1); + const finding = advisory.findings[0]!; + expect(finding.code).toBe("backtest_regression"); + expect(finding.severity).toBe("warning"); + expect(finding.detail).toContain("linked_issue_scope_mismatch"); + expect(finding.publicText).not.toMatch(/reward|payout|trust|wallet|hotkey/i); + }); + + it("block pushes nothing when the change IMPROVES the backtest (raising the floor catches the reversed call)", async () => { + const env = createTestEnv(); + await seedReversedHistory(env); + const files = [fileRecord("src/services/linked-issue-satisfaction.ts", thresholdPatch("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.2", "0.5"))]; + const advisory = { findings: [] }; + const result = await resolveThresholdBacktestAdvisory(env, "acme/widgets", { number: 7 }, files, { mode: "block", advisory }); + expect(result).toContain("improved"); + expect(advisory.findings).toEqual([]); + }); + + it("block without an advisory sink degrades gracefully: section renders, nothing crashes", async () => { + const env = createTestEnv(); + await seedReversedHistory(env); + const result = await resolveThresholdBacktestAdvisory(env, "acme/widgets", { number: 7 }, regressingFiles(), { mode: "block" }); + expect(result).toContain("REGRESSED"); + }); +}); diff --git a/test/unit/unified-comment-parity.test.ts b/test/unit/unified-comment-parity.test.ts index 403a723e52..8185262870 100644 --- a/test/unit/unified-comment-parity.test.ts +++ b/test/unit/unified-comment-parity.test.ts @@ -64,6 +64,7 @@ const settings: RepositorySettings = { selfAuthoredLinkedIssueGateMode: "advisory", linkedIssueSatisfactionGateMode: "off", contentLaneDeliverableGateMode: "off", + backtestRegressionGateMode: "advisory", slopAiAdvisory: false, qualityGateMinScore: null, autoLabelEnabled: true,