From 4d05fc52c9d01363d2a62f4d524d4e5a941a1f94 Mon Sep 17 00:00:00 2001
From: JSONbored <49853598+JSONbored@users.noreply.github.com>
Date: Wed, 22 Jul 2026 23:49:56 -0700
Subject: [PATCH] feat(gate): config-driven backtest-regression gate mode,
default advisory (#8105)
Resolves #8105's Phase-2 decision as a CONDITIONAL policy: the full gating
mechanism ships now as config-as-code, defaulting to the shipped advisory-only
behavior, so the deploy is behavior-identical and the future flip to block is
a one-line config change made only once #8140's persisted track record
supports it -- honoring the issue's own do-not-gate-before-data boundary
while completing the epic's mechanical scope end to end.
backtestRegressionGateMode (off|advisory|block; DB column + dashboard field +
.loopover.yml gate.backtestRegression override) is wired through every
per-repo-setting site: types, schema + migration 0174, repositories
(defaults/parse/normalize/both write paths), settings-preview, focus-manifest
(host + engine manifest schema), openapi schemas + regenerated artifact,
routes PATCH schema, gate-check policy passthrough, isConfiguredGateBlocker
(host + engine gate-decision twin), and the example yml. Enforcement lives in
resolveThresholdBacktestAdvisory: off silences the advisory entirely; block
pushes a backtest_regression finding when any comparison is REGRESSED; the
finding only exists in block mode, so advisory stays byte-identical.
Advances #8105.
---
.loopover.yml.example | 10 +++
.../content/docs/backtest-calibration.mdx | 7 +-
apps/loopover-ui/content/docs/tuning.mdx | 62 +++++++++------
apps/loopover-ui/public/openapi.json | 18 +++++
config/examples/loopover.full.yml | 10 +++
migrations/0174_gate_backtest_regression.sql | 6 ++
.../src/advisory/gate-advisory.ts | 7 ++
.../loopover-engine/src/focus-manifest.ts | 9 +++
scripts/check-docs-drift.ts | 1 +
src/api/routes.ts | 1 +
src/db/repositories.ts | 7 ++
src/db/schema.ts | 2 +
src/openapi/schemas.ts | 2 +
src/queue/gate-checks.ts | 1 +
src/queue/processors.ts | 32 +++++++-
src/rules/advisory.ts | 8 ++
src/signals/focus-manifest.ts | 1 +
src/signals/settings-preview.ts | 2 +
src/types.ts | 8 ++
.../backtest-regression-gate-setting.test.ts | 47 ++++++++++++
test/unit/check-docs-drift-script.test.ts | 8 +-
test/unit/focus-manifest.test.ts | 5 +-
test/unit/gate-check-policy.test.ts | 27 +++++++
test/unit/maintainer-activation.test.ts | 1 +
test/unit/policy-sanitizer.test.ts | 1 +
...dicted-gate-engine-branch-coverage.test.ts | 4 +
test/unit/registration-readiness.test.ts | 1 +
test/unit/repo-policy-readiness.test.ts | 1 +
.../repository-settings-enforcement.test.ts | 1 +
.../self-dogfood-registration-pack.test.ts | 1 +
test/unit/settings-preview.test.ts | 1 +
test/unit/signals-coverage.test.ts | 1 +
test/unit/signals-v2.test.ts | 1 +
test/unit/signals.test.ts | 4 +
...threshold-backtest-advisory-wiring.test.ts | 76 ++++++++++++++++++-
test/unit/unified-comment-parity.test.ts | 1 +
36 files changed, 339 insertions(+), 36 deletions(-)
create mode 100644 migrations/0174_gate_backtest_regression.sql
create mode 100644 test/unit/backtest-regression-gate-setting.test.ts
diff --git a/.loopover.yml.example b/.loopover.yml.example
index d2e3077b13..5358bb6861 100644
--- a/.loopover.yml.example
+++ b/.loopover.yml.example
@@ -329,6 +329,16 @@ gate:
# value.
contentLaneDeliverable: off
+ # Backtest-regression gate (#8105). Governs what a REGRESSED verdict from the
+ # pre-merge backtest (threshold #8138 / logic #8139) does: advisory (default)
+ # renders the comparison in the review comment but never blocks; block
+ # escalates it into a backtest_regression hard blocker; off silences the
+ # backtest advisory entirely. Flip to block only once the persisted track
+ # record (scripts/backtest-track-record.ts, #8140) supports it.
+ # off | advisory | block. Default: advisory. DB-backed (dashboard-settable
+ # too); this overrides the stored value.
+ backtestRegression: advisory
+
# Gate-check dry-run. When true, the posted check conclusion remains the real
# non-enforcing verdict while comments/check text may also show the would-be
# stricter verdict for AI-review blocker mode. It does not disable downstream
diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx
index cf2a56776c..8d0d5e1ec9 100644
--- a/apps/loopover-ui/content/docs/backtest-calibration.mdx
+++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx
@@ -15,9 +15,10 @@ threshold or rewrites detection logic gets scored against the real recorded hist
targets, the same raw inputs — instead of being eyeballed.
- Everything on this page is advisory-only today. A backtest verdict — even a regression — never
- blocks a merge. Whether a REGRESSED verdict should ever gate merges is a tracked, deliberately
- separate decision that waits for real production track-record data.
+ Everything on this page is advisory-only by default. A REGRESSED verdict blocks a merge only if a
+ repo explicitly opts in via `backtestRegressionGateMode: block` (`.loopover.yml
+ gate.backtestRegression`) — the shipped default is `advisory`, and the flip is meant to wait for
+ real production track-record data.
## The corpus
diff --git a/apps/loopover-ui/content/docs/tuning.mdx b/apps/loopover-ui/content/docs/tuning.mdx
index 2a1e0486e1..6bd9f6b058 100644
--- a/apps/loopover-ui/content/docs/tuning.mdx
+++ b/apps/loopover-ui/content/docs/tuning.mdx
@@ -259,6 +259,12 @@ forward and wins whenever both are set for the same field.
`advisory` renders the assessment in the review comment without blocking;
`block` additionally lets a confidence-floor-passing "unaddressed" verdict
become a blocker.
+- `gate.backtestRegression` — governs what a REGRESSED verdict from the
+ [pre-merge backtest](/docs/backtest-calibration) does. Default `advisory`:
+ the comparison renders in the review comment but never blocks. `block`
+ escalates a REGRESSED verdict into a hard blocker — flip it only once the
+ persisted track record supports gating. `off` silences the backtest advisory
+ entirely.
- `gate.contentLaneDeliverable` — only meaningful for a repo with a registry
content-lane spec configured (`contentLane`); a no-op otherwise. Fully
deterministic (a text/path match, no AI call): when the PR's primary linked
@@ -381,36 +387,42 @@ testExpectations:
linkedIssuePolicy: preferred
# Gate policy — checkMode is set explicitly (not the legacy, ambiguous "enabled" alias)
+
gate:
- checkMode: visible
- pack: gittensor
- duplicates: block
- linkedIssue: advisory
- readiness:
- mode: advisory
- minScore: 70
- slop:
- mode: block
- minScore: 60
- aiAdvisory: true
- mergeReadiness: advisory
- manifestPolicy: block
- aiReview:
- mode: advisory
- byok: true
- provider: anthropic
- model: claude-3-5-sonnet-latest
+checkMode: visible
+pack: gittensor
+duplicates: block
+linkedIssue: advisory
+readiness:
+mode: advisory
+minScore: 70
+slop:
+mode: block
+minScore: 60
+aiAdvisory: true
+mergeReadiness: advisory
+manifestPolicy: block
+aiReview:
+mode: advisory
+byok: true
+provider: anthropic
+model: claude-3-5-sonnet-latest
# Generic settings overrides -- commentMode/checkRunMode/checkRunDetailLevel/badgeEnabled are
+
# config-as-code only (no DB column or dashboard toggle); this file is their sole source.
+
settings:
- commentMode: detected_contributors_only
- checkRunMode: enabled
- checkRunDetailLevel: standard
- badgeEnabled: true
- # Optional path holds. Omitted or [] means no path guardrails.
- # hardGuardrailGlobs:
- # - "src/selfhost/**"`}
+commentMode: detected_contributors_only
+checkRunMode: enabled
+checkRunDetailLevel: standard
+badgeEnabled: true
+
+# Optional path holds. Omitted or [] means no path guardrails.
+
+# hardGuardrailGlobs:
+
+# - "src/selfhost/\*\*"`}
/>
diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json
index 06f0ddd1f9..4ef08d1c6a 100644
--- a/apps/loopover-ui/public/openapi.json
+++ b/apps/loopover-ui/public/openapi.json
@@ -9761,6 +9761,14 @@
"advisory",
"block"
]
+ },
+ "backtestRegressionGateMode": {
+ "type": "string",
+ "enum": [
+ "off",
+ "advisory",
+ "block"
+ ]
}
},
"required": [
@@ -9782,6 +9790,7 @@
"selfAuthoredLinkedIssueGateMode",
"linkedIssueSatisfactionGateMode",
"contentLaneDeliverableGateMode",
+ "backtestRegressionGateMode",
"slopAiAdvisory",
"aiReviewMode",
"aiReviewByok",
@@ -10492,6 +10501,14 @@
"advisory",
"block"
]
+ },
+ "backtestRegressionGateMode": {
+ "type": "string",
+ "enum": [
+ "off",
+ "advisory",
+ "block"
+ ]
}
},
"required": [
@@ -10513,6 +10530,7 @@
"selfAuthoredLinkedIssueGateMode",
"linkedIssueSatisfactionGateMode",
"contentLaneDeliverableGateMode",
+ "backtestRegressionGateMode",
"autoLabelEnabled",
"typeLabelsEnabled",
"gittensorLabel",
diff --git a/config/examples/loopover.full.yml b/config/examples/loopover.full.yml
index 1ee4ca0690..a7cfe2ea61 100644
--- a/config/examples/loopover.full.yml
+++ b/config/examples/loopover.full.yml
@@ -343,6 +343,16 @@ gate:
# value.
contentLaneDeliverable: off
+ # Backtest-regression gate (#8105). Governs what a REGRESSED verdict from the
+ # pre-merge backtest (threshold #8138 / logic #8139) does: advisory (default)
+ # renders the comparison in the review comment but never blocks; block
+ # escalates it into a backtest_regression hard blocker; off silences the
+ # backtest advisory entirely. Flip to block only once the persisted track
+ # record (scripts/backtest-track-record.ts, #8140) supports it.
+ # off | advisory | block. Default: advisory. DB-backed (dashboard-settable
+ # too); this overrides the stored value.
+ backtestRegression: advisory
+
# Gate-check dry-run. When true, the posted check conclusion remains the real
# non-enforcing verdict while comments/check text may also show the would-be
# stricter verdict for AI-review blocker mode. It does not disable downstream
diff --git a/migrations/0174_gate_backtest_regression.sql b/migrations/0174_gate_backtest_regression.sql
new file mode 100644
index 0000000000..45a4b4290e
--- /dev/null
+++ b/migrations/0174_gate_backtest_regression.sql
@@ -0,0 +1,6 @@
+-- Backtest-regression gate (#8105, epic #8082): advisory by default -- byte-identical behavior for every
+-- existing row (the shipped #8138/#8142 comment-only advisory). block escalates a REGRESSED pre-merge
+-- backtest verdict into a backtest_regression hard blocker; off silences the backtest advisory entirely.
+-- The flip to block is deliberately a config change made only once #8140's persisted track record supports
+-- it -- see #8105's own do-not-gate-before-data boundary.
+ALTER TABLE repository_settings ADD COLUMN backtest_regression_gate_mode TEXT NOT NULL DEFAULT 'advisory';
diff --git a/packages/loopover-engine/src/advisory/gate-advisory.ts b/packages/loopover-engine/src/advisory/gate-advisory.ts
index 95116fbcd9..f346fb21d6 100644
--- a/packages/loopover-engine/src/advisory/gate-advisory.ts
+++ b/packages/loopover-engine/src/advisory/gate-advisory.ts
@@ -99,6 +99,10 @@ export type GateCheckPolicy = {
* on this mode; see runContentLaneDeliverableCheckForAdvisory, src/queue/processors.ts on the host side),
* so this branch only matters once a repo has explicitly opted into `advisory`/`block`. */
contentLaneDeliverableGateMode?: GateRuleMode | undefined;
+ /** Backtest-regression gate (#8105): `block` turns a `backtest_regression` finding into a hard blocker.
+ * Default `advisory` — the finding only exists in block mode (the host resolver pushes it there), so
+ * this branch is defense-in-depth, mirroring content_lane_deliverable_missing's above. */
+ backtestRegressionGateMode?: GateRuleMode | undefined;
/** CLA / license-compatibility gate (#2564). When `block`, a `cla_consent_missing` finding — raised when
* neither configured detection method (a consent phrase in the PR body, or a named CLA-bot check-run
* conclusion) confirms consent — becomes a hard blocker. `off` (default) = no finding at all; `advisory` =
@@ -636,6 +640,9 @@ function isConfiguredGateBlocker(finding: AdvisoryFinding, policy: GateCheckPoli
// opts in with `block`. Fully deterministic (no AI judgment involved), so it is exempt from the
// close-precision circuit breaker on the host side.
if (code === "content_lane_deliverable_missing") return gatePolicyBlocks(policy.contentLaneDeliverableGateMode, "off");
+ // Backtest-regression gate (#8105): blocks only under an explicit opt-in; default advisory (the shipped
+ // pre-#8105 behavior). The finding itself only exists in block mode -- see the host-side resolver.
+ if (code === "backtest_regression") return gatePolicyBlocks(policy.backtestRegressionGateMode, "advisory");
// Lockfile-tamper-risk gate (#2563): blocks only when the maintainer opts in with `block`. Defaults to `off`
// (the finding is never even produced — see maybeAddLockfileTamperFinding's mode gate in queue/processors.ts),
// so this branch only matters once a repo has explicitly turned the scan on.
diff --git a/packages/loopover-engine/src/focus-manifest.ts b/packages/loopover-engine/src/focus-manifest.ts
index b7288d5072..676ae101c8 100644
--- a/packages/loopover-engine/src/focus-manifest.ts
+++ b/packages/loopover-engine/src/focus-manifest.ts
@@ -165,6 +165,12 @@ export type FocusManifestGateConfig = {
* `linkedIssueSatisfaction` immediately above in shape, but is a purely structural check, not an AI
* opinion, so it carries none of that feature's AI-budget/confidence-floor machinery. */
contentLaneDeliverable: GateRuleMode | null;
+ /** `gate.backtestRegression` (#8105, epic #8082): off|advisory|block, advisory by default — governs what a
+ * REGRESSED pre-merge backtest verdict (#8138 threshold / #8139 logic) does. `advisory` renders the
+ * comparison but never blocks (the shipped pre-#8105 behavior); `block` escalates it into a
+ * `backtest_regression` hard blocker; `off` silences the backtest advisory entirely. DB-backed
+ * (dashboard-settable too); this overrides the stored value like every other `gate:` field. */
+ backtestRegression: GateRuleMode | null;
dryRun: boolean | null;
/** `gate.premergeContentRecheck` (#2550): for a PR touching `migrations/**`, re-verify against a live,
* freshly-fetched tip of the base branch — unioned with this PR's own new migration filenames — for a
@@ -1303,6 +1309,7 @@ const EMPTY_GATE_CONFIG: FocusManifestGateConfig = {
selfAuthoredLinkedIssue: null,
linkedIssueSatisfaction: null,
contentLaneDeliverable: null,
+ backtestRegression: null,
dryRun: null,
premergeContentRecheck: null,
requireFreshRebaseWindowMinutes: null,
@@ -1788,6 +1795,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu
selfAuthoredLinkedIssue: normalizeOptionalGateMode(record.selfAuthoredLinkedIssue, "gate.selfAuthoredLinkedIssue", warnings),
linkedIssueSatisfaction: normalizeOptionalGateMode(record.linkedIssueSatisfaction, "gate.linkedIssueSatisfaction", warnings),
contentLaneDeliverable: normalizeOptionalGateMode(record.contentLaneDeliverable, "gate.contentLaneDeliverable", warnings),
+ backtestRegression: normalizeOptionalGateMode(record.backtestRegression, "gate.backtestRegression", warnings),
dryRun: normalizeOptionalBoolean(record.dryRun, "gate.dryRun", warnings),
premergeContentRecheck: normalizeOptionalBoolean(record.premergeContentRecheck, "gate.premergeContentRecheck", warnings),
requireFreshRebaseWindowMinutes: normalizeOptionalPositiveInteger(record.requireFreshRebaseWindow, "gate.requireFreshRebaseWindow", warnings),
@@ -1845,6 +1853,7 @@ function parseGateConfig(value: JsonValue | undefined, warnings: string[]): Focu
gate.selfAuthoredLinkedIssue !== null ||
gate.linkedIssueSatisfaction !== null ||
gate.contentLaneDeliverable !== null ||
+ gate.backtestRegression !== null ||
gate.dryRun !== null ||
gate.premergeContentRecheck !== null ||
gate.requireFreshRebaseWindowMinutes !== null ||
diff --git a/scripts/check-docs-drift.ts b/scripts/check-docs-drift.ts
index be2d5eb908..7f2d3bf321 100644
--- a/scripts/check-docs-drift.ts
+++ b/scripts/check-docs-drift.ts
@@ -240,6 +240,7 @@ export const GATE_MODE_MANIFEST: GateModeManifestRow[] = [
{ field: "selfAuthoredLinkedIssueGateMode", aliases: ["selfAuthoredLinkedIssueGateMode", "gate.selfAuthoredLinkedIssue"], pages: ["how-reviews-work.mdx", "tuning.mdx", "github-app.mdx"] },
{ field: "linkedIssueSatisfactionGateMode", aliases: ["linkedIssueSatisfactionGateMode", "gate.linkedIssueSatisfaction"], pages: ["how-reviews-work.mdx", "tuning.mdx", "github-app.mdx"] },
{ field: "contentLaneDeliverableGateMode", aliases: ["contentLaneDeliverableGateMode", "gate.contentLaneDeliverable"], pages: ["how-reviews-work.mdx", "tuning.mdx", "github-app.mdx"] },
+ { field: "backtestRegressionGateMode", aliases: ["backtestRegressionGateMode", "gate.backtestRegression"], pages: ["backtest-calibration.mdx", "tuning.mdx"] },
{ field: "moderationGateMode", aliases: ["moderationGateMode", "settings.moderationGateMode"], pages: ["how-reviews-work.mdx", "tuning.mdx", "github-app.mdx"] },
];
diff --git a/src/api/routes.ts b/src/api/routes.ts
index 74caa6eef3..858d66860c 100644
--- a/src/api/routes.ts
+++ b/src/api/routes.ts
@@ -922,6 +922,7 @@ const maintainerSettingsSchema = z
manifestPolicyGateMode: z.enum(["off", "advisory", "block"]),
linkedIssueSatisfactionGateMode: z.enum(["off", "advisory", "block"]),
contentLaneDeliverableGateMode: z.enum(["off", "advisory", "block"]),
+ backtestRegressionGateMode: z.enum(["off", "advisory", "block"]),
// #6443: mergeTrainMode/gittensorLabel/blacklistLabel/createMissingLabel removed -- no longer DB-backed,
// config-as-code only via .loopover.yml's settings: block now.
// #6446: firstTimeContributorGrace removed -- a dead, never-wired RESERVED/INERT field (#2266); deleted
diff --git a/src/db/repositories.ts b/src/db/repositories.ts
index 5c2a7ee91c..b547c3ec9d 100644
--- a/src/db/repositories.ts
+++ b/src/db/repositories.ts
@@ -624,6 +624,7 @@ export async function getRepositorySettings(env: Env, fullName: string): Promise
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopGateMinScore: null,
slopAiAdvisory: false,
aiReviewMode: "off",
@@ -720,6 +721,9 @@ export async function getRepositorySettings(env: Env, fullName: string): Promise
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: parseGateRuleMode(row.linkedIssueSatisfactionGateMode),
contentLaneDeliverableGateMode: parseGateRuleMode(row.contentLaneDeliverableGateMode),
+ // parseGateRuleMode's fallback IS this field's default ("advisory"), so an unrecognized stored value
+ // degrades to the safe shipped behavior rather than silently blocking (#8105).
+ backtestRegressionGateMode: parseGateRuleMode(row.backtestRegressionGateMode),
slopGateMinScore: normalizeQualityGateMinScore(row.slopGateMinScore),
slopAiAdvisory: row.slopAiAdvisory,
aiReviewMode: "off",
@@ -865,6 +869,7 @@ export async function upsertRepositorySettings(env: Env, settings: Partial>,
+ // #8105: what a REGRESSED verdict does. Optional + defaulted so every existing caller/test keeps the
+ // shipped advisory-only behavior; the live gate call site threads settings.backtestRegressionGateMode.
+ options: { mode?: GateRuleMode; advisory?: { findings: AdvisoryFinding[] } } = {},
): Promise {
+ const mode = options.mode ?? "advisory";
+ // `off` silences the whole advisory (no D1 read, no persist, no comment section) -- the same short-circuit
+ // shape contentLaneDeliverableGateMode's "off" uses. advisory/block both still render the section.
+ if (mode === "off") return "";
if (!files.some((file) => THRESHOLD_BACKTEST_WATCHED_PATHS.has(file.path))) return "";
try {
const { changed, comparisons } = await runThresholdBacktestAdvisory(env, buildSecretScanDiff(files));
if (comparisons.length === 0) return "";
await persistThresholdBacktestRuns(env, repoFullName, pr.number, changed, comparisons);
+ // #8105 block mode: a REGRESSED verdict becomes a configured gate blocker. The finding only exists in
+ // block mode (advisory keeps today's comment-only behavior byte-identically), and
+ // isConfiguredGateBlocker's own `backtest_regression` branch is the defense-in-depth mirror of this
+ // push -- same split as linked_issue_scope_mismatch's block-mode wiring above.
+ const regressed = comparisons.filter((comparison) => comparison.verdict === "regressed");
+ if (mode === "block" && regressed.length > 0 && options.advisory) {
+ const ruleIds = regressed.map((comparison) => comparison.ruleId).join(", ");
+ options.advisory.findings.push({
+ code: "backtest_regression",
+ severity: "warning",
+ title: "Backtest regression against recorded history",
+ detail: `Backtesting this PR's threshold change against real recorded history REGRESSED ${ruleIds} on at least one axis (see the Threshold backtest section).`,
+ action: "Revisit the changed threshold so no backtest axis regresses, or ask a maintainer to override.",
+ publicText: `Backtest: this change scores worse than the current value against recorded review history (${ruleIds}). See the Threshold backtest section for the comparison.`,
+ });
+ }
return thresholdBacktestBlock(comparisons);
} catch (error) {
console.error(JSON.stringify({ level: "error", event: "threshold_backtest_failed", repoFullName, pullNumber: pr.number, error: errorMessage(error) }));
@@ -9485,8 +9508,13 @@ async function maybePublishPrPublicSurface(
}
}
// Threshold-only backtest advisory (#8138, epic #8082): independent of linkedIssueSatisfactionGateMode
- // above -- see resolveThresholdBacktestAdvisory's own doc comment for why.
- thresholdBacktest = await resolveThresholdBacktestAdvisory(env, repoFullName, pr, await getReviewFiles());
+ // above -- see resolveThresholdBacktestAdvisory's own doc comment for why. #8105 threads the repo's
+ // backtestRegressionGateMode: advisory (default) keeps the shipped comment-only behavior; block lets a
+ // REGRESSED verdict push a configured gate blocker; off silences the section.
+ thresholdBacktest = await resolveThresholdBacktestAdvisory(env, repoFullName, pr, await getReviewFiles(), {
+ mode: settings.backtestRegressionGateMode,
+ advisory,
+ });
// Content-lane linked-issue deliverable check (#content-lane-deliverable, opt-in via
// contentLaneDeliverableGateMode). Independent gate mode from the AI-based satisfaction assessment above --
// `off` (default) short-circuits before any fetch, so this is byte-identical to before this feature existed
diff --git a/src/rules/advisory.ts b/src/rules/advisory.ts
index e1fae729c8..6f933e6017 100644
--- a/src/rules/advisory.ts
+++ b/src/rules/advisory.ts
@@ -100,6 +100,11 @@ export type GateCheckPolicy = {
* on this mode; see runContentLaneDeliverableCheckForAdvisory, src/queue/processors.ts), so this branch
* only matters once a repo has explicitly opted into `advisory`/`block`. */
contentLaneDeliverableGateMode?: GateRuleMode | undefined;
+ /** Backtest-regression gate (#8105): `block` turns a `backtest_regression` finding (pushed by
+ * resolveThresholdBacktestAdvisory when a REGRESSED verdict exists) into a hard blocker. Default
+ * `advisory` — the finding never even exists below `block` (the resolver only pushes it in block mode),
+ * so this branch is defense-in-depth like content_lane_deliverable_missing's. */
+ backtestRegressionGateMode?: GateRuleMode | undefined;
/** CLA / license-compatibility gate (#2564). When `block`, a `cla_consent_missing` finding — raised when
* neither configured detection method (a consent phrase in the PR body, or a named CLA-bot check-run
* conclusion) confirms consent — becomes a hard blocker. `off` (default) = no finding at all; `advisory` =
@@ -1043,6 +1048,9 @@ function isConfiguredGateBlocker(finding: AdvisoryFinding, policy: GateCheckPoli
// "off" mode (see runContentLaneDeliverableCheckForAdvisory), so this is a defense-in-depth mirror of that
// gate, not the primary enforcement point -- mirrors linked_issue_scope_mismatch immediately above.
if (code === "content_lane_deliverable_missing") return gateMode(policy.contentLaneDeliverableGateMode ?? "off") === "block";
+ // Backtest-regression gate (#8105): blocks only under an explicit opt-in, mirroring the content-lane gate
+ // above -- the finding itself only exists in block mode (see resolveThresholdBacktestAdvisory).
+ if (code === "backtest_regression") return gateMode(policy.backtestRegressionGateMode ?? "advisory") === "block";
// Lockfile-tamper-risk gate (#2563): blocks only when the maintainer opts in with `block`. Defaults to `off`
// (the finding is never even produced — see maybeAddLockfileTamperFinding's mode gate in queue/processors.ts),
// so this branch only matters once a repo has explicitly turned the scan on.
diff --git a/src/signals/focus-manifest.ts b/src/signals/focus-manifest.ts
index 2fb63f6ea9..0306bdd53e 100644
--- a/src/signals/focus-manifest.ts
+++ b/src/signals/focus-manifest.ts
@@ -525,6 +525,7 @@ function applyGateConfigOverrides(effective: RepositorySettings, gate: FocusMani
if (gate.selfAuthoredLinkedIssue !== null) effective.selfAuthoredLinkedIssueGateMode = gate.selfAuthoredLinkedIssue;
if (gate.linkedIssueSatisfaction !== null) effective.linkedIssueSatisfactionGateMode = gate.linkedIssueSatisfaction;
if (gate.contentLaneDeliverable !== null) effective.contentLaneDeliverableGateMode = gate.contentLaneDeliverable;
+ if (gate.backtestRegression !== null) effective.backtestRegressionGateMode = gate.backtestRegression;
if (gate.dryRun !== null) effective.gateDryRun = gate.dryRun;
if (gate.premergeContentRecheck !== null) effective.premergeContentRecheck = gate.premergeContentRecheck;
if (gate.requireFreshRebaseWindowMinutes !== null) effective.requireFreshRebaseWindowMinutes = gate.requireFreshRebaseWindowMinutes;
diff --git a/src/signals/settings-preview.ts b/src/signals/settings-preview.ts
index dbae539a89..9e9c2292a9 100644
--- a/src/signals/settings-preview.ts
+++ b/src/signals/settings-preview.ts
@@ -247,6 +247,7 @@ export type RepoSettingsPreview = {
selfAuthoredLinkedIssueGateMode: RepositorySettings["selfAuthoredLinkedIssueGateMode"];
linkedIssueSatisfactionGateMode: RepositorySettings["linkedIssueSatisfactionGateMode"];
contentLaneDeliverableGateMode: RepositorySettings["contentLaneDeliverableGateMode"];
+ backtestRegressionGateMode: RepositorySettings["backtestRegressionGateMode"];
slopGateMinScore?: number | null | undefined;
autoLabelEnabled: boolean;
typeLabelsEnabled: boolean;
@@ -384,6 +385,7 @@ export function buildRepoSettingsPreview(args: {
selfAuthoredLinkedIssueGateMode: settings.selfAuthoredLinkedIssueGateMode,
linkedIssueSatisfactionGateMode: settings.linkedIssueSatisfactionGateMode,
contentLaneDeliverableGateMode: settings.contentLaneDeliverableGateMode,
+ backtestRegressionGateMode: settings.backtestRegressionGateMode,
slopGateMinScore: settings.slopGateMinScore ?? null,
autoLabelEnabled: settings.autoLabelEnabled,
typeLabelsEnabled: settings.typeLabelsEnabled ?? true,
diff --git a/src/types.ts b/src/types.ts
index bb5d88271e..36a5b32025 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -976,6 +976,14 @@ export type RepositorySettings = {
* exactly like every other `gate:` field overrides its `RepositorySettings` counterpart. Default `off` —
* opt-in. */
contentLaneDeliverableGateMode: GateRuleMode;
+ /** Backtest-regression gate (#8105, epic #8082). Governs what a REGRESSED verdict from the pre-merge
+ * backtest (#8138 threshold / #8139 logic) does: `advisory` (default — today's shipped behavior) renders
+ * the comparison but never blocks; `block` additionally pushes a `backtest_regression` finding that
+ * isConfiguredGateBlocker treats as a hard blocker; `off` skips the backtest advisory entirely. Default
+ * deliberately stays `advisory` until the persisted track record (#8140) supports flipping — see #8105's
+ * own do-not-gate-before-data boundary. `.loopover.yml gate.backtestRegression` overrides it exactly like
+ * every other `gate:` field. */
+ backtestRegressionGateMode: GateRuleMode;
/** Slop-risk threshold (0-100) at/above which `slopGateMode: block` blocks. Default 60 (the `high` band). */
slopGateMinScore?: number | null | undefined;
/** AI-assisted slop advisory (the `slopAiAdvisory` capability). When true AND `slopGateMode != off`, a
diff --git a/test/unit/backtest-regression-gate-setting.test.ts b/test/unit/backtest-regression-gate-setting.test.ts
new file mode 100644
index 0000000000..e0d69960d7
--- /dev/null
+++ b/test/unit/backtest-regression-gate-setting.test.ts
@@ -0,0 +1,47 @@
+import { describe, expect, it } from "vitest";
+import { getRepositorySettings, upsertRepositoryFromGitHub, upsertRepositorySettings } from "../../src/db/repositories";
+import { createTestEnv } from "../helpers/d1";
+
+// backtestRegressionGateMode persistence wiring (#8105): defaults, round-trip, and the unrecognized-value
+// fallback. The behavioral (blocker/enforcement) sides live in gate-check-policy.test.ts and
+// threshold-backtest-advisory-wiring.test.ts — this file pins ONLY the storage layer's three paths.
+
+async function seedRepo(env: Env, fullName: string) {
+ await upsertRepositoryFromGitHub(env, {
+ id: 1,
+ full_name: fullName,
+ name: fullName.split("/")[1]!,
+ owner: { login: fullName.split("/")[0]! },
+ private: false,
+ default_branch: "main",
+ } as never);
+}
+
+describe("backtestRegressionGateMode setting (#8105)", () => {
+ it("defaults to advisory — the shipped pre-#8105 behavior — for an unconfigured repo", async () => {
+ const env = createTestEnv();
+ const settings = await getRepositorySettings(env, "acme/unconfigured");
+ expect(settings.backtestRegressionGateMode).toBe("advisory");
+ });
+
+ it("round-trips an explicit block, and an upsert omitting the field keeps the advisory default", async () => {
+ const env = createTestEnv();
+ await seedRepo(env, "acme/widgets");
+ await upsertRepositorySettings(env, { repoFullName: "acme/widgets", backtestRegressionGateMode: "block" });
+ expect((await getRepositorySettings(env, "acme/widgets")).backtestRegressionGateMode).toBe("block");
+
+ await seedRepo(env, "acme/other");
+ await upsertRepositorySettings(env, { repoFullName: "acme/other" });
+ expect((await getRepositorySettings(env, "acme/other")).backtestRegressionGateMode).toBe("advisory");
+ });
+
+ it("degrades an unrecognized stored value to advisory (parseGateRuleMode's fallback), never silently to block", async () => {
+ const env = createTestEnv();
+ await seedRepo(env, "acme/widgets");
+ await upsertRepositorySettings(env, { repoFullName: "acme/widgets", backtestRegressionGateMode: "advisory" });
+ await env.DB.prepare("UPDATE repository_settings SET backtest_regression_gate_mode = 'bogus' WHERE repo_full_name = ?")
+ .bind("acme/widgets")
+ .run();
+ expect((await getRepositorySettings(env, "acme/widgets")).backtestRegressionGateMode).toBe("advisory");
+ });
+});
diff --git a/test/unit/check-docs-drift-script.test.ts b/test/unit/check-docs-drift-script.test.ts
index d0f10cbe71..d28a889209 100644
--- a/test/unit/check-docs-drift-script.test.ts
+++ b/test/unit/check-docs-drift-script.test.ts
@@ -310,6 +310,8 @@ describe("check-docs-drift script", () => {
'import { PUBLIC_COMMAND_ENTRIES, MAINTAINER_COMMAND_ENTRIES, ACTION_COMMAND_ENTRIES } from "@/lib/command-reference";',
"apps/loopover-ui/content/docs/how-reviews-work.mdx": buildGateModePageText(),
"apps/loopover-ui/content/docs/github-app.mdx": buildGateModePageText(),
+ // #8105: backtestRegressionGateMode's manifest row names this page too.
+ "apps/loopover-ui/content/docs/backtest-calibration.mdx": buildGateModePageText(),
};
return files;
}
@@ -327,10 +329,10 @@ describe("check-docs-drift script", () => {
const result = checkDocsDrift({ root: "/fake", readFile: makeReadFile(files) });
expect(result.failures).toEqual([]);
- // gateModes bumped 13 -> 14 for contentLaneDeliverableGateMode (#content-lane-deliverable).
- // settingsFields = 14 GATE_MODE_MANIFEST fields + 20 synthetic extras; focusManifestFields = 18
+ // gateModes bumped 14 -> 15 for backtestRegressionGateMode (#8105).
+ // settingsFields = 15 GATE_MODE_MANIFEST fields + 20 synthetic extras; focusManifestFields = 18
// synthetic review fields + the nested review.visual.productionUrl leaf (#4617).
- expect(result.counts).toEqual({ flags: 10, commands: 19, gateModes: 14, settingsFields: 34, focusManifestFields: 19 });
+ expect(result.counts).toEqual({ flags: 10, commands: 19, gateModes: 15, settingsFields: 35, focusManifestFields: 19 });
});
it("catches an unmapped *GateMode field missing from GATE_MODE_MANIFEST", () => {
diff --git a/test/unit/focus-manifest.test.ts b/test/unit/focus-manifest.test.ts
index 5b2de4d0fe..6ef8c3fd71 100644
--- a/test/unit/focus-manifest.test.ts
+++ b/test/unit/focus-manifest.test.ts
@@ -287,6 +287,7 @@ describe(".loopover.yml.example field-exhaustiveness (#1670)", () => {
selfAuthoredLinkedIssue: "selfAuthoredLinkedIssue:",
linkedIssueSatisfaction: "linkedIssueSatisfaction:",
contentLaneDeliverable: "contentLaneDeliverable:",
+ backtestRegression: "backtestRegression:",
dryRun: "dryRun:",
premergeContentRecheck: "premergeContentRecheck:",
requireFreshRebaseWindowMinutes: "requireFreshRebaseWindow:",
@@ -943,7 +944,7 @@ describe("compileFocusManifestPolicy", () => {
issueDiscoveryPolicy: "neutral",
maintainerNotes: [],
publicNotes: ["Keep PRs focused.", "Maximize your reward payout"],
- gate: { present: false, enabled: null, checkMode: null, pack: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, sizeMaxFiles: null, sizeMaxLines: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewLowConfidenceDisposition: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, linkedIssueSatisfaction: null, contentLaneDeliverable: null, manifestPolicy: null, dryRun: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, staleBaseAheadByThreshold: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, expectedCiContexts: null, advisoryCheckRuns: null, aiJudgmentBlockersMode: null, copycatMode: null, copycatMinScore: null },
+ gate: { present: false, enabled: null, checkMode: null, pack: null, linkedIssue: null, duplicates: null, readinessMode: null, readinessMinScore: null, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, sizeMaxFiles: null, sizeMaxLines: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewLowConfidenceDisposition: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, linkedIssueSatisfaction: null, contentLaneDeliverable: null, backtestRegression: null, manifestPolicy: null, dryRun: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, staleBaseAheadByThreshold: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, expectedCiContexts: null, advisoryCheckRuns: null, aiJudgmentBlockersMode: null, copycatMode: null, copycatMinScore: null },
settings: {},
review: { present: false, footerText: null, note: null, fields: {}, enrichmentAnalyzers: {}, profile: null, tone: null, securityFocus: null, inlineComments: null, fixHandoff: null, autoMergeSummary: null, suggestions: null, changedFilesSummary: null, effortScore: null, impactMap: null, cultureProfile: null, selftune: null, sweepWatchdog: null, prReconciliation: null, activeReviewReconciliation: null, reviewMemory: null, findingCategories: null, inlineCommentsPerCategory: null, minFindingSeverity: null, maxFindings: { blockers: null, nits: null }, commentVerbosity: null, e2eTestDelivery: null, e2eTestAutoTrigger: null, pathInstructions: [], instructions: null, excludePaths: [], pathFilters: [], preMergeChecks: [], autoReview: { ...EMPTY_AUTO_REVIEW_CONFIG }, aiModel: { ...EMPTY_SELF_HOST_AI_MODEL_CONFIG }, visual: { ...EMPTY_VISUAL_CONFIG }, linkedIssueSatisfaction: null, sharedConfigSource: null },
features: { present: false, rag: null, reputation: null, safety: null, grounding: null, e2eTests: null, screenshots: null, improvementSignal: null, amsReputationBridge: null },
@@ -1110,7 +1111,7 @@ describe("parseFocusManifest gate config", () => {
// the block→advisory deprecation-downgrade behavior itself is covered separately below.
const m = parseFocusManifest({ gate: { linkedIssue: "block", duplicates: "advisory", readiness: { mode: "advisory", minScore: 70 } } });
expect(m.present).toBe(true);
- expect(m.gate).toEqual({ present: true, enabled: null, checkMode: null, pack: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "advisory", readinessMinScore: 70, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, sizeMaxFiles: null, sizeMaxLines: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewLowConfidenceDisposition: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, linkedIssueSatisfaction: null, contentLaneDeliverable: null, manifestPolicy: null, dryRun: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, staleBaseAheadByThreshold: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, expectedCiContexts: null, advisoryCheckRuns: null, aiJudgmentBlockersMode: null, copycatMode: null, copycatMinScore: null });
+ expect(m.gate).toEqual({ present: true, enabled: null, checkMode: null, pack: null, linkedIssue: "block", duplicates: "advisory", readinessMode: "advisory", readinessMinScore: 70, slopMode: null, slopMinScore: null, slopAiAdvisory: null, sizeMode: null, sizeMaxFiles: null, sizeMaxLines: null, lockfileIntegrityMode: null, aiReviewMode: null, aiReviewByok: null, aiReviewProvider: null, aiReviewModel: null, aiReviewAllAuthors: null, aiReviewCloseConfidence: null, aiReviewLowConfidenceDisposition: null, aiReviewCombine: null, aiReviewOnMerge: null, aiReviewReviewers: null, mergeReadiness: null, selfAuthoredLinkedIssue: null, linkedIssueSatisfaction: null, contentLaneDeliverable: null, backtestRegression: null, manifestPolicy: null, dryRun: null, premergeContentRecheck: null, requireFreshRebaseWindowMinutes: null, staleBaseAheadByThreshold: null, claMode: null, claConsentPhrase: null, claCheckRunName: null, claCheckRunAppSlug: null, expectedCiContexts: null, advisoryCheckRuns: null, aiJudgmentBlockersMode: null, copycatMode: null, copycatMinScore: null });
});
it("parses gate.mergeReadiness, round-trips it, and warns on a bad value (#822)", () => {
diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts
index fddc61d92a..77f0eb59e8 100644
--- a/test/unit/gate-check-policy.test.ts
+++ b/test/unit/gate-check-policy.test.ts
@@ -989,6 +989,33 @@ describe("linked-issue satisfaction gate blocker (#1961/#3906)", () => {
});
});
+describe("backtest-regression gate blocker (#8105)", () => {
+ const backtestAdvisory = (): Advisory => ({
+ ...missingIssueAdvisory(),
+ findings: [{ code: "backtest_regression", title: "Backtest regression against recorded history", severity: "warning", detail: "Backtesting this PR's threshold change REGRESSED linked_issue_scope_mismatch on at least one axis.", action: "Revisit the changed threshold so no backtest axis regresses, or ask a maintainer to override." }],
+ });
+
+ it("blocks (failure) under backtestRegressionGateMode: block", () => {
+ const result = evaluateGateCheck(backtestAdvisory(), { backtestRegressionGateMode: "block", confirmedContributor: true });
+ expect(result.conclusion).toBe("failure");
+ expect(result.blockers.map((b) => b.code)).toContain("backtest_regression");
+ });
+
+ it("stays advisory (never blocks) under unset (default advisory), explicit advisory, and off — the shipped pre-#8105 behavior", () => {
+ expect(evaluateGateCheck(backtestAdvisory(), {}).conclusion).toBe("success"); // unset ⇒ defaults to advisory
+ expect(evaluateGateCheck(backtestAdvisory(), { backtestRegressionGateMode: "advisory" }).conclusion).toBe("success");
+ expect(evaluateGateCheck(backtestAdvisory(), { backtestRegressionGateMode: "off" }).conclusion).toBe("success");
+ const advisoryResult = evaluateGateCheck(backtestAdvisory(), { backtestRegressionGateMode: "advisory" });
+ expect(advisoryResult.warnings.map((w) => w.code)).toContain("backtest_regression");
+ });
+
+ it("resolveEffectiveSettings maps gate.backtestRegression → backtestRegressionGateMode, and gateCheckPolicy threads it", () => {
+ const eff = resolveEffectiveSettings(settings({}), parseFocusManifest({ gate: { backtestRegression: "block" } }));
+ expect(eff.backtestRegressionGateMode).toBe("block");
+ expect(gateCheckPolicy(settings({ backtestRegressionGateMode: "block" }), null, true).backtestRegressionGateMode).toBe("block");
+ });
+});
+
describe("content-lane linked-issue deliverable gate blocker (#content-lane-deliverable)", () => {
const deliverableAdvisory = (): Advisory => ({
...missingIssueAdvisory(),
diff --git a/test/unit/maintainer-activation.test.ts b/test/unit/maintainer-activation.test.ts
index ae8335ba5e..d334b0ad93 100644
--- a/test/unit/maintainer-activation.test.ts
+++ b/test/unit/maintainer-activation.test.ts
@@ -39,6 +39,7 @@ function settings(overrides: Partial = {}): RepositorySettin
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,
diff --git a/test/unit/policy-sanitizer.test.ts b/test/unit/policy-sanitizer.test.ts
index baefe4bc77..4e3c8103b7 100644
--- a/test/unit/policy-sanitizer.test.ts
+++ b/test/unit/policy-sanitizer.test.ts
@@ -72,6 +72,7 @@ function settingsFor(repoFullName: string, overrides: Partial {
expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("duplicate_pr_risk"), block)).toBe(true);
expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("ai_consensus_defect"), advisory)).toBe(false);
expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("ai_consensus_defect"), block)).toBe(true);
+ // Backtest-regression gate (#8105): absent (defaults to advisory), explicit advisory, and block.
+ expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("backtest_regression"), advisory)).toBe(false);
+ expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("backtest_regression"), { ...advisory, backtestRegressionGateMode: "advisory" as const })).toBe(false);
+ expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("backtest_regression"), { ...block, backtestRegressionGateMode: "block" as const })).toBe(true);
expect(gateAdvisoryInternals.isConfiguredGateBlocker(finding("ai_review_split"), advisory)).toBe(false);
// #4603: aiReviewLowConfidenceDisposition branches — the default ("hold_for_review", exercised by the
// bare `block` policy above with no confidence set) and "one_shot" both ignore confidence entirely and
diff --git a/test/unit/registration-readiness.test.ts b/test/unit/registration-readiness.test.ts
index b5a47a8a44..edef6dd721 100644
--- a/test/unit/registration-readiness.test.ts
+++ b/test/unit/registration-readiness.test.ts
@@ -53,6 +53,7 @@ function settingsFor(repoFullName: string, overrides: Partial = {}): RepositorySettin
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,
diff --git a/test/unit/repository-settings-enforcement.test.ts b/test/unit/repository-settings-enforcement.test.ts
index 8dc00807a5..c306da409b 100644
--- a/test/unit/repository-settings-enforcement.test.ts
+++ b/test/unit/repository-settings-enforcement.test.ts
@@ -27,6 +27,7 @@ function settings(over: Partial = {}): RepositorySettings {
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,
diff --git a/test/unit/self-dogfood-registration-pack.test.ts b/test/unit/self-dogfood-registration-pack.test.ts
index a22ba76e9b..aaca6de720 100644
--- a/test/unit/self-dogfood-registration-pack.test.ts
+++ b/test/unit/self-dogfood-registration-pack.test.ts
@@ -66,6 +66,7 @@ function settingsFor(repoFullName: string, overrides: Partial = {}): RepositorySettin
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,
diff --git a/test/unit/signals-coverage.test.ts b/test/unit/signals-coverage.test.ts
index 778af041ee..cc1e6b67f6 100644
--- a/test/unit/signals-coverage.test.ts
+++ b/test/unit/signals-coverage.test.ts
@@ -2389,6 +2389,7 @@ function repoSettings(repoFullName: string): RepositorySettings {
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,
diff --git a/test/unit/signals-v2.test.ts b/test/unit/signals-v2.test.ts
index 3d981461af..51e61bb91f 100644
--- a/test/unit/signals-v2.test.ts
+++ b/test/unit/signals-v2.test.ts
@@ -2116,6 +2116,7 @@ describe("v2 signal builders", () => {
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,
diff --git a/test/unit/signals.test.ts b/test/unit/signals.test.ts
index cab3c6edb0..e5b40757fb 100644
--- a/test/unit/signals.test.ts
+++ b/test/unit/signals.test.ts
@@ -543,6 +543,7 @@ describe("world-class backend signals", () => {
selfAuthoredLinkedIssueGateMode: "advisory" as const,
linkedIssueSatisfactionGateMode: "off" as const,
contentLaneDeliverableGateMode: "off" as const,
+ backtestRegressionGateMode: "advisory" as const,
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,
@@ -604,6 +605,7 @@ describe("world-class backend signals", () => {
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,
@@ -687,6 +689,7 @@ describe("world-class backend signals", () => {
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,
@@ -803,6 +806,7 @@ describe("world-class backend signals", () => {
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,
diff --git a/test/unit/threshold-backtest-advisory-wiring.test.ts b/test/unit/threshold-backtest-advisory-wiring.test.ts
index cb58b478ee..bb7f5f5f75 100644
--- a/test/unit/threshold-backtest-advisory-wiring.test.ts
+++ b/test/unit/threshold-backtest-advisory-wiring.test.ts
@@ -4,7 +4,7 @@ import { createSignalStore } from "../../src/review/signal-tracking-wire";
import { listAuditEventsByType } from "../../src/db/repositories";
import * as thresholdBacktestRun from "../../src/services/threshold-backtest-run";
import { THRESHOLD_BACKTEST_EVENT_TYPE } from "../../src/services/threshold-backtest-run";
-import type { PullRequestFileRecord } from "../../src/types";
+import type { AdvisoryFinding, PullRequestFileRecord } from "../../src/types";
import { createTestEnv } from "../helpers/d1";
afterEach(() => {
@@ -72,3 +72,77 @@ describe("resolveThresholdBacktestAdvisory (processor wiring, #8138)", () => {
expect(result).toBe("");
});
});
+
+describe("resolveThresholdBacktestAdvisory backtestRegressionGateMode (#8105)", () => {
+ // The shipped fixture shape: lowering the floor 0.5 -> 0.2 against a reversed firing recorded at
+ // confidence 0.35 REGRESSES recall (the old floor caught the bad call, the new one would not).
+ async function seedReversedHistory(env: ReturnType) {
+ const now = Date.now();
+ await createSignalStore(env).recordRuleFired({
+ ruleId: "linked_issue_scope_mismatch",
+ targetKey: "acme/widgets#1",
+ outcome: "unaddressed",
+ occurredAt: new Date(now - 1000).toISOString(),
+ metadata: { confidence: 0.35 },
+ });
+ await createSignalStore(env).recordHumanOverride({
+ ruleId: "linked_issue_scope_mismatch",
+ targetKey: "acme/widgets#1",
+ verdict: "reversed",
+ occurredAt: new Date(now).toISOString(),
+ });
+ return now;
+ }
+ const regressingFiles = () => [
+ fileRecord("src/services/linked-issue-satisfaction.ts", thresholdPatch("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.5", "0.2")),
+ ];
+
+ it("off silences the whole advisory: no section, no persisted run, even for a real threshold change", async () => {
+ const env = createTestEnv();
+ const now = await seedReversedHistory(env);
+ const result = await resolveThresholdBacktestAdvisory(env, "acme/widgets", { number: 7 }, regressingFiles(), { mode: "off" });
+ expect(result).toBe("");
+ expect(await listAuditEventsByType(env, THRESHOLD_BACKTEST_EVENT_TYPE, new Date(now - 60_000).toISOString())).toHaveLength(0);
+ });
+
+ it("advisory (explicit AND defaulted) renders the section but never pushes a finding, even on a REGRESSED verdict", async () => {
+ for (const options of [{ mode: "advisory" as const, advisory: { findings: [] } }, {}]) {
+ const env = createTestEnv();
+ await seedReversedHistory(env);
+ const result = await resolveThresholdBacktestAdvisory(env, "acme/widgets", { number: 7 }, regressingFiles(), options);
+ expect(result).toContain("REGRESSED");
+ if ("advisory" in options) expect(options.advisory.findings).toEqual([]);
+ }
+ });
+
+ it("block escalates a REGRESSED verdict into a backtest_regression finding and still renders the section", async () => {
+ const env = createTestEnv();
+ await seedReversedHistory(env);
+ const advisory: { findings: AdvisoryFinding[] } = { findings: [] };
+ const result = await resolveThresholdBacktestAdvisory(env, "acme/widgets", { number: 7 }, regressingFiles(), { mode: "block", advisory });
+ expect(result).toContain("REGRESSED");
+ expect(advisory.findings).toHaveLength(1);
+ const finding = advisory.findings[0]!;
+ expect(finding.code).toBe("backtest_regression");
+ expect(finding.severity).toBe("warning");
+ expect(finding.detail).toContain("linked_issue_scope_mismatch");
+ expect(finding.publicText).not.toMatch(/reward|payout|trust|wallet|hotkey/i);
+ });
+
+ it("block pushes nothing when the change IMPROVES the backtest (raising the floor catches the reversed call)", async () => {
+ const env = createTestEnv();
+ await seedReversedHistory(env);
+ const files = [fileRecord("src/services/linked-issue-satisfaction.ts", thresholdPatch("LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR", "0.2", "0.5"))];
+ const advisory = { findings: [] };
+ const result = await resolveThresholdBacktestAdvisory(env, "acme/widgets", { number: 7 }, files, { mode: "block", advisory });
+ expect(result).toContain("improved");
+ expect(advisory.findings).toEqual([]);
+ });
+
+ it("block without an advisory sink degrades gracefully: section renders, nothing crashes", async () => {
+ const env = createTestEnv();
+ await seedReversedHistory(env);
+ const result = await resolveThresholdBacktestAdvisory(env, "acme/widgets", { number: 7 }, regressingFiles(), { mode: "block" });
+ expect(result).toContain("REGRESSED");
+ });
+});
diff --git a/test/unit/unified-comment-parity.test.ts b/test/unit/unified-comment-parity.test.ts
index 403a723e52..8185262870 100644
--- a/test/unit/unified-comment-parity.test.ts
+++ b/test/unit/unified-comment-parity.test.ts
@@ -64,6 +64,7 @@ const settings: RepositorySettings = {
selfAuthoredLinkedIssueGateMode: "advisory",
linkedIssueSatisfactionGateMode: "off",
contentLaneDeliverableGateMode: "off",
+ backtestRegressionGateMode: "advisory",
slopAiAdvisory: false,
qualityGateMinScore: null,
autoLabelEnabled: true,