From a50b863d11a1c51f8e0779d2cf6d36d5d96878a4 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 01:21:10 -0700 Subject: [PATCH] feat(review): surface backtest-cleared loosening proposals in the tuning advisor (#8160) computeTuningRecommendations still emits loosening advice as prose with no payload -- and the #8121 loop that can now actually measure a loosening was invisible to the advisor. buildSatisfactionFloorLooseningRecs (pure) turns the loop's state into ranked TuningRecs: a backtest-cleared proposal surfaces as a good-severity rec carrying both split verdicts, sample sizes, and the precision movement inline, with the action line matched to the flag state; a recently applied loosening surfaces as info pointing at the #8161 status surface. HARD BOUNDARY, tested: these recs never carry an overridePayload -- that field is the tightening-only auto-apply channel, so the advisor's apply path provably cannot promote a loosening. loadSatisfactionFloorRecState (evaluate-ONLY, fail-safe) feeds it from the same corpus + current floor the applying tick would use, and runSelfTune appends the recs exactly once per pass on the first repo's iteration (deployment-global state, never once per repo -- pinned by a spy test). 100% line+branch coverage on the new pure module; state-read paths (override- adjusted floor, empty corpus, broken DB, flag-off advice) all pinned. Closes #8160. --- src/review/loosening-recs.ts | 61 ++++++++++++++++ src/review/selftune-wire.ts | 7 ++ .../satisfaction-floor-loosening-run.ts | 39 ++++++++++ test/unit/loosening-recs.test.ts | 73 +++++++++++++++++++ .../satisfaction-floor-loosening-run.test.ts | 42 +++++++++++ test/unit/selftune-wiring.test.ts | 20 ++++- 6 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 src/review/loosening-recs.ts create mode 100644 test/unit/loosening-recs.test.ts diff --git a/src/review/loosening-recs.ts b/src/review/loosening-recs.ts new file mode 100644 index 0000000000..f8baa3c73e --- /dev/null +++ b/src/review/loosening-recs.ts @@ -0,0 +1,61 @@ +// Loosening recommendations for the tuning advisor (#8160, sub-issue of epic #8121). auto-tune.ts's +// computeTuningRecommendations deliberately emits loosening advice as prose with no payload (autonomous +// loosening was the regression risk the loop existed to avoid — see OverridePayload's own doc). The #8121 +// narrow start made ONE loosening measurable (the satisfaction floor, backtest-gated); this module surfaces +// that loop's state as TuningRec entries alongside the tightening recs, so the advisor's reader sees a +// backtest-cleared loosening opportunity — or a recently-applied one — in the same ranked list. +// +// HARD BOUNDARY (the issue's own): these recs NEVER carry an `overridePayload`. That field is the +// tightening-only auto-apply channel (runAutoApplyRecommendations consumes it); a loosening must only ever +// be applied by the flag-gated loop itself (satisfaction-floor-loosening-run.ts), never promoted by the +// advisor's apply path. PURE — no IO; the caller supplies the loop's state. +import type { TuningRec } from "./auto-tune"; +import type { SatisfactionFloorLooseningProposal } from "../services/satisfaction-floor-loosening"; + +/** The advisor list is per-project elsewhere; the satisfaction floor is deployment-global, so its recs use + * this fixed pseudo-project label rather than impersonating any repo. */ +export const LOOSENING_REC_PROJECT = "global:satisfaction-floor"; + +export type SatisfactionFloorRecInput = { + flagEnabled: boolean; + proposal: SatisfactionFloorLooseningProposal | null; + /** created_at of the most recent applied loosening (calibration.satisfaction_floor_loosened), or null. */ + lastAppliedAt: string | null; +}; + +const pct = (value: number | null): string => (value == null ? "—" : `${Math.round(value * 100)}%`); + +/** + * Build the loosening TuningRecs from the loop's current state. At most two entries, never a payload: + * • a backtest-cleared PROPOSAL → severity `good` — the positive "evidence says this can loosen" signal, + * with both split verdicts + sample sizes inline and the action that matches the flag state (flip the + * flag vs. wait for the hourly tick); + * • a recently APPLIED loosening → severity `info`, pointing at the operator status surface (#8161). + * No state ⇒ []. Pure and deterministic. + */ +export function buildSatisfactionFloorLooseningRecs(input: SatisfactionFloorRecInput): TuningRec[] { + const recs: TuningRec[] = []; + if (input.proposal) { + const { proposal } = input; + const action = input.flagEnabled + ? "The autotune flag is ON — the hourly tick will apply this step automatically." + : "The autotune flag is OFF — set SATISFACTION_FLOOR_AUTOTUNE_ENABLED (or POST /v1/internal/calibration/loosen-satisfaction-floor after flipping it) to let the loop act."; + recs.push({ + project: LOOSENING_REC_PROJECT, + severity: "good", + message: + `Backtest-cleared LOOSENING available: satisfaction confidence floor ${proposal.currentFloor} → ${proposal.proposedFloor}. ` + + `Visible split ${proposal.visible.verdict} (${proposal.visibleCases} case(s), precision ${pct(proposal.visible.baseline.precision)} → ${pct(proposal.visible.candidate.precision)}); ` + + `held-out split ${proposal.heldOut.verdict} (${proposal.heldOutCases} case(s)). ${action}`, + // Deliberately NO overridePayload: that channel is tightening-only (see the module doc). + }); + } + if (input.lastAppliedAt) { + recs.push({ + project: LOOSENING_REC_PROJECT, + severity: "info", + message: `A backtest-gated loosening was applied at ${input.lastAppliedAt} — see GET /v1/internal/calibration/satisfaction-floor for the live floor and full evidence history.`, + }); + } + return recs; +} diff --git a/src/review/selftune-wire.ts b/src/review/selftune-wire.ts index 68c3677e08..9ad301b859 100644 --- a/src/review/selftune-wire.ts +++ b/src/review/selftune-wire.ts @@ -41,6 +41,8 @@ import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; import { loadRepoFocusManifest } from "../signals/focus-manifest-loader"; import { errorMessage } from "../utils/json"; import { computeTuningRecommendations, type GateEvalReport, type GateEvalRow } from "./auto-tune"; +import { buildSatisfactionFloorLooseningRecs } from "./loosening-recs"; +import { loadSatisfactionFloorRecState } from "../services/satisfaction-floor-loosening-run"; import { runAutoApplyRecommendations, type StorageEnv } from "./auto-apply"; /** True when the self-improvement loop is enabled. Flag-OFF (default) → every export below is a no-op. Truthy @@ -155,6 +157,11 @@ export async function runSelfTune(env: Env): Promise { const row = await buildEvalRow(env, repoFullName); const report: GateEvalReport = { rows: [row], hasSignal: row.decided >= 10 }; const recs = computeTuningRecommendations(report); + // #8160: surface the backtest-gated loosening loop's state in the same ranked list. Payload-less by + // design — runAutoApplyRecommendations below only ever consumes recs carrying a TIGHTENING + // overridePayload, so these are report-only here and can never be promoted by the apply path. + // Appended once per pass (deployment-global state), on the first repo's iteration. + if (repoFullName === repos[0]) recs.push(...buildSatisfactionFloorLooseningRecs(await loadSatisfactionFloorRecState(env, nowMs))); // runAutoApplyRecommendations only ever consumes recs that carry a TIGHTENING overridePayload, shadow- // soaks them, and promotes a soaked override only when isStrictlyTightening + evidence + soak pass. await runAutoApplyRecommendations(env as unknown as StorageEnv, { diff --git a/src/services/satisfaction-floor-loosening-run.ts b/src/services/satisfaction-floor-loosening-run.ts index 93f9b342d2..159e81d827 100644 --- a/src/services/satisfaction-floor-loosening-run.ts +++ b/src/services/satisfaction-floor-loosening-run.ts @@ -133,3 +133,42 @@ export async function runScheduledSatisfactionFloorLoosening(env: Env): Promise< return null; } } + +/** The loop state the tuning advisor surfaces (#8160). Structurally what + * src/review/loosening-recs.ts's builder consumes — kept here so the advisor wire needs one read. */ +export type SatisfactionFloorRecState = { + flagEnabled: boolean; + proposal: import("./satisfaction-floor-loosening").SatisfactionFloorLooseningProposal | null; + lastAppliedAt: string | null; +}; + +/** + * Evaluate-ONLY state read for the advisor (#8160): the current proposal (from the same corpus + current + * floor the applying tick would use — but never writing anything) plus the newest applied-loosening + * timestamp. Fail-safe on every read: a corpus/history error degrades to a null section rather than + * breaking the advisor surface that embeds this. + */ +export async function loadSatisfactionFloorRecState(env: Env, nowMs: number = Date.now()): Promise { + const flagEnabled = isSatisfactionFloorAutotuneEnabled(env); + + let proposal: SatisfactionFloorRecState["proposal"] = null; + try { + const currentFloor = (await getSatisfactionFloorOverride(env)) ?? LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR; + const { fired, overrides } = await createSignalStore(env).queryRuleHistory(SATISFACTION_FLOOR_RULE_ID, nowMs - CORPUS_LOOKBACK_MS); + proposal = evaluateSatisfactionFloorLoosening(buildBacktestCorpus(SATISFACTION_FLOOR_RULE_ID, fired, overrides), currentFloor); + } catch { + proposal = null; + } + + let lastAppliedAt: string | null = null; + try { + const row = await env.DB.prepare("SELECT created_at FROM audit_events WHERE event_type = ? ORDER BY created_at DESC LIMIT 1") + .bind(SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE) + .first<{ created_at: string }>(); + lastAppliedAt = row?.created_at ?? null; + } catch { + lastAppliedAt = null; + } + + return { flagEnabled, proposal, lastAppliedAt }; +} diff --git a/test/unit/loosening-recs.test.ts b/test/unit/loosening-recs.test.ts new file mode 100644 index 0000000000..5b8b76215b --- /dev/null +++ b/test/unit/loosening-recs.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, it } from "vitest"; +import { buildSatisfactionFloorLooseningRecs, LOOSENING_REC_PROJECT } from "../../src/review/loosening-recs"; +import type { SatisfactionFloorLooseningProposal } from "../../src/services/satisfaction-floor-loosening"; + +function proposal(overrides: Partial = {}): SatisfactionFloorLooseningProposal { + return { + ruleId: "linked_issue_scope_mismatch", + currentFloor: 0.5, + proposedFloor: 0.45, + visibleCases: 24, + heldOutCases: 7, + visible: { + ruleId: "linked_issue_scope_mismatch", + baseline: { ruleId: "linked_issue_scope_mismatch", caseCount: 24, truePositive: 1, falsePositive: 4, trueNegative: 19, falseNegative: 0, precision: 0.2, recall: 1 }, + candidate: { ruleId: "linked_issue_scope_mismatch", caseCount: 24, truePositive: 1, falsePositive: 0, trueNegative: 23, falseNegative: 0, precision: 1, recall: 1 }, + regressedAxes: [], + improvedAxes: ["precision"], + verdict: "improved", + }, + heldOut: { + ruleId: "linked_issue_scope_mismatch", + baseline: { ruleId: "linked_issue_scope_mismatch", caseCount: 7, truePositive: 0, falsePositive: 0, trueNegative: 7, falseNegative: 0, precision: null, recall: null }, + candidate: { ruleId: "linked_issue_scope_mismatch", caseCount: 7, truePositive: 0, falsePositive: 0, trueNegative: 7, falseNegative: 0, precision: null, recall: null }, + regressedAxes: [], + improvedAxes: [], + verdict: "unchanged", + }, + ...overrides, + }; +} + +describe("buildSatisfactionFloorLooseningRecs (#8160)", () => { + it("returns [] with no proposal and no applied history", () => { + expect(buildSatisfactionFloorLooseningRecs({ flagEnabled: false, proposal: null, lastAppliedAt: null })).toEqual([]); + }); + + it("surfaces a backtest-cleared proposal as a good-severity rec with both split verdicts, sample sizes, and precision movement", () => { + const recs = buildSatisfactionFloorLooseningRecs({ flagEnabled: false, proposal: proposal(), lastAppliedAt: null }); + expect(recs).toHaveLength(1); + const [rec] = recs; + expect(rec!.project).toBe(LOOSENING_REC_PROJECT); + expect(rec!.severity).toBe("good"); + expect(rec!.message).toContain("0.5 → 0.45"); + expect(rec!.message).toContain("Visible split improved (24 case(s), precision 20% → 100%)"); + expect(rec!.message).toContain("held-out split unchanged (7 case(s))"); + // Null precision renders as the em-dash convention, never a fake number. + const nullPrecision = buildSatisfactionFloorLooseningRecs({ + flagEnabled: false, + proposal: proposal({ visible: { ...proposal().visible, baseline: { ...proposal().visible.baseline, precision: null } } }), + lastAppliedAt: null, + }); + expect(nullPrecision[0]!.message).toContain("precision — →"); + }); + + it("the action line matches the flag state, and the rec NEVER carries an overridePayload (the tightening-only channel)", () => { + const off = buildSatisfactionFloorLooseningRecs({ flagEnabled: false, proposal: proposal(), lastAppliedAt: null }); + expect(off[0]!.message).toContain("SATISFACTION_FLOOR_AUTOTUNE_ENABLED"); + const on = buildSatisfactionFloorLooseningRecs({ flagEnabled: true, proposal: proposal(), lastAppliedAt: null }); + expect(on[0]!.message).toContain("hourly tick will apply"); + for (const rec of [...off, ...on]) expect(rec.overridePayload).toBeUndefined(); + }); + + it("reports a recently applied loosening as an info rec pointing at the operator status surface, alongside a proposal when both exist", () => { + const both = buildSatisfactionFloorLooseningRecs({ flagEnabled: true, proposal: proposal(), lastAppliedAt: "2026-07-23T05:00:00.000Z" }); + expect(both.map((rec) => rec.severity)).toEqual(["good", "info"]); + expect(both[1]!.message).toContain("2026-07-23T05:00:00.000Z"); + expect(both[1]!.message).toContain("/v1/internal/calibration/satisfaction-floor"); + + const appliedOnly = buildSatisfactionFloorLooseningRecs({ flagEnabled: true, proposal: null, lastAppliedAt: "2026-07-23T05:00:00.000Z" }); + expect(appliedOnly).toHaveLength(1); + expect(appliedOnly[0]!.severity).toBe("info"); + }); +}); diff --git a/test/unit/satisfaction-floor-loosening-run.test.ts b/test/unit/satisfaction-floor-loosening-run.test.ts index 805358250b..29b21ca1a2 100644 --- a/test/unit/satisfaction-floor-loosening-run.test.ts +++ b/test/unit/satisfaction-floor-loosening-run.test.ts @@ -3,6 +3,7 @@ import { splitBacktestCorpus } from "@loopover/engine"; import { getSatisfactionFloorOverride, isSatisfactionFloorAutotuneEnabled, + loadSatisfactionFloorRecState, runSatisfactionFloorLoosening, runScheduledSatisfactionFloorLoosening, SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE, @@ -225,3 +226,44 @@ describe("runScheduledSatisfactionFloorLoosening + queue wiring (#8158)", () => expect(await getSatisfactionFloorOverride(onEnv)).toBe(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0]); }); }); + +describe("loadSatisfactionFloorRecState (#8160)", () => { + it("reports flag state, a live proposal from the current corpus, and null lastAppliedAt on a fresh deployment", async () => { + const env = enabledEnv(); + await seedLooseningFriendlyHistory(env); + const state = await loadSatisfactionFloorRecState(env); + expect(state.flagEnabled).toBe(true); + expect(state.proposal?.proposedFloor).toBe(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0]); + expect(state.lastAppliedAt).toBeNull(); + }); + + it("evaluates from the OVERRIDDEN floor when one is live, and reads the newest applied timestamp", async () => { + const env = enabledEnv(); + await seedLooseningFriendlyHistory(env); + // Apply once for real: the next state read must evaluate from 0.45 (no further proposal on this corpus) + // and report the applied event's timestamp. + expect((await runSatisfactionFloorLoosening(env)).applied).toBe(true); + const state = await loadSatisfactionFloorRecState(env); + expect(state.proposal).toBeNull(); + expect(state.lastAppliedAt).not.toBeNull(); + }); + + it("stays proposal-null on an empty corpus and fails safe to nulls on a broken DB", async () => { + const fresh = await loadSatisfactionFloorRecState(enabledEnv()); + expect(fresh.proposal).toBeNull(); + expect(fresh.lastAppliedAt).toBeNull(); + + const env = enabledEnv(); + env.DB = { prepare: () => { throw new Error("boom"); } } as never; + const broken = await loadSatisfactionFloorRecState(env); + expect(broken).toEqual({ flagEnabled: true, proposal: null, lastAppliedAt: null }); + }); + + it("flag off: reports flagEnabled false while still evaluating from the shipped floor (advice can precede the opt-in)", async () => { + const env = createTestEnv(); + await seedLooseningFriendlyHistory(env); + const state = await loadSatisfactionFloorRecState(env); + expect(state.flagEnabled).toBe(false); + expect(state.proposal?.currentFloor).toBe(0.5); + }); +}); diff --git a/test/unit/selftune-wiring.test.ts b/test/unit/selftune-wiring.test.ts index c55e71e106..cdd5dc5aea 100644 --- a/test/unit/selftune-wiring.test.ts +++ b/test/unit/selftune-wiring.test.ts @@ -45,7 +45,7 @@ async function seedRegisteredRepo(env: Env, fullName: string, autonomyJson: stri // Seed N resolved recommendation outcomes for a repo: `negative` rejected/closed (the dangerous error a // tightening fixes) + `positive` accepted. Inserted directly (FKs off) — buildRepoOutcomeCalibration reads // the outcome_state split, which is all the eval mapping needs. -async function seedRecommendationOutcomes(env: Env, repoFullName: string, positive: number, negative: number, maintainerLane = true): Promise { +async function seedRecommendationOutcomes(env: Env, repoFullName: string, positive: number, negative: number, maintainerLane = true, idPrefix = ""): Promise { await env.DB.prepare("PRAGMA foreign_keys=OFF").run(); let i = 0; const insert = async (state: string) => { @@ -56,7 +56,7 @@ async function seedRecommendationOutcomes(env: Env, repoFullName: string, positi maintainer_lane, confidence, reason, source, updated_at) VALUES (?, ?, ?, ?, 'review', ?, 'pull_request', ?, ?, 'medium', 'seed', 'inferred', CURRENT_TIMESTAMP)`, ) - .bind(`o${i}`, `a${i}`, `r${i}`, "bot", state, repoFullName, maintainerLane ? 1 : 0) + .bind(`${idPrefix}o${i}`, `${idPrefix}a${i}`, `${idPrefix}r${i}`, "bot", state, repoFullName, maintainerLane ? 1 : 0) .run(); }; for (let n = 0; n < positive; n += 1) await insert("accepted"); @@ -234,6 +234,22 @@ describe("runSelfTune — shadow-soak over loopover's own outcome data", () => { expect((await listOverrideAudit(env as never, "owner/repo")).length).toBe(0); }); + it("appends the loosening-loop recs exactly once per pass, on the first repo only (#8160)", async () => { + const state = await import("../../src/services/satisfaction-floor-loosening-run"); + const spy = vi.spyOn(state, "loadSatisfactionFloorRecState"); + const env = createTestEnv({ LOOPOVER_REVIEW_SELFTUNE: "true" }); + await seedRegisteredRepo(env, "owner/repo", ACTING_AUTONOMY); + await seedRegisteredRepo(env, "owner/other", ACTING_AUTONOMY); + await seedRecommendationOutcomes(env, "owner/repo", 5, 10); + await seedRecommendationOutcomes(env, "owner/other", 5, 10, true, "b-"); + + await processJob(env, { type: "selftune", requestedBy: "schedule" }); + + // Two repos in the pass, ONE deployment-global loosening-state read: the recs are appended on the + // first repo's iteration only, never once per repo. + expect(spy).toHaveBeenCalledTimes(1); + }); + it("FLAG-ON via the processor: a stale in-flight selftune job runs the tick (defense-in-depth gate)", async () => { const env = createTestEnv({ LOOPOVER_REVIEW_SELFTUNE: "true" }); await seedRegisteredRepo(env, "owner/repo", ACTING_AUTONOMY);