Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions src/review/loosening-recs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// Loosening recommendations for the tuning advisor (#8160, sub-issue of epic #8121). auto-tune.ts's
// computeTuningRecommendations deliberately emits loosening advice as prose with no payload (autonomous
// loosening was the regression risk the loop existed to avoid — see OverridePayload's own doc). The #8121
// narrow start made ONE loosening measurable (the satisfaction floor, backtest-gated); this module surfaces
// that loop's state as TuningRec entries alongside the tightening recs, so the advisor's reader sees a
// backtest-cleared loosening opportunity — or a recently-applied one — in the same ranked list.
//
// HARD BOUNDARY (the issue's own): these recs NEVER carry an `overridePayload`. That field is the
// tightening-only auto-apply channel (runAutoApplyRecommendations consumes it); a loosening must only ever
// be applied by the flag-gated loop itself (satisfaction-floor-loosening-run.ts), never promoted by the
// advisor's apply path. PURE — no IO; the caller supplies the loop's state.
import type { TuningRec } from "./auto-tune";
import type { SatisfactionFloorLooseningProposal } from "../services/satisfaction-floor-loosening";

/** The advisor list is per-project elsewhere; the satisfaction floor is deployment-global, so its recs use
* this fixed pseudo-project label rather than impersonating any repo. */
export const LOOSENING_REC_PROJECT = "global:satisfaction-floor";

export type SatisfactionFloorRecInput = {
flagEnabled: boolean;
proposal: SatisfactionFloorLooseningProposal | null;
/** created_at of the most recent applied loosening (calibration.satisfaction_floor_loosened), or null. */
lastAppliedAt: string | null;
};

const pct = (value: number | null): string => (value == null ? "—" : `${Math.round(value * 100)}%`);

/**
* Build the loosening TuningRecs from the loop's current state. At most two entries, never a payload:
* • a backtest-cleared PROPOSAL → severity `good` — the positive "evidence says this can loosen" signal,
* with both split verdicts + sample sizes inline and the action that matches the flag state (flip the
* flag vs. wait for the hourly tick);
* • a recently APPLIED loosening → severity `info`, pointing at the operator status surface (#8161).
* No state ⇒ []. Pure and deterministic.
*/
export function buildSatisfactionFloorLooseningRecs(input: SatisfactionFloorRecInput): TuningRec[] {
const recs: TuningRec[] = [];
if (input.proposal) {
const { proposal } = input;
const action = input.flagEnabled
? "The autotune flag is ON — the hourly tick will apply this step automatically."
: "The autotune flag is OFF — set SATISFACTION_FLOOR_AUTOTUNE_ENABLED (or POST /v1/internal/calibration/loosen-satisfaction-floor after flipping it) to let the loop act.";
recs.push({
project: LOOSENING_REC_PROJECT,
severity: "good",
message:
`Backtest-cleared LOOSENING available: satisfaction confidence floor ${proposal.currentFloor} → ${proposal.proposedFloor}. ` +
`Visible split ${proposal.visible.verdict} (${proposal.visibleCases} case(s), precision ${pct(proposal.visible.baseline.precision)} → ${pct(proposal.visible.candidate.precision)}); ` +
`held-out split ${proposal.heldOut.verdict} (${proposal.heldOutCases} case(s)). ${action}`,
// Deliberately NO overridePayload: that channel is tightening-only (see the module doc).
});
}
if (input.lastAppliedAt) {
recs.push({
project: LOOSENING_REC_PROJECT,
severity: "info",
message: `A backtest-gated loosening was applied at ${input.lastAppliedAt} — see GET /v1/internal/calibration/satisfaction-floor for the live floor and full evidence history.`,
});
}
return recs;
}
7 changes: 7 additions & 0 deletions src/review/selftune-wire.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ import { buildRepoOutcomeCalibration } from "../services/outcome-calibration";
import { loadRepoFocusManifest } from "../signals/focus-manifest-loader";
import { errorMessage } from "../utils/json";
import { computeTuningRecommendations, type GateEvalReport, type GateEvalRow } from "./auto-tune";
import { buildSatisfactionFloorLooseningRecs } from "./loosening-recs";
import { loadSatisfactionFloorRecState } from "../services/satisfaction-floor-loosening-run";
import { runAutoApplyRecommendations, type StorageEnv } from "./auto-apply";

/** True when the self-improvement loop is enabled. Flag-OFF (default) → every export below is a no-op. Truthy
Expand Down Expand Up @@ -155,6 +157,11 @@ export async function runSelfTune(env: Env): Promise<void> {
const row = await buildEvalRow(env, repoFullName);
const report: GateEvalReport = { rows: [row], hasSignal: row.decided >= 10 };
const recs = computeTuningRecommendations(report);
// #8160: surface the backtest-gated loosening loop's state in the same ranked list. Payload-less by
// design — runAutoApplyRecommendations below only ever consumes recs carrying a TIGHTENING
// overridePayload, so these are report-only here and can never be promoted by the apply path.
// Appended once per pass (deployment-global state), on the first repo's iteration.
if (repoFullName === repos[0]) recs.push(...buildSatisfactionFloorLooseningRecs(await loadSatisfactionFloorRecState(env, nowMs)));
// runAutoApplyRecommendations only ever consumes recs that carry a TIGHTENING overridePayload, shadow-
// soaks them, and promotes a soaked override only when isStrictlyTightening + evidence + soak pass.
await runAutoApplyRecommendations(env as unknown as StorageEnv, {
Expand Down
39 changes: 39 additions & 0 deletions src/services/satisfaction-floor-loosening-run.ts
Original file line number Diff line number Diff line change
Expand Up @@ -133,3 +133,42 @@ export async function runScheduledSatisfactionFloorLoosening(env: Env): Promise<
return null;
}
}

/** The loop state the tuning advisor surfaces (#8160). Structurally what
* src/review/loosening-recs.ts's builder consumes — kept here so the advisor wire needs one read. */
export type SatisfactionFloorRecState = {
flagEnabled: boolean;
proposal: import("./satisfaction-floor-loosening").SatisfactionFloorLooseningProposal | null;
lastAppliedAt: string | null;
};

/**
* Evaluate-ONLY state read for the advisor (#8160): the current proposal (from the same corpus + current
* floor the applying tick would use — but never writing anything) plus the newest applied-loosening
* timestamp. Fail-safe on every read: a corpus/history error degrades to a null section rather than
* breaking the advisor surface that embeds this.
*/
export async function loadSatisfactionFloorRecState(env: Env, nowMs: number = Date.now()): Promise<SatisfactionFloorRecState> {
const flagEnabled = isSatisfactionFloorAutotuneEnabled(env);

let proposal: SatisfactionFloorRecState["proposal"] = null;
try {
const currentFloor = (await getSatisfactionFloorOverride(env)) ?? LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR;
const { fired, overrides } = await createSignalStore(env).queryRuleHistory(SATISFACTION_FLOOR_RULE_ID, nowMs - CORPUS_LOOKBACK_MS);
proposal = evaluateSatisfactionFloorLoosening(buildBacktestCorpus(SATISFACTION_FLOOR_RULE_ID, fired, overrides), currentFloor);
} catch {
proposal = null;
}

let lastAppliedAt: string | null = null;
try {
const row = await env.DB.prepare("SELECT created_at FROM audit_events WHERE event_type = ? ORDER BY created_at DESC LIMIT 1")
.bind(SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE)
.first<{ created_at: string }>();
lastAppliedAt = row?.created_at ?? null;
} catch {
lastAppliedAt = null;
}

return { flagEnabled, proposal, lastAppliedAt };
}
73 changes: 73 additions & 0 deletions test/unit/loosening-recs.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
import { describe, expect, it } from "vitest";
import { buildSatisfactionFloorLooseningRecs, LOOSENING_REC_PROJECT } from "../../src/review/loosening-recs";
import type { SatisfactionFloorLooseningProposal } from "../../src/services/satisfaction-floor-loosening";

function proposal(overrides: Partial<SatisfactionFloorLooseningProposal> = {}): SatisfactionFloorLooseningProposal {
return {
ruleId: "linked_issue_scope_mismatch",
currentFloor: 0.5,
proposedFloor: 0.45,
visibleCases: 24,
heldOutCases: 7,
visible: {
ruleId: "linked_issue_scope_mismatch",
baseline: { ruleId: "linked_issue_scope_mismatch", caseCount: 24, truePositive: 1, falsePositive: 4, trueNegative: 19, falseNegative: 0, precision: 0.2, recall: 1 },
candidate: { ruleId: "linked_issue_scope_mismatch", caseCount: 24, truePositive: 1, falsePositive: 0, trueNegative: 23, falseNegative: 0, precision: 1, recall: 1 },
regressedAxes: [],
improvedAxes: ["precision"],
verdict: "improved",
},
heldOut: {
ruleId: "linked_issue_scope_mismatch",
baseline: { ruleId: "linked_issue_scope_mismatch", caseCount: 7, truePositive: 0, falsePositive: 0, trueNegative: 7, falseNegative: 0, precision: null, recall: null },
candidate: { ruleId: "linked_issue_scope_mismatch", caseCount: 7, truePositive: 0, falsePositive: 0, trueNegative: 7, falseNegative: 0, precision: null, recall: null },
regressedAxes: [],
improvedAxes: [],
verdict: "unchanged",
},
...overrides,
};
}

describe("buildSatisfactionFloorLooseningRecs (#8160)", () => {
it("returns [] with no proposal and no applied history", () => {
expect(buildSatisfactionFloorLooseningRecs({ flagEnabled: false, proposal: null, lastAppliedAt: null })).toEqual([]);
});

it("surfaces a backtest-cleared proposal as a good-severity rec with both split verdicts, sample sizes, and precision movement", () => {
const recs = buildSatisfactionFloorLooseningRecs({ flagEnabled: false, proposal: proposal(), lastAppliedAt: null });
expect(recs).toHaveLength(1);
const [rec] = recs;
expect(rec!.project).toBe(LOOSENING_REC_PROJECT);
expect(rec!.severity).toBe("good");
expect(rec!.message).toContain("0.5 → 0.45");
expect(rec!.message).toContain("Visible split improved (24 case(s), precision 20% → 100%)");
expect(rec!.message).toContain("held-out split unchanged (7 case(s))");
// Null precision renders as the em-dash convention, never a fake number.
const nullPrecision = buildSatisfactionFloorLooseningRecs({
flagEnabled: false,
proposal: proposal({ visible: { ...proposal().visible, baseline: { ...proposal().visible.baseline, precision: null } } }),
lastAppliedAt: null,
});
expect(nullPrecision[0]!.message).toContain("precision — →");
});

it("the action line matches the flag state, and the rec NEVER carries an overridePayload (the tightening-only channel)", () => {
const off = buildSatisfactionFloorLooseningRecs({ flagEnabled: false, proposal: proposal(), lastAppliedAt: null });
expect(off[0]!.message).toContain("SATISFACTION_FLOOR_AUTOTUNE_ENABLED");
const on = buildSatisfactionFloorLooseningRecs({ flagEnabled: true, proposal: proposal(), lastAppliedAt: null });
expect(on[0]!.message).toContain("hourly tick will apply");
for (const rec of [...off, ...on]) expect(rec.overridePayload).toBeUndefined();
});

it("reports a recently applied loosening as an info rec pointing at the operator status surface, alongside a proposal when both exist", () => {
const both = buildSatisfactionFloorLooseningRecs({ flagEnabled: true, proposal: proposal(), lastAppliedAt: "2026-07-23T05:00:00.000Z" });
expect(both.map((rec) => rec.severity)).toEqual(["good", "info"]);
expect(both[1]!.message).toContain("2026-07-23T05:00:00.000Z");
expect(both[1]!.message).toContain("/v1/internal/calibration/satisfaction-floor");

const appliedOnly = buildSatisfactionFloorLooseningRecs({ flagEnabled: true, proposal: null, lastAppliedAt: "2026-07-23T05:00:00.000Z" });
expect(appliedOnly).toHaveLength(1);
expect(appliedOnly[0]!.severity).toBe("info");
});
});
42 changes: 42 additions & 0 deletions test/unit/satisfaction-floor-loosening-run.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { splitBacktestCorpus } from "@loopover/engine";
import {
getSatisfactionFloorOverride,
isSatisfactionFloorAutotuneEnabled,
loadSatisfactionFloorRecState,
runSatisfactionFloorLoosening,
runScheduledSatisfactionFloorLoosening,
SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE,
Expand Down Expand Up @@ -225,3 +226,44 @@ describe("runScheduledSatisfactionFloorLoosening + queue wiring (#8158)", () =>
expect(await getSatisfactionFloorOverride(onEnv)).toBe(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0]);
});
});

describe("loadSatisfactionFloorRecState (#8160)", () => {
it("reports flag state, a live proposal from the current corpus, and null lastAppliedAt on a fresh deployment", async () => {
const env = enabledEnv();
await seedLooseningFriendlyHistory(env);
const state = await loadSatisfactionFloorRecState(env);
expect(state.flagEnabled).toBe(true);
expect(state.proposal?.proposedFloor).toBe(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0]);
expect(state.lastAppliedAt).toBeNull();
});

it("evaluates from the OVERRIDDEN floor when one is live, and reads the newest applied timestamp", async () => {
const env = enabledEnv();
await seedLooseningFriendlyHistory(env);
// Apply once for real: the next state read must evaluate from 0.45 (no further proposal on this corpus)
// and report the applied event's timestamp.
expect((await runSatisfactionFloorLoosening(env)).applied).toBe(true);
const state = await loadSatisfactionFloorRecState(env);
expect(state.proposal).toBeNull();
expect(state.lastAppliedAt).not.toBeNull();
});

it("stays proposal-null on an empty corpus and fails safe to nulls on a broken DB", async () => {
const fresh = await loadSatisfactionFloorRecState(enabledEnv());
expect(fresh.proposal).toBeNull();
expect(fresh.lastAppliedAt).toBeNull();

const env = enabledEnv();
env.DB = { prepare: () => { throw new Error("boom"); } } as never;
const broken = await loadSatisfactionFloorRecState(env);
expect(broken).toEqual({ flagEnabled: true, proposal: null, lastAppliedAt: null });
});

it("flag off: reports flagEnabled false while still evaluating from the shipped floor (advice can precede the opt-in)", async () => {
const env = createTestEnv();
await seedLooseningFriendlyHistory(env);
const state = await loadSatisfactionFloorRecState(env);
expect(state.flagEnabled).toBe(false);
expect(state.proposal?.currentFloor).toBe(0.5);
});
});
20 changes: 18 additions & 2 deletions test/unit/selftune-wiring.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ async function seedRegisteredRepo(env: Env, fullName: string, autonomyJson: stri
// Seed N resolved recommendation outcomes for a repo: `negative` rejected/closed (the dangerous error a
// tightening fixes) + `positive` accepted. Inserted directly (FKs off) — buildRepoOutcomeCalibration reads
// the outcome_state split, which is all the eval mapping needs.
async function seedRecommendationOutcomes(env: Env, repoFullName: string, positive: number, negative: number, maintainerLane = true): Promise<void> {
async function seedRecommendationOutcomes(env: Env, repoFullName: string, positive: number, negative: number, maintainerLane = true, idPrefix = ""): Promise<void> {
await env.DB.prepare("PRAGMA foreign_keys=OFF").run();
let i = 0;
const insert = async (state: string) => {
Expand All @@ -56,7 +56,7 @@ async function seedRecommendationOutcomes(env: Env, repoFullName: string, positi
maintainer_lane, confidence, reason, source, updated_at)
VALUES (?, ?, ?, ?, 'review', ?, 'pull_request', ?, ?, 'medium', 'seed', 'inferred', CURRENT_TIMESTAMP)`,
)
.bind(`o${i}`, `a${i}`, `r${i}`, "bot", state, repoFullName, maintainerLane ? 1 : 0)
.bind(`${idPrefix}o${i}`, `${idPrefix}a${i}`, `${idPrefix}r${i}`, "bot", state, repoFullName, maintainerLane ? 1 : 0)
.run();
};
for (let n = 0; n < positive; n += 1) await insert("accepted");
Expand Down Expand Up @@ -234,6 +234,22 @@ describe("runSelfTune — shadow-soak over loopover's own outcome data", () => {
expect((await listOverrideAudit(env as never, "owner/repo")).length).toBe(0);
});

it("appends the loosening-loop recs exactly once per pass, on the first repo only (#8160)", async () => {
const state = await import("../../src/services/satisfaction-floor-loosening-run");
const spy = vi.spyOn(state, "loadSatisfactionFloorRecState");
const env = createTestEnv({ LOOPOVER_REVIEW_SELFTUNE: "true" });
await seedRegisteredRepo(env, "owner/repo", ACTING_AUTONOMY);
await seedRegisteredRepo(env, "owner/other", ACTING_AUTONOMY);
await seedRecommendationOutcomes(env, "owner/repo", 5, 10);
await seedRecommendationOutcomes(env, "owner/other", 5, 10, true, "b-");

await processJob(env, { type: "selftune", requestedBy: "schedule" });

// Two repos in the pass, ONE deployment-global loosening-state read: the recs are appended on the
// first repo's iteration only, never once per repo.
expect(spy).toHaveBeenCalledTimes(1);
});

it("FLAG-ON via the processor: a stale in-flight selftune job runs the tick (defense-in-depth gate)", async () => {
const env = createTestEnv({ LOOPOVER_REVIEW_SELFTUNE: "true" });
await seedRegisteredRepo(env, "owner/repo", ACTING_AUTONOMY);
Expand Down