From 614638b594d61537e45dc21a7a471721925bcc07 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 00:58:43 -0700 Subject: [PATCH] feat(calibration): schedule the satisfaction-floor loosening tick with a one-shot apply alert (#8158) The #8121 narrow start shipped the backtest-gated loosening loop behind a manual internal trigger only. This puts it on the existing cron surface as a sibling of the selftune tick: enqueued hourly ONLY when SATISFACTION_FLOOR_AUTOTUNE_ENABLED is on (flag-OFF the enqueued set is byte-identical), with the same defense-in-depth flag re-check in the queue consumer so a stale in-flight job after a flag flip still no-ops. An APPLIED loosening emits ONE structured error-level alert on the same Workers-Logs + Sentry notify path runOpsAlerts documents (event satisfaction_floor_loosened, ev-fingerprinted, carrying the floors and split sample sizes) -- and cannot re-alert on later ticks by construction, since the next evaluation starts from the already-loosened floor. The tick itself fails safe: a thrown evaluation is logged and swallowed, never poisoning the queue. 100% line+branch coverage held on the run module; enqueue gating (on/off/ non-hourly), processor defense-in-depth, one-shot alerting, and both failure-formatting arms are all pinned. Closes #8158. --- src/index.ts | 7 + src/queue/job-dispatch.ts | 7 + .../satisfaction-floor-loosening-run.ts | 32 +++++ src/types.ts | 9 ++ test/unit/index.test.ts | 34 +++++ .../satisfaction-floor-loosening-run.test.ts | 131 ++++++++++++------ 6 files changed, 180 insertions(+), 40 deletions(-) diff --git a/src/index.ts b/src/index.ts index a691ed6ef7..7656e5e919 100644 --- a/src/index.ts +++ b/src/index.ts @@ -15,6 +15,7 @@ import { isPrReconciliationEnabled, resolvePrReconciliationManifestOverride } fr import { isActiveReviewReconciliationEnabled, resolveActiveReviewReconciliationManifestOverride } from "./review/active-review-reconciliation"; import { isRagEnabled } from "./review/rag-wire"; import { isSelfTuneEnabled } from "./review/selftune-wire"; +import { isSatisfactionFloorAutotuneEnabled } from "./services/satisfaction-floor-loosening-run"; import { githubRateLimitAdmissionKeyForJob, isGitHubBudgetBackgroundJob, @@ -274,6 +275,12 @@ async function enqueueScheduledJobs(env: Env, controller: ScheduledController): // Enqueued ONLY when the flag is ON — flag-OFF (default) this job is never created, so the cron tick does // ZERO new tuning work and the enqueued set is byte-identical to today. if (selfHostedReviews && isSelfTuneEnabled(env)) jobs.push({ type: "selftune", requestedBy: "schedule" }); + // Backtest-gated satisfaction-floor loosening (#8158/#8121, flag SATISFACTION_FLOOR_AUTOTUNE_ENABLED). + // Hourly evaluation of the one approved loosenable knob; applies at most one candidate step per tick and + // alerts once on apply. Same runtime gate as its selftune sibling (the acting-autonomy surface). Enqueued + // ONLY when the flag is ON — flag-OFF (default) this job is never created, so the cron tick does ZERO new + // work and the enqueued set is byte-identical to today. + if (selfHostedReviews && isSatisfactionFloorAutotuneEnabled(env)) jobs.push({ type: "satisfaction-floor-loosening", requestedBy: "schedule" }); // APR repo-transfer acceptance/expiry detection (#7741, flag LOOPOVER_APR_TRANSFER_POLL). Hourly poll that // resolves each pending APR transfer (accepted / accepted-and-departed / expired at 7 days) and reconciles // the per-repo AMS pause. Enqueued ONLY when the flag is ON — flag-OFF (default) this job is never created, diff --git a/src/queue/job-dispatch.ts b/src/queue/job-dispatch.ts index 9c99180152..6d750e8003 100644 --- a/src/queue/job-dispatch.ts +++ b/src/queue/job-dispatch.ts @@ -30,6 +30,7 @@ import { isLoopEscalationSweepEnabled, resolveLoopEscalationManifestOverride, ru import { isPrReconciliationEnabled, resolvePrReconciliationManifestOverride, runOpenPrReconciliation } from "../review/pr-reconciliation"; import { isActiveReviewReconciliationEnabled, resolveActiveReviewReconciliationManifestOverride, runActiveReviewReconciliation } from "../review/active-review-reconciliation"; import { isSelfTuneEnabled, runSelfTune } from "../review/selftune-wire"; +import { isSatisfactionFloorAutotuneEnabled, runScheduledSatisfactionFloorLoosening } from "../services/satisfaction-floor-loosening-run"; import { runSelfTuneBreaker } from "../review/outcomes-wire"; import { isRagEnabled } from "../review/rag-wire"; import { processSubmitDraft } from "../services/draft"; @@ -342,6 +343,12 @@ export async function processJob(env: Env, message: JobMessage): Promise { if (isActiveReviewReconciliationEnabled(env, activeReviewReconciliationManifestOverride)) await runActiveReviewReconciliation(env); } return; + case "satisfaction-floor-loosening": + // #8158: defense-in-depth mirror of the selftune case below — the cron only ENQUEUES this when the + // flag is ON, but a stale in-flight job landing after a flag-flip must still no-op. Never throws into + // the queue (the scheduled wrapper fails safe). + if (isSatisfactionFloorAutotuneEnabled(env)) await runScheduledSatisfactionFloorLoosening(env); + return; case "selftune": // Convergence (self-improve / auto-tune, flag LOOPOVER_REVIEW_SELFTUNE). Defense-in-depth: the cron only // ENQUEUES this when the flag is ON, but a stale in-flight job that lands after a flag-flip must still diff --git a/src/services/satisfaction-floor-loosening-run.ts b/src/services/satisfaction-floor-loosening-run.ts index 809f3b6781..93f9b342d2 100644 --- a/src/services/satisfaction-floor-loosening-run.ts +++ b/src/services/satisfaction-floor-loosening-run.ts @@ -101,3 +101,35 @@ export async function runSatisfactionFloorLoosening(env: Env, nowMs: number = Da return { applied: true, proposal }; } + +/** + * The cron-tick wrapper (#8158): one loosening evaluation, failing SAFE (a thrown evaluation is logged and + * swallowed — the queue consumer must never poison-pill on telemetry work). An APPLIED loosening emits ONE + * structured error-level alert — the same Workers-Logs + Sentry notify path runOpsAlerts documents (the + * `ev` sub-field keeps distinct rules from collapsing into one Sentry issue) — and by construction cannot + * re-alert on later ticks: the next evaluation starts from the already-loosened floor and returns + * no_proposal until the corpus justifies another step. + */ +export async function runScheduledSatisfactionFloorLoosening(env: Env): Promise { + try { + const result = await runSatisfactionFloorLoosening(env); + if (result.applied) { + console.error( + JSON.stringify({ + level: "error", + event: "satisfaction_floor_loosened", + ev: SATISFACTION_FLOOR_RULE_ID, + at: new Date().toISOString(), + currentFloor: result.proposal.currentFloor, + proposedFloor: result.proposal.proposedFloor, + visibleCases: result.proposal.visibleCases, + heldOutCases: result.proposal.heldOutCases, + }), + ); + } + return result; + } catch (error) { + console.warn(JSON.stringify({ level: "warn", event: "satisfaction_floor_loosening_tick_failed", error: error instanceof Error ? error.message : "unknown error" })); + return null; + } +} diff --git a/src/types.ts b/src/types.ts index 36a5b32025..1e96b51db8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -249,6 +249,15 @@ export type JobMessage = type: "selftune"; requestedBy: "schedule" | "api" | "test"; } + | { + // Backtest-gated satisfaction-floor loosening tick (#8158/#8121, flag + // SATISFACTION_FLOOR_AUTOTUNE_ENABLED). One evaluation of the approved loosenable knob per hour: + // proposes/applies at most one candidate step when the backtest clears both splits, alerts once on + // apply. Enqueued hourly by the cron ONLY when the flag is ON (index.ts), so flag-OFF this job + // never exists. + type: "satisfaction-floor-loosening"; + requestedBy: "schedule" | "api" | "test"; + } | { // Convergence (RAG / codebase index — Layer C, flag-gated by LOOPOVER_REVIEW_RAG). Populate + maintain the // vector index that retrieval reads. diff --git a/test/unit/index.test.ts b/test/unit/index.test.ts index 21f61c79fe..8ff4d42f82 100644 --- a/test/unit/index.test.ts +++ b/test/unit/index.test.ts @@ -1036,6 +1036,40 @@ describe("worker entrypoint", () => { ]); }); + it("enqueues the satisfaction-floor-loosening tick hourly only when SATISFACTION_FLOOR_AUTOTUNE_ENABLED is ON (#8158)", async () => { + const sentFor = async (flag?: string): Promise> => { + const sent: Array = []; + const env = createTestEnv({ + ...(flag === undefined ? {} : ({ SATISFACTION_FLOOR_AUTOTUNE_ENABLED: flag } as Partial)), + JOBS: { + async send(message: import("../../src/types").JobMessage) { + sent.push(message); + }, + } as unknown as Queue, + }); + const waitUntil: Promise[] = []; + await worker.scheduled(controllerFor("2026-05-25T05:00:00.000Z"), env, executionContext(waitUntil)); + await Promise.all(waitUntil); + return sent; + }; + + // createTestEnv defaults the flag to "false" — the undefined case exercises exactly that shipped default. + expect((await sentFor()).some((m) => m.type === "satisfaction-floor-loosening")).toBe(false); + expect((await sentFor("true")).filter((m) => m.type === "satisfaction-floor-loosening")).toEqual([ + { type: "satisfaction-floor-loosening", requestedBy: "schedule" }, + ]); + // Off the hourly boundary, flag-ON still enqueues nothing. + const sent: Array = []; + const env = createTestEnv({ + SATISFACTION_FLOOR_AUTOTUNE_ENABLED: "true" as never, + JOBS: { async send(message: import("../../src/types").JobMessage) { sent.push(message); } } as unknown as Queue, + }); + const waitUntil: Promise[] = []; + await worker.scheduled(controllerFor("2026-05-25T05:14:00.000Z"), env, executionContext(waitUntil)); + await Promise.all(waitUntil); + expect(sent.some((m) => m.type === "satisfaction-floor-loosening")).toBe(false); + }); + it("enqueues the rag-index-repo fan-out in the full-sync window ONLY when LOOPOVER_REVIEW_RAG is ON (flag-OFF is byte-identical)", async () => { const sentFor = async (ragFlag?: string): Promise> => { const sent: Array = []; diff --git a/test/unit/satisfaction-floor-loosening-run.test.ts b/test/unit/satisfaction-floor-loosening-run.test.ts index 2186f4d7b3..805358250b 100644 --- a/test/unit/satisfaction-floor-loosening-run.test.ts +++ b/test/unit/satisfaction-floor-loosening-run.test.ts @@ -4,9 +4,11 @@ import { getSatisfactionFloorOverride, isSatisfactionFloorAutotuneEnabled, runSatisfactionFloorLoosening, + runScheduledSatisfactionFloorLoosening, SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE, SATISFACTION_FLOOR_OVERRIDE_FLAG_KEY, } from "../../src/services/satisfaction-floor-loosening-run"; +import { processJob } from "../../src/queue/processors"; import * as core from "../../src/services/satisfaction-floor-loosening"; import { LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR } from "../../src/services/linked-issue-satisfaction"; import { createSignalStore } from "../../src/review/signal-tracking-wire"; @@ -66,51 +68,51 @@ describe("getSatisfactionFloorOverride (#8121)", () => { }); }); -describe("runSatisfactionFloorLoosening (#8121)", () => { - async function seedLooseningFriendlyHistory(env: Env) { - // Same membership-probe technique as the pure-core suite: ask the real splitter which keys land where, - // then seed borderline-confirmed history in both slices so 0.5 -> 0.45 clears both gates. - const pool = Array.from({ length: 120 }, (_, i) => `acme/widgets#${i + 1}`); - const probe = pool.map((targetKey) => ({ +async function seedLooseningFriendlyHistory(env: Env) { + // Same membership-probe technique as the pure-core suite: ask the real splitter which keys land where, + // then seed borderline-confirmed history in both slices so 0.5 -> 0.45 clears both gates. + const pool = Array.from({ length: 120 }, (_, i) => `acme/widgets#${i + 1}`); + const probe = pool.map((targetKey) => ({ + ruleId: core.SATISFACTION_FLOOR_RULE_ID, + targetKey, + outcome: "unaddressed", + label: "confirmed" as const, + firedAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + })); + const { visible, heldOut } = splitBacktestCorpus(probe, core.SATISFACTION_FLOOR_HELD_OUT_FRACTION, core.SATISFACTION_FLOOR_SPLIT_SEED); + const store = createSignalStore(env); + const now = Date.now(); + const keys = [ + ...visible.slice(0, core.SATISFACTION_FLOOR_MIN_VISIBLE_CASES + 4).map((c) => c.targetKey), + ...heldOut.slice(0, core.SATISFACTION_FLOOR_MIN_HELD_OUT_CASES + 2).map((c) => c.targetKey), + ]; + for (const [i, targetKey] of keys.entries()) { + await store.recordRuleFired({ ruleId: core.SATISFACTION_FLOOR_RULE_ID, targetKey, outcome: "unaddressed", - label: "confirmed" as const, - firedAt: "2026-07-01T00:00:00.000Z", - decidedAt: "2026-07-02T00:00:00.000Z", - })); - const { visible, heldOut } = splitBacktestCorpus(probe, core.SATISFACTION_FLOOR_HELD_OUT_FRACTION, core.SATISFACTION_FLOOR_SPLIT_SEED); - const store = createSignalStore(env); - const now = Date.now(); - const keys = [ - ...visible.slice(0, core.SATISFACTION_FLOOR_MIN_VISIBLE_CASES + 4).map((c) => c.targetKey), - ...heldOut.slice(0, core.SATISFACTION_FLOOR_MIN_HELD_OUT_CASES + 2).map((c) => c.targetKey), - ]; - for (const [i, targetKey] of keys.entries()) { - await store.recordRuleFired({ - ruleId: core.SATISFACTION_FLOOR_RULE_ID, - targetKey, - outcome: "unaddressed", - occurredAt: new Date(now - 10_000 - i).toISOString(), - metadata: { confidence: 0.47 }, - }); - await store.recordHumanOverride({ ruleId: core.SATISFACTION_FLOOR_RULE_ID, targetKey, verdict: "confirmed", occurredAt: new Date(now - i).toISOString() }); - } - // One genuinely-reversed deep-low-confidence firing per slice, mirroring the pure-core fixture: a true - // positive must exist on both sides of the comparison or the candidate's precision denominator is 0 - // (null) and the improvement can't register on any axis. - for (const targetKey of [visible[core.SATISFACTION_FLOOR_MIN_VISIBLE_CASES + 5]!.targetKey, heldOut[core.SATISFACTION_FLOOR_MIN_HELD_OUT_CASES + 3]!.targetKey]) { - await store.recordRuleFired({ - ruleId: core.SATISFACTION_FLOOR_RULE_ID, - targetKey, - outcome: "unaddressed", - occurredAt: new Date(now - 20_000).toISOString(), - metadata: { confidence: 0.1 }, - }); - await store.recordHumanOverride({ ruleId: core.SATISFACTION_FLOOR_RULE_ID, targetKey, verdict: "reversed", occurredAt: new Date(now - 5000).toISOString() }); - } + occurredAt: new Date(now - 10_000 - i).toISOString(), + metadata: { confidence: 0.47 }, + }); + await store.recordHumanOverride({ ruleId: core.SATISFACTION_FLOOR_RULE_ID, targetKey, verdict: "confirmed", occurredAt: new Date(now - i).toISOString() }); + } + // One genuinely-reversed deep-low-confidence firing per slice, mirroring the pure-core fixture: a true + // positive must exist on both sides of the comparison or the candidate's precision denominator is 0 + // (null) and the improvement can't register on any axis. + for (const targetKey of [visible[core.SATISFACTION_FLOOR_MIN_VISIBLE_CASES + 5]!.targetKey, heldOut[core.SATISFACTION_FLOOR_MIN_HELD_OUT_CASES + 3]!.targetKey]) { + await store.recordRuleFired({ + ruleId: core.SATISFACTION_FLOOR_RULE_ID, + targetKey, + outcome: "unaddressed", + occurredAt: new Date(now - 20_000).toISOString(), + metadata: { confidence: 0.1 }, + }); + await store.recordHumanOverride({ ruleId: core.SATISFACTION_FLOOR_RULE_ID, targetKey, verdict: "reversed", occurredAt: new Date(now - 5000).toISOString() }); } +} +describe("runSatisfactionFloorLoosening (#8121)", () => { it("returns flag_off without touching anything when the autotune flag is not set", async () => { expect(await runSatisfactionFloorLoosening(createTestEnv())).toEqual({ applied: false, reason: "flag_off" }); }); @@ -174,3 +176,52 @@ describe("runSatisfactionFloorLoosening (#8121)", () => { expect(await getSatisfactionFloorOverride(env)).toBeNull(); }); }); + +describe("runScheduledSatisfactionFloorLoosening + queue wiring (#8158)", () => { + it("flag off: returns the flag_off result and emits NO alert", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + expect(await runScheduledSatisfactionFloorLoosening(createTestEnv())).toEqual({ applied: false, reason: "flag_off" }); + expect(error.mock.calls.some((c) => String(c[0]).includes("satisfaction_floor_loosened"))).toBe(false); + }); + + it("alerts exactly once on an applied loosening — the next tick is no_proposal and stays silent", async () => { + const error = vi.spyOn(console, "error").mockImplementation(() => undefined); + const env = enabledEnv(); + await seedLooseningFriendlyHistory(env); + const first = await runScheduledSatisfactionFloorLoosening(env); + expect(first?.applied).toBe(true); + const alerts = () => error.mock.calls.map((c) => String(c[0])).filter((line) => line.includes('"event":"satisfaction_floor_loosened"')); + expect(alerts()).toHaveLength(1); + expect(alerts()[0]).toContain('"ev":"' + core.SATISFACTION_FLOOR_RULE_ID + '"'); + expect(alerts()[0]).toContain('"proposedFloor":' + String(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0])); + + expect(await runScheduledSatisfactionFloorLoosening(env)).toEqual({ applied: false, reason: "no_proposal" }); + expect(alerts()).toHaveLength(1); // still exactly one + }); + + it("fails safe: a thrown evaluation (broken DB with the flag on) returns null and warns, never throws into the queue", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const env = enabledEnv(); + env.DB = { prepare: () => { throw new Error("boom"); } } as never; + expect(await runScheduledSatisfactionFloorLoosening(env)).toBeNull(); + expect(warn.mock.calls.some((c) => String(c[0]).includes("satisfaction_floor_loosening_tick_failed"))).toBe(true); + + // A thrown NON-Error degrades to the generic message instead of crashing the formatter. + const stringThrowEnv = enabledEnv(); + stringThrowEnv.DB = { prepare: () => { throw "string boom"; } } as never; + expect(await runScheduledSatisfactionFloorLoosening(stringThrowEnv)).toBeNull(); + expect(warn.mock.calls.some((c) => String(c[0]).includes('"error":"unknown error"'))).toBe(true); + }); + + it("processor wiring: the job runs the tick when the flag is ON and no-ops (defense-in-depth) when OFF", async () => { + const offEnv = createTestEnv(); + await seedLooseningFriendlyHistory(offEnv); + await processJob(offEnv, { type: "satisfaction-floor-loosening", requestedBy: "schedule" }); + expect(await getSatisfactionFloorOverride(createTestEnv({ ...({ SATISFACTION_FLOOR_AUTOTUNE_ENABLED: "true" } as Partial), DB: offEnv.DB }))).toBeNull(); + + const onEnv = enabledEnv(); + await seedLooseningFriendlyHistory(onEnv); + await processJob(onEnv, { type: "satisfaction-floor-loosening", requestedBy: "schedule" }); + expect(await getSatisfactionFloorOverride(onEnv)).toBe(core.SATISFACTION_FLOOR_LOOSENING_CANDIDATES[0]); + }); +});