From d2a733cbc054140237d05b185d3fcf3bb21b5a06 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 03:18:39 -0700 Subject: [PATCH] feat(calibration): consume the ai_review_close_confidence override and flip the knob to live MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #8176 gate-authority change, mirroring the satisfaction floor's consumption discipline exactly: - One bounds-validated resolution point: getKnobOverride reads the knob's system_flags row only when its own wrangler var (AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED, false by default) is ON, and refuses anything not strictly below the shipped 0.93 or under the 0.85 hard minimum. gateCheckPolicy threads it as the LAST-resort default — an explicit per-repo gate.aiReview.closeConfidence setting always wins. - Generic apply machinery (knob-loosening-run.ts): the satisfaction run/status/tick generalized over the registry entry rather than duplicated. The satisfaction floor keeps its legacy module and event shape; the shared cron tick now runs every OTHER live knob, double-gated per knob. Report-only refusal stays enforced in the write path itself. - Registry entries carry their apply plumbing (override flag key, event type, autotune var), pinned against the legacy constants by invariant tests; the close-confidence knob flips to applyMode live. - Operator surface: GET /v1/internal/calibration/knobs renders every live knob's flag state, shipped/live/override values, and applied history through one projector that reads both proposal spellings. Closes #8176 --- src/api/routes.ts | 6 + src/queue/gate-checks.ts | 9 +- src/queue/job-dispatch.ts | 6 + src/queue/processors.ts | 12 +- src/services/knob-loosening-run.ts | 238 ++++++++++++++ src/services/loosening-knobs.ts | 27 +- .../satisfaction-floor-loosening-run.ts | 11 +- test/helpers/d1.ts | 1 + test/unit/gate-check-policy.test.ts | 12 + test/unit/knob-loosening-run.test.ts | 310 ++++++++++++++++++ test/unit/loosening-knobs.test.ts | 11 +- .../satisfaction-floor-loosening-run.test.ts | 8 +- worker-configuration.d.ts | 4 +- wrangler.jsonc | 1 + 14 files changed, 637 insertions(+), 19 deletions(-) create mode 100644 src/services/knob-loosening-run.ts create mode 100644 test/unit/knob-loosening-run.test.ts diff --git a/src/api/routes.ts b/src/api/routes.ts index 0fb4c9ddf..f389c382a 100644 --- a/src/api/routes.ts +++ b/src/api/routes.ts @@ -321,6 +321,7 @@ import { getPublicStats, isPublicStatsEnabled, resolvePublicStatsManifestOverrid import { loadPublicAccuracyTrend } from "../services/public-accuracy-trend"; import { loadCalibrationTrend } from "../services/rule-calibration-trend"; import { isSatisfactionFloorAutotuneEnabled, loadSatisfactionFloorStatus, runSatisfactionFloorLoosening } from "../services/satisfaction-floor-loosening-run"; +import { loadLiveKnobStatuses } from "../services/knob-loosening-run"; import { loadPublicReuseRateTrend } from "../services/public-reuse-rate-trend"; import { loadPublicReviewVolumeTrend } from "../services/public-review-volume-trend"; import { buildMaintainerQualityDashboard, isMaintainerQualityDataStale } from "../services/maintainer-quality-dashboard"; @@ -4832,6 +4833,11 @@ export function createApp() { // off. Same INTERNAL_JOB_TOKEN gate via the /v1/internal/* middleware; aggregate numbers/verdicts only. app.get("/v1/internal/calibration/satisfaction-floor", async (c) => c.json(await loadSatisfactionFloorStatus(c.env))); + // The #8161 surface generalized across EVERY live registry knob (#8176): one endpoint, one projector, + // per-knob flag state + shipped/live/override values + applied history (both split verdicts). Same + // deliberate non-flag-gating and INTERNAL_JOB_TOKEN posture as the satisfaction-floor read above. + app.get("/v1/internal/calibration/knobs", async (c) => c.json({ knobs: await loadLiveKnobStatuses(c.env) })); + app.post("/v1/internal/jobs/refresh-registry", async (c) => { const message: JobMessage = { type: "refresh-registry", requestedBy: "api" }; await c.env.JOBS.send(message); diff --git a/src/queue/gate-checks.ts b/src/queue/gate-checks.ts index 632ddb6a6..7997958e2 100644 --- a/src/queue/gate-checks.ts +++ b/src/queue/gate-checks.ts @@ -66,6 +66,10 @@ export function gateCheckPolicy( guardrailHit: boolean; guardrailMatches?: ReturnType | undefined; }, + // #8176: the backtest-gated GLOBAL default-override for the AI close-confidence floor, resolved by the + // env-bearing caller (getAiReviewCloseConfidenceOverride — flag-gated + bounds-validated). It only fills + // the DEFAULT: an explicit per-repo `gate.aiReview.closeConfidence` setting always wins below. + aiReviewCloseConfidenceOverride?: number | null, ) { // `settings` is already the EFFECTIVE config (`.loopover.yml` > DB > defaults), resolved upstream by // resolveRepositorySettings, so the blocker modes here reflect the repo's config file directly. @@ -81,8 +85,9 @@ export function gateCheckPolicy( qualityGateMinScore: settings.qualityGateMinScore ?? null, aiReviewGateMode: settings.aiReviewMode, // Calibrated AI close-confidence floor (#7) — config-as-code via `.loopover.yml gate.aiReview.closeConfidence`, - // resolved into settings upstream. `null`/undefined ⇒ advisory.ts applies the 0.93 default. - aiReviewCloseConfidence: settings.aiReviewCloseConfidence ?? null, + // resolved into settings upstream. When the repo has no explicit setting, the #8176 backtest-gated + // global override (if any) becomes the default; `null` ⇒ advisory.ts applies the 0.93 shipped default. + aiReviewCloseConfidence: settings.aiReviewCloseConfidence ?? aiReviewCloseConfidenceOverride ?? null, // Sub-floor AI-judgment disposition (#4603) — DB-backed (dashboard-settable) + `.loopover.yml // gate.aiReview.lowConfidenceDisposition` override, resolved into settings upstream. `null`/undefined ⇒ // advisory.ts applies the "hold_for_review" default. diff --git a/src/queue/job-dispatch.ts b/src/queue/job-dispatch.ts index 6d750e800..b63c3819c 100644 --- a/src/queue/job-dispatch.ts +++ b/src/queue/job-dispatch.ts @@ -31,6 +31,7 @@ import { isPrReconciliationEnabled, resolvePrReconciliationManifestOverride, run import { isActiveReviewReconciliationEnabled, resolveActiveReviewReconciliationManifestOverride, runActiveReviewReconciliation } from "../review/active-review-reconciliation"; import { isSelfTuneEnabled, runSelfTune } from "../review/selftune-wire"; import { isSatisfactionFloorAutotuneEnabled, runScheduledSatisfactionFloorLoosening } from "../services/satisfaction-floor-loosening-run"; +import { GENERIC_LIVE_KNOBS, isKnobAutotuneEnabled, runScheduledKnobLoosening } from "../services/knob-loosening-run"; import { runSelfTuneBreaker } from "../review/outcomes-wire"; import { isRagEnabled } from "../review/rag-wire"; import { processSubmitDraft } from "../services/draft"; @@ -348,6 +349,11 @@ export async function processJob(env: Env, message: JobMessage): Promise { // flag is ON, but a stale in-flight job landing after a flag-flip must still no-op. Never throws into // the queue (the scheduled wrapper fails safe). if (isSatisfactionFloorAutotuneEnabled(env)) await runScheduledSatisfactionFloorLoosening(env); + // #8176: every LATER live registry knob rides the same tick through the generic runner — each knob + // is double-gated on its OWN wrangler var, so an un-flagged knob does zero work here. + for (const knob of GENERIC_LIVE_KNOBS) { + if (isKnobAutotuneEnabled(env, knob)) await runScheduledKnobLoosening(env, knob); + } return; case "selftune": // Convergence (self-improve / auto-tune, flag LOOPOVER_REVIEW_SELFTUNE). Defense-in-depth: the cron only diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 3d3d8fce5..83f525bbf 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -426,6 +426,7 @@ export { runRetentionPrune } from "./retention"; // test/unit/gate-check-policy.test.ts and test/unit/repository-settings-enforcement.test.ts's existing // `import { gateCheckPolicy } from "../../src/queue/processors"` keeps working unchanged. import { auditGateCheckPermissionMissing, gateCheckPolicy, recordPublishedGateCheckSummary } from "./gate-checks"; +import { getAiReviewCloseConfidenceOverride } from "../services/knob-loosening-run"; export { gateCheckPolicy } from "./gate-checks"; // #4013 step 9: same shim shape for the AI-review-orchestration functions -- imported here for this file's // own remaining internal callers, and re-exported so the many existing tests importing claimAiReviewLock, @@ -1538,6 +1539,9 @@ export async function sweepRepoRegate( // isScheduledRegateSweepJob (queue-common.ts) misclassifies it as background maintenance and it inherits the // exact starvation this priority mechanism exists to avoid. Ordinary stale candidates keep the sweep prefix // unchanged. + // #8176: the global close-confidence default-override, resolved once for the sweep (same value the main + // webhook path threads; an explicit per-repo setting still wins inside gateCheckPolicy). + const sweepCloseConfidenceOverride = await getAiReviewCloseConfidenceOverride(env); for (const [index, pr] of candidates.entries()) { const others = openPullRequests.filter( (other) => other.number !== pr.number, @@ -1561,7 +1565,7 @@ export async function sweepRepoRegate( }); const gate = evaluateGateCheck( advisory, - gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null), + gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, sweepCloseConfidenceOverride), ); verdicts[String(pr.number)] = gate.conclusion; if (gate.conclusion === "failure" || gate.conclusion === "action_required") @@ -10381,6 +10385,8 @@ async function maybePublishPrPublicSurface( slopRisk, authorHistory, gateSizeContext, + // #8176: backtest-gated global default for the close-confidence floor (explicit per-repo wins inside). + await getAiReviewCloseConfidenceOverride(env), ); gateEvaluation = await withReviewPipelineSpan( "selfhost.review.gate", @@ -11871,7 +11877,7 @@ async function maybeProcessResolveCommand(env: Env, deliveryId: string, payload: if (!findingRef.ok) { await recordAuditEvent(env, { eventType: "github_app.finding_resolved_skipped", actor: req.actor, targetKey, outcome: "completed", detail: findingRef.reason, metadata: { deliveryId, repoFullName: req.repoFullName, reason: findingRef.reason } }); await recordGithubProductUsage(env, "finding_resolved_skipped", { actor: req.actor, repoFullName: req.repoFullName, targetKey, outcome: "skipped", metadata: { reason: findingRef.reason } }); return true; } const { advisory } = await buildAuthorizedPrActionAdvisory(env, req.repoFullName, pr, settings); await appendPublishedAiReviewFindingsForResolve(env, req.repoFullName, pr, settings.aiReviewMode, advisory); - const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null)); + const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, await getAiReviewCloseConfidenceOverride(env))); const selection = selectWarningsForResolve(gate.warnings, findingRef); if (selection.reason === "finding_not_found") { await recordAuditEvent(env, { eventType: "github_app.finding_resolved_skipped", actor: req.actor, targetKey, outcome: "completed", detail: selection.reason, metadata: { deliveryId, repoFullName: req.repoFullName, reason: selection.reason } }); await recordGithubProductUsage(env, "finding_resolved_skipped", { actor: req.actor, repoFullName: req.repoFullName, targetKey, outcome: "skipped", metadata: { reason: selection.reason } }); return true; } const mode = resolveAgentActionMode({ globalPaused: isGlobalAgentPause(env) || (await isGlobalAgentFrozen(env)), agentPaused: settings.agentPaused, agentDryRun: settings.agentDryRun }); @@ -12102,7 +12108,7 @@ async function maybeProcessExplainCommand(env: Env, deliveryId: string, payload: } const { advisory } = await buildAuthorizedPrActionAdvisory(env, req.repoFullName, pr, settings); await appendPublishedAiReviewFindingsForResolve(env, req.repoFullName, pr, settings.aiReviewMode, advisory); - const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null)); + const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, await getAiReviewCloseConfidenceOverride(env))); const selection = selectWarningsForResolve(gate.warnings, findingRef); if (selection.reason === "finding_not_found") { const notFound = sanitizePublicComment([AGENT_COMMAND_COMMENT_MARKER, "", "> [!NOTE]", `> **No review finding \`${findingRef.findingCode}\` on this PR**`, "> That id is not among this PR's current review findings — re-run `@loopover explain ` with an id from the review summary.", "", "---", loopoverFooter(env)].join("\n")); diff --git a/src/services/knob-loosening-run.ts b/src/services/knob-loosening-run.ts new file mode 100644 index 000000000..325494088 --- /dev/null +++ b/src/services/knob-loosening-run.ts @@ -0,0 +1,238 @@ +// Generic IO orchestration for LIVE registry knobs (#8176) — the satisfaction machinery +// (satisfaction-floor-loosening-run.ts, #8121/#8158/#8161) generalized over a LoosenableKnob entry instead +// of duplicated per knob. The satisfaction floor itself stays on its original module (its event/metadata +// field names — currentFloor/proposedFloor — are a load-bearing legacy shape its operator surfaces parse); +// every LATER live knob runs through here, and the generic status projector reads BOTH field spellings so +// one endpoint can render all live knobs' histories. +// +// Invariants carried over verbatim from the narrow start: +// • double gating — the knob's own truthy-string wrangler var must be ON for the loop AND the override +// read, so flipping the var off instantly restores the shipped default with no cleanup; +// • the write path independently refuses anything that isn't a strict, bounded loosening; +// • the override write is NOT best-effort (an unrecorded change is worse than none) — the audit trail is; +// • one structured error-level alert per applied step, never re-alerting (the next run starts from the +// already-loosened value and proposes nothing until the corpus justifies another step). +import { buildBacktestCorpus } from "@loopover/engine"; +import { createSignalStore } from "../review/signal-tracking-wire"; +import { recordAuditEvent } from "../db/repositories"; +import { evaluateKnobLoosening, LOOSENABLE_KNOBS, type KnobLooseningProposal, type LoosenableKnob } from "./loosening-knobs"; + +const CORPUS_LOOKBACK_MS = 90 * 24 * 60 * 60 * 1000; // mirrors threshold-backtest-run's 90-day window + +/** Live knobs the GENERIC loop owns — the satisfaction floor is excluded because its own module + * (satisfaction-floor-loosening-run.ts) already runs it with its legacy event shape. Parameterized for + * tests; production callers use the frozen registry-derived constant below. */ +export function genericLiveKnobs(knobs: readonly LoosenableKnob[] = Object.values(LOOSENABLE_KNOBS)): LoosenableKnob[] { + return knobs.filter((knob) => knob.applyMode === "live" && knob.knobId !== "satisfaction_floor"); +} +export const GENERIC_LIVE_KNOBS: readonly LoosenableKnob[] = Object.freeze(genericLiveKnobs()); + +/** Truthy-string env flag for `knob`, matching the repo's flag convention (mirrors outcomes-wire's flagTruthy). */ +export function isKnobAutotuneEnabled(env: Env, knob: LoosenableKnob): boolean { + const raw = (env as unknown as Record)[knob.autotuneEnvVar]; + const value = (typeof raw === "string" ? raw : "").trim().toLowerCase(); + return value === "1" || value === "true" || value === "on" || value === "yes"; +} + +/** + * Read a knob's live override. Null (caller uses the shipped default) when: the knob's autotune flag is + * off, no override row exists, or the stored value fails validation — an override may only ever sit BELOW + * the shipped value and AT/ABOVE the hard minimum, so a corrupted/hand-edited row can never tighten the + * knob or loosen it past safety. Fail-safe null on any DB error. + */ +export async function getKnobOverride(env: Env, knob: LoosenableKnob): Promise { + if (!isKnobAutotuneEnabled(env, knob)) return null; + try { + const row = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?").bind(knob.overrideFlagKey).first<{ value: string }>(); + if (!row) return null; + const parsed = Number(row.value); + if (!Number.isFinite(parsed) || parsed >= knob.shippedValue || parsed < knob.hardMinimum) return null; + return parsed; + } catch { + return null; + } +} + +/** The #8176 consumption read: the validated global default-override for the AI close-confidence floor. + * Threaded into gateCheckPolicy as its LAST-resort default — an explicit per-repo setting always wins. */ +export async function getAiReviewCloseConfidenceOverride(env: Env): Promise { + return getKnobOverride(env, LOOSENABLE_KNOBS.ai_review_close_confidence!); +} + +export type KnobLooseningRunResult = + | { applied: false; reason: "flag_off" | "report_only" | "no_proposal" | "already_applied" } + | { applied: true; proposal: KnobLooseningProposal }; + +/** + * Evaluate and (when justified) apply a backtest-gated loosening of `knob` — the generic form of + * runSatisfactionFloorLoosening, with one extra refusal: a report-only knob can NEVER write, whatever its + * evidence says (#8159's applyMode contract). Persists the override plus the knob's own audit event type + * carrying both split comparisons. Audit write is best-effort; the override write throws to the caller. + */ +export async function runKnobLoosening(env: Env, knob: LoosenableKnob, nowMs: number = Date.now()): Promise { + if (knob.applyMode !== "live") return { applied: false, reason: "report_only" }; + if (!isKnobAutotuneEnabled(env, knob)) return { applied: false, reason: "flag_off" }; + + const currentValue = (await getKnobOverride(env, knob)) ?? knob.shippedValue; + if (currentValue <= knob.hardMinimum) return { applied: false, reason: "already_applied" }; + + const { fired, overrides } = await createSignalStore(env).queryRuleHistory(knob.ruleId, nowMs - CORPUS_LOOKBACK_MS); + const proposal = evaluateKnobLoosening(knob, buildBacktestCorpus(knob.ruleId, fired, overrides), currentValue); + if (!proposal) return { applied: false, reason: "no_proposal" }; + // Defense in depth: the write path independently refuses anything that isn't a strict, bounded loosening. + if (proposal.proposedValue >= currentValue || proposal.proposedValue < knob.hardMinimum) { + return { applied: false, reason: "no_proposal" }; + } + + await env.DB.prepare( + "INSERT INTO system_flags (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at", + ) + .bind(knob.overrideFlagKey, String(proposal.proposedValue)) + .run(); + + await recordAuditEvent(env, { + eventType: knob.looseningEventType, + actor: "loopover", + targetKey: knob.ruleId, + outcome: "completed", + detail: `${knob.knobId} loosened ${proposal.currentValue} -> ${proposal.proposedValue} (backtest-gated, visible improved + held-out non-regressed)`, + metadata: { proposal }, + }).catch(() => undefined); + + return { applied: true, proposal }; +} + +/** The cron-tick wrapper — one evaluation per knob, failing SAFE; an applied step emits ONE structured + * error-level alert on the same Workers-Logs + Sentry notify path the #8158 satisfaction wrapper uses. */ +export async function runScheduledKnobLoosening(env: Env, knob: LoosenableKnob): Promise { + try { + const result = await runKnobLoosening(env, knob); + if (result.applied) { + console.error( + JSON.stringify({ + level: "error", + event: "calibration_knob_loosened", + ev: knob.knobId, + at: new Date().toISOString(), + currentValue: result.proposal.currentValue, + proposedValue: result.proposal.proposedValue, + visibleCases: result.proposal.visibleCases, + heldOutCases: result.proposal.heldOutCases, + }), + ); + } + return result; + } catch (error) { + console.warn( + JSON.stringify({ level: "warn", event: "knob_loosening_tick_failed", ev: knob.knobId, error: error instanceof Error ? error.message : "unknown error" }), + ); + return null; + } +} + +// ── Operator status (the #8161 surface generalized across live knobs) ──────────────────────────────────── + +export type KnobAppliedEntry = { + at: string; + currentValue: number | null; + proposedValue: number | null; + visibleCases: number | null; + heldOutCases: number | null; + visibleVerdict: string | null; + heldOutVerdict: string | null; +}; + +export type KnobStatus = { + knobId: string; + flagEnabled: boolean; + shippedValue: number; + /** The value the live consumption actually uses right now: the validated override when the flag is on, + * else the shipped constant. */ + liveValue: number; + /** The RAW stored override row (validated), reported even when the flag is off — an operator needs to + * see a lingering row that would take effect the moment the flag flips. */ + storedOverride: number | null; + applied: KnobAppliedEntry[]; +}; + +const KNOB_STATUS_HISTORY_LIMIT = 25; + +function numberOrNull(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) ? value : null; +} + +function verdictOrNull(value: unknown): string | null { + const verdict = (value as { verdict?: unknown } | undefined)?.verdict; + return typeof verdict === "string" ? verdict : null; +} + +/** + * One live knob's operator status: flag state, shipped vs live value, the stored override row (validated, + * shown regardless of flag state), and the applied history projected from the knob's own audit events, + * newest first. Reads BOTH proposal field spellings (currentValue/proposedValue and the satisfaction + * floor's legacy currentFloor/proposedFloor) so every live knob renders through one projector. Aggregate + * numbers and verdicts only — no corpus content. Fail-safe: a read error degrades the affected section. + */ +export async function loadKnobStatus(env: Env, knob: LoosenableKnob): Promise { + const flagEnabled = isKnobAutotuneEnabled(env, knob); + + let storedOverride: number | null = null; + try { + const row = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?").bind(knob.overrideFlagKey).first<{ value: string }>(); + if (row) { + const parsed = Number(row.value); + if (Number.isFinite(parsed) && parsed < knob.shippedValue && parsed >= knob.hardMinimum) storedOverride = parsed; + } + } catch { + storedOverride = null; + } + + const applied: KnobAppliedEntry[] = []; + try { + const rows = await env.DB.prepare("SELECT created_at, metadata_json FROM audit_events WHERE event_type = ? ORDER BY created_at DESC LIMIT ?") + .bind(knob.looseningEventType, KNOB_STATUS_HISTORY_LIMIT) + .all<{ created_at: string; metadata_json: string }>(); + /* v8 ignore next -- .all() over a live D1/TestD1 always yields a defined results array; the ?? [] guards + * a future driver-shape change, mirroring loadSatisfactionFloorStatus's identical note. */ + for (const row of rows.results ?? []) { + let proposal: Record = {}; + try { + const metadata = JSON.parse(row.metadata_json) as { proposal?: Record }; + proposal = metadata.proposal && typeof metadata.proposal === "object" ? metadata.proposal : {}; + } catch { + /* corrupt row -- keep the entry with nulls rather than hiding that an apply happened */ + } + applied.push({ + at: row.created_at, + currentValue: numberOrNull(proposal.currentValue) ?? numberOrNull(proposal.currentFloor), + proposedValue: numberOrNull(proposal.proposedValue) ?? numberOrNull(proposal.proposedFloor), + visibleCases: numberOrNull(proposal.visibleCases), + heldOutCases: numberOrNull(proposal.heldOutCases), + visibleVerdict: verdictOrNull(proposal.visible), + heldOutVerdict: verdictOrNull(proposal.heldOut), + }); + } + } catch { + /* degrade to an empty history -- the endpoint must not throw on a read blip */ + } + + return { + knobId: knob.knobId, + flagEnabled, + shippedValue: knob.shippedValue, + liveValue: flagEnabled && storedOverride !== null ? storedOverride : knob.shippedValue, + storedOverride, + applied, + }; +} + +/** Every live knob's status (satisfaction floor included — the generic projector reads its legacy + * proposal spelling), for GET /v1/internal/calibration/knobs. */ +export async function loadLiveKnobStatuses(env: Env, knobs: readonly LoosenableKnob[] = Object.values(LOOSENABLE_KNOBS)): Promise { + const statuses: KnobStatus[] = []; + for (const knob of knobs) { + if (knob.applyMode !== "live") continue; + statuses.push(await loadKnobStatus(env, knob)); + } + return statuses; +} diff --git a/src/services/loosening-knobs.ts b/src/services/loosening-knobs.ts index 502f9b4eb..7714b0fbb 100644 --- a/src/services/loosening-knobs.ts +++ b/src/services/loosening-knobs.ts @@ -39,6 +39,12 @@ export type LoosenableKnob = { * (advisor/status) but the apply path REFUSES — flipping a knob to live requires shipping its * consumption plumbing first, reviewed on its own. */ applyMode: "live" | "report_only"; + /** system_flags key holding this knob's live override (migration 0054's operational-flag table). */ + overrideFlagKey: string; + /** Audit event type an apply writes — a knob's evidence trail keeps ONE stable type forever. */ + looseningEventType: string; + /** Truthy-string wrangler var double-gating this knob's autotune loop AND its override read. */ + autotuneEnvVar: string; }; export const LOOSENABLE_KNOBS: Readonly> = Object.freeze({ @@ -56,12 +62,18 @@ export const LOOSENABLE_KNOBS: Readonly> = Object heldOutFraction: 0.25, splitSeed: "satisfaction-floor-loosening-v1", applyMode: "live", + // Literals (not imports) to keep this registry dependency-light; the invariant test pins them to the + // run module's exported constants so they can never drift. + overrideFlagKey: "satisfaction_floor_override", + looseningEventType: "calibration.satisfaction_floor_loosened", + autotuneEnvVar: "SATISFACTION_FLOOR_AUTOTUNE_ENABLED", }, - // The AI close-confidence floor (#8159's second knob). Its corpus (ai_consensus_defect — including the - // #8157 backfilled decision-level history) is real TODAY, so proposals carry evidence now — but - // loosening it means MORE auto-closes, a direct gate-authority change, so it enters REPORT-ONLY: no - // override consumer exists yet, and the apply path refuses until that plumbing ships as its own - // reviewed change. Tight bounds by design: two small steps, hard floor 0.85. + // The AI close-confidence floor (#8159's second knob), LIVE since #8176: the override is consumed as the + // gate policy's DEFAULT (gate-checks.ts threads it under `settings.aiReviewCloseConfidence ?? override`), + // so an explicit per-repo `gate.aiReview.closeConfidence` ALWAYS wins — the knob only moves the default. + // Loosening it means MORE auto-closes (a direct gate-authority change), hence the double gating: the + // AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED var must be ON for both the loop and the override read, + // and its corpus floors are the registry's strictest. Tight bounds by design: two steps, hard floor 0.85. ai_review_close_confidence: { knobId: "ai_review_close_confidence", ruleId: "ai_consensus_defect", @@ -72,7 +84,10 @@ export const LOOSENABLE_KNOBS: Readonly> = Object minHeldOutCases: 12, heldOutFraction: 0.25, splitSeed: "ai-close-confidence-loosening-v1", - applyMode: "report_only", + applyMode: "live", + overrideFlagKey: "ai_review_close_confidence_override", + looseningEventType: "calibration.ai_review_close_confidence_loosened", + autotuneEnvVar: "AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED", }, }); diff --git a/src/services/satisfaction-floor-loosening-run.ts b/src/services/satisfaction-floor-loosening-run.ts index 1a443273a..79169f133 100644 --- a/src/services/satisfaction-floor-loosening-run.ts +++ b/src/services/satisfaction-floor-loosening-run.ts @@ -281,9 +281,16 @@ export async function loadSatisfactionFloorStatus(env: Env): Promise { +export async function loadReportOnlyKnobProposals( + env: Env, + nowMs: number = Date.now(), + // Parameterized for tests: with #8176 flipping ai_review_close_confidence to live, the SHIPPED registry + // currently has no report-only knob — this path stays, fully covered, for the next knob that enters + // report-only (every new knob starts there per the registry's applyMode contract). + knobs: readonly (typeof LOOSENABLE_KNOBS)[string][] = Object.values(LOOSENABLE_KNOBS), +): Promise { const proposals: KnobLooseningProposal[] = []; - for (const knob of Object.values(LOOSENABLE_KNOBS)) { + for (const knob of knobs) { if (knob.applyMode !== "report_only") continue; try { const { fired, overrides } = await createSignalStore(env).queryRuleHistory(knob.ruleId, nowMs - CORPUS_LOOKBACK_MS); diff --git a/test/helpers/d1.ts b/test/helpers/d1.ts index 6bba3ad9c..cc63ee72d 100644 --- a/test/helpers/d1.ts +++ b/test/helpers/d1.ts @@ -88,6 +88,7 @@ export function createTestEnv(overrides: Partial = {}): Env { GITTENSOR_REGISTRY_URL: "https://raw.githubusercontent.com/entrius/gittensor/test/gittensor/validator/weights/master_repositories.json", LOOPOVER_AUTO_FILE_DRIFT_ISSUES: "false", SATISFACTION_FLOOR_AUTOTUNE_ENABLED: "false", + AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: "false", // Deliberately NOT "JSONbored/gittensory" (the old pre-rename repo name most test fixtures use as their // generic placeholder repoFullName) and NOT "JSONbored/loopover" (the real self-repo default) -- either // would make isLoopOverSelfRepo() accidentally match a fixture that has no intent to exercise self-repo diff --git a/test/unit/gate-check-policy.test.ts b/test/unit/gate-check-policy.test.ts index 77f0eb59e..8255cda29 100644 --- a/test/unit/gate-check-policy.test.ts +++ b/test/unit/gate-check-policy.test.ts @@ -21,6 +21,18 @@ function settings(over: Partial = {}): RepositorySettings { } as unknown as RepositorySettings; } +describe("gateCheckPolicy — #8176 global close-confidence default-override", () => { + it("fills the default only when the repo has no explicit setting, and the explicit setting always wins", () => { + // No explicit setting: the backtest-gated override becomes the default. + expect(gateCheckPolicy(settings(), null, undefined, null, undefined, undefined, 0.9).aiReviewCloseConfidence).toBe(0.9); + // Explicit per-repo setting: the override must NEVER displace it. + expect(gateCheckPolicy(settings({ aiReviewCloseConfidence: 0.97 }), null, undefined, null, undefined, undefined, 0.9).aiReviewCloseConfidence).toBe(0.97); + // Neither: null, so advisory.ts applies its shipped default. + expect(gateCheckPolicy(settings(), null, undefined, null, undefined, undefined, null).aiReviewCloseConfidence).toBeNull(); + expect(gateCheckPolicy(settings()).aiReviewCloseConfidence).toBeNull(); + }); +}); + function missingIssueAdvisory(): Advisory { return { id: "advisory-policy", diff --git a/test/unit/knob-loosening-run.test.ts b/test/unit/knob-loosening-run.test.ts new file mode 100644 index 000000000..3eff7e391 --- /dev/null +++ b/test/unit/knob-loosening-run.test.ts @@ -0,0 +1,310 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { splitBacktestCorpus } from "@loopover/engine"; +import * as looseningKnobs from "../../src/services/loosening-knobs"; +import { LOOSENABLE_KNOBS, type LoosenableKnob } from "../../src/services/loosening-knobs"; +import { + GENERIC_LIVE_KNOBS, + genericLiveKnobs, + getAiReviewCloseConfidenceOverride, + getKnobOverride, + isKnobAutotuneEnabled, + loadKnobStatus, + loadLiveKnobStatuses, + runKnobLoosening, + runScheduledKnobLoosening, +} from "../../src/services/knob-loosening-run"; +import { + SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE, + SATISFACTION_FLOOR_OVERRIDE_FLAG_KEY, +} from "../../src/services/satisfaction-floor-loosening-run"; +import { createSignalStore } from "../../src/review/signal-tracking-wire"; +import { recordAuditEvent } from "../../src/db/repositories"; +import { processJob } from "../../src/queue/processors"; +import { createApp } from "../../src/api/routes"; +import { createTestEnv } from "../helpers/d1"; + +// #8176: the generic live-knob machinery. The evaluator itself is #8159's (own suite); these tests pin the +// double gating, the validated override read the gate policy consumes, the apply/alert path, and the +// generalized #8161 status projector (including the satisfaction floor's legacy proposal spelling). + +const AI_KNOB = LOOSENABLE_KNOBS.ai_review_close_confidence!; +const SATISFACTION_KNOB = LOOSENABLE_KNOBS.satisfaction_floor!; + +// cf-typegen types the var as the literal "false" from wrangler.jsonc's default — same `as never` +// escape hatch the satisfaction suite's enabledEnv uses. +const enabledEnv = (overrides: Partial = {}) => createTestEnv({ AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: "true" as never, ...overrides }); + +async function setOverrideRow(env: Env, key: string, value: string): Promise { + await env.DB.prepare("INSERT INTO system_flags (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP) ON CONFLICT(key) DO UPDATE SET value = excluded.value") + .bind(key, value) + .run(); +} + +// Membership-probe seeding (same technique as the satisfaction suites) sized for the AI knob's stricter +// floors: borderline-confirmed history between the first candidate (0.9) and the shipped 0.93 in both +// slices, plus one genuinely-reversed deep-low firing per slice so precision has a denominator. +async function seedAiLooseningFriendlyHistory(env: Env): Promise { + const pool = Array.from({ length: 400 }, (_, i) => `acme/widgets#${i + 1}`); + const probe = pool.map((targetKey) => ({ + ruleId: AI_KNOB.ruleId, + targetKey, + outcome: "unaddressed", + label: "confirmed" as const, + firedAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + })); + const { visible, heldOut } = splitBacktestCorpus(probe, AI_KNOB.heldOutFraction, AI_KNOB.splitSeed); + const store = createSignalStore(env); + const now = Date.now(); + const keys = [ + ...visible.slice(0, AI_KNOB.minVisibleCases + 4).map((c) => c.targetKey), + ...heldOut.slice(0, AI_KNOB.minHeldOutCases + 2).map((c) => c.targetKey), + ]; + for (const [i, targetKey] of keys.entries()) { + await store.recordRuleFired({ + ruleId: AI_KNOB.ruleId, + targetKey, + outcome: "unaddressed", + occurredAt: new Date(now - 10_000 - i).toISOString(), + metadata: { confidence: 0.91 }, + }); + await store.recordHumanOverride({ ruleId: AI_KNOB.ruleId, targetKey, verdict: "confirmed", occurredAt: new Date(now - i).toISOString() }); + } + for (const targetKey of [visible[AI_KNOB.minVisibleCases + 5]!.targetKey, heldOut[AI_KNOB.minHeldOutCases + 3]!.targetKey]) { + await store.recordRuleFired({ + ruleId: AI_KNOB.ruleId, + targetKey, + outcome: "unaddressed", + occurredAt: new Date(now - 20_000).toISOString(), + metadata: { confidence: 0.2 }, + }); + await store.recordHumanOverride({ ruleId: AI_KNOB.ruleId, targetKey, verdict: "reversed", occurredAt: new Date(now - 5000).toISOString() }); + } +} + +afterEach(() => { + vi.restoreAllMocks(); +}); + +describe("registry ↔ run-module invariants (#8176)", () => { + it("pins the satisfaction knob's registry literals to the legacy module's exported constants — they can never drift", () => { + expect(SATISFACTION_KNOB.overrideFlagKey).toBe(SATISFACTION_FLOOR_OVERRIDE_FLAG_KEY); + expect(SATISFACTION_KNOB.looseningEventType).toBe(SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE); + expect(SATISFACTION_KNOB.autotuneEnvVar).toBe("SATISFACTION_FLOOR_AUTOTUNE_ENABLED"); + }); + + it("GENERIC_LIVE_KNOBS owns every live knob EXCEPT the satisfaction floor (its own module runs it)", () => { + expect(GENERIC_LIVE_KNOBS.map((knob) => knob.knobId)).toEqual(["ai_review_close_confidence"]); + // Parameterized form: a report-only knob is excluded even when not the satisfaction floor. + const reportOnly = { ...AI_KNOB, knobId: "future_knob", applyMode: "report_only" as const }; + expect(genericLiveKnobs([reportOnly, SATISFACTION_KNOB, AI_KNOB]).map((knob) => knob.knobId)).toEqual(["ai_review_close_confidence"]); + }); +}); + +describe("isKnobAutotuneEnabled / getKnobOverride (#8176 double gating)", () => { + it("parses the knob's own truthy-string var; unset/false/non-string are OFF", () => { + for (const value of ["1", "true", "on", "yes", " TRUE "]) { + expect(isKnobAutotuneEnabled({ AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: value } as unknown as Env, AI_KNOB)).toBe(true); + } + for (const value of ["false", "0", "", undefined]) { + expect(isKnobAutotuneEnabled({ AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: value } as unknown as Env, AI_KNOB)).toBe(false); + } + expect(isKnobAutotuneEnabled({ AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: 1 } as unknown as Env, AI_KNOB)).toBe(false); + }); + + it("returns the stored override only when the flag is ON and the value is a strict, bounded loosening", async () => { + const env = enabledEnv(); + expect(await getKnobOverride(env, AI_KNOB)).toBeNull(); // no row + await setOverrideRow(env, AI_KNOB.overrideFlagKey, "0.9"); + expect(await getKnobOverride(env, AI_KNOB)).toBe(0.9); + expect(await getAiReviewCloseConfidenceOverride(env)).toBe(0.9); // the gate policy's convenience read + + // Flag off: the same valid row is IGNORED — flipping the var restores the shipped default instantly. + expect(await getKnobOverride(createTestEnv({ ...env, AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: "false" as never }), AI_KNOB)).toBeNull(); + + // Validation: at/above shipped (tightening-disguised-as-loosening), below hard minimum, non-numeric. + for (const bad of [String(AI_KNOB.shippedValue), "0.97", String(AI_KNOB.hardMinimum - 0.01), "not-a-number"]) { + await setOverrideRow(env, AI_KNOB.overrideFlagKey, bad); + expect(await getKnobOverride(env, AI_KNOB)).toBeNull(); + } + }); + + it("fails safe (null) when the flag-store read throws", async () => { + const env = enabledEnv(); + env.DB = { prepare: () => { throw new Error("boom"); } } as never; + expect(await getKnobOverride(env, AI_KNOB)).toBeNull(); + }); +}); + +describe("runKnobLoosening (#8176)", () => { + it("REFUSES a report-only knob before anything else — applyMode is the registry's hard contract", async () => { + const reportOnly = { ...AI_KNOB, applyMode: "report_only" as const }; + expect(await runKnobLoosening(enabledEnv(), reportOnly)).toEqual({ applied: false, reason: "report_only" }); + }); + + it("returns flag_off / no_proposal / already_applied on the corresponding states without writing", async () => { + expect(await runKnobLoosening(createTestEnv(), AI_KNOB)).toEqual({ applied: false, reason: "flag_off" }); + expect(await runKnobLoosening(enabledEnv(), AI_KNOB)).toEqual({ applied: false, reason: "no_proposal" }); // empty corpus + const env = enabledEnv(); + await setOverrideRow(env, AI_KNOB.overrideFlagKey, String(AI_KNOB.hardMinimum)); + expect(await runKnobLoosening(env, AI_KNOB)).toEqual({ applied: false, reason: "already_applied" }); + }); + + it("applies a backtest-cleared loosening: writes the override row + the knob's own audit event", async () => { + const env = enabledEnv(); + await seedAiLooseningFriendlyHistory(env); + const result = await runKnobLoosening(env, AI_KNOB); + expect(result.applied).toBe(true); + if (!result.applied) throw new Error("unreachable"); + expect(result.proposal.proposedValue).toBe(AI_KNOB.candidates[0]); + + expect(await getKnobOverride(env, AI_KNOB)).toBe(AI_KNOB.candidates[0]); + const events = await env.DB.prepare("SELECT metadata_json FROM audit_events WHERE event_type = ?") + .bind(AI_KNOB.looseningEventType) + .all<{ metadata_json: string }>(); + expect(events.results).toHaveLength(1); + const proposal = (JSON.parse(events.results![0]!.metadata_json) as { proposal: { currentValue: number; proposedValue: number } }).proposal; + expect(proposal).toMatchObject({ currentValue: AI_KNOB.shippedValue, proposedValue: AI_KNOB.candidates[0] }); + }); + + it("defense in depth: the write path independently refuses a non-loosening or below-minimum proposal, whatever the evaluator claims", async () => { + const env = enabledEnv(); + const base = { + knobId: AI_KNOB.knobId, + ruleId: AI_KNOB.ruleId, + visibleCases: 60, + heldOutCases: 15, + visible: {} as never, + heldOut: {} as never, + }; + const spy = vi.spyOn(looseningKnobs, "evaluateKnobLoosening"); + spy.mockReturnValueOnce({ ...base, currentValue: AI_KNOB.shippedValue, proposedValue: AI_KNOB.shippedValue }); + expect(await runKnobLoosening(env, AI_KNOB)).toEqual({ applied: false, reason: "no_proposal" }); + spy.mockReturnValueOnce({ ...base, currentValue: AI_KNOB.shippedValue, proposedValue: AI_KNOB.hardMinimum - 0.1 }); + expect(await runKnobLoosening(env, AI_KNOB)).toEqual({ applied: false, reason: "no_proposal" }); + expect(await getKnobOverride(env, AI_KNOB)).toBeNull(); // nothing was written either time + }); +}); + +describe("runScheduledKnobLoosening (#8176 tick wrapper)", () => { + it("emits exactly ONE structured error-level alert on an applied step, and none otherwise", async () => { + const env = enabledEnv(); + await seedAiLooseningFriendlyHistory(env); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + const applied = await runScheduledKnobLoosening(env, AI_KNOB); + expect(applied?.applied).toBe(true); + const alerts = errorSpy.mock.calls.map((call) => String(call[0])).filter((line) => line.includes("calibration_knob_loosened")); + expect(alerts).toHaveLength(1); + expect(alerts[0]).toContain('"ev":"ai_review_close_confidence"'); + + errorSpy.mockClear(); + const second = await runScheduledKnobLoosening(env, AI_KNOB); // starts from the loosened value + expect(second?.applied).toBe(false); + expect(errorSpy.mock.calls.filter((call) => String(call[0]).includes("calibration_knob_loosened"))).toHaveLength(0); + }); + + it("fails SAFE: a thrown evaluation is warned and swallowed (null), never rethrown into the queue", async () => { + const env = enabledEnv(); + env.DB = { prepare: () => { throw new Error("store down"); } } as never; + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); + expect(await runScheduledKnobLoosening(env, AI_KNOB)).toBeNull(); + expect(warnSpy.mock.calls.some((call) => String(call[0]).includes("knob_loosening_tick_failed"))).toBe(true); + + // A thrown NON-Error degrades to the generic message instead of crashing the formatter. + const stringThrowEnv = enabledEnv(); + stringThrowEnv.DB = { prepare: () => { throw "string boom"; } } as never; + expect(await runScheduledKnobLoosening(stringThrowEnv, AI_KNOB)).toBeNull(); + expect(warnSpy.mock.calls.some((call) => String(call[0]).includes('"error":"unknown error"'))).toBe(true); + }); + + it("the audit-event write is best-effort: a rejecting recordAuditEvent still applies the override (the catch arm)", async () => { + const env = enabledEnv(); + await seedAiLooseningFriendlyHistory(env); + const repositories = await import("../../src/db/repositories"); + vi.spyOn(repositories, "recordAuditEvent").mockRejectedValue(new Error("audit write down")); + const result = await runKnobLoosening(env, AI_KNOB); + expect(result.applied).toBe(true); + expect(await getKnobOverride(env, AI_KNOB)).toBe(AI_KNOB.candidates[0]); // the override write was NOT sacrificed + }); +}); + +describe("processor + endpoint wiring (#8176)", () => { + it("the shared loosening tick job runs every generic live knob when ITS flag is ON and no-ops when OFF", async () => { + const offEnv = createTestEnv(); + await seedAiLooseningFriendlyHistory(offEnv); + await processJob(offEnv, { type: "satisfaction-floor-loosening", requestedBy: "schedule" }); + expect(await getKnobOverride({ ...offEnv, AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: "true" as never } as Env, AI_KNOB)).toBeNull(); + + const onEnv = enabledEnv(); + await seedAiLooseningFriendlyHistory(onEnv); + await processJob(onEnv, { type: "satisfaction-floor-loosening", requestedBy: "schedule" }); + expect(await getKnobOverride(onEnv, AI_KNOB)).toBe(AI_KNOB.candidates[0]); + }); + + it("GET /v1/internal/calibration/knobs: 401 without the internal token; lists every live knob, NOT flag-gated, no private terms", async () => { + const app = createApp(); + const env = createTestEnv(); + expect((await app.request("/v1/internal/calibration/knobs", {}, env)).status).toBe(401); + const res = await app.request("/v1/internal/calibration/knobs", { headers: { authorization: `Bearer ${env.INTERNAL_JOB_TOKEN}` } }, env); + expect(res.status).toBe(200); + const body = (await res.json()) as { knobs: Array<{ knobId: string; flagEnabled: boolean }> }; + expect(body.knobs.map((knob) => knob.knobId).sort()).toEqual(["ai_review_close_confidence", "satisfaction_floor"]); + expect(body.knobs.every((knob) => knob.flagEnabled === false)).toBe(true); + expect(JSON.stringify(body)).not.toMatch(/reward|payout|trust|wallet|hotkey|issueText|modelResponse/i); + }); +}); + +describe("loadKnobStatus / loadLiveKnobStatuses (#8161 generalized)", () => { + it("reports a lingering override row even with the flag OFF, and the live value only when ON", async () => { + const env = createTestEnv(); + await setOverrideRow(env, AI_KNOB.overrideFlagKey, "0.9"); + const off = await loadKnobStatus(env, AI_KNOB); + expect(off).toMatchObject({ knobId: "ai_review_close_confidence", flagEnabled: false, storedOverride: 0.9, liveValue: AI_KNOB.shippedValue }); + + const on = await loadKnobStatus({ ...env, AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: "true" as never } as Env, AI_KNOB); + expect(on).toMatchObject({ flagEnabled: true, liveValue: 0.9 }); + + // An out-of-bounds row is reported as no override at all (same validation as the consumption read). + await setOverrideRow(env, AI_KNOB.overrideFlagKey, "0.99"); + expect((await loadKnobStatus(env, AI_KNOB)).storedOverride).toBeNull(); + }); + + it("projects applied history from the knob's events — reading BOTH proposal spellings — and keeps corrupt rows visible as nulls", async () => { + const env = enabledEnv(); + await seedAiLooseningFriendlyHistory(env); + await runKnobLoosening(env, AI_KNOB); + const status = await loadKnobStatus(env, AI_KNOB); + expect(status.applied).toHaveLength(1); + expect(status.applied[0]!).toMatchObject({ currentValue: AI_KNOB.shippedValue, proposedValue: AI_KNOB.candidates[0], visibleVerdict: "improved" }); + + // The satisfaction floor's legacy spelling renders through the same projector. + await recordAuditEvent(env, { + eventType: SATISFACTION_FLOOR_LOOSENING_EVENT_TYPE, + targetKey: SATISFACTION_KNOB.ruleId, + outcome: "completed", + metadata: { proposal: { currentFloor: 0.5, proposedFloor: 0.45, visibleCases: 24, heldOutCases: 7, visible: { verdict: "improved" }, heldOut: { verdict: "unchanged" } } }, + }); + const legacy = await loadKnobStatus(env, SATISFACTION_KNOB); + expect(legacy.applied[0]!).toMatchObject({ currentValue: 0.5, proposedValue: 0.45, visibleVerdict: "improved", heldOutVerdict: "unchanged" }); + + // A corrupt metadata row stays visible (an apply happened) with null fields. + await env.DB.prepare("UPDATE audit_events SET metadata_json = 'corrupt' WHERE event_type = ?").bind(AI_KNOB.looseningEventType).run(); + const corrupt = await loadKnobStatus(env, AI_KNOB); + expect(corrupt.applied).toHaveLength(1); + expect(corrupt.applied[0]!.proposedValue).toBeNull(); + }); + + it("degrades on a broken DB (null override, empty history) and lists every live registry knob", async () => { + const broken = createTestEnv(); + broken.DB = { prepare: () => { throw new Error("boom"); } } as never; + const status = await loadKnobStatus(broken, AI_KNOB); + expect(status).toMatchObject({ storedOverride: null, applied: [], liveValue: AI_KNOB.shippedValue }); + + const statuses = await loadLiveKnobStatuses(createTestEnv()); + expect(statuses.map((s) => s.knobId).sort()).toEqual(["ai_review_close_confidence", "satisfaction_floor"]); + // Parameterized: a report-only knob is excluded from the surface. + const reportOnly = { ...AI_KNOB, knobId: "future_knob", applyMode: "report_only" as const }; + expect((await loadLiveKnobStatuses(createTestEnv(), [reportOnly])).map((s) => s.knobId)).toEqual([]); + }); +}); diff --git a/test/unit/loosening-knobs.test.ts b/test/unit/loosening-knobs.test.ts index 695b064f6..6adf514e4 100644 --- a/test/unit/loosening-knobs.test.ts +++ b/test/unit/loosening-knobs.test.ts @@ -29,14 +29,21 @@ describe("LOOSENABLE_KNOBS registry invariants (#8159)", () => { heldOutFraction: SATISFACTION_FLOOR_HELD_OUT_FRACTION, splitSeed: SATISFACTION_FLOOR_SPLIT_SEED, applyMode: "live", + // #8176's apply plumbing — pinned to the legacy run module's constants by knob-loosening-run.test.ts. + overrideFlagKey: "satisfaction_floor_override", + looseningEventType: "calibration.satisfaction_floor_loosened", + autotuneEnvVar: "SATISFACTION_FLOOR_AUTOTUNE_ENABLED", }); }); - it("pins the close-confidence knob to the shipped default, tight bounds, and REPORT-ONLY apply mode", () => { + it("pins the close-confidence knob to the shipped default, tight bounds, and its LIVE apply plumbing (#8176)", () => { expect(AI_KNOB.shippedValue).toBe(DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE); expect(AI_KNOB.ruleId).toBe("ai_consensus_defect"); - expect(AI_KNOB.applyMode).toBe("report_only"); + expect(AI_KNOB.applyMode).toBe("live"); // flipped by #8176 — the override consumer ships with it expect(AI_KNOB.hardMinimum).toBe(0.85); + expect(AI_KNOB.overrideFlagKey).toBe("ai_review_close_confidence_override"); + expect(AI_KNOB.looseningEventType).toBe("calibration.ai_review_close_confidence_loosened"); + expect(AI_KNOB.autotuneEnvVar).toBe("AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED"); }); it("every entry satisfies the structural safety invariants: candidates strictly below shipped, at/above the hard minimum, descending; ids and seeds unique", () => { diff --git a/test/unit/satisfaction-floor-loosening-run.test.ts b/test/unit/satisfaction-floor-loosening-run.test.ts index 44609fe4f..05d8ac534 100644 --- a/test/unit/satisfaction-floor-loosening-run.test.ts +++ b/test/unit/satisfaction-floor-loosening-run.test.ts @@ -274,9 +274,11 @@ describe("loadReportOnlyKnobProposals (#8159)", () => { expect(await loadReportOnlyKnobProposals(enabledEnv())).toEqual([]); }); - it("surfaces a proposal for the close-confidence knob from its own (backfill-shaped) corpus", async () => { + it("surfaces a proposal for a report-only knob from its own corpus (the shipped registry is all-live since #8176, so the knob is crafted)", async () => { const env = enabledEnv(); - const knob = (await import("../../src/services/loosening-knobs")).LOOSENABLE_KNOBS.ai_review_close_confidence!; + // #8176 flipped ai_review_close_confidence to live; a report-only twin of it keeps this path pinned + // for the next knob that enters report-only (every new knob starts there). + const knob = { ...(await import("../../src/services/loosening-knobs")).LOOSENABLE_KNOBS.ai_review_close_confidence!, applyMode: "report_only" as const }; const { splitBacktestCorpus } = await import("@loopover/engine"); const pool = Array.from({ length: 400 }, (_, i) => ({ ruleId: knob.ruleId, @@ -298,7 +300,7 @@ describe("loadReportOnlyKnobProposals (#8159)", () => { await seed(visible[knob.minVisibleCases + 10]!.targetKey, 0.5, "reversed"); await seed(heldOut[knob.minHeldOutCases + 6]!.targetKey, 0.5, "reversed"); - const proposals = await loadReportOnlyKnobProposals(env); + const proposals = await loadReportOnlyKnobProposals(env, Date.now(), [knob]); expect(proposals).toHaveLength(1); expect(proposals[0]!.knobId).toBe("ai_review_close_confidence"); expect(proposals[0]!.proposedValue).toBe(0.9); diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 3889c5c7e..20b0a8cfc 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: 2992b08ac2db8a03b3ac164a3b2d0eed) +// Generated by Wrangler by running `wrangler types` (hash: 7729263a42b8a5cfc3dabe8c4db7b526) // Runtime types generated with workerd@1.20260701.1 2026-05-28 nodejs_compat interface __BaseEnv_Env { REVIEW_AUDIT: R2Bucket; @@ -13,6 +13,7 @@ interface __BaseEnv_Env { GITTENSOR_REGISTRY_URL: "https://raw.githubusercontent.com/entrius/gittensor/test/gittensor/validator/weights/master_repositories.json"; SCORING_TIME_DECAY_ENABLED: "true"; SATISFACTION_FLOOR_AUTOTUNE_ENABLED: "false"; + AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: "false"; LOOPOVER_AUTO_FILE_DRIFT_ISSUES: "false"; LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover"; PUBLIC_API_ORIGIN: "https://api.loopover.ai"; @@ -68,6 +69,7 @@ type StringifyValues> = { declare namespace NodeJS { interface ProcessEnv extends StringifyValues