diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx index aede3e6e7..844b534a1 100644 --- a/apps/loopover-ui/content/docs/backtest-calibration.mdx +++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx @@ -96,6 +96,18 @@ Every earned scope applies the identical loosening-only validation (strictly bel the hard minimum), and the knob's autotune flag gates *all* earned scopes at once — turning the flag off restores shipped behavior everywhere instantly, no cleanup. The knobs status endpoint lists each knob's per-repo overrides so a lingering row is always visible. +## Config-drift sentinel + +Stale configuration — rules that were right when set and wrong after the repo's reality moved — is the +largest empirically-measured wrongness source in the historical ledger. The drift sentinel closes that +gap: on the calibration tick (behind `CONFIG_DRIFT_SENTINEL_ENABLED`, default off) it replays each live +knob's **current** value against the trailing corpus and alerts when a *tighter* alternative (or a revert +to the shipped default) Pareto-dominates what is live — same splits, same floors, same never-on-noise +minimums as every other calibration verdict. Looser-dominating findings are deliberately suppressed (the +loosening loop owns that direction). One alert per drift episode: a standing unchanged drift never +re-alerts, a changed one does, and recovery clears the episode. The sentinel is **alert-only** — it never +writes a knob value — and the knobs status endpoint shows each knob's current drift report regardless of +the flag, so a standing drift is always visible. ## Counterfactual replay (design) diff --git a/src/queue/job-dispatch.ts b/src/queue/job-dispatch.ts index b63c3819c..87fd3f3e7 100644 --- a/src/queue/job-dispatch.ts +++ b/src/queue/job-dispatch.ts @@ -31,7 +31,7 @@ import { isPrReconciliationEnabled, resolvePrReconciliationManifestOverride, run import { isActiveReviewReconciliationEnabled, resolveActiveReviewReconciliationManifestOverride, runActiveReviewReconciliation } from "../review/active-review-reconciliation"; import { isSelfTuneEnabled, runSelfTune } from "../review/selftune-wire"; import { isSatisfactionFloorAutotuneEnabled, runScheduledSatisfactionFloorLoosening } from "../services/satisfaction-floor-loosening-run"; -import { GENERIC_LIVE_KNOBS, isKnobAutotuneEnabled, runScheduledKnobLoosening } from "../services/knob-loosening-run"; +import { GENERIC_LIVE_KNOBS, isConfigDriftSentinelEnabled, isKnobAutotuneEnabled, runConfigDriftSentinel, runScheduledKnobLoosening } from "../services/knob-loosening-run"; import { runSelfTuneBreaker } from "../review/outcomes-wire"; import { isRagEnabled } from "../review/rag-wire"; import { processSubmitDraft } from "../services/draft"; @@ -354,6 +354,9 @@ export async function processJob(env: Env, message: JobMessage): Promise { for (const knob of GENERIC_LIVE_KNOBS) { if (isKnobAutotuneEnabled(env, knob)) await runScheduledKnobLoosening(env, knob); } + // #8213: the drift sentinel rides the same calibration tick, behind its own default-off flag. + // Alert-only — it never writes a knob value — and internally fail-safe per knob. + if (isConfigDriftSentinelEnabled(env)) await runConfigDriftSentinel(env); return; case "selftune": // Convergence (self-improve / auto-tune, flag LOOPOVER_REVIEW_SELFTUNE). Defense-in-depth: the cron only diff --git a/src/services/knob-loosening-run.ts b/src/services/knob-loosening-run.ts index a66afa123..0d74b344b 100644 --- a/src/services/knob-loosening-run.ts +++ b/src/services/knob-loosening-run.ts @@ -15,7 +15,7 @@ import { buildBacktestCorpus } from "@loopover/engine"; import { createSignalStore } from "../review/signal-tracking-wire"; import { recordAuditEvent } from "../db/repositories"; -import { evaluateKnobLoosening, LOOSENABLE_KNOBS, type KnobLooseningProposal, type LoosenableKnob } from "./loosening-knobs"; +import { evaluateKnobDrift, evaluateKnobLoosening, LOOSENABLE_KNOBS, type KnobDriftReport, type KnobLooseningProposal, type LoosenableKnob } from "./loosening-knobs"; const CORPUS_LOOKBACK_MS = 90 * 24 * 60 * 60 * 1000; // mirrors threshold-backtest-run's 90-day window @@ -161,6 +161,81 @@ export async function runScheduledKnobLoosening(env: Env, knob: LoosenableKnob): } } +// ── Config-drift sentinel (#8213, epic #8211 track A) ──────────────────────────────────────────────────── + +/** Truthy-string flag for the drift sentinel — default off, so a deploy is byte-identical until opted in. */ +export function isConfigDriftSentinelEnabled(env: Env): boolean { + const value = ((env as unknown as Record).CONFIG_DRIFT_SENTINEL_ENABLED as string | undefined ?? "").trim().toLowerCase(); + return value === "1" || value === "true" || value === "on" || value === "yes"; +} + +const DRIFT_FINGERPRINT_FLAG_PREFIX = "config_drift_fingerprint:"; + +export type ConfigDriftTickResult = { knobId: string; state: "alerted" | "standing" | "suppressed_looser" | "clean" }; + +/** + * One sentinel pass over every live knob (#8213): replay the CURRENT live value against the knob's + * trailing corpus and alert when a TIGHTER (or shipped-revert) alternative Pareto-dominates it — the + * stale-config signal the #8170 retro analysis proved is the operator's largest wrongness source. A + * LOOSER winner is suppressed (the loosening loop's own surfacing owns that direction). Episode dedup: + * the last-alerted fingerprint (knob + direction + dominating value) persists in system_flags; a standing + * unchanged drift never re-alerts, a CHANGED drift does, and a cleared drift clears the fingerprint. + * ALERT-ONLY authority — the sentinel never writes a knob value. Fail-safe per knob. + */ +export async function runConfigDriftSentinel(env: Env, knobs: readonly LoosenableKnob[] = Object.values(LOOSENABLE_KNOBS)): Promise { + const results: ConfigDriftTickResult[] = []; + for (const knob of knobs) { + if (knob.applyMode !== "live") continue; + try { + const liveValue = (await getKnobOverride(env, knob)) ?? knob.shippedValue; + const { fired, overrides } = await createSignalStore(env).queryRuleHistory(knob.ruleId, Date.now() - CORPUS_LOOKBACK_MS); + const report = evaluateKnobDrift(knob, buildBacktestCorpus(knob.ruleId, fired, overrides), liveValue); + const fingerprintKey = `${DRIFT_FINGERPRINT_FLAG_PREFIX}${knob.knobId}`; + const stored = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?").bind(fingerprintKey).first<{ value: string }>(); + + if (!report || report.direction === "looser") { + if (stored) { + await env.DB.prepare("DELETE FROM system_flags WHERE key = ?").bind(fingerprintKey).run(); + } + results.push({ knobId: knob.knobId, state: report ? "suppressed_looser" : "clean" }); + continue; + } + + const fingerprint = `${knob.knobId}:${report.direction}:${report.dominatingValue}`; + if (stored?.value === fingerprint) { + results.push({ knobId: knob.knobId, state: "standing" }); + continue; + } + await env.DB.prepare( + "INSERT INTO system_flags (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP) ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.updated_at", + ) + .bind(fingerprintKey, fingerprint) + .run(); + // Same Workers-Logs + Sentry notify path as the loosening alert; `ev` keeps knobs distinct. + console.error( + JSON.stringify({ + level: "error", + event: "config_drift_detected", + ev: knob.knobId, + at: new Date().toISOString(), + direction: report.direction, + liveValue: report.liveValue, + dominatingValue: report.dominatingValue, + visibleCases: report.visibleCases, + heldOutCases: report.heldOutCases, + }), + ); + results.push({ knobId: knob.knobId, state: "alerted" }); + } catch (error) { + console.warn( + JSON.stringify({ level: "warn", event: "config_drift_tick_failed", ev: knob.knobId, error: error instanceof Error ? error.message : "unknown error" }), + ); + results.push({ knobId: knob.knobId, state: "clean" }); + } + } + return results; +} + // ── Operator status (the #8161 surface generalized across live knobs) ──────────────────────────────────── export type KnobAppliedEntry = { @@ -188,6 +263,9 @@ export type KnobStatus = { /** Per-repo earned overrides (#8216), validated rows only, sorted by repo — an operator must see every * scope that would take effect the moment the flag is on. */ repoOverrides: KnobRepoOverride[]; + /** The CURRENT drift report at the live value (#8213) — computed on read, deliberately NOT flag-gated + * (an operator must see a standing drift even while the sentinel is off); null when clean/insufficient. */ + drift: KnobDriftReport | null; applied: KnobAppliedEntry[]; }; @@ -240,6 +318,15 @@ export async function loadKnobStatus(env: Env, knob: LoosenableKnob): Promise = {}): Env { LOOPOVER_AUTO_FILE_DRIFT_ISSUES: "false", SATISFACTION_FLOOR_AUTOTUNE_ENABLED: "false", AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: "false", + CONFIG_DRIFT_SENTINEL_ENABLED: "false", // Deliberately NOT "JSONbored/gittensory" (the old pre-rename repo name most test fixtures use as their // generic placeholder repoFullName) and NOT "JSONbored/loopover" (the real self-repo default) -- either // would make isLoopOverSelfRepo() accidentally match a fixture that has no intent to exercise self-repo diff --git a/test/unit/knob-loosening-run.test.ts b/test/unit/knob-loosening-run.test.ts index ffa0646aa..d02711a5a 100644 --- a/test/unit/knob-loosening-run.test.ts +++ b/test/unit/knob-loosening-run.test.ts @@ -10,8 +10,10 @@ import { getKnobOverrideForRepo, repoKnobOverrideFlagKey, isKnobAutotuneEnabled, + isConfigDriftSentinelEnabled, loadKnobStatus, loadLiveKnobStatuses, + runConfigDriftSentinel, runKnobLoosening, runScheduledKnobLoosening, } from "../../src/services/knob-loosening-run"; @@ -277,6 +279,114 @@ describe("processor + endpoint wiring (#8176)", () => { }); }); +describe("runConfigDriftSentinel (#8213)", () => { + const driftEnv = (base?: Env) => ({ ...(base ?? enabledEnv()), CONFIG_DRIFT_SENTINEL_ENABLED: "true" as never }) as Env; + + // A corpus where TIGHTENING wins: reversed cases sit at 0.91 — a 0.85 live floor misses them, the + // shipped 0.93 catches them (recall up, no precision loss) — the stale-config shape the sentinel exists + // to flag. Same membership-probe technique as the loosening seeder. + async function seedDriftFriendlyHistory(env: Env): Promise { + const pool = Array.from({ length: 400 }, (_, i) => `acme/widgets#${i + 1}`); + const probe = pool.map((targetKey) => ({ + ruleId: AI_KNOB.ruleId, targetKey, outcome: "unaddressed", label: "confirmed" as const, + firedAt: "2026-07-01T00:00:00.000Z", decidedAt: "2026-07-02T00:00:00.000Z", + })); + const { visible, heldOut } = splitBacktestCorpus(probe, AI_KNOB.heldOutFraction, AI_KNOB.splitSeed); + const store = createSignalStore(env); + const now = Date.now(); + const seed = async (targetKey: string, confidence: number, verdict: "confirmed" | "reversed", i: number) => { + await store.recordRuleFired({ ruleId: AI_KNOB.ruleId, targetKey, outcome: "unaddressed", occurredAt: new Date(now - 10_000 - i).toISOString(), metadata: { confidence } }); + await store.recordHumanOverride({ ruleId: AI_KNOB.ruleId, targetKey, verdict, occurredAt: new Date(now - i).toISOString() }); + }; + let i = 0; + for (const c of visible.slice(0, AI_KNOB.minVisibleCases + 4)) await seed(c.targetKey, 0.91, "reversed", i++); + for (const c of heldOut.slice(0, AI_KNOB.minHeldOutCases + 2)) await seed(c.targetKey, 0.91, "reversed", i++); + await seed(visible[AI_KNOB.minVisibleCases + 5]!.targetKey, 0.2, "reversed", i++); + await seed(heldOut[AI_KNOB.minHeldOutCases + 3]!.targetKey, 0.2, "reversed", i++); + } + + it("flag parse mirrors the house convention", () => { + expect(isConfigDriftSentinelEnabled({ CONFIG_DRIFT_SENTINEL_ENABLED: "true" } as unknown as Env)).toBe(true); + expect(isConfigDriftSentinelEnabled({ CONFIG_DRIFT_SENTINEL_ENABLED: "false" } as unknown as Env)).toBe(false); + expect(isConfigDriftSentinelEnabled({} as unknown as Env)).toBe(false); + }); + + it("alerts ONCE per drift episode, stays silent while it stands, re-alerts on change, clears on recovery", async () => { + const env = driftEnv(); + // A live override at the hard minimum with a corpus proving the SHIPPED value dominates it → a + // 'shipped'-direction (revert) drift, the actionable class. + await setOverrideRow(env, AI_KNOB.overrideFlagKey, String(AI_KNOB.hardMinimum)); + await seedDriftFriendlyHistory(env); // reversed cluster at 0.91: shipped 0.93 dominates a 0.85 live floor + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + + const first = await runConfigDriftSentinel(env); + expect(first.find((r) => r.knobId === AI_KNOB.knobId)?.state).toBe("alerted"); + expect(errorSpy.mock.calls.filter((c) => String(c[0]).includes("config_drift_detected"))).toHaveLength(1); + + errorSpy.mockClear(); + const second = await runConfigDriftSentinel(env); + expect(second.find((r) => r.knobId === AI_KNOB.knobId)?.state).toBe("standing"); + expect(errorSpy.mock.calls.filter((c) => String(c[0]).includes("config_drift_detected"))).toHaveLength(0); + + // Recovery: remove the override -- live returns to shipped, nothing dominates, fingerprint clears. + await env.DB.prepare("DELETE FROM system_flags WHERE key = ?").bind(AI_KNOB.overrideFlagKey).run(); + const third = await runConfigDriftSentinel(env); + expect(third.find((r) => r.knobId === AI_KNOB.knobId)?.state).toBe("clean"); + // And a NEW episode after recovery alerts again. + await setOverrideRow(env, AI_KNOB.overrideFlagKey, String(AI_KNOB.hardMinimum)); + const fourth = await runConfigDriftSentinel(env); + expect(fourth.find((r) => r.knobId === AI_KNOB.knobId)?.state).toBe("alerted"); + }); + + it("suppresses a LOOSER-dominating result (the loosening loop owns that direction) and clears any stale fingerprint", async () => { + const env = driftEnv(); + await seedAiLooseningFriendlyHistory(env); // from shipped 0.93, candidate 0.9 dominates → looser + await env.DB.prepare("INSERT INTO system_flags (key, value, updated_at) VALUES (?, ?, CURRENT_TIMESTAMP)") + .bind(`config_drift_fingerprint:${AI_KNOB.knobId}`, "stale") + .run(); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + const results = await runConfigDriftSentinel(env); + expect(results.find((r) => r.knobId === AI_KNOB.knobId)?.state).toBe("suppressed_looser"); + expect(errorSpy.mock.calls.filter((c) => String(c[0]).includes("config_drift_detected"))).toHaveLength(0); + const fp = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?").bind(`config_drift_fingerprint:${AI_KNOB.knobId}`).first(); + expect(fp ?? null).toBeNull(); // TestD1 returns undefined where live D1 returns null + }); + + it("is clean on an empty corpus and fail-safe per knob on a broken store", async () => { + const clean = await runConfigDriftSentinel(driftEnv()); + expect(clean.every((r) => r.state === "clean")).toBe(true); + + const broken = driftEnv(); + broken.DB = { prepare: () => { throw new Error("boom"); } } as never; + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => undefined); + const results = await runConfigDriftSentinel(broken); + expect(results.every((r) => r.state === "clean")).toBe(true); + expect(warnSpy.mock.calls.some((c) => String(c[0]).includes("config_drift_tick_failed"))).toBe(true); + + // Non-Error throw degrades to the generic message; a report-only knob is skipped entirely. + const stringThrow = driftEnv(); + stringThrow.DB = { prepare: () => { throw "string boom"; } } as never; + await runConfigDriftSentinel(stringThrow); + expect(warnSpy.mock.calls.some((c) => String(c[0]).includes('"error":"unknown error"'))).toBe(true); + const reportOnly = { ...AI_KNOB, knobId: "future_knob", applyMode: "report_only" as const }; + expect(await runConfigDriftSentinel(driftEnv(), [reportOnly])).toEqual([]); + }); + + it("the calibration tick job runs the sentinel only when its flag is ON (dispatch wiring)", async () => { + const env = driftEnv(); + await setOverrideRow(env, AI_KNOB.overrideFlagKey, String(AI_KNOB.hardMinimum)); + await seedDriftFriendlyHistory(env); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => undefined); + await processJob(env, { type: "satisfaction-floor-loosening", requestedBy: "schedule" }); + expect(errorSpy.mock.calls.some((c) => String(c[0]).includes("config_drift_detected"))).toBe(true); + + errorSpy.mockClear(); + const off = { ...enabledEnv(), DB: env.DB } as Env; // sentinel flag unset + await processJob(off, { type: "satisfaction-floor-loosening", requestedBy: "schedule" }); + expect(errorSpy.mock.calls.some((c) => String(c[0]).includes("config_drift_detected"))).toBe(false); + }); +}); + describe("loadKnobStatus / loadLiveKnobStatuses (#8161 generalized)", () => { it("reports a lingering override row even with the flag OFF, and the live value only when ON", async () => { const env = createTestEnv(); @@ -331,7 +441,7 @@ describe("loadKnobStatus / loadLiveKnobStatuses (#8161 generalized)", () => { const broken = createTestEnv(); broken.DB = { prepare: () => { throw new Error("boom"); } } as never; const status = await loadKnobStatus(broken, AI_KNOB); - expect(status).toMatchObject({ storedOverride: null, applied: [], liveValue: AI_KNOB.shippedValue }); + expect(status).toMatchObject({ storedOverride: null, applied: [], liveValue: AI_KNOB.shippedValue, drift: null }); const statuses = await loadLiveKnobStatuses(createTestEnv()); expect(statuses.map((s) => s.knobId).sort()).toEqual(["ai_review_close_confidence", "satisfaction_floor"]); diff --git a/worker-configuration.d.ts b/worker-configuration.d.ts index 20b0a8cfc..d5fdd90e6 100644 --- a/worker-configuration.d.ts +++ b/worker-configuration.d.ts @@ -1,5 +1,5 @@ /* eslint-disable */ -// Generated by Wrangler by running `wrangler types` (hash: 7729263a42b8a5cfc3dabe8c4db7b526) +// Generated by Wrangler by running `wrangler types` (hash: 63dada085784011bb62d8a1c75f64252) // Runtime types generated with workerd@1.20260701.1 2026-05-28 nodejs_compat interface __BaseEnv_Env { REVIEW_AUDIT: R2Bucket; @@ -14,6 +14,7 @@ interface __BaseEnv_Env { SCORING_TIME_DECAY_ENABLED: "true"; SATISFACTION_FLOOR_AUTOTUNE_ENABLED: "false"; AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: "false"; + CONFIG_DRIFT_SENTINEL_ENABLED: "false"; LOOPOVER_AUTO_FILE_DRIFT_ISSUES: "false"; LOOPOVER_DRIFT_ISSUE_REPO: "JSONbored/loopover"; PUBLIC_API_ORIGIN: "https://api.loopover.ai"; @@ -70,6 +71,7 @@ declare namespace NodeJS { interface ProcessEnv extends StringifyValues