From 557ac15a054f2e09a8a09b84162f84076d7797c9 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 08:48:41 -0700 Subject: [PATCH] =?UTF-8?q?feat(calibration):=20reliability=20curves=20bes?= =?UTF-8?q?ide=20the=20ladder=20=E2=80=94=20knobs=20status=20+=20advisor?= =?UTF-8?q?=20surfacing=20(#8227)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Epic #8211 track E capstone over the contributor-delivered curve math (#8226): every live knob's status now carries its rule's claimed-confidence reliability curve over the trailing corpus plus the floor DERIVED from it at the documented 0.9 precision bar (respecting the knob's own hard minimum), and the advisor shows the derived suggestion next to the ladder proposal whenever they differ. Surfacing-only by design: the bounded candidate ladder still owns movement, and the ladder-replacement question is the issue's recorded soak decision — a persistent curve-vs-ladder gap on real corpora IS that soak evidence, now visible where operators look. Fail-safe: an empty corpus or a read blip degrades the block to null, never a fabricated curve. Closes #8227 --- .../content/docs/backtest-calibration.mdx | 8 ++++++ src/review/loosening-recs.ts | 22 ++++++++++++++++ src/review/selftune-wire.ts | 5 +++- src/services/knob-loosening-run.ts | 26 ++++++++++++++++++- test/unit/knob-loosening-run.test.ts | 20 ++++++++++++++ test/unit/loosening-knobs.test.ts | 25 +++++++++++++++++- 6 files changed, 103 insertions(+), 3 deletions(-) diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx index 844b534a1..e90ba2a8f 100644 --- a/apps/loopover-ui/content/docs/backtest-calibration.mdx +++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx @@ -108,6 +108,14 @@ loosening loop owns that direction). One alert per drift episode: a standing unc re-alerts, a changed one does, and recovery clears the episode. The sentinel is **alert-only** — it never writes a knob value — and the knobs status endpoint shows each knob's current drift report regardless of the flag, so a standing drift is always visible. +## Reliability curves + +Beside the candidate-ladder machinery, each live knob's status now carries its rule's +claimed-confidence **reliability curve** (per-bucket empirical precision over the trailing corpus, null +below the sample floor) and the floor that *falls out* of it at the 0.9 precision bar. The advisor shows +the derived suggestion next to the ladder proposal whenever they differ. Curve-derived suggestions are +surfacing-only: the bounded ladder still owns any movement, and replacing it is a recorded soak decision +(#8227), not a side effect of the curves existing. ## Counterfactual replay (design) diff --git a/src/review/loosening-recs.ts b/src/review/loosening-recs.ts index 06972f979..b39708b83 100644 --- a/src/review/loosening-recs.ts +++ b/src/review/loosening-recs.ts @@ -67,6 +67,28 @@ export function buildSatisfactionFloorLooseningRecs(input: SatisfactionFloorRecI * reviewed decision (its consumption plumbing changes real authority), never a flag flip. Same hard * boundary: no overridePayload, ever. */ +/** Reliability-curve view beside the ladder (#8227): one info-severity rec per live knob whose DERIVED + * floor suggestion differs from its live value — evidence shown, authority unchanged (the ladder machinery + * still owns movement; ladder replacement is #8227's recorded soak decision, not this rec). */ +export function buildKnobReliabilityRecs( + statuses: readonly { knobId: string; liveValue: number; reliability: { suggestion: number | null } | null }[], +): TuningRec[] { + const recs: TuningRec[] = []; + for (const status of statuses) { + const suggestion = status.reliability?.suggestion ?? null; + if (suggestion === null || suggestion === status.liveValue) continue; + recs.push({ + project: `global:${status.knobId}`, + severity: "info", + message: + `Reliability-curve view for ${status.knobId}: the derived floor at the 0.9 precision bar is ${suggestion} ` + + `vs live ${status.liveValue}. Curve-derived suggestions are SURFACING ONLY — the bounded candidate ladder ` + + "still owns any movement; treat a persistent gap as soak evidence for the ladder-replacement decision.", + }); + } + return recs; +} + export function buildReportOnlyKnobRecs(proposals: readonly KnobLooseningProposal[]): TuningRec[] { return proposals.map((proposal) => ({ project: `global:${proposal.knobId}`, diff --git a/src/review/selftune-wire.ts b/src/review/selftune-wire.ts index e6055a9bb..830d55e81 100644 --- a/src/review/selftune-wire.ts +++ b/src/review/selftune-wire.ts @@ -41,8 +41,9 @@ import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; import { loadRepoFocusManifest } from "../signals/focus-manifest-loader"; import { errorMessage } from "../utils/json"; import { computeTuningRecommendations, type GateEvalReport, type GateEvalRow } from "./auto-tune"; -import { buildReportOnlyKnobRecs, buildSatisfactionFloorLooseningRecs } from "./loosening-recs"; +import { buildKnobReliabilityRecs, buildReportOnlyKnobRecs, buildSatisfactionFloorLooseningRecs } from "./loosening-recs"; import { loadReportOnlyKnobProposals, loadSatisfactionFloorRecState } from "../services/satisfaction-floor-loosening-run"; +import { loadLiveKnobStatuses } from "../services/knob-loosening-run"; import { runAutoApplyRecommendations, type StorageEnv } from "./auto-apply"; /** True when the self-improvement loop is enabled. Flag-OFF (default) → every export below is a no-op. Truthy @@ -166,6 +167,8 @@ export async function runSelfTune(env: Env): Promise { // #8159: report-only registry knobs surface their evidence in the same pass -- payload-less, so // the apply path below ignores them identically. recs.push(...buildReportOnlyKnobRecs(await loadReportOnlyKnobProposals(env, nowMs))); + // #8227: the curve-derived view beside the ladder, one line per live knob with a differing suggestion. + recs.push(...buildKnobReliabilityRecs(await loadLiveKnobStatuses(env))); } // runAutoApplyRecommendations only ever consumes recs that carry a TIGHTENING overridePayload, shadow- // soaks them, and promotes a soaked override only when isStrictlyTightening + evidence + soak pass. diff --git a/src/services/knob-loosening-run.ts b/src/services/knob-loosening-run.ts index 0d74b344b..4fa89efbe 100644 --- a/src/services/knob-loosening-run.ts +++ b/src/services/knob-loosening-run.ts @@ -12,7 +12,7 @@ // • the override write is NOT best-effort (an unrecorded change is worse than none) — the audit trail is; // • one structured error-level alert per applied step, never re-alerting (the next run starts from the // already-loosened value and proposes nothing until the corpus justifies another step). -import { buildBacktestCorpus } from "@loopover/engine"; +import { buildBacktestCorpus, computeReliabilityCurve, deriveThresholdSuggestion, type ReliabilityCurve } from "@loopover/engine"; import { createSignalStore } from "../review/signal-tracking-wire"; import { recordAuditEvent } from "../db/repositories"; import { evaluateKnobDrift, evaluateKnobLoosening, LOOSENABLE_KNOBS, type KnobDriftReport, type KnobLooseningProposal, type LoosenableKnob } from "./loosening-knobs"; @@ -266,9 +266,20 @@ export type KnobStatus = { /** The CURRENT drift report at the live value (#8213) — computed on read, deliberately NOT flag-gated * (an operator must see a standing drift even while the sentinel is off); null when clean/insufficient. */ drift: KnobDriftReport | null; + /** Reliability view (#8227): the rule's claimed-confidence curve over the trailing corpus plus the + * DERIVED floor suggestion at {@link KNOB_SUGGESTION_TARGET_PRECISION} — surfaced NEXT TO the + * ladder-based machinery, never replacing it (the ladder-replacement decision is #8227's recorded + * soak, not this field). Null when the corpus read fails or is empty. */ + reliability: { curve: ReliabilityCurve; suggestion: number | null } | null; applied: KnobAppliedEntry[]; }; +/** The precision bar a derived-floor suggestion must clear (#8227): 0.9 — the floors exist so the + * at-or-above class is trustworthy enough for autonomous disposition, and nine-in-ten human-confirmed + * is the same order the close-confidence knob's own tight ladder implies. Surfacing-only: no evaluator + * consumes this constant. */ +export const KNOB_SUGGESTION_TARGET_PRECISION = 0.9; + const KNOB_STATUS_HISTORY_LIMIT = 25; function numberOrNull(value: unknown): number | null { @@ -327,6 +338,18 @@ export async function loadKnobStatus(env: Env, knob: LoosenableKnob): Promise 0) { + const curve = computeReliabilityCurve(cases); + reliability = { curve, suggestion: deriveThresholdSuggestion(curve, KNOB_SUGGESTION_TARGET_PRECISION, knob.hardMinimum) }; + } + } catch { + reliability = null; // degrade -- the endpoint must not throw on a read blip + } + const applied: KnobAppliedEntry[] = []; try { const rows = await env.DB.prepare("SELECT created_at, metadata_json FROM audit_events WHERE event_type = ? ORDER BY created_at DESC LIMIT ?") @@ -364,6 +387,7 @@ export async function loadKnobStatus(env: Env, knob: LoosenableKnob): Promise { expect(corrupt.applied[0]!.proposedValue).toBeNull(); }); + it("reliability view (#8227): curve + derived suggestion ride the status; empty corpus and read blips degrade to null", async () => { + expect(KNOB_SUGGESTION_TARGET_PRECISION).toBe(0.9); + const empty = await loadKnobStatus(createTestEnv(), AI_KNOB); + expect(empty.reliability).toBeNull(); // no cases at all -> no curve, never a fake one + + const env = enabledEnv(); + await seedAiLooseningFriendlyHistory(env); + const status = await loadKnobStatus(env, AI_KNOB); + expect(status.reliability).not.toBeNull(); + expect(status.reliability!.curve.buckets.length).toBeGreaterThan(0); + const decided = status.reliability!.curve.buckets.reduce((sum, b) => sum + b.cases, 0); + expect(decided).toBeGreaterThan(0); + // The suggestion, when present, respects the knob's own hard minimum — same bound as every evaluator. + if (status.reliability!.suggestion !== null) { + expect(status.reliability!.suggestion).toBeGreaterThanOrEqual(AI_KNOB.hardMinimum); + } + }); + it("degrades on a broken DB (null override, empty history) and lists every live registry knob", async () => { const broken = createTestEnv(); broken.DB = { prepare: () => { throw new Error("boom"); } } as never; const status = await loadKnobStatus(broken, AI_KNOB); expect(status).toMatchObject({ storedOverride: null, applied: [], liveValue: AI_KNOB.shippedValue, drift: null }); + expect(status).toMatchObject({ storedOverride: null, applied: [], liveValue: AI_KNOB.shippedValue, reliability: null }); const statuses = await loadLiveKnobStatuses(createTestEnv()); expect(statuses.map((s) => s.knobId).sort()).toEqual(["ai_review_close_confidence", "satisfaction_floor"]); diff --git a/test/unit/loosening-knobs.test.ts b/test/unit/loosening-knobs.test.ts index 2e82acbb8..174b2d141 100644 --- a/test/unit/loosening-knobs.test.ts +++ b/test/unit/loosening-knobs.test.ts @@ -12,7 +12,7 @@ import { } from "../../src/services/satisfaction-floor-loosening"; import { LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR } from "../../src/services/linked-issue-satisfaction"; import { DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE } from "../../src/rules/advisory"; -import { buildReportOnlyKnobRecs } from "../../src/review/loosening-recs"; +import { buildKnobReliabilityRecs, buildReportOnlyKnobRecs } from "../../src/review/loosening-recs"; const AI_KNOB = LOOSENABLE_KNOBS.ai_review_close_confidence!; @@ -121,6 +121,29 @@ describe("evaluateKnobLoosening on the close-confidence knob (#8159)", () => { }); }); +describe("buildKnobReliabilityRecs (#8227)", () => { + const status = (over: Partial<{ knobId: string; liveValue: number; reliability: { suggestion: number | null } | null }> = {}) => ({ + knobId: "ai_review_close_confidence", + liveValue: 0.93, + reliability: { suggestion: 0.9 }, + ...over, + }); + + it("emits one info rec per live knob whose derived suggestion differs from live — and nothing otherwise", () => { + const recs = buildKnobReliabilityRecs([ + status(), // differs -> rec + status({ knobId: "satisfaction_floor", liveValue: 0.5, reliability: { suggestion: 0.5 } }), // equal -> silent + status({ knobId: "third", reliability: { suggestion: null } }), // no suggestion -> silent + status({ knobId: "fourth", reliability: null }), // no curve -> silent + ]); + expect(recs).toHaveLength(1); + expect(recs[0]!.project).toBe("global:ai_review_close_confidence"); + expect(recs[0]!.severity).toBe("info"); + expect(recs[0]!.message).toContain("0.9"); + expect(recs[0]!.message).toContain("SURFACING ONLY"); + }); +}); + describe("buildReportOnlyKnobRecs (#8159)", () => { it("surfaces the evidence with the report-only action line and NEVER a payload", () => { const proposal = evaluateKnobLoosening(AI_KNOB, aiLooseningFriendlyCorpus())!;