feat(calibration): reliability curves beside the ladder — knobs status + advisor surfacing

apps/loopover-ui/content/docs/backtest-calibration.mdx

@@ -108,6 +108,14 @@ loosening loop owns that direction). One alert per drift episode: a standing unc
 re-alerts, a changed one does, and recovery clears the episode. The sentinel is **alert-only** — it never
 writes a knob value — and the knobs status endpoint shows each knob's current drift report regardless of
 the flag, so a standing drift is always visible.
+## Reliability curves
+
+Beside the candidate-ladder machinery, each live knob's status now carries its rule's
+claimed-confidence **reliability curve** (per-bucket empirical precision over the trailing corpus, null
+below the sample floor) and the floor that *falls out* of it at the 0.9 precision bar. The advisor shows
+the derived suggestion next to the ladder proposal whenever they differ. Curve-derived suggestions are
+surfacing-only: the bounded ladder still owns any movement, and replacing it is a recorded soak decision
+(#8227), not a side effect of the curves existing.
 
 ## Counterfactual replay (design)
 

src/review/loosening-recs.ts

@@ -67,6 +67,28 @@ export function buildSatisfactionFloorLooseningRecs(input: SatisfactionFloorRecI
  * reviewed decision (its consumption plumbing changes real authority), never a flag flip. Same hard
  * boundary: no overridePayload, ever.
  */
+/** Reliability-curve view beside the ladder (#8227): one info-severity rec per live knob whose DERIVED
+ *  floor suggestion differs from its live value — evidence shown, authority unchanged (the ladder machinery
+ *  still owns movement; ladder replacement is #8227's recorded soak decision, not this rec). */
+export function buildKnobReliabilityRecs(
+  statuses: readonly { knobId: string; liveValue: number; reliability: { suggestion: number | null } | null }[],
+): TuningRec[] {
+  const recs: TuningRec[] = [];
+  for (const status of statuses) {
+    const suggestion = status.reliability?.suggestion ?? null;
+    if (suggestion === null || suggestion === status.liveValue) continue;
+    recs.push({
+      project: `global:${status.knobId}`,
+      severity: "info",
+      message:
+        `Reliability-curve view for ${status.knobId}: the derived floor at the 0.9 precision bar is ${suggestion} ` +
+        `vs live ${status.liveValue}. Curve-derived suggestions are SURFACING ONLY — the bounded candidate ladder ` +
+        "still owns any movement; treat a persistent gap as soak evidence for the ladder-replacement decision.",
+    });
+  }
+  return recs;
+}
+
 export function buildReportOnlyKnobRecs(proposals: readonly KnobLooseningProposal[]): TuningRec[] {
   return proposals.map((proposal) => ({
     project: `global:${proposal.knobId}`,

src/review/selftune-wire.ts

@@ -41,8 +41,9 @@ import { buildRepoOutcomeCalibration } from "../services/outcome-calibration";
 import { loadRepoFocusManifest } from "../signals/focus-manifest-loader";
 import { errorMessage } from "../utils/json";
 import { computeTuningRecommendations, type GateEvalReport, type GateEvalRow } from "./auto-tune";
-import { buildReportOnlyKnobRecs, buildSatisfactionFloorLooseningRecs } from "./loosening-recs";
+import { buildKnobReliabilityRecs, buildReportOnlyKnobRecs, buildSatisfactionFloorLooseningRecs } from "./loosening-recs";
 import { loadReportOnlyKnobProposals, loadSatisfactionFloorRecState } from "../services/satisfaction-floor-loosening-run";
+import { loadLiveKnobStatuses } from "../services/knob-loosening-run";
 import { runAutoApplyRecommendations, type StorageEnv } from "./auto-apply";
 
 /** True when the self-improvement loop is enabled. Flag-OFF (default) → every export below is a no-op. Truthy
@@ -166,6 +167,8 @@ export async function runSelfTune(env: Env): Promise<void> {
           // #8159: report-only registry knobs surface their evidence in the same pass -- payload-less, so
           // the apply path below ignores them identically.
           recs.push(...buildReportOnlyKnobRecs(await loadReportOnlyKnobProposals(env, nowMs)));
+          // #8227: the curve-derived view beside the ladder, one line per live knob with a differing suggestion.
+          recs.push(...buildKnobReliabilityRecs(await loadLiveKnobStatuses(env)));
         }
         // runAutoApplyRecommendations only ever consumes recs that carry a TIGHTENING overridePayload, shadow-
         // soaks them, and promotes a soaked override only when isStrictlyTightening + evidence + soak pass.

src/services/knob-loosening-run.ts

@@ -12,7 +12,7 @@
 //   • the override write is NOT best-effort (an unrecorded change is worse than none) — the audit trail is;
 //   • one structured error-level alert per applied step, never re-alerting (the next run starts from the
 //     already-loosened value and proposes nothing until the corpus justifies another step).
-import { buildBacktestCorpus } from "@loopover/engine";
+import { buildBacktestCorpus, computeReliabilityCurve, deriveThresholdSuggestion, type ReliabilityCurve } from "@loopover/engine";
 import { createSignalStore } from "../review/signal-tracking-wire";
 import { recordAuditEvent } from "../db/repositories";
 import { evaluateKnobDrift, evaluateKnobLoosening, LOOSENABLE_KNOBS, type KnobDriftReport, type KnobLooseningProposal, type LoosenableKnob } from "./loosening-knobs";
@@ -266,9 +266,20 @@ export type KnobStatus = {
   /** The CURRENT drift report at the live value (#8213) — computed on read, deliberately NOT flag-gated
    *  (an operator must see a standing drift even while the sentinel is off); null when clean/insufficient. */
   drift: KnobDriftReport | null;
+  /** Reliability view (#8227): the rule's claimed-confidence curve over the trailing corpus plus the
+   *  DERIVED floor suggestion at {@link KNOB_SUGGESTION_TARGET_PRECISION} — surfaced NEXT TO the
+   *  ladder-based machinery, never replacing it (the ladder-replacement decision is #8227's recorded
+   *  soak, not this field). Null when the corpus read fails or is empty. */
+  reliability: { curve: ReliabilityCurve; suggestion: number | null } | null;
   applied: KnobAppliedEntry[];
 };
 
+/** The precision bar a derived-floor suggestion must clear (#8227): 0.9 — the floors exist so the
+ *  at-or-above class is trustworthy enough for autonomous disposition, and nine-in-ten human-confirmed
+ *  is the same order the close-confidence knob's own tight ladder implies. Surfacing-only: no evaluator
+ *  consumes this constant. */
+export const KNOB_SUGGESTION_TARGET_PRECISION = 0.9;
+
 const KNOB_STATUS_HISTORY_LIMIT = 25;
 
 function numberOrNull(value: unknown): number | null {
@@ -327,6 +338,18 @@ export async function loadKnobStatus(env: Env, knob: LoosenableKnob): Promise<Kn
     drift = null; // degrade -- the endpoint must not throw on a read blip
   }
 
+  let reliability: KnobStatus["reliability"] = null;
+  try {
+    const { fired, overrides } = await createSignalStore(env).queryRuleHistory(knob.ruleId, Date.now() - CORPUS_LOOKBACK_MS);
+    const cases = buildBacktestCorpus(knob.ruleId, fired, overrides);
+    if (cases.length > 0) {
+      const curve = computeReliabilityCurve(cases);
+      reliability = { curve, suggestion: deriveThresholdSuggestion(curve, KNOB_SUGGESTION_TARGET_PRECISION, knob.hardMinimum) };
+    }
+  } catch {
+    reliability = null; // degrade -- the endpoint must not throw on a read blip
+  }
+
   const applied: KnobAppliedEntry[] = [];
   try {
     const rows = await env.DB.prepare("SELECT created_at, metadata_json FROM audit_events WHERE event_type = ? ORDER BY created_at DESC LIMIT ?")
@@ -364,6 +387,7 @@ export async function loadKnobStatus(env: Env, knob: LoosenableKnob): Promise<Kn
     storedOverride,
     repoOverrides,
     drift,
+    reliability,
     applied,
   };
 }

test/unit/knob-loosening-run.test.ts

@@ -11,6 +11,7 @@ import {
   repoKnobOverrideFlagKey,
   isKnobAutotuneEnabled,
   isConfigDriftSentinelEnabled,
+  KNOB_SUGGESTION_TARGET_PRECISION,
   loadKnobStatus,
   loadLiveKnobStatuses,
   runConfigDriftSentinel,
@@ -437,11 +438,30 @@ describe("loadKnobStatus / loadLiveKnobStatuses (#8161 generalized)", () => {
     expect(corrupt.applied[0]!.proposedValue).toBeNull();
   });
 
+  it("reliability view (#8227): curve + derived suggestion ride the status; empty corpus and read blips degrade to null", async () => {
+    expect(KNOB_SUGGESTION_TARGET_PRECISION).toBe(0.9);
+    const empty = await loadKnobStatus(createTestEnv(), AI_KNOB);
+    expect(empty.reliability).toBeNull(); // no cases at all -> no curve, never a fake one
+
+    const env = enabledEnv();
+    await seedAiLooseningFriendlyHistory(env);
+    const status = await loadKnobStatus(env, AI_KNOB);
+    expect(status.reliability).not.toBeNull();
+    expect(status.reliability!.curve.buckets.length).toBeGreaterThan(0);
+    const decided = status.reliability!.curve.buckets.reduce((sum, b) => sum + b.cases, 0);
+    expect(decided).toBeGreaterThan(0);
+    // The suggestion, when present, respects the knob's own hard minimum — same bound as every evaluator.
+    if (status.reliability!.suggestion !== null) {
+      expect(status.reliability!.suggestion).toBeGreaterThanOrEqual(AI_KNOB.hardMinimum);
+    }
+  });
+
   it("degrades on a broken DB (null override, empty history) and lists every live registry knob", async () => {
     const broken = createTestEnv();
     broken.DB = { prepare: () => { throw new Error("boom"); } } as never;
     const status = await loadKnobStatus(broken, AI_KNOB);
     expect(status).toMatchObject({ storedOverride: null, applied: [], liveValue: AI_KNOB.shippedValue, drift: null });
+    expect(status).toMatchObject({ storedOverride: null, applied: [], liveValue: AI_KNOB.shippedValue, reliability: null });
 
     const statuses = await loadLiveKnobStatuses(createTestEnv());
     expect(statuses.map((s) => s.knobId).sort()).toEqual(["ai_review_close_confidence", "satisfaction_floor"]);

test/unit/loosening-knobs.test.ts

@@ -12,7 +12,7 @@ import {
 } from "../../src/services/satisfaction-floor-loosening";
 import { LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR } from "../../src/services/linked-issue-satisfaction";
 import { DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE } from "../../src/rules/advisory";
-import { buildReportOnlyKnobRecs } from "../../src/review/loosening-recs";
+import { buildKnobReliabilityRecs, buildReportOnlyKnobRecs } from "../../src/review/loosening-recs";
 
 const AI_KNOB = LOOSENABLE_KNOBS.ai_review_close_confidence!;
 
@@ -121,6 +121,29 @@ describe("evaluateKnobLoosening on the close-confidence knob (#8159)", () => {
   });
 });
 
+describe("buildKnobReliabilityRecs (#8227)", () => {
+  const status = (over: Partial<{ knobId: string; liveValue: number; reliability: { suggestion: number | null } | null }> = {}) => ({
+    knobId: "ai_review_close_confidence",
+    liveValue: 0.93,
+    reliability: { suggestion: 0.9 },
+    ...over,
+  });
+
+  it("emits one info rec per live knob whose derived suggestion differs from live — and nothing otherwise", () => {
+    const recs = buildKnobReliabilityRecs([
+      status(), // differs -> rec
+      status({ knobId: "satisfaction_floor", liveValue: 0.5, reliability: { suggestion: 0.5 } }), // equal -> silent
+      status({ knobId: "third", reliability: { suggestion: null } }), // no suggestion -> silent
+      status({ knobId: "fourth", reliability: null }), // no curve -> silent
+    ]);
+    expect(recs).toHaveLength(1);
+    expect(recs[0]!.project).toBe("global:ai_review_close_confidence");
+    expect(recs[0]!.severity).toBe("info");
+    expect(recs[0]!.message).toContain("0.9");
+    expect(recs[0]!.message).toContain("SURFACING ONLY");
+  });
+});
+
 describe("buildReportOnlyKnobRecs (#8159)", () => {
   it("surfaces the evidence with the report-only action line and NEVER a payload", () => {
     const proposal = evaluateKnobLoosening(AI_KNOB, aiLooseningFriendlyCorpus())!;