diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx index 844b534a1..e90ba2a8f 100644 --- a/apps/loopover-ui/content/docs/backtest-calibration.mdx +++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx @@ -108,6 +108,14 @@ loosening loop owns that direction). One alert per drift episode: a standing unc re-alerts, a changed one does, and recovery clears the episode. The sentinel is **alert-only** — it never writes a knob value — and the knobs status endpoint shows each knob's current drift report regardless of the flag, so a standing drift is always visible. +## Reliability curves + +Beside the candidate-ladder machinery, each live knob's status now carries its rule's +claimed-confidence **reliability curve** (per-bucket empirical precision over the trailing corpus, null +below the sample floor) and the floor that *falls out* of it at the 0.9 precision bar. The advisor shows +the derived suggestion next to the ladder proposal whenever they differ. Curve-derived suggestions are +surfacing-only: the bounded ladder still owns any movement, and replacing it is a recorded soak decision +(#8227), not a side effect of the curves existing. ## Counterfactual replay (design) diff --git a/src/review/loosening-recs.ts b/src/review/loosening-recs.ts index 06972f979..b39708b83 100644 --- a/src/review/loosening-recs.ts +++ b/src/review/loosening-recs.ts @@ -67,6 +67,28 @@ export function buildSatisfactionFloorLooseningRecs(input: SatisfactionFloorRecI * reviewed decision (its consumption plumbing changes real authority), never a flag flip. Same hard * boundary: no overridePayload, ever. */ +/** Reliability-curve view beside the ladder (#8227): one info-severity rec per live knob whose DERIVED + * floor suggestion differs from its live value — evidence shown, authority unchanged (the ladder machinery + * still owns movement; ladder replacement is #8227's recorded soak decision, not this rec). */ +export function buildKnobReliabilityRecs( + statuses: readonly { knobId: string; liveValue: number; reliability: { suggestion: number | null } | null }[], +): TuningRec[] { + const recs: TuningRec[] = []; + for (const status of statuses) { + const suggestion = status.reliability?.suggestion ?? null; + if (suggestion === null || suggestion === status.liveValue) continue; + recs.push({ + project: `global:${status.knobId}`, + severity: "info", + message: + `Reliability-curve view for ${status.knobId}: the derived floor at the 0.9 precision bar is ${suggestion} ` + + `vs live ${status.liveValue}. Curve-derived suggestions are SURFACING ONLY — the bounded candidate ladder ` + + "still owns any movement; treat a persistent gap as soak evidence for the ladder-replacement decision.", + }); + } + return recs; +} + export function buildReportOnlyKnobRecs(proposals: readonly KnobLooseningProposal[]): TuningRec[] { return proposals.map((proposal) => ({ project: `global:${proposal.knobId}`, diff --git a/src/review/selftune-wire.ts b/src/review/selftune-wire.ts index e6055a9bb..830d55e81 100644 --- a/src/review/selftune-wire.ts +++ b/src/review/selftune-wire.ts @@ -41,8 +41,9 @@ import { buildRepoOutcomeCalibration } from "../services/outcome-calibration"; import { loadRepoFocusManifest } from "../signals/focus-manifest-loader"; import { errorMessage } from "../utils/json"; import { computeTuningRecommendations, type GateEvalReport, type GateEvalRow } from "./auto-tune"; -import { buildReportOnlyKnobRecs, buildSatisfactionFloorLooseningRecs } from "./loosening-recs"; +import { buildKnobReliabilityRecs, buildReportOnlyKnobRecs, buildSatisfactionFloorLooseningRecs } from "./loosening-recs"; import { loadReportOnlyKnobProposals, loadSatisfactionFloorRecState } from "../services/satisfaction-floor-loosening-run"; +import { loadLiveKnobStatuses } from "../services/knob-loosening-run"; import { runAutoApplyRecommendations, type StorageEnv } from "./auto-apply"; /** True when the self-improvement loop is enabled. Flag-OFF (default) → every export below is a no-op. Truthy @@ -166,6 +167,8 @@ export async function runSelfTune(env: Env): Promise { // #8159: report-only registry knobs surface their evidence in the same pass -- payload-less, so // the apply path below ignores them identically. recs.push(...buildReportOnlyKnobRecs(await loadReportOnlyKnobProposals(env, nowMs))); + // #8227: the curve-derived view beside the ladder, one line per live knob with a differing suggestion. + recs.push(...buildKnobReliabilityRecs(await loadLiveKnobStatuses(env))); } // runAutoApplyRecommendations only ever consumes recs that carry a TIGHTENING overridePayload, shadow- // soaks them, and promotes a soaked override only when isStrictlyTightening + evidence + soak pass. diff --git a/src/services/knob-loosening-run.ts b/src/services/knob-loosening-run.ts index 0d74b344b..4fa89efbe 100644 --- a/src/services/knob-loosening-run.ts +++ b/src/services/knob-loosening-run.ts @@ -12,7 +12,7 @@ // • the override write is NOT best-effort (an unrecorded change is worse than none) — the audit trail is; // • one structured error-level alert per applied step, never re-alerting (the next run starts from the // already-loosened value and proposes nothing until the corpus justifies another step). -import { buildBacktestCorpus } from "@loopover/engine"; +import { buildBacktestCorpus, computeReliabilityCurve, deriveThresholdSuggestion, type ReliabilityCurve } from "@loopover/engine"; import { createSignalStore } from "../review/signal-tracking-wire"; import { recordAuditEvent } from "../db/repositories"; import { evaluateKnobDrift, evaluateKnobLoosening, LOOSENABLE_KNOBS, type KnobDriftReport, type KnobLooseningProposal, type LoosenableKnob } from "./loosening-knobs"; @@ -266,9 +266,20 @@ export type KnobStatus = { /** The CURRENT drift report at the live value (#8213) — computed on read, deliberately NOT flag-gated * (an operator must see a standing drift even while the sentinel is off); null when clean/insufficient. */ drift: KnobDriftReport | null; + /** Reliability view (#8227): the rule's claimed-confidence curve over the trailing corpus plus the + * DERIVED floor suggestion at {@link KNOB_SUGGESTION_TARGET_PRECISION} — surfaced NEXT TO the + * ladder-based machinery, never replacing it (the ladder-replacement decision is #8227's recorded + * soak, not this field). Null when the corpus read fails or is empty. */ + reliability: { curve: ReliabilityCurve; suggestion: number | null } | null; applied: KnobAppliedEntry[]; }; +/** The precision bar a derived-floor suggestion must clear (#8227): 0.9 — the floors exist so the + * at-or-above class is trustworthy enough for autonomous disposition, and nine-in-ten human-confirmed + * is the same order the close-confidence knob's own tight ladder implies. Surfacing-only: no evaluator + * consumes this constant. */ +export const KNOB_SUGGESTION_TARGET_PRECISION = 0.9; + const KNOB_STATUS_HISTORY_LIMIT = 25; function numberOrNull(value: unknown): number | null { @@ -327,6 +338,18 @@ export async function loadKnobStatus(env: Env, knob: LoosenableKnob): Promise 0) { + const curve = computeReliabilityCurve(cases); + reliability = { curve, suggestion: deriveThresholdSuggestion(curve, KNOB_SUGGESTION_TARGET_PRECISION, knob.hardMinimum) }; + } + } catch { + reliability = null; // degrade -- the endpoint must not throw on a read blip + } + const applied: KnobAppliedEntry[] = []; try { const rows = await env.DB.prepare("SELECT created_at, metadata_json FROM audit_events WHERE event_type = ? ORDER BY created_at DESC LIMIT ?") @@ -364,6 +387,7 @@ export async function loadKnobStatus(env: Env, knob: LoosenableKnob): Promise { expect(corrupt.applied[0]!.proposedValue).toBeNull(); }); + it("reliability view (#8227): curve + derived suggestion ride the status; empty corpus and read blips degrade to null", async () => { + expect(KNOB_SUGGESTION_TARGET_PRECISION).toBe(0.9); + const empty = await loadKnobStatus(createTestEnv(), AI_KNOB); + expect(empty.reliability).toBeNull(); // no cases at all -> no curve, never a fake one + + const env = enabledEnv(); + await seedAiLooseningFriendlyHistory(env); + const status = await loadKnobStatus(env, AI_KNOB); + expect(status.reliability).not.toBeNull(); + expect(status.reliability!.curve.buckets.length).toBeGreaterThan(0); + const decided = status.reliability!.curve.buckets.reduce((sum, b) => sum + b.cases, 0); + expect(decided).toBeGreaterThan(0); + // The suggestion, when present, respects the knob's own hard minimum — same bound as every evaluator. + if (status.reliability!.suggestion !== null) { + expect(status.reliability!.suggestion).toBeGreaterThanOrEqual(AI_KNOB.hardMinimum); + } + }); + it("degrades on a broken DB (null override, empty history) and lists every live registry knob", async () => { const broken = createTestEnv(); broken.DB = { prepare: () => { throw new Error("boom"); } } as never; const status = await loadKnobStatus(broken, AI_KNOB); expect(status).toMatchObject({ storedOverride: null, applied: [], liveValue: AI_KNOB.shippedValue, drift: null }); + expect(status).toMatchObject({ storedOverride: null, applied: [], liveValue: AI_KNOB.shippedValue, reliability: null }); const statuses = await loadLiveKnobStatuses(createTestEnv()); expect(statuses.map((s) => s.knobId).sort()).toEqual(["ai_review_close_confidence", "satisfaction_floor"]); diff --git a/test/unit/loosening-knobs.test.ts b/test/unit/loosening-knobs.test.ts index 2e82acbb8..174b2d141 100644 --- a/test/unit/loosening-knobs.test.ts +++ b/test/unit/loosening-knobs.test.ts @@ -12,7 +12,7 @@ import { } from "../../src/services/satisfaction-floor-loosening"; import { LINKED_ISSUE_SATISFACTION_CONFIDENCE_FLOOR } from "../../src/services/linked-issue-satisfaction"; import { DEFAULT_AI_REVIEW_CLOSE_CONFIDENCE } from "../../src/rules/advisory"; -import { buildReportOnlyKnobRecs } from "../../src/review/loosening-recs"; +import { buildKnobReliabilityRecs, buildReportOnlyKnobRecs } from "../../src/review/loosening-recs"; const AI_KNOB = LOOSENABLE_KNOBS.ai_review_close_confidence!; @@ -121,6 +121,29 @@ describe("evaluateKnobLoosening on the close-confidence knob (#8159)", () => { }); }); +describe("buildKnobReliabilityRecs (#8227)", () => { + const status = (over: Partial<{ knobId: string; liveValue: number; reliability: { suggestion: number | null } | null }> = {}) => ({ + knobId: "ai_review_close_confidence", + liveValue: 0.93, + reliability: { suggestion: 0.9 }, + ...over, + }); + + it("emits one info rec per live knob whose derived suggestion differs from live — and nothing otherwise", () => { + const recs = buildKnobReliabilityRecs([ + status(), // differs -> rec + status({ knobId: "satisfaction_floor", liveValue: 0.5, reliability: { suggestion: 0.5 } }), // equal -> silent + status({ knobId: "third", reliability: { suggestion: null } }), // no suggestion -> silent + status({ knobId: "fourth", reliability: null }), // no curve -> silent + ]); + expect(recs).toHaveLength(1); + expect(recs[0]!.project).toBe("global:ai_review_close_confidence"); + expect(recs[0]!.severity).toBe("info"); + expect(recs[0]!.message).toContain("0.9"); + expect(recs[0]!.message).toContain("SURFACING ONLY"); + }); +}); + describe("buildReportOnlyKnobRecs (#8159)", () => { it("surfaces the evidence with the report-only action line and NEVER a payload", () => { const proposal = evaluateKnobLoosening(AI_KNOB, aiLooseningFriendlyCorpus())!;