From 9d92d3d36eeb103ba3485499d06c37f9e106bc3a Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 08:25:12 -0700 Subject: [PATCH] =?UTF-8?q?feat(calibration):=20per-repo=20knob=20override?= =?UTF-8?q?s=20=E2=80=94=20storage=20scheme,=20resolution=20seam,=20status?= =?UTF-8?q?=20listing=20(#8216)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Epic #8211 track B: repos can now carry individually-earned knob values. - Storage: one system_flags key per (knob, repo) — :repo: beside the global key; migration-free on the schemaless flag table and enumerable with one LIKE for the status surface. - ONE resolution seam (getKnobOverrideForRepo) with the documented precedence: explicit yml setting > per-repo earned > global earned > shipped. Identical loosening-only validation per scope; the knob's autotune flag gates every scope — off restores shipped everywhere instantly. The gate-policy threading gains the repo dimension at all four call sites; the pure twins are untouched (they still consume one resolved number). - GET /v1/internal/calibration/knobs lists each knob's validated per-repo overrides, sorted, invalid rows excluded — a lingering row is always visible to the operator. - Docs: the precedence table on the backtest-calibration page. Evaluation stays global-only by design — the per-repo LOOP is #8217. Closes #8216 --- .../content/docs/backtest-calibration.mdx | 17 +++++ src/queue/processors.ts | 9 +-- src/services/knob-loosening-run.ts | 64 +++++++++++++++++-- test/unit/knob-loosening-run.test.ts | 32 ++++++++++ 4 files changed, 113 insertions(+), 9 deletions(-) diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx index 38bfe1f65..aede3e6e7 100644 --- a/apps/loopover-ui/content/docs/backtest-calibration.mdx +++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx @@ -80,6 +80,23 @@ npx tsx scripts/backtest-corpus-export.ts --rule-id --output corpus.jso Both CLIs are strictly read-only against the database. +## Override precedence (per-repo autonomy) + +Earned knob overrides resolve through one seam with one precedence order — most specific wins, and an +explicit configuration choice always beats anything the system earned on its own: + +| Priority | Source | Where it lives | +| --- | --- | --- | +| 1 | Explicit per-repo `.loopover.yml` setting (e.g. `gate.aiReview.closeConfidence`) | config-as-code, resolved into settings | +| 2 | Per-repo **earned** override | `system_flags` key `:repo:` | +| 3 | Global **earned** override | `system_flags` key `` | +| 4 | Shipped default | the constant beside the rule | + +Every earned scope applies the identical loosening-only validation (strictly below shipped, at or above +the hard minimum), and the knob's autotune flag gates *all* earned scopes at once — turning the flag off +restores shipped behavior everywhere instantly, no cleanup. The knobs status endpoint lists each knob's +per-repo overrides so a lingering row is always visible. + ## Counterfactual replay (design) Deterministic rule changes replay against history before they land; the same discipline is being diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 600879a82..2fc2f399a 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -1542,7 +1542,7 @@ export async function sweepRepoRegate( // unchanged. // #8176: the global close-confidence default-override, resolved once for the sweep (same value the main // webhook path threads; an explicit per-repo setting still wins inside gateCheckPolicy). - const sweepCloseConfidenceOverride = await getAiReviewCloseConfidenceOverride(env); + const sweepCloseConfidenceOverride = await getAiReviewCloseConfidenceOverride(env, repoFullName); for (const [index, pr] of candidates.entries()) { const others = openPullRequests.filter( (other) => other.number !== pr.number, @@ -10387,7 +10387,8 @@ async function maybePublishPrPublicSurface( authorHistory, gateSizeContext, // #8176: backtest-gated global default for the close-confidence floor (explicit per-repo wins inside). - await getAiReviewCloseConfidenceOverride(env), + // #8216: the repo's own earned override outranks the global one; explicit settings still win inside. + await getAiReviewCloseConfidenceOverride(env, repoFullName), ); gateEvaluation = await withReviewPipelineSpan( "selfhost.review.gate", @@ -11880,7 +11881,7 @@ async function maybeProcessResolveCommand(env: Env, deliveryId: string, payload: if (!findingRef.ok) { await recordAuditEvent(env, { eventType: "github_app.finding_resolved_skipped", actor: req.actor, targetKey, outcome: "completed", detail: findingRef.reason, metadata: { deliveryId, repoFullName: req.repoFullName, reason: findingRef.reason } }); await recordGithubProductUsage(env, "finding_resolved_skipped", { actor: req.actor, repoFullName: req.repoFullName, targetKey, outcome: "skipped", metadata: { reason: findingRef.reason } }); return true; } const { advisory } = await buildAuthorizedPrActionAdvisory(env, req.repoFullName, pr, settings); await appendPublishedAiReviewFindingsForResolve(env, req.repoFullName, pr, settings.aiReviewMode, advisory); - const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, await getAiReviewCloseConfidenceOverride(env))); + const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, await getAiReviewCloseConfidenceOverride(env, req.repoFullName))); const selection = selectWarningsForResolve(gate.warnings, findingRef); if (selection.reason === "finding_not_found") { await recordAuditEvent(env, { eventType: "github_app.finding_resolved_skipped", actor: req.actor, targetKey, outcome: "completed", detail: selection.reason, metadata: { deliveryId, repoFullName: req.repoFullName, reason: selection.reason } }); await recordGithubProductUsage(env, "finding_resolved_skipped", { actor: req.actor, repoFullName: req.repoFullName, targetKey, outcome: "skipped", metadata: { reason: selection.reason } }); return true; } const mode = resolveAgentActionMode({ globalPaused: isGlobalAgentPause(env) || (await isGlobalAgentFrozen(env)), agentPaused: settings.agentPaused, agentDryRun: settings.agentDryRun }); @@ -12111,7 +12112,7 @@ async function maybeProcessExplainCommand(env: Env, deliveryId: string, payload: } const { advisory } = await buildAuthorizedPrActionAdvisory(env, req.repoFullName, pr, settings); await appendPublishedAiReviewFindingsForResolve(env, req.repoFullName, pr, settings.aiReviewMode, advisory); - const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, await getAiReviewCloseConfidenceOverride(env))); + const gate = evaluateGateCheck(advisory, gateCheckPolicy(settings, null, undefined, pr.slopRisk ?? null, undefined, undefined, await getAiReviewCloseConfidenceOverride(env, req.repoFullName))); const selection = selectWarningsForResolve(gate.warnings, findingRef); if (selection.reason === "finding_not_found") { const notFound = sanitizePublicComment([AGENT_COMMAND_COMMENT_MARKER, "", "> [!NOTE]", `> **No review finding \`${findingRef.findingCode}\` on this PR**`, "> That id is not among this PR's current review findings — re-run `@loopover explain ` with an id from the review summary.", "", "---", loopoverFooter(env)].join("\n")); diff --git a/src/services/knob-loosening-run.ts b/src/services/knob-loosening-run.ts index 325494088..a66afa123 100644 --- a/src/services/knob-loosening-run.ts +++ b/src/services/knob-loosening-run.ts @@ -42,8 +42,38 @@ export function isKnobAutotuneEnabled(env: Env, knob: LoosenableKnob): boolean { */ export async function getKnobOverride(env: Env, knob: LoosenableKnob): Promise { if (!isKnobAutotuneEnabled(env, knob)) return null; + return readValidatedOverrideRow(env, knob, knob.overrideFlagKey); +} + +/** Per-repo override storage (#8216): one system_flags key per (knob, repo) beside the global key. The + * repo rides inside the key — migration-free on the schemaless flag table, and trivially enumerable + * with one LIKE for the status surface. */ +export function repoKnobOverrideFlagKey(knob: LoosenableKnob, repoFullName: string): string { + return `${knob.overrideFlagKey}:repo:${repoFullName}`; +} + +/** + * The EARNED-override resolution seam (#8216) — one function, one precedence order: + * explicit per-repo `.loopover.yml` setting (resolved upstream into settings; callers apply it FIRST + * via the `settings.x ?? override` chain in gateCheckPolicy — it never reaches this function) + * > per-repo earned override (this function, when `repoFullName` is given and its row validates) + * > global earned override (this function's fallback) + * > shipped default (the caller's final ?? in the pure twins). + * Validation is identical per scope (strictly below shipped, at/above the hard minimum), and the knob's + * autotune flag gates EVERY scope — flipping it off restores shipped behavior everywhere instantly. + */ +export async function getKnobOverrideForRepo(env: Env, knob: LoosenableKnob, repoFullName: string | null): Promise { + if (!isKnobAutotuneEnabled(env, knob)) return null; + if (repoFullName !== null) { + const repoValue = await readValidatedOverrideRow(env, knob, repoKnobOverrideFlagKey(knob, repoFullName)); + if (repoValue !== null) return repoValue; + } + return readValidatedOverrideRow(env, knob, knob.overrideFlagKey); +} + +async function readValidatedOverrideRow(env: Env, knob: LoosenableKnob, key: string): Promise { try { - const row = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?").bind(knob.overrideFlagKey).first<{ value: string }>(); + const row = await env.DB.prepare("SELECT value FROM system_flags WHERE key = ?").bind(key).first<{ value: string }>(); if (!row) return null; const parsed = Number(row.value); if (!Number.isFinite(parsed) || parsed >= knob.shippedValue || parsed < knob.hardMinimum) return null; @@ -53,10 +83,11 @@ export async function getKnobOverride(env: Env, knob: LoosenableKnob): Promise { - return getKnobOverride(env, LOOSENABLE_KNOBS.ai_review_close_confidence!); +/** The #8176 consumption read: the validated default-override for the AI close-confidence floor. + * Threaded into gateCheckPolicy as its LAST-resort default — an explicit per-repo setting always wins. + * With a `repoFullName` (#8216) the repo's own earned override outranks the global one. */ +export async function getAiReviewCloseConfidenceOverride(env: Env, repoFullName: string | null = null): Promise { + return getKnobOverrideForRepo(env, LOOSENABLE_KNOBS.ai_review_close_confidence!, repoFullName); } export type KnobLooseningRunResult = @@ -142,6 +173,8 @@ export type KnobAppliedEntry = { heldOutVerdict: string | null; }; +export type KnobRepoOverride = { repoFullName: string; value: number }; + export type KnobStatus = { knobId: string; flagEnabled: boolean; @@ -152,6 +185,9 @@ export type KnobStatus = { /** The RAW stored override row (validated), reported even when the flag is off — an operator needs to * see a lingering row that would take effect the moment the flag flips. */ storedOverride: number | null; + /** Per-repo earned overrides (#8216), validated rows only, sorted by repo — an operator must see every + * scope that would take effect the moment the flag is on. */ + repoOverrides: KnobRepoOverride[]; applied: KnobAppliedEntry[]; }; @@ -187,6 +223,23 @@ export async function loadKnobStatus(env: Env, knob: LoosenableKnob): Promise(); + /* v8 ignore next -- same defined-results note as the applied-history read below. */ + for (const row of rows.results ?? []) { + const parsed = Number(row.value); + if (!Number.isFinite(parsed) || parsed >= knob.shippedValue || parsed < knob.hardMinimum) continue; + repoOverrides.push({ repoFullName: row.key.slice(prefix.length), value: parsed }); + } + repoOverrides.sort((a, b) => a.repoFullName.localeCompare(b.repoFullName)); + } catch { + /* degrade to an empty listing -- the endpoint must not throw on a read blip */ + } + const applied: KnobAppliedEntry[] = []; try { const rows = await env.DB.prepare("SELECT created_at, metadata_json FROM audit_events WHERE event_type = ? ORDER BY created_at DESC LIMIT ?") @@ -222,6 +275,7 @@ export async function loadKnobStatus(env: Env, knob: LoosenableKnob): Promise } }); + it("per-repo overrides (#8216): the repo's earned row outranks global, invalid repo rows fall through, flag-off zeroes every scope", async () => { + const env = enabledEnv(); + await setOverrideRow(env, AI_KNOB.overrideFlagKey, "0.9"); // global + await setOverrideRow(env, repoKnobOverrideFlagKey(AI_KNOB, "acme/widgets"), "0.85"); // repo-earned + await setOverrideRow(env, repoKnobOverrideFlagKey(AI_KNOB, "acme/broken"), "0.99"); // invalid: above shipped + + // Repo row wins for its repo; other repos inherit global; invalid repo row falls through to global. + expect(await getKnobOverrideForRepo(env, AI_KNOB, "acme/widgets")).toBe(0.85); + expect(await getKnobOverrideForRepo(env, AI_KNOB, "acme/other")).toBe(0.9); + expect(await getKnobOverrideForRepo(env, AI_KNOB, "acme/broken")).toBe(0.9); + // Null repo = the plain global read; convenience wrapper threads the repo. + expect(await getKnobOverrideForRepo(env, AI_KNOB, null)).toBe(0.9); + expect(await getAiReviewCloseConfidenceOverride(env, "acme/widgets")).toBe(0.85); + expect(await getAiReviewCloseConfidenceOverride(env)).toBe(0.9); + + // The knob's flag gates EVERY scope. + const off = { ...env, AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: "false" as never } as Env; + expect(await getKnobOverrideForRepo(off, AI_KNOB, "acme/widgets")).toBeNull(); + }); + it("fails safe (null) when the flag-store read throws", async () => { const env = enabledEnv(); env.DB = { prepare: () => { throw new Error("boom"); } } as never; @@ -261,6 +283,7 @@ describe("loadKnobStatus / loadLiveKnobStatuses (#8161 generalized)", () => { await setOverrideRow(env, AI_KNOB.overrideFlagKey, "0.9"); const off = await loadKnobStatus(env, AI_KNOB); expect(off).toMatchObject({ knobId: "ai_review_close_confidence", flagEnabled: false, storedOverride: 0.9, liveValue: AI_KNOB.shippedValue }); + expect(off.repoOverrides).toEqual([]); const on = await loadKnobStatus({ ...env, AI_REVIEW_CLOSE_CONFIDENCE_AUTOTUNE_ENABLED: "true" as never } as Env, AI_KNOB); expect(on).toMatchObject({ flagEnabled: true, liveValue: 0.9 }); @@ -268,6 +291,15 @@ describe("loadKnobStatus / loadLiveKnobStatuses (#8161 generalized)", () => { // An out-of-bounds row is reported as no override at all (same validation as the consumption read). await setOverrideRow(env, AI_KNOB.overrideFlagKey, "0.99"); expect((await loadKnobStatus(env, AI_KNOB)).storedOverride).toBeNull(); + + // Per-repo listing (#8216): validated rows only, sorted by repo, invalid rows silently excluded. + await setOverrideRow(env, repoKnobOverrideFlagKey(AI_KNOB, "zeta/repo"), "0.9"); + await setOverrideRow(env, repoKnobOverrideFlagKey(AI_KNOB, "acme/widgets"), "0.85"); + await setOverrideRow(env, repoKnobOverrideFlagKey(AI_KNOB, "bad/row"), "not-a-number"); + expect((await loadKnobStatus(env, AI_KNOB)).repoOverrides).toEqual([ + { repoFullName: "acme/widgets", value: 0.85 }, + { repoFullName: "zeta/repo", value: 0.9 }, + ]); }); it("projects applied history from the knob's events — reading BOTH proposal spellings — and keeps corrupt rows visible as nulls", async () => {