From cb6ed76f0d4e6088c7803fc222f2ec1a517a1255 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 08:21:25 -0700 Subject: [PATCH] =?UTF-8?q?feat(calibration):=20reason-code=20enrichment?= =?UTF-8?q?=20pass=20=E2=80=94=20the=20flat-confidence=20era's=20only=20di?= =?UTF-8?q?scriminator=20(#8243)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Investigation result first: the constant-1.0 decision confidence has NO live writer to fix — those decisions came from the retired legacy content gate (review_targets decisions stopped 2026-06-22); the live path records finding-level confidences that genuinely vary. What the backfill era needs instead is segmentability: pass C copies the ledger's own decision reasonCode onto each phase-1 fired row (DB-only, idempotent, distinct provenance), separating AI-judgment closes (dual_review_declined, 48%) from deterministic ones. Applied to both stores (460/460 each) and immediately decisive: thin_description closes reversed 100% (20/20 — pure friction), checks_failed 47%, dual_review_declined 30%, strict_duplicate 15%. Per-reason reversal rates are exactly the evidence the disposition and drift work (#8211 tracks A/D) consume. Closes #8243 --- ...backfill-calibration-corpus-phase2-core.ts | 21 +++++++++- scripts/backfill-calibration-corpus-phase2.ts | 42 +++++++++++++++++-- ...backfill-calibration-corpus-phase2.test.ts | 20 +++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/scripts/backfill-calibration-corpus-phase2-core.ts b/scripts/backfill-calibration-corpus-phase2-core.ts index a44e8b6ad..3f9c2ed4f 100644 --- a/scripts/backfill-calibration-corpus-phase2-core.ts +++ b/scripts/backfill-calibration-corpus-phase2-core.ts @@ -20,6 +20,11 @@ export const RETRO_SUCCESSOR_PROVENANCE = "github_successor_scan"; export const RETRO_SAME_PR_MERGED_PROVENANCE = "github_same_pr_merged"; /** Distinct provenance for pass B's re-fetched raw context. */ export const RAW_CONTEXT_REFETCH_PROVENANCE = "github_raw_context_refetch"; +/** Provenance for pass C's reason-code enrichment (#8243): the ledger's own decision reasonCode copied + * onto the fired row so AI-judgment closes (dual_review_declined) are segmentable from deterministic + * ones — the backfill era's confidence axis is flat (a constant 1.0 from the retired legacy writer), + * so reason class is the only within-era discriminator the corpus has. */ +export const REASON_CODE_ENRICHMENT_PROVENANCE = "review_targets_reason_code"; /** A phase-1 backfilled close decision, hydrated with the GitHub truth the wrapper fetched. */ export type HistoricalCloseSide = { @@ -131,7 +136,7 @@ export function patchFiredMetadataWithDiff(metadataJson: string, diff: string): } export type Phase2Report = { - pass: "successors" | "raw-context"; + pass: "successors" | "raw-context" | "reason-codes"; scanned: number; patched: number; alreadyPatched: number; @@ -153,7 +158,7 @@ export type Phase2Report = { export function renderPhase2Report(report: Phase2Report, mode: "dry-run" | "apply"): string { const lines = [ `Calibration corpus backfill phase 2 (${mode}) — pass ${report.pass}, provenance ${ - report.pass === "successors" ? RETRO_SUCCESSOR_PROVENANCE : RAW_CONTEXT_REFETCH_PROVENANCE + report.pass === "successors" ? RETRO_SUCCESSOR_PROVENANCE : report.pass === "raw-context" ? RAW_CONTEXT_REFETCH_PROVENANCE : REASON_CODE_ENRICHMENT_PROVENANCE }`, ` scanned: ${report.scanned} patched: ${report.patched} already-patched: ${report.alreadyPatched} no-match/skipped: ${report.noMatch}`, ...(report.pass === "successors" @@ -167,6 +172,18 @@ export function renderPhase2Report(report: Phase2Report, mode: "dry-run" | "appl return lines.join("\n"); } +/** + * Patch a phase-1 fired row's metadata with the ledger's own decision reasonCode (#8243). Idempotent + * (already-tagged rows return null) and never guesses (unparseable metadata or a blank code return null). + */ +export function patchFiredMetadataWithReasonCode(metadataJson: string, reasonCode: string): string | null { + const metadata = parseObject(metadataJson); + if (!metadata) return null; + if (typeof metadata.reasonCode === "string") return null; + if (reasonCode.trim() === "") return null; + return JSON.stringify({ ...metadata, reasonCode: reasonCode.trim(), reasonCodeProvenance: REASON_CODE_ENRICHMENT_PROVENANCE }); +} + function parseObject(json: string): Record | null { try { const parsed: unknown = JSON.parse(json); diff --git a/scripts/backfill-calibration-corpus-phase2.ts b/scripts/backfill-calibration-corpus-phase2.ts index 9b3096555..004952805 100644 --- a/scripts/backfill-calibration-corpus-phase2.ts +++ b/scripts/backfill-calibration-corpus-phase2.ts @@ -24,6 +24,7 @@ import { patchFiredMetadataWithDiff, patchOverrideMetadataToReversed, patchOverrideMetadataToSamePrMerged, + patchFiredMetadataWithReasonCode, renderPhase2Report, type HistoricalCloseSide, type Phase2Report, @@ -31,7 +32,7 @@ import { } from "./backfill-calibration-corpus-phase2-core.js"; import { BACKFILL_RULE_ID } from "./backfill-calibration-corpus-core.js"; -type Pass = "successors" | "raw-context"; +type Pass = "successors" | "raw-context" | "reason-codes"; type Args = { db: string; remote: boolean; @@ -61,7 +62,7 @@ function parseArgs(argv: string[]): Args { } else if (flag === "--pass") { const value = argv[++i]; - if (value !== "successors" && value !== "raw-context") throw new Error(`--pass must be successors or raw-context, got ${value}`); + if (value !== "successors" && value !== "raw-context" && value !== "reason-codes") throw new Error(`--pass must be successors, raw-context, or reason-codes, got ${value}`); args.pass = value; } else if (flag === "--max-requests") args.maxRequests = Number(argv[++i]); else if (flag === "--state-file") args.stateFile = argv[++i]!; @@ -461,6 +462,39 @@ async function runRawContextPass(args: Args, budget: RequestBudget, state: Curso return report; } +/** Pass C (#8243): copy the ledger's own decision reasonCode onto each phase-1 fired row — DB-only, no + * GitHub. The backfill era's confidence axis is flat (constant 1.0 from the retired legacy writer), so + * reason class (AI-judgment dual_review_declined vs deterministic codes) is the era's only + * within-corpus discriminator. Idempotent via the patcher's already-tagged null. */ +async function runReasonCodesPass(args: Args): Promise { + const report: Phase2Report = { pass: "reason-codes", scanned: 0, patched: 0, alreadyPatched: 0, noMatch: 0, matchedSameAuthor: 0, matchedSharedIssueOnly: 0, matchedSamePrMerged: 0, requestsUsed: 0, exhaustedBudget: false, resumeFrom: null }; + const decisionRows = await executeSql( + args, + `SELECT repo, number, json_extract(decision_json, '$.reasonCode') AS reason FROM review_targets WHERE kind = 'pull_request' AND decision_json IS NOT NULL`, + ); + const reasonByTarget = new Map(); + for (const row of decisionRows) { + if (typeof row.repo === "string" && typeof row.reason === "string" && row.reason !== "") { + reasonByTarget.set(`${row.repo}#${row.number}`, row.reason); + } + } + for (const row of await loadBackfillRows(args, "fired")) { + report.scanned += 1; + const reason = reasonByTarget.get(row.target_key); + if (!reason) { + report.noMatch += 1; + continue; + } + const patched = patchFiredMetadataWithReasonCode(row.metadata_json, reason); + if (patched === null) report.alreadyPatched += 1; + else if (args.apply) { + await applyMetadataUpdate(args, backfillFiredId(row.target_key), patched); + report.patched += 1; + } else report.patched += 1; + } + return report; +} + async function main(): Promise { const args = parseArgs(process.argv.slice(2)); const pgConnection = resolvePgConnection(args.pgPresent, args.pgValue, process.env.DATABASE_URL); @@ -473,7 +507,9 @@ async function main(): Promise { ? args.planIn ? await runSuccessorsFromPlan(args) : await runSuccessorsPass(args, budget, state) - : await runRawContextPass(args, budget, state); + : args.pass === "raw-context" + ? await runRawContextPass(args, budget, state) + : await runReasonCodesPass(args); await pgSession?.close(); writeFileSync(args.stateFile, `${JSON.stringify(state, null, 2)}\n`); console.log(renderPhase2Report(report, args.apply ? "apply" : "dry-run")); diff --git a/test/unit/backfill-calibration-corpus-phase2.test.ts b/test/unit/backfill-calibration-corpus-phase2.test.ts index 4b1ca7019..921e8eaf0 100644 --- a/test/unit/backfill-calibration-corpus-phase2.test.ts +++ b/test/unit/backfill-calibration-corpus-phase2.test.ts @@ -8,9 +8,11 @@ import { patchFiredMetadataWithDiff, patchOverrideMetadataToReversed, patchOverrideMetadataToSamePrMerged, + patchFiredMetadataWithReasonCode, renderPhase2Report, RETRO_SUCCESSOR_PROVENANCE, RETRO_SAME_PR_MERGED_PROVENANCE, + REASON_CODE_ENRICHMENT_PROVENANCE, RAW_CONTEXT_REFETCH_PROVENANCE, type HistoricalCloseSide, type Phase2Report, @@ -131,6 +133,22 @@ describe("metadata patchers (#8170)", () => { }); }); +describe("patchFiredMetadataWithReasonCode (#8243)", () => { + it("tags the fired row with the ledger's reasonCode + provenance, exactly once, never guessing", () => { + const original = JSON.stringify({ confidence: 1, backfilled: true }); + const patched = JSON.parse(patchFiredMetadataWithReasonCode(original, "dual_review_declined")!) as Record; + expect(patched.reasonCode).toBe("dual_review_declined"); + expect(patched.reasonCodeProvenance).toBe(REASON_CODE_ENRICHMENT_PROVENANCE); + expect(patched.backfilled).toBe(true); // phase-1 fields survive + // Idempotent + never-guess arms. + expect(patchFiredMetadataWithReasonCode(JSON.stringify(patched), "checks_failed")).toBeNull(); + expect(patchFiredMetadataWithReasonCode(original, " ")).toBeNull(); + expect(patchFiredMetadataWithReasonCode("not-json", "checks_failed")).toBeNull(); + // Whitespace-trimmed code. + expect((JSON.parse(patchFiredMetadataWithReasonCode(original, " scope_failure ")!) as Record).reasonCode).toBe("scope_failure"); + }); +}); + describe("ids + report rendering (#8170)", () => { it("derives the deterministic phase-1 row ids (the only rows the passes may touch)", () => { expect(backfillOverrideId("acme/widgets#7")).toBe("backfill:ai_consensus_defect:acme/widgets#7:override"); @@ -149,6 +167,8 @@ describe("ids + report rendering (#8170)", () => { "apply", ); expect(exhausted).toContain(RAW_CONTEXT_REFETCH_PROVENANCE); + const reasonPass = renderPhase2Report({ ...base, pass: "reason-codes" }, "dry-run"); + expect(reasonPass).toContain(REASON_CODE_ENRICHMENT_PROVENANCE); expect(exhausted).toContain("budget exhausted"); expect(exhausted).toContain("resume from: acme/widgets#7"); });