From 66f414de74d4551dacf9a7bd0442013d6d1689c1 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:20:33 -0700 Subject: [PATCH] feat(miner): AMS min-rank calibration loop over the local event ledger (#8184, #8185, #8186, #8187) The full #8172 arc in one coherent change, transposing the ORB backtest discipline onto the miner's own ledgers with the engine primitives reused untouched: - engine: buildAmsRankCorpus + runAmsMinRankBacktest (fixed-seed split, symmetric Pareto floor via compareBacktestScores; closed take = reversed, merged = confirmed, rank score as replay confidence) - pr_outcome rows now carry the claimed issueNumber so the corpus can join discovery-time rank records to realized outcomes (older rows never join) - calibration backtest-threshold --candidate: advisory replay, shared renderer output, persisted ams_threshold_backtest_run events; exit never reflects verdict - calibration report: REGRESSED-verdict track record (shared engine aggregation) + backtest-cleared proposals with the explicit no-autonomy line; doctor mirrors the proposals; the Phase-7 snapshot payload gains an optional backtestTrackRecord section - double-gated min-rank self-adjustment: .loopover-ams.yml minRankAutotuneEnabled (default OFF) AND per-apply --approve, hard bounds (0, 0.5] validated on every read, evidence required per apply, typed apply/revert ledger events, one-command revert; consumed at discover's single enqueue point, fail-open to shipped --- .../loopover-engine/src/ams-policy-spec.ts | 22 +- .../src/calibration/ams-rank-corpus.ts | 107 ++++++ packages/loopover-engine/src/index.ts | 1 + .../test/ams-rank-corpus.test.ts | 30 ++ .../docs/miner-selfimprove-calibration.md | 24 ++ .../loopover-miner/lib/ams-calibration.ts | 338 ++++++++++++++++++ .../loopover-miner/lib/calibration-cli.ts | 154 +++++++- .../loopover-miner/lib/calibration-run.ts | 19 + packages/loopover-miner/lib/cli.ts | 3 + packages/loopover-miner/lib/discover-cli.ts | 24 +- packages/loopover-miner/lib/loop-cli.ts | 3 + packages/loopover-miner/lib/pr-outcome.ts | 8 + packages/loopover-miner/lib/status.ts | 29 +- test/unit/ams-policy-spec-parser.test.ts | 14 + test/unit/ams-rank-corpus-engine.test.ts | 91 +++++ test/unit/miner-ams-calibration.test.ts | 218 +++++++++++ test/unit/miner-calibration-cli.test.ts | 145 +++++++- test/unit/miner-calibration-run.test.ts | 47 +++ test/unit/miner-discover-cli.test.ts | 54 +++ test/unit/miner-loop-cli.test.ts | 2 + test/unit/miner-orb-export.test.ts | 6 +- test/unit/miner-pr-outcome.test.ts | 20 +- test/unit/miner-status.test.ts | 40 +++ 23 files changed, 1382 insertions(+), 17 deletions(-) create mode 100644 packages/loopover-engine/src/calibration/ams-rank-corpus.ts create mode 100644 packages/loopover-engine/test/ams-rank-corpus.test.ts create mode 100644 packages/loopover-miner/lib/ams-calibration.ts create mode 100644 test/unit/ams-rank-corpus-engine.test.ts create mode 100644 test/unit/miner-ams-calibration.test.ts diff --git a/packages/loopover-engine/src/ams-policy-spec.ts b/packages/loopover-engine/src/ams-policy-spec.ts index 21cf14f27d..f437c28565 100644 --- a/packages/loopover-engine/src/ams-policy-spec.ts +++ b/packages/loopover-engine/src/ams-policy-spec.ts @@ -105,6 +105,10 @@ export type AmsPolicySpec = { * -- no additions beyond the always-on OS-registry/git-remote defaults. INERT until #7857's OS-level * enforcement mechanism is built; see {@link AmsNetworkAllowlist}'s own doc comment. */ networkAllowlist: AmsNetworkAllowlist; + /** Whether the min-rank skip threshold may self-adjust from backtest evidence (#8187, epic #8172). The + * FIRST of the double gates: with this OFF (the default) the apply/revert commands refuse and any + * previously-applied override reads as absent; the second gate is the per-apply `--approve` flag. */ + minRankAutotuneEnabled: boolean; }; /** The tolerant parser result for `.loopover-ams.yml`. Mirrors `ParsedMinerGoalSpec`'s present/warnings shape. */ @@ -127,6 +131,7 @@ export const DEFAULT_AMS_POLICY_SPEC: Readonly = Object.freeze({ maxTurnsPerIteration: 6, selfLoopAutonomy: "auto", networkAllowlist: Object.freeze({ ecosystems: [], extraHosts: [] }), + minRankAutotuneEnabled: false, }); const MAX_AMS_POLICY_SPEC_BYTES = 8_192; @@ -144,6 +149,7 @@ function cloneDefaultAmsPolicySpec(): AmsPolicySpec { ecosystems: [...DEFAULT_AMS_POLICY_SPEC.networkAllowlist.ecosystems], extraHosts: [...DEFAULT_AMS_POLICY_SPEC.networkAllowlist.extraHosts], }, + minRankAutotuneEnabled: DEFAULT_AMS_POLICY_SPEC.minRankAutotuneEnabled, }; } @@ -151,6 +157,13 @@ function emptyAmsPolicySpec(warnings: string[] = []): ParsedAmsPolicySpec { return { present: false, spec: cloneDefaultAmsPolicySpec(), warnings }; } +function normalizeBooleanFlag(value: unknown, field: string, fallback: boolean, warnings: string[]): boolean { + if (value === undefined || value === null) return fallback; + if (typeof value === "boolean") return value; + warnings.push(`AmsPolicySpec field "${field}" must be a boolean; falling back to ${fallback}.`); + return fallback; +} + function normalizeSubmissionMode(value: unknown, fallback: AmsSubmissionMode, warnings: string[]): AmsSubmissionMode { if (value === undefined || value === null) return fallback; if (value === "observe" || value === "enforce") return value; @@ -307,7 +320,8 @@ function hasConfiguredPolicyFields(spec: AmsPolicySpec): boolean { // means the operator configured something, so length alone is the right "differs from default" check; // no need to compare contents. spec.networkAllowlist.ecosystems.length > 0 || - spec.networkAllowlist.extraHosts.length > 0 + spec.networkAllowlist.extraHosts.length > 0 || + spec.minRankAutotuneEnabled !== DEFAULT_AMS_POLICY_SPEC.minRankAutotuneEnabled ); } @@ -356,6 +370,12 @@ export function parseAmsPolicySpec(raw: unknown): ParsedAmsPolicySpec { warnings, ), networkAllowlist: normalizeNetworkAllowlist(record.networkAllowlist, DEFAULT_AMS_POLICY_SPEC.networkAllowlist, warnings), + minRankAutotuneEnabled: normalizeBooleanFlag( + record.minRankAutotuneEnabled, + "minRankAutotuneEnabled", + DEFAULT_AMS_POLICY_SPEC.minRankAutotuneEnabled, + warnings, + ), }; if (!hasConfiguredPolicyFields(spec)) { warnings.push("AmsPolicySpec contained no recognized non-default policy fields; falling back to safe defaults."); diff --git a/packages/loopover-engine/src/calibration/ams-rank-corpus.ts b/packages/loopover-engine/src/calibration/ams-rank-corpus.ts new file mode 100644 index 0000000000..d63ed7782b --- /dev/null +++ b/packages/loopover-engine/src/calibration/ams-rank-corpus.ts @@ -0,0 +1,107 @@ +// AMS min-rank corpus + advisory backtest (#8184, epic #8172 phase 2) -- the miner-side twin of the ORB +// threshold backtest (#8138), over the miner's OWN taken-opportunity history instead of ORB signal events. +// The opportunity ranker's score is an equal-weight clamped product in [0, 1] (opportunity-ranker.ts -- no +// scalar weights), so the SKIP THRESHOLD is the tunable, and the counterfactual question is: "had the +// min-rank floor been X, which taken opportunities would have been skipped, and were those the ones that +// went badly?" Labels come from realized outcomes: a MERGED take was a good take (label "confirmed"); a +// CLOSED take was a bad one -- skipping it would have been right (label "reversed"). That polarity lines +// up exactly with buildConfidenceThresholdClassifier's positive class ("predicted reversed" when the score +// sits below the threshold), so the whole ORB replay stack -- splitBacktestCorpus, runThresholdBacktest's +// scoreBacktest + compareBacktestScores Pareto floor -- is reused untouched, zero new math. +// +// Same purity contract as the rest of this module family: no IO, no randomness, no wall-clock reads. + +import type { BacktestCase } from "./backtest-corpus.js"; +import type { BacktestComparison } from "./backtest-compare.js"; +import { runThresholdBacktest } from "./backtest-threshold.js"; +import { splitBacktestCorpus } from "./backtest-split.js"; + +/** The synthetic rule id AMS min-rank replay cases carry (namespaced away from every ORB rule id). */ +export const AMS_MIN_RANK_RULE_ID = "ams_min_rank_skip"; + +// The #8121 split discipline transposed: a fixed seed so held-out membership never reshuffles between +// evaluations, and never-on-noise sample floors sized like the satisfaction floor's (the miner's local +// history is closer to that corpus's scale than to the AI knob's firehose). +export const AMS_MIN_RANK_SPLIT_SEED = "ams-min-rank-skip-v1"; +export const AMS_MIN_RANK_HELD_OUT_FRACTION = 0.25; +export const AMS_MIN_RANK_MIN_VISIBLE_CASES = 20; +export const AMS_MIN_RANK_MIN_HELD_OUT_CASES = 5; + +/** One taken opportunity with a realized terminal outcome -- the join of a `discovered_issue` rank record + * and the miner's own `pr_outcome` for the PR that issue produced. Assembled miner-side (the ledger join + * lives in @loopover/miner's ams-calibration module); this module only replays. */ +export type AmsTakenOpportunity = { + repoFullName: string; + issueNumber: number; + /** The ranker's clamped-product score at discovery time, in [0, 1]. */ + rankScore: number; + realizedDecision: "merged" | "closed"; + /** When the opportunity was discovered/ranked (ISO). */ + discoveredAt: string; + /** When the terminal outcome was recorded (ISO). */ + decidedAt: string; +}; + +/** + * Shape taken opportunities into {@link BacktestCase}s for the min-rank replay: `metadata.confidence` + * carries the rank score (the value the threshold classifier replays against), a CLOSED take labels + * "reversed" (skipping would have been right), a MERGED take labels "confirmed". Records with a + * non-finite or out-of-[0,1] rank score are dropped -- a case the classifier cannot honestly replay must + * not default to confidence 1. Deterministic order (repo#issue, then discoveredAt), so downstream splits + * see a stable corpus. + */ +export function buildAmsRankCorpus(takes: readonly AmsTakenOpportunity[]): BacktestCase[] { + const cases: BacktestCase[] = []; + for (const take of takes) { + if (!Number.isFinite(take.rankScore) || take.rankScore < 0 || take.rankScore > 1) continue; + cases.push({ + ruleId: AMS_MIN_RANK_RULE_ID, + targetKey: `${take.repoFullName}#issue-${take.issueNumber}`, + outcome: "take", + label: take.realizedDecision === "closed" ? "reversed" : "confirmed", + firedAt: take.discoveredAt, + decidedAt: take.decidedAt, + metadata: { confidence: take.rankScore }, + }); + } + cases.sort((left, right) => { + const key = left.targetKey.localeCompare(right.targetKey); + return key !== 0 ? key : left.firedAt.localeCompare(right.firedAt); + }); + return cases; +} + +export type AmsMinRankBacktestResult = { + ruleId: string; + currentThreshold: number; + candidateThreshold: number; + visibleCases: number; + heldOutCases: number; + visible: BacktestComparison; + heldOut: BacktestComparison; +}; + +/** + * Advisory replay of a candidate min-rank skip threshold against the taken-opportunity corpus -- the + * #8138 discipline verbatim: the fixed-seed split, then {@link runThresholdBacktest} (scoreBacktest + + * the symmetric compareBacktestScores Pareto floor, per #8184's required pattern) on EACH slice. Null -- + * never a guess -- when either slice misses its sample floor. Report-only by construction: this function + * returns comparisons; it never moves a knob. + */ +export function runAmsMinRankBacktest( + cases: readonly BacktestCase[], + currentThreshold: number, + candidateThreshold: number, +): AmsMinRankBacktestResult | null { + const { visible, heldOut } = splitBacktestCorpus(cases, AMS_MIN_RANK_HELD_OUT_FRACTION, AMS_MIN_RANK_SPLIT_SEED); + if (visible.length < AMS_MIN_RANK_MIN_VISIBLE_CASES || heldOut.length < AMS_MIN_RANK_MIN_HELD_OUT_CASES) return null; + return { + ruleId: AMS_MIN_RANK_RULE_ID, + currentThreshold, + candidateThreshold, + visibleCases: visible.length, + heldOutCases: heldOut.length, + visible: runThresholdBacktest(AMS_MIN_RANK_RULE_ID, visible, currentThreshold, candidateThreshold), + heldOut: runThresholdBacktest(AMS_MIN_RANK_RULE_ID, heldOut, currentThreshold, candidateThreshold), + }; +} diff --git a/packages/loopover-engine/src/index.ts b/packages/loopover-engine/src/index.ts index 64a9b130a5..5ed9bb76da 100644 --- a/packages/loopover-engine/src/index.ts +++ b/packages/loopover-engine/src/index.ts @@ -166,6 +166,7 @@ export * from "./calibration/signal-tracking.js"; export * from "./calibration/backtest-corpus.js"; export * from "./calibration/repo-corpus-slice.js"; export * from "./calibration/ams-prediction-corpus.js"; +export * from "./calibration/ams-rank-corpus.js"; export * from "./calibration/counterfactual-contract.js"; export * from "./calibration/counterfactual-fixtures.js"; export * from "./calibration/backtest-score.js"; diff --git a/packages/loopover-engine/test/ams-rank-corpus.test.ts b/packages/loopover-engine/test/ams-rank-corpus.test.ts new file mode 100644 index 0000000000..e095e5074a --- /dev/null +++ b/packages/loopover-engine/test/ams-rank-corpus.test.ts @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; + +import { AMS_MIN_RANK_RULE_ID, buildAmsRankCorpus, runAmsMinRankBacktest, type AmsTakenOpportunity } from "../dist/index.js"; + +function take(issueNumber: number, rankScore: number, realizedDecision: "merged" | "closed"): AmsTakenOpportunity { + return { + repoFullName: "acme/widgets", + issueNumber, + rankScore, + realizedDecision, + discoveredAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + }; +} + +test("barrel: the public entrypoint re-exports the AMS min-rank corpus + backtest (#8184)", () => { + assert.equal(typeof buildAmsRankCorpus, "function"); + assert.equal(typeof runAmsMinRankBacktest, "function"); + assert.equal(AMS_MIN_RANK_RULE_ID, "ams_min_rank_skip"); +}); + +test("runAmsMinRankBacktest: a raise that skips exactly the bad takes is improved on both slices", () => { + const takes = Array.from({ length: 60 }, (_, i) => take(i + 1, 0.15, "closed")); + takes.push(take(101, 0.4, "merged"), take(102, 0.4, "merged")); + const result = runAmsMinRankBacktest(buildAmsRankCorpus(takes), 0, 0.2); + assert.ok(result); + assert.equal(result.visible.verdict, "improved"); + assert.equal(result.heldOut.verdict, "improved"); +}); diff --git a/packages/loopover-miner/docs/miner-selfimprove-calibration.md b/packages/loopover-miner/docs/miner-selfimprove-calibration.md index fda816832e..2269abecde 100644 --- a/packages/loopover-miner/docs/miner-selfimprove-calibration.md +++ b/packages/loopover-miner/docs/miner-selfimprove-calibration.md @@ -110,6 +110,30 @@ From any contributor-safe tool in this batch: auto-tune hold-only flag, or any autonomy configuration. - **No de-anonymizing or write-capable telemetry** — local-only or HMAC-anonymized, read-only. +## The min-rank calibration loop (#8172: #8184-#8187) + +The first knob to graduate past measure-only: the **min-rank skip threshold** (portfolio-discovery's +`minRankScore`, shipped default 0). The loop is the ORB backtest discipline transposed onto the miner's own +event ledger, end to end: + +1. **Corpus** — `discovered_issue` rank records joined to the miner's own `pr_outcome` events by + `(repo, issueNumber)` (the pairing rides on outcome rows written since #8184; older rows never join). A + MERGED take labels `confirmed`, a CLOSED take labels `reversed` — "skipping it would have been right". +2. **Advisory backtest** — `loopover-miner calibration backtest-threshold --candidate `: fixed-seed + held-out split, Pareto floor via `compareBacktestScores`, rendered with the shared comparison renderer, + persisted as an `ams_threshold_backtest_run` ledger event. Exit code never reflects the verdict. +3. **Track record + proposals** — the calibration report prints the REGRESSED-verdict track record over + every persisted run (`computeRegressedVerdictTrackRecord`, the same aggregation ORB uses) and any + backtest-cleared proposals; `doctor` mirrors the proposals line. Display only. +4. **Double-gated self-adjustment (#8187)** — `calibration apply-min-rank --candidate --approve` + moves the knob ONLY when: `.loopover-ams.yml` sets `minRankAutotuneEnabled: true` (gate one, default + OFF), the per-apply `--approve` is passed (gate two), the candidate sits inside the hard bounds + `(0, 0.5]` declared next to the shipped constant, AND a recent persisted run actually cleared that exact + candidate — evidence is not optional. Every apply/revert is a typed ledger event carrying the evidence; + `calibration revert-min-rank --approve` is the one-command revert. Consumption happens at exactly one + point (discover's enqueue), re-validating bounds and the flag on every read, so flipping the flag off + restores the shipped default on the very next run. + ## The neighborhood - Contributor-safe (this batch): drift detector, calibration dashboard + extension panel, prediction ledger, metrics diff --git a/packages/loopover-miner/lib/ams-calibration.ts b/packages/loopover-miner/lib/ams-calibration.ts new file mode 100644 index 0000000000..aa75e06b23 --- /dev/null +++ b/packages/loopover-miner/lib/ams-calibration.ts @@ -0,0 +1,338 @@ +// AMS calibration loop over the miner's own event ledger (#8184/#8185/#8186/#8187, epic #8172) -- the +// miner-side transposition of ORB's backtest loop, reusing the engine primitives untouched. This module is +// the ledger seam: derive taken opportunities (a `discovered_issue` rank record joined to the miner's own +// `pr_outcome` for the issue's PR -- the pairing #8184 added to the outcome payload), persist advisory +// backtest runs as typed events, aggregate their REGRESSED-verdict track record (#8185), project current +// backtest-cleared proposals for the status surface (#8186), and hold the DOUBLE-GATED min-rank override +// (#8187: the `.loopover-ams.yml` `minRankAutotuneEnabled` flag AND a per-apply `--approve`, loosening +// nothing by itself -- every consumer resolves the value through readMinRankOverride, which validates the +// hard bounds on every read so a corrupted ledger row can never move the knob past safety). +// +// Same layering as pr-outcome.ts: typed event constants + inject-ledger record/read helpers over the +// generic append-only event-ledger.js. Everything here is local-node-only. + +import { + AMS_MIN_RANK_RULE_ID, + buildAmsRankCorpus, + runAmsMinRankBacktest, + type AmsMinRankBacktestResult, + type AmsTakenOpportunity, + type BacktestComparison, + computeRegressedVerdictTrackRecord, + type RegressedVerdictTrackRecord, +} from "@loopover/engine"; +import { existsSync as fsExistsSync, readFileSync as fsReadFileSync } from "node:fs"; +import { parseAmsPolicySpecContent } from "@loopover/engine"; +import type { AppendEventInput, LedgerEntry } from "./event-ledger.js"; +import { resolveAmsPolicyConfigPath } from "./ams-policy.js"; +import { MINER_PR_OUTCOME_EVENT } from "./pr-outcome.js"; + +/** Event-ledger vocabulary for one persisted advisory min-rank backtest run (the AMS analog of ORB's + * `calibration.threshold_backtest_run` -- the #8185 track record aggregates over these). */ +export const MINER_AMS_THRESHOLD_BACKTEST_EVENT = "ams_threshold_backtest_run"; +/** Event-ledger vocabulary for an approved min-rank override apply (#8187). */ +export const MINER_AMS_MIN_RANK_APPLIED_EVENT = "ams_min_rank_override_applied"; +/** Event-ledger vocabulary for a min-rank override reversion (#8187's one-command revert). */ +export const MINER_AMS_MIN_RANK_REVERTED_EVENT = "ams_min_rank_override_reverted"; + +/** The shipped min-rank skip threshold: portfolio-discovery's normalizeMinRankScore default (0 -- nothing + * skipped). Declared here, next to the hard bound, per the #8121 discipline. */ +export const AMS_MIN_RANK_SHIPPED = 0; +/** No evidence, however good, may raise the skip floor past this -- above it the miner would starve. */ +export const AMS_MIN_RANK_HARD_MAXIMUM = 0.5; +/** Proposals older than this are stale and drop out of the status projection (#8186). */ +export const AMS_BACKTEST_PROPOSAL_LOOKBACK_MS = 90 * 24 * 60 * 60 * 1000; + +type LedgerReader = { readEvents(filter?: { since?: number | null; repoFullName?: string | null }): unknown[] }; +type LedgerWriter = { appendEvent(event: AppendEventInput): LedgerEntry }; + +function isFiniteNumber(value: unknown): value is number { + return typeof value === "number" && Number.isFinite(value); +} + +function readAll(eventLedger: LedgerReader): Array> { + const events = eventLedger && typeof eventLedger.readEvents === "function" ? eventLedger.readEvents() : []; + return (Array.isArray(events) ? events : []).filter( + (event): event is Record => !!event && typeof event === "object", + ); +} + +/** + * PURE: join `discovered_issue` rank records to the miner's own `pr_outcome` events (#8184). The pairing + * key is (repoFullName, issueNumber) -- outcome rows carry `issueNumber` since this issue's capture-time + * addition; older rows without it simply never join (no fabricated pairs). Latest rank per issue wins + * (re-discovery refreshes the score); latest outcome per issue wins (a reopened-then-merged PR settles on + * its final decision, the same latest-wins discipline as calibration-cli's outcome reduction). + */ +export function deriveTakenOpportunities(events: readonly unknown[]): AmsTakenOpportunity[] { + const ranks = new Map(); + const outcomes = new Map(); + for (const event of Array.isArray(events) ? events : []) { + const record = event as Record | null | undefined; + if (!record || typeof record !== "object") continue; + const repoFullName = typeof record.repoFullName === "string" ? record.repoFullName : null; + const payload = record.payload as Record | null | undefined; + if (!repoFullName || !payload || typeof payload !== "object") continue; + if (record.type === "discovered_issue") { + if (!Number.isInteger(payload.issueNumber) || !isFiniteNumber(payload.rankScore)) continue; + // Ledger order is append order -- a later discovery overwrites, so the newest rank wins. + ranks.set(`${repoFullName}#${payload.issueNumber}`, { + rankScore: payload.rankScore, + discoveredAt: typeof record.createdAt === "string" ? record.createdAt : "", + }); + } else if (record.type === MINER_PR_OUTCOME_EVENT) { + if (!Number.isInteger(payload.issueNumber)) continue; // pre-pairing rows cannot join + const decision = payload.decision; + if (decision !== "merged" && decision !== "closed") continue; + const key = `${repoFullName}#${payload.issueNumber}`; + const seq = Number.isInteger(record.seq) ? (record.seq as number) : 0; + const prior = outcomes.get(key); + if (prior && prior.seq > seq) continue; + outcomes.set(key, { + decision, + decidedAt: + typeof payload.closedAt === "string" && payload.closedAt + ? payload.closedAt + : typeof record.createdAt === "string" + ? record.createdAt + : "", + seq, + }); + } + } + const takes: AmsTakenOpportunity[] = []; + for (const [key, outcome] of outcomes) { + const rank = ranks.get(key); + if (!rank) continue; + const separator = key.lastIndexOf("#"); + takes.push({ + repoFullName: key.slice(0, separator), + issueNumber: Number(key.slice(separator + 1)), + rankScore: rank.rankScore, + realizedDecision: outcome.decision, + discoveredAt: rank.discoveredAt, + decidedAt: outcome.decidedAt, + }); + } + takes.sort((a, b) => a.repoFullName.localeCompare(b.repoFullName) || a.issueNumber - b.issueNumber); + return takes; +} + +/** Convenience composition: ledger events -> taken opportunities -> replay corpus -> advisory result. */ +export function backtestMinRankCandidate( + events: readonly unknown[], + currentThreshold: number, + candidateThreshold: number, +): AmsMinRankBacktestResult | null { + return runAmsMinRankBacktest(buildAmsRankCorpus(deriveTakenOpportunities(events)), currentThreshold, candidateThreshold); +} + +/** Persist one advisory backtest run (#8184's third deliverable): the full comparison metadata, same shape + * ORB persists, so #8185's aggregation is byte-compatible with `computeRegressedVerdictTrackRecord`. */ +export function recordAmsThresholdBacktestRun(result: AmsMinRankBacktestResult, options: { eventLedger?: LedgerWriter } = {}): LedgerEntry { + const eventLedger = options.eventLedger; + if (!eventLedger || typeof eventLedger.appendEvent !== "function") throw new Error("invalid_event_ledger"); + return eventLedger.appendEvent({ + type: MINER_AMS_THRESHOLD_BACKTEST_EVENT, + payload: { + ruleId: result.ruleId, + currentThreshold: result.currentThreshold, + candidateThreshold: result.candidateThreshold, + visibleCases: result.visibleCases, + heldOutCases: result.heldOutCases, + visible: result.visible as unknown as Record, + heldOut: result.heldOut as unknown as Record, + }, + }); +} + +export type PersistedAmsBacktestRun = { + createdAt: string | null; + currentThreshold: number; + candidateThreshold: number; + visibleCases: number; + heldOutCases: number; + visible: BacktestComparison; + heldOut: BacktestComparison; +}; + +function isComparison(value: unknown): value is BacktestComparison { + const record = value as Record | null | undefined; + return ( + !!record && + typeof record === "object" && + typeof record.ruleId === "string" && + (record.verdict === "improved" || record.verdict === "regressed" || record.verdict === "unchanged") + ); +} + +/** Read every persisted backtest run, oldest first; foreign types and malformed payloads are skipped (the + * pr-outcome read discipline -- a corrupt row can neither be written nor read back). */ +export function readAmsThresholdBacktestRuns(eventLedger: LedgerReader): PersistedAmsBacktestRun[] { + const runs: PersistedAmsBacktestRun[] = []; + for (const record of readAll(eventLedger)) { + if (record.type !== MINER_AMS_THRESHOLD_BACKTEST_EVENT) continue; + const payload = record.payload as Record | null | undefined; + if (!payload || typeof payload !== "object") continue; + if (!isFiniteNumber(payload.currentThreshold) || !isFiniteNumber(payload.candidateThreshold)) continue; + if (!isComparison(payload.visible) || !isComparison(payload.heldOut)) continue; + runs.push({ + createdAt: typeof record.createdAt === "string" ? record.createdAt : null, + currentThreshold: payload.currentThreshold, + candidateThreshold: payload.candidateThreshold, + visibleCases: Number.isInteger(payload.visibleCases) ? (payload.visibleCases as number) : 0, + heldOutCases: Number.isInteger(payload.heldOutCases) ? (payload.heldOutCases as number) : 0, + visible: payload.visible, + heldOut: payload.heldOut, + }); + } + return runs; +} + +/** #8185: the REGRESSED-verdict track record over every persisted run's comparisons -- the SAME aggregation + * ORB uses (`computeRegressedVerdictTrackRecord`), zero new math. Both slices count: a held-out REGRESSED + * is exactly as real a verdict as a visible one. */ +export function computeAmsBacktestTrackRecord(runs: readonly PersistedAmsBacktestRun[]): RegressedVerdictTrackRecord { + return computeRegressedVerdictTrackRecord(runs.flatMap((run) => [run.visible, run.heldOut])); +} + +export type AmsBacktestProposal = { + candidateThreshold: number; + currentThreshold: number; + visibleCases: number; + heldOutCases: number; + visibleVerdict: string; + heldOutVerdict: string; + at: string | null; +}; + +/** #8186: the current backtest-CLEARED proposals -- latest run per candidate inside the lookback whose + * visible slice is strictly `improved` and whose held-out slice is non-`regressed` (the Pareto discipline + * the run was scored under). Deterministic order (candidate ascending), bounded by construction (one per + * candidate). Display-only: nothing here applies anything. */ +export function buildAmsBacktestProposals( + runs: readonly PersistedAmsBacktestRun[], + nowMs: number, + lookbackMs: number = AMS_BACKTEST_PROPOSAL_LOOKBACK_MS, +): AmsBacktestProposal[] { + const latest = new Map(); + for (const run of runs) { + const at = run.createdAt ? Date.parse(run.createdAt) : Number.NaN; + if (!Number.isFinite(at) || nowMs - at > lookbackMs) continue; // stale or undatable runs never propose + latest.set(run.candidateThreshold, run); // ledger order is append order -- the newest run wins + } + const proposals: AmsBacktestProposal[] = []; + for (const run of latest.values()) { + if (run.visible.verdict !== "improved" || run.heldOut.verdict === "regressed") continue; + proposals.push({ + candidateThreshold: run.candidateThreshold, + currentThreshold: run.currentThreshold, + visibleCases: run.visibleCases, + heldOutCases: run.heldOutCases, + visibleVerdict: run.visible.verdict, + heldOutVerdict: run.heldOut.verdict, + at: run.createdAt, + }); + } + proposals.sort((a, b) => a.candidateThreshold - b.candidateThreshold); + return proposals; +} + +/** Sync read of the operator's `.loopover-ams.yml` `minRankAutotuneEnabled` flag (#8187's gate one). The + * async resolveAmsPolicy wrapper exists for attempt-time policy; the calibration commands and discover's + * consumption point need only this one boolean and must stay synchronous, so this reuses the same path + * resolution + tolerant parser. Fail CLOSED: an unreadable policy file never enables autonomy. */ +export function readMinRankAutotuneEnabled( + env: Record, + deps: { readFileSync?: typeof fsReadFileSync; existsSync?: typeof fsExistsSync } = {}, +): boolean { + try { + const path = resolveAmsPolicyConfigPath(env); + const exists = deps.existsSync ?? fsExistsSync; + if (!exists(path)) return false; + const read = deps.readFileSync ?? fsReadFileSync; + return parseAmsPolicySpecContent(String(read(path, "utf8"))).spec.minRankAutotuneEnabled; + } catch { + return false; + } +} + +/** Bounds check shared by the apply path and every read: strictly above shipped (a "raise" to shipped is + * meaningless), at/below the hard maximum. */ +export function isValidMinRankOverride(value: unknown): value is number { + return isFiniteNumber(value) && value > AMS_MIN_RANK_SHIPPED && value <= AMS_MIN_RANK_HARD_MAXIMUM; +} + +/** + * #8187: resolve the effective min-rank override by replaying apply/revert events, latest wins. Gated on + * the `.loopover-ams.yml` flag at EVERY read -- flipping `minRankAutotuneEnabled` off instantly restores + * the shipped default with no cleanup, exactly like ORB's autotune vars. Bounds re-validate on every read, + * so a hand-edited ledger row can never move the knob past safety. Null means "use the shipped default". + */ +export function readMinRankOverride(eventLedger: LedgerReader, options: { enabled: boolean }): number | null { + if (!options.enabled) return null; + let override: number | null = null; + for (const record of readAll(eventLedger)) { + if (record.type === MINER_AMS_MIN_RANK_REVERTED_EVENT) { + override = null; + } else if (record.type === MINER_AMS_MIN_RANK_APPLIED_EVENT) { + const payload = record.payload as Record | null | undefined; + const value = payload && typeof payload === "object" ? payload.value : undefined; + if (isValidMinRankOverride(value)) override = value; + } + } + return override; +} + +export type ApplyMinRankOverrideResult = + | { applied: true; entry: LedgerEntry } + | { applied: false; reason: "flag_off" | "not_approved" | "out_of_bounds" | "no_supporting_run" }; + +/** + * #8187: the double-gated apply. Refuses unless the config flag is ON (gate one), the caller passed the + * explicit per-apply approval (gate two), the value sits inside the hard bounds, AND a persisted run + * inside the lookback actually cleared this exact candidate (evidence is not optional -- an operator + * cannot approve a number no backtest earned). The evidence rides the apply event verbatim. + */ +export function applyMinRankOverride( + value: number, + options: { eventLedger: LedgerWriter & LedgerReader; enabled: boolean; approved: boolean; nowMs?: number }, +): ApplyMinRankOverrideResult { + const eventLedger = options.eventLedger; + if (!eventLedger || typeof eventLedger.appendEvent !== "function") throw new Error("invalid_event_ledger"); + if (!options.enabled) return { applied: false, reason: "flag_off" }; + if (!options.approved) return { applied: false, reason: "not_approved" }; + if (!isValidMinRankOverride(value)) return { applied: false, reason: "out_of_bounds" }; + const nowMs = options.nowMs ?? Date.now(); + const supporting = buildAmsBacktestProposals(readAmsThresholdBacktestRuns(eventLedger), nowMs).find( + (proposal) => proposal.candidateThreshold === value, + ); + if (!supporting) return { applied: false, reason: "no_supporting_run" }; + const entry = eventLedger.appendEvent({ + type: MINER_AMS_MIN_RANK_APPLIED_EVENT, + payload: { + ruleId: AMS_MIN_RANK_RULE_ID, + value, + shipped: AMS_MIN_RANK_SHIPPED, + hardMaximum: AMS_MIN_RANK_HARD_MAXIMUM, + evidence: supporting as unknown as Record, + }, + }); + return { applied: true, entry }; +} + +/** #8187's one-command revert: appends the reversion event (readMinRankOverride then resolves to shipped). + * Approval-gated like the apply -- reverting is also a knob movement, just a safe-ward one. */ +export function revertMinRankOverride(options: { + eventLedger: LedgerWriter; + approved: boolean; +}): { reverted: boolean; reason?: "not_approved" } { + const eventLedger = options.eventLedger; + if (!eventLedger || typeof eventLedger.appendEvent !== "function") throw new Error("invalid_event_ledger"); + if (!options.approved) return { reverted: false, reason: "not_approved" }; + eventLedger.appendEvent({ + type: MINER_AMS_MIN_RANK_REVERTED_EVENT, + payload: { ruleId: AMS_MIN_RANK_RULE_ID, restoredValue: AMS_MIN_RANK_SHIPPED }, + }); + return { reverted: true }; +} diff --git a/packages/loopover-miner/lib/calibration-cli.ts b/packages/loopover-miner/lib/calibration-cli.ts index c1e885f999..2341d38985 100644 --- a/packages/loopover-miner/lib/calibration-cli.ts +++ b/packages/loopover-miner/lib/calibration-cli.ts @@ -2,8 +2,22 @@ // verdicts (prediction-ledger) with the realized PR outcomes it later observed (event-ledger `pr_outcome` // events), via the pure buildCalibrationReport join. Opens both local stores, maps their rows to the // calibration record shapes, renders, and closes. Never modifies the live scoring/calibration logic. -import { AMS_GATE_PREDICTION_RULE_ID, buildAmsPredictionCorpus, computeAmsCorpusStats } from "@loopover/engine"; +import type { existsSync, readFileSync } from "node:fs"; +import { AMS_GATE_PREDICTION_RULE_ID, buildAmsPredictionCorpus, computeAmsCorpusStats, renderBacktestComparison } from "@loopover/engine"; import type { AmsPredictionRecord, AmsRealizedOutcome } from "@loopover/engine"; +import { + AMS_MIN_RANK_HARD_MAXIMUM, + AMS_MIN_RANK_SHIPPED, + applyMinRankOverride, + backtestMinRankCandidate, + buildAmsBacktestProposals, + computeAmsBacktestTrackRecord, + readAmsThresholdBacktestRuns, + readMinRankAutotuneEnabled, + readMinRankOverride, + recordAmsThresholdBacktestRun, + revertMinRankOverride, +} from "./ams-calibration.js"; import { buildCalibrationReport } from "./calibration.js"; import { initEventLedger, resolveEventLedgerDbPath } from "./event-ledger.js"; import type { LedgerEntry } from "./event-ledger.js"; @@ -13,7 +27,21 @@ import type { PredictionLedgerEntry } from "./prediction-ledger.js"; import type { PredictedVerdictRecord, ObservedOutcomeRecord, CalibrationReport } from "./calibration-types.js"; import { reportCliFailure, describeCliError } from "./cli-error.js"; -const CALIBRATION_USAGE = "Usage: loopover-miner calibration [--json]"; +const CALIBRATION_USAGE = + "Usage: loopover-miner calibration [--json] | calibration backtest-threshold --candidate [--json] | calibration apply-min-rank --candidate --approve [--json] | calibration revert-min-rank --approve [--json]"; + +export type CalibrationCliDeps = { + readFileSync?: typeof readFileSync; + existsSync?: typeof existsSync; + nowMs?: number; +}; + +function parseCandidate(args: string[]): number | null { + const index = args.indexOf("--candidate"); + if (index === -1 || index + 1 >= args.length) return null; + const value = Number(args[index + 1]); + return Number.isFinite(value) ? value : null; +} /** Map prediction-ledger rows to predicted-verdict records: the target id becomes a string key and the recorded * prediction verdict is the `conclusion`. Exported so callers other than this CLI (the MCP calibration-report @@ -97,12 +125,96 @@ function renderReportText(report: CalibrationReport): void { } } +/** `calibration backtest-threshold --candidate ` (#8184): advisory replay of a candidate min-rank skip + * threshold against the taken-opportunity corpus. Prints the shared comparison renderer's report and + * persists the run event. Exit is nonzero ONLY on operational error -- never on verdict (the #8138 + * advisory guarantee); an under-floored corpus prints an explicit line and exits 0. */ +function runBacktestThreshold(args: string[], env: Record, deps: CalibrationCliDeps): number { + const json = args.includes("--json"); + const candidate = parseCandidate(args); + if (candidate === null) return reportCliFailure(json, `Missing or invalid --candidate. ${CALIBRATION_USAGE}`, 1); + + let eventLedger; + try { + eventLedger = initEventLedger(resolveEventLedgerDbPath(env)); + const events = eventLedger.readEvents(); + const enabled = readMinRankAutotuneEnabled(env, deps); + const current = readMinRankOverride(eventLedger, { enabled }) ?? AMS_MIN_RANK_SHIPPED; + const result = backtestMinRankCandidate(events, current, candidate); + if (!result) { + const message = "backtest-threshold: not enough labeled taken opportunities yet (both splits must clear their sample floors); no verdict, nothing persisted."; + console.log(json ? JSON.stringify({ ran: false, reason: "insufficient_corpus" }) : message); + return 0; + } + recordAmsThresholdBacktestRun(result, { eventLedger }); + if (json) { + console.log(JSON.stringify({ ran: true, ...result }, null, 2)); + } else { + console.log(`min-rank skip threshold: current ${result.currentThreshold} -> candidate ${result.candidateThreshold}`); + console.log(`visible split (${result.visibleCases} case(s)):`); + console.log(renderBacktestComparison(result.visible)); + console.log(`held-out split (${result.heldOutCases} case(s)):`); + console.log(renderBacktestComparison(result.heldOut)); + console.log("advisory only: no threshold changed; the run event was persisted for the track record."); + } + return 0; + } catch (error) { + return reportCliFailure(json, describeCliError(error)); + } finally { + eventLedger?.close(); + } +} + +/** `calibration apply-min-rank --candidate --approve` / `calibration revert-min-rank --approve` + * (#8187): the double-gated apply and its one-command revert. Exit 1 when the command did NOT move the + * knob (refusal or usage error) so scripts can tell; 0 only on a real apply/revert. */ +function runMinRankMutation(kind: "apply" | "revert", args: string[], env: Record, deps: CalibrationCliDeps): number { + const json = args.includes("--json"); + const approved = args.includes("--approve"); + let eventLedger; + try { + eventLedger = initEventLedger(resolveEventLedgerDbPath(env)); + const enabled = readMinRankAutotuneEnabled(env, deps); + if (kind === "revert") { + const result = revertMinRankOverride({ eventLedger, approved }); + if (!result.reverted) { + return reportCliFailure(json, "revert-min-rank refused: pass --approve to confirm (the reversion is itself a knob movement).", 1); + } + console.log(json ? JSON.stringify({ reverted: true, restoredValue: AMS_MIN_RANK_SHIPPED }) : `min-rank override reverted; shipped default ${AMS_MIN_RANK_SHIPPED} is live again.`); + return 0; + } + const candidate = parseCandidate(args); + if (candidate === null) return reportCliFailure(json, `Missing or invalid --candidate. ${CALIBRATION_USAGE}`, 1); + const result = applyMinRankOverride(candidate, { eventLedger, enabled, approved, ...(deps.nowMs !== undefined ? { nowMs: deps.nowMs } : {}) }); + if (!result.applied) { + const detail: Record = { + flag_off: "set minRankAutotuneEnabled: true in .loopover-ams.yml first (gate one of two).", + not_approved: "pass --approve to confirm (gate two of two).", + out_of_bounds: `candidate must sit inside (${AMS_MIN_RANK_SHIPPED}, ${AMS_MIN_RANK_HARD_MAXIMUM}] -- no evidence may cross the hard bounds.`, + no_supporting_run: "no recent persisted backtest run cleared this exact candidate -- run calibration backtest-threshold first; evidence is not optional.", + }; + return reportCliFailure(json, `apply-min-rank refused (${result.reason}): ${detail[result.reason]}`, 1); + } + console.log(json ? JSON.stringify({ applied: true, value: candidate }) : `min-rank override ${candidate} applied (evidence recorded on the ledger event); revert any time with calibration revert-min-rank --approve.`); + return 0; + } catch (error) { + return reportCliFailure(json, describeCliError(error)); + } finally { + eventLedger?.close(); + } +} + /** - * Run `loopover-miner calibration [--json]`. Reads the prediction ledger + PR-outcome events, joins them into a - * calibration report, and prints it (a JSON dump under `--json`, else a per-project text summary). Returns the - * process exit code: 0 on success, 1 on an unknown option. + * Run `loopover-miner calibration [--json]` (or one of the #8184/#8187 subcommands -- see + * CALIBRATION_USAGE). The bare form reads the prediction ledger + PR-outcome events, joins them into a + * calibration report, and prints it (a JSON dump under `--json`, else a per-project text summary) along + * with the corpus stats, the backtest track record (#8185), and any current backtest-cleared proposals + * (#8186). Returns the process exit code: 0 on success, 1 on an unknown option. */ -export function runCalibrationCli(args: string[] = [], env: Record = process.env): number { +export function runCalibrationCli(args: string[] = [], env: Record = process.env, deps: CalibrationCliDeps = {}): number { + if (args[0] === "backtest-threshold") return runBacktestThreshold(args.slice(1), env, deps); + if (args[0] === "apply-min-rank") return runMinRankMutation("apply", args.slice(1), env, deps); + if (args[0] === "revert-min-rank") return runMinRankMutation("revert", args.slice(1), env, deps); const json = args.includes("--json"); // This command takes no positional arguments, so anything that is not `--json` is a mistake -- including a // bare positional (`calibration foo`), which a `startsWith("-")` check silently let through (#5834). Mirrors @@ -123,8 +235,23 @@ export function runCalibrationCli(args: string[] = [], env: Record ${proposal.candidateThreshold} | visible ${proposal.visibleVerdict} (${proposal.visibleCases}) | held-out ${proposal.heldOutVerdict} (${proposal.heldOutCases}) | ${proposal.at ?? "undated"}`, + ); + } + if (proposals.length > 0) console.log("nothing applies automatically: apply-min-rank requires the config flag AND --approve."); } return 0; } catch (error) { diff --git a/packages/loopover-miner/lib/calibration-run.ts b/packages/loopover-miner/lib/calibration-run.ts index 5ff4f0106e..400a5d2b7c 100644 --- a/packages/loopover-miner/lib/calibration-run.ts +++ b/packages/loopover-miner/lib/calibration-run.ts @@ -82,12 +82,17 @@ export interface CalibrationSnapshotPayload { replayRunId: string | null; observedAt: string | null; replaySampleSize: number; + /** #8185: the AMS backtest loop's REGRESSED-verdict track record at snapshot time (aggregated from the + * persisted ams_threshold_backtest_run events), so authority-earning history is queryable from the + * ledger alone. Null when the writer had no track record to attach (including every pre-#8185 row). */ + backtestTrackRecord: { totalRuns: number; regressedRuns: number; regressedRate: number | null } | null; } export interface SnapshotMeta { replayRunId?: string | null; observedAt?: string | null; sampleSize?: number; + backtestTrackRecord?: { totalRuns: number; regressedRuns: number; regressedRate: number | null } | null; } export interface RecordCalibrationSnapshotOptions { @@ -236,9 +241,22 @@ export function snapshotPayloadFromResult(result: Phase7CalibrationLoopResult, m replayRunId: optionalString(meta.replayRunId), observedAt: optionalString(meta.observedAt), replaySampleSize: Number.isInteger(meta.sampleSize) && (meta.sampleSize as number) >= 0 ? (meta.sampleSize as number) : 0, + backtestTrackRecord: normalizeBacktestTrackRecord(meta.backtestTrackRecord), }; } +/** Tolerant #8185 section normalizer: a malformed shape degrades to null (the pre-#8185 reading), never + * rejecting the whole snapshot -- the Phase 7 metric is still real without the AMS track record. */ +function normalizeBacktestTrackRecord(value: unknown): CalibrationSnapshotPayload["backtestTrackRecord"] { + const record = value as Record | null | undefined; + if (!record || typeof record !== "object") return null; + if (!Number.isInteger(record.totalRuns) || (record.totalRuns as number) < 0) return null; + if (!Number.isInteger(record.regressedRuns) || (record.regressedRuns as number) < 0) return null; + const rate = record.regressedRate; + if (rate !== null && !(typeof rate === "number" && Number.isFinite(rate))) return null; + return { totalRuns: record.totalRuns as number, regressedRuns: record.regressedRuns as number, regressedRate: rate as number | null }; +} + /** * Validate + normalize a calibration-snapshot payload, returning `null` on any malformed shape (mirrors * pr-outcome.js's `normalizePrOutcomePayload`, so a corrupted row can neither be written nor read back). Skipped @@ -274,6 +292,7 @@ export function normalizeCalibrationSnapshotPayload(payload: unknown): Calibrati observedAt: optionalString(record.observedAt), replaySampleSize: Number.isInteger(record.replaySampleSize) && (record.replaySampleSize as number) >= 0 ? (record.replaySampleSize as number) : 0, + backtestTrackRecord: normalizeBacktestTrackRecord(record.backtestTrackRecord), }; } diff --git a/packages/loopover-miner/lib/cli.ts b/packages/loopover-miner/lib/cli.ts index 276a2e9fdb..6e22b653a5 100644 --- a/packages/loopover-miner/lib/cli.ts +++ b/packages/loopover-miner/lib/cli.ts @@ -53,6 +53,9 @@ export function printHelp(input: { packageName: string }): void { " loopover-miner governor status [--json] Show whether the governor is paused", " loopover-miner governor metrics Print governor rate-limit/cap-usage counters in Prometheus text format", " loopover-miner calibration [--json] Report predicted-vs-realized gate accuracy", + " loopover-miner calibration backtest-threshold --candidate Advisory min-rank replay against realized outcomes (#8184)", + " loopover-miner calibration apply-min-rank --candidate --approve Apply a backtest-cleared min-rank override (double-gated, #8187)", + " loopover-miner calibration revert-min-rank --approve Restore the shipped min-rank default", " loopover-miner pr-outcomes --miner-login [--limit ] [--json] Show your own hosted post-merge PR outcomes", " loopover-miner feasibility [--not-found] [--json]", " loopover-miner idea-feasibility [--not-resolvable] [--hint ]... [--json]", diff --git a/packages/loopover-miner/lib/discover-cli.ts b/packages/loopover-miner/lib/discover-cli.ts index 17e831933c..f7014cf7f6 100644 --- a/packages/loopover-miner/lib/discover-cli.ts +++ b/packages/loopover-miner/lib/discover-cli.ts @@ -23,6 +23,8 @@ import type { PolicyDocCacheStore } from "./policy-doc-cache.js"; import { initPolicyVerdictCacheStore } from "./policy-verdict-cache.js"; import type { PolicyVerdictCacheStore } from "./policy-verdict-cache.js"; import { enqueueRankedDiscovery } from "./portfolio-discovery.js"; +import { AMS_MIN_RANK_SHIPPED, readMinRankAutotuneEnabled, readMinRankOverride } from "./ams-calibration.js"; +import { initEventLedger, resolveEventLedgerDbPath } from "./event-ledger.js"; import type { EnqueueRankedDiscoverySummary } from "./portfolio-discovery.js"; import { initPortfolioQueueStore } from "./portfolio-queue.js"; import type { PortfolioQueueStore } from "./portfolio-queue.js"; @@ -426,6 +428,25 @@ export async function runDiscover(args: string[], options: RunDiscoverOptions = const searchTargets = options.searchCandidateIssuesWithSummary ?? searchCandidateIssuesWithSummary; const rankIssues = options.rankCandidateIssuesWithSummary ?? rankCandidateIssuesWithSummary; const enqueue = options.enqueueRankedDiscovery ?? enqueueRankedDiscovery; + // #8187: THE single consumption point for the earned min-rank override -- resolved fresh per run through + // readMinRankOverride (which re-validates the hard bounds and gates on the config flag at every read), so + // flipping minRankAutotuneEnabled off restores the shipped default on the very next discover. Fail-open to + // shipped on any read error: a broken ledger must never change what discover enqueues. Resolved once and + // passed to BOTH the real enqueue and the dry-run preview, so a dry run shows the exact skip set a real + // run would produce. + let minRankScore: number = AMS_MIN_RANK_SHIPPED; + { + let overrideLedger = null; + try { + const ledgerEnv = options.env ?? process.env; + overrideLedger = initEventLedger(resolveEventLedgerDbPath(ledgerEnv)); + minRankScore = readMinRankOverride(overrideLedger, { enabled: readMinRankAutotuneEnabled(ledgerEnv) }) ?? AMS_MIN_RANK_SHIPPED; + } catch { + minRankScore = AMS_MIN_RANK_SHIPPED; + } finally { + overrideLedger?.close(); + } + } // Eligibility filtering (#6798): resolve each candidate repo's ContributionProfile and drop candidates the // repo's own conventions would reject, BEFORE ranking. Safe by default -- see resolveContributionProfilesForDiscover. const resolveProfiles = options.resolveContributionProfiles ?? resolveContributionProfilesForDiscover; @@ -477,7 +498,7 @@ export async function runDiscover(args: string[], options: RunDiscoverOptions = : {}), }); const noopQueueStore = { enqueue: () => {} } as unknown as PortfolioQueueStore; - const enqueueSummary = enqueue(rankedSummary.issues, { queueStore: noopQueueStore }); + const enqueueSummary = enqueue(rankedSummary.issues, { queueStore: noopQueueStore, minRankScore }); const result = { outcome: "dry_run", fanOutCount: fanOut.issues.length, @@ -605,6 +626,7 @@ export async function runDiscover(args: string[], options: RunDiscoverOptions = }); const enqueueSummary = enqueue(rankedSummary.issues, { queueStore: portfolioQueue, + minRankScore, ...(apiBaseUrl !== undefined ? { apiBaseUrl } : {}), }); diff --git a/packages/loopover-miner/lib/loop-cli.ts b/packages/loopover-miner/lib/loop-cli.ts index 7b8037047e..5efcdd66bf 100644 --- a/packages/loopover-miner/lib/loop-cli.ts +++ b/packages/loopover-miner/lib/loop-cli.ts @@ -566,6 +566,9 @@ export async function runLoop(args: string[], options: RunLoopOptions = {}): Pro prNumber, decision: prDisposition.merged ? "merged" : "closed", closedAt: prDisposition.closedAt, + // #8184: pair the outcome with the claimed issue so the AMS min-rank corpus can join it + // to the discovery-time rank record. Null when the identifier isn't issue-shaped. + issueNumber: parseIssueNumberFromIdentifier(claimedEntry.identifier), }, { eventLedger, recipientLogin: parsed.minerLogin }, ); diff --git a/packages/loopover-miner/lib/pr-outcome.ts b/packages/loopover-miner/lib/pr-outcome.ts index ad78ddaf32..48001cf390 100644 --- a/packages/loopover-miner/lib/pr-outcome.ts +++ b/packages/loopover-miner/lib/pr-outcome.ts @@ -30,6 +30,9 @@ export type NormalizedPrOutcomePayload = { decision: MinerPrOutcomeDecision; closedAt: string | null; reason: string | null; + /** The claimed issue this PR addressed (#8184) -- the pairing key the AMS min-rank corpus joins on. + * Null on rows written before the pairing existed (those simply never join; nothing is fabricated). */ + issueNumber: number | null; }; export type PrOutcomeInput = { @@ -38,6 +41,7 @@ export type PrOutcomeInput = { decision?: unknown; closedAt?: unknown; reason?: unknown; + issueNumber?: unknown; }; export type RecordPrOutcomeOptions = { @@ -87,6 +91,9 @@ export function normalizePrOutcomePayload(payload: unknown): NormalizedPrOutcome decision: decision as MinerPrOutcomeDecision, closedAt: optionalString(record.closedAt), reason, + // Tolerant, not gating (#8184): a malformed issueNumber degrades to null rather than rejecting the + // whole outcome row -- the outcome itself is still real even when the pairing is unusable. + issueNumber: Number.isInteger(record.issueNumber) && (record.issueNumber as number) > 0 ? (record.issueNumber as number) : null, }; } @@ -107,6 +114,7 @@ export function recordPrOutcomeSnapshot(input: PrOutcomeInput, options: RecordPr decision: input.decision, closedAt: input.closedAt, reason: input.reason, + issueNumber: input.issueNumber, }); if (!payload) return null; const entry = eventLedger.appendEvent({ type: MINER_PR_OUTCOME_EVENT, repoFullName, payload }); diff --git a/packages/loopover-miner/lib/status.ts b/packages/loopover-miner/lib/status.ts index 4b096b54b2..6b32fd91b1 100644 --- a/packages/loopover-miner/lib/status.ts +++ b/packages/loopover-miner/lib/status.ts @@ -13,7 +13,8 @@ import { } from "./laptop-init.js"; import { resolveMinerVersion } from "./version.js"; import { checkStoreIntegrity, describeError } from "./store-maintenance.js"; -import { resolveEventLedgerDbPath } from "./event-ledger.js"; +import { initEventLedger, resolveEventLedgerDbPath } from "./event-ledger.js"; +import { buildAmsBacktestProposals, readAmsThresholdBacktestRuns } from "./ams-calibration.js"; import { resolveGovernorLedgerDbPath } from "./governor-ledger.js"; import { hasGitHubTokenSource } from "./github-token-resolution.js"; import { resolvePredictionLedgerDbPath } from "./prediction-ledger.js"; @@ -486,6 +487,31 @@ export function checkCodingAgentCredential( /** Run the doctor checks. Returns an array of { name, ok, detail }; only writes a transient probe in the state dir, * never touches the network. */ +/** #8186: current backtest-cleared min-rank proposals, mirrored from the ORB advisor's posture -- full + * evidence per line and an explicit nothing-applies-automatically stance baked into the detail. Always + * ok:true (informational -- a proposal is an opportunity, not a fault) and fail-open on a ledger blip + * (doctor must keep working on a box whose ledger is broken; store integrity has its own check). */ +export function checkAmsBacktestProposals(env: Record = process.env, nowMs: number = Date.now()): DoctorCheck { + try { + const eventLedger = initEventLedger(resolveEventLedgerDbPath(env)); + try { + const proposals = buildAmsBacktestProposals(readAmsThresholdBacktestRuns(eventLedger), nowMs); + if (proposals.length === 0) { + return { name: "ams-backtest-proposals", ok: true, detail: "no backtest-cleared min-rank proposals (nothing applies automatically)" }; + } + const lines = proposals.map( + (proposal) => + `min-rank ${proposal.currentThreshold} -> ${proposal.candidateThreshold} (visible ${proposal.visibleVerdict}/${proposal.visibleCases}, held-out ${proposal.heldOutVerdict}/${proposal.heldOutCases})`, + ); + return { name: "ams-backtest-proposals", ok: true, detail: `${lines.join("; ")} -- nothing applies automatically (apply-min-rank needs the config flag AND --approve)` }; + } finally { + eventLedger.close(); + } + } catch { + return { name: "ams-backtest-proposals", ok: true, detail: "event ledger unreadable; proposals unavailable" }; + } +} + export function runDoctorChecks( env: Record = process.env, cwd: string = process.cwd(), @@ -514,6 +540,7 @@ export function runDoctorChecks( checkGitHubTokenPresent(env), checkCodingAgentCredential(env), checkConfigContent(cwd), + checkAmsBacktestProposals(env), ...storeIntegrityChecks(env), ]; } diff --git a/test/unit/ams-policy-spec-parser.test.ts b/test/unit/ams-policy-spec-parser.test.ts index c8ffcf8ebd..cda09b041b 100644 --- a/test/unit/ams-policy-spec-parser.test.ts +++ b/test/unit/ams-policy-spec-parser.test.ts @@ -55,6 +55,7 @@ describe("AmsPolicySpec parser (#5132)", () => { maxTurnsPerIteration: 10, selfLoopAutonomy: "observe", networkAllowlist: { ecosystems: ["npm", "pypi"], extraHosts: ["api.example.com"] }, + minRankAutotuneEnabled: false, }); expect(parsed.warnings).toEqual([]); }); @@ -380,3 +381,16 @@ describe("AmsPolicySpec parser (#5132)", () => { expect(withMultiByteChars.spec.submissionMode).toBe("enforce"); }); }); + +describe("minRankAutotuneEnabled (#8187 gate one)", () => { + it("defaults OFF, accepts booleans, and warns + falls back on non-boolean values", () => { + expect(DEFAULT_AMS_POLICY_SPEC.minRankAutotuneEnabled).toBe(false); + expect(parseAmsPolicySpec({}).spec.minRankAutotuneEnabled).toBe(false); + const on = parseAmsPolicySpec({ minRankAutotuneEnabled: true }); + expect(on.spec.minRankAutotuneEnabled).toBe(true); + expect(on.present).toBe(true); // a non-default flag counts as configured + const bad = parseAmsPolicySpec({ minRankAutotuneEnabled: "yes" }); + expect(bad.spec.minRankAutotuneEnabled).toBe(false); + expect(bad.warnings.some((w) => w.includes("minRankAutotuneEnabled"))).toBe(true); + }); +}); diff --git a/test/unit/ams-rank-corpus-engine.test.ts b/test/unit/ams-rank-corpus-engine.test.ts new file mode 100644 index 0000000000..c048ecac9d --- /dev/null +++ b/test/unit/ams-rank-corpus-engine.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, it } from "vitest"; + +// Engine SOURCE import (not dist) -- coverage.include lists packages/loopover-engine/src/**, so only a +// source-path import exercises these branches; the dist twin in packages/loopover-engine/test/ covers the +// built barrel for the workspace suite. Same pattern as backtest-compare-engine.test.ts. +import { + AMS_MIN_RANK_MIN_HELD_OUT_CASES, + AMS_MIN_RANK_MIN_VISIBLE_CASES, + AMS_MIN_RANK_RULE_ID, + AMS_MIN_RANK_SPLIT_SEED, + buildAmsRankCorpus, + runAmsMinRankBacktest, + type AmsTakenOpportunity, +} from "../../packages/loopover-engine/src/calibration/ams-rank-corpus"; + +function take(issueNumber: number, rankScore: number, realizedDecision: "merged" | "closed"): AmsTakenOpportunity { + return { + repoFullName: "acme/widgets", + issueNumber, + rankScore, + realizedDecision, + discoveredAt: "2026-07-01T00:00:00.000Z", + decidedAt: "2026-07-02T00:00:00.000Z", + }; +} + +// Uniform corpus: 60 closed takes at rank 0.15 (a 0.2 floor would have skipped exactly the bad takes) plus +// two merged anchors at 0.4 that stay above any candidate here. Deterministic: the split hashes fixed keys. +function skipFriendlyTakes(): AmsTakenOpportunity[] { + const takes = Array.from({ length: 60 }, (_, i) => take(i + 1, 0.15, "closed")); + takes.push(take(101, 0.4, "merged"), take(102, 0.4, "merged")); + return takes; +} + +describe("buildAmsRankCorpus (#8184)", () => { + it("maps takes to cases with the skip polarity: closed -> reversed, merged -> confirmed, rank as confidence", () => { + const cases = buildAmsRankCorpus([take(7, 0.3, "closed"), take(8, 0.9, "merged")]); + expect(cases).toHaveLength(2); + expect(cases[0]).toMatchObject({ + ruleId: AMS_MIN_RANK_RULE_ID, + targetKey: "acme/widgets#issue-7", + label: "reversed", + metadata: { confidence: 0.3 }, + }); + expect(cases[1]).toMatchObject({ targetKey: "acme/widgets#issue-8", label: "confirmed", metadata: { confidence: 0.9 } }); + }); + + it("drops non-replayable rank scores -- a case the classifier cannot honestly replay never defaults to confidence 1", () => { + expect(buildAmsRankCorpus([take(1, Number.NaN, "closed"), take(2, -0.1, "merged"), take(3, 1.5, "closed")])).toEqual([]); + // Boundary values ARE replayable. + expect(buildAmsRankCorpus([take(4, 0, "closed"), take(5, 1, "merged")])).toHaveLength(2); + }); + + it("orders deterministically by target key then discoveredAt", () => { + const shuffled = [take(20, 0.5, "merged"), take(3, 0.5, "closed"), take(11, 0.5, "merged")]; + const keys = buildAmsRankCorpus(shuffled).map((c) => c.targetKey); + expect(keys).toEqual([...keys].sort()); + // Same key, different firedAt: earlier discovery sorts first. + const twice = buildAmsRankCorpus([ + { ...take(9, 0.5, "closed"), discoveredAt: "2026-07-05T00:00:00.000Z" }, + { ...take(9, 0.5, "closed"), discoveredAt: "2026-07-01T00:00:00.000Z" }, + ]); + expect(twice.map((c) => c.firedAt)).toEqual(["2026-07-01T00:00:00.000Z", "2026-07-05T00:00:00.000Z"]); + }); +}); + +describe("runAmsMinRankBacktest (#8184)", () => { + it("replays a candidate raise with the fixed-seed split and the symmetric Pareto floor on each slice", () => { + const result = runAmsMinRankBacktest(buildAmsRankCorpus(skipFriendlyTakes()), 0, 0.2); + expect(result).not.toBeNull(); + expect(result!.ruleId).toBe(AMS_MIN_RANK_RULE_ID); + expect(result!.currentThreshold).toBe(0); + expect(result!.candidateThreshold).toBe(0.2); + expect(result!.visibleCases).toBeGreaterThanOrEqual(AMS_MIN_RANK_MIN_VISIBLE_CASES); + expect(result!.heldOutCases).toBeGreaterThanOrEqual(AMS_MIN_RANK_MIN_HELD_OUT_CASES); + // Raising 0 -> 0.2 catches every bad take (recall up) and skips no good one -- improved on both slices. + expect(result!.visible.verdict).toBe("improved"); + expect(result!.heldOut.verdict).toBe("improved"); + }); + + it("returns null -- never a guess -- when either slice misses its sample floor", () => { + expect(runAmsMinRankBacktest(buildAmsRankCorpus(skipFriendlyTakes().slice(0, 5)), 0, 0.2)).toBeNull(); + expect(runAmsMinRankBacktest([], 0, 0.2)).toBeNull(); + }); + + it("pins the split constants the miner replays under -- held-out membership must never reshuffle", () => { + expect(AMS_MIN_RANK_SPLIT_SEED).toBe("ams-min-rank-skip-v1"); + expect(AMS_MIN_RANK_MIN_VISIBLE_CASES).toBe(20); + expect(AMS_MIN_RANK_MIN_HELD_OUT_CASES).toBe(5); + }); +}); diff --git a/test/unit/miner-ams-calibration.test.ts b/test/unit/miner-ams-calibration.test.ts new file mode 100644 index 0000000000..512dba8cc1 --- /dev/null +++ b/test/unit/miner-ams-calibration.test.ts @@ -0,0 +1,218 @@ +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + AMS_MIN_RANK_HARD_MAXIMUM, + AMS_MIN_RANK_SHIPPED, + applyMinRankOverride, + backtestMinRankCandidate, + buildAmsBacktestProposals, + computeAmsBacktestTrackRecord, + deriveTakenOpportunities, + isValidMinRankOverride, + MINER_AMS_MIN_RANK_APPLIED_EVENT, + MINER_AMS_MIN_RANK_REVERTED_EVENT, + MINER_AMS_THRESHOLD_BACKTEST_EVENT, + readAmsThresholdBacktestRuns, + readMinRankAutotuneEnabled, + readMinRankOverride, + recordAmsThresholdBacktestRun, + revertMinRankOverride, + type PersistedAmsBacktestRun, +} from "../../packages/loopover-miner/lib/ams-calibration.js"; +import { initEventLedger, resolveEventLedgerDbPath, type EventLedger } from "../../packages/loopover-miner/lib/event-ledger.js"; + +// #8184-#8187: the miner-side calibration seam. The engine replay math has its own suite +// (ams-rank-corpus-engine.test.ts); these tests pin the ledger join, run persistence + read-back, the +// track-record/proposals projections, and every arm of the double-gated min-rank override. + +const tempDirs: string[] = []; +const ledgers: EventLedger[] = []; +afterEach(() => { + for (const ledger of ledgers.splice(0)) ledger.close(); + for (const dir of tempDirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function tempLedger(): EventLedger { + const dir = mkdtempSync(join(tmpdir(), "miner-ams-calibration-")); + tempDirs.push(dir); + const ledger = initEventLedger(resolveEventLedgerDbPath({ LOOPOVER_MINER_CONFIG_DIR: dir })); + ledgers.push(ledger); + return ledger; +} + +function seedTake(ledger: EventLedger, issueNumber: number, rankScore: number, decision: "merged" | "closed"): void { + ledger.appendEvent({ type: "discovered_issue", repoFullName: "acme/widgets", payload: { issueNumber, rankScore, title: "t", labels: [] } }); + ledger.appendEvent({ + type: "pr_outcome", + repoFullName: "acme/widgets", + payload: { prNumber: 1000 + issueNumber, decision, closedAt: "2026-07-10T00:00:00Z", reason: null, issueNumber }, + }); +} + +function seedSkipFriendlyHistory(ledger: EventLedger): void { + for (let i = 1; i <= 60; i += 1) seedTake(ledger, i, 0.15, "closed"); + seedTake(ledger, 101, 0.4, "merged"); + seedTake(ledger, 102, 0.4, "merged"); +} + +describe("deriveTakenOpportunities (#8184 ledger join)", () => { + it("joins discovered_issue to pr_outcome by (repo, issueNumber); unpaired rows on either side never fabricate a take", () => { + const ledger = tempLedger(); + seedTake(ledger, 7, 0.3, "closed"); + ledger.appendEvent({ type: "discovered_issue", repoFullName: "acme/widgets", payload: { issueNumber: 8, rankScore: 0.9, title: "t", labels: [] } }); // no outcome + ledger.appendEvent({ type: "pr_outcome", repoFullName: "acme/widgets", payload: { prNumber: 99, decision: "merged", closedAt: null, reason: null, issueNumber: 55 } }); // no rank + ledger.appendEvent({ type: "pr_outcome", repoFullName: "acme/widgets", payload: { prNumber: 98, decision: "merged", closedAt: null, reason: null } }); // pre-pairing row: no issueNumber + const takes = deriveTakenOpportunities(ledger.readEvents()); + expect(takes).toHaveLength(1); + expect(takes[0]).toMatchObject({ repoFullName: "acme/widgets", issueNumber: 7, rankScore: 0.3, realizedDecision: "closed", decidedAt: "2026-07-10T00:00:00Z" }); + }); + + it("latest wins on BOTH sides: a re-discovery refreshes the rank, a later outcome supersedes (reopened-then-merged)", () => { + const ledger = tempLedger(); + seedTake(ledger, 7, 0.3, "closed"); + ledger.appendEvent({ type: "discovered_issue", repoFullName: "acme/widgets", payload: { issueNumber: 7, rankScore: 0.45, title: "t", labels: [] } }); + ledger.appendEvent({ type: "pr_outcome", repoFullName: "acme/widgets", payload: { prNumber: 1007, decision: "merged", closedAt: "2026-07-12T00:00:00Z", reason: null, issueNumber: 7 } }); + const takes = deriveTakenOpportunities(ledger.readEvents()); + expect(takes).toHaveLength(1); + expect(takes[0]).toMatchObject({ rankScore: 0.45, realizedDecision: "merged", decidedAt: "2026-07-12T00:00:00Z" }); + }); + + it("tolerates malformed rows: non-object events, missing repo/payload, bad issueNumber/rankScore, foreign decisions", () => { + const ledger = tempLedger(); + ledger.appendEvent({ type: "discovered_issue", repoFullName: "acme/widgets", payload: { issueNumber: 1.5, rankScore: 0.5 } }); + ledger.appendEvent({ type: "discovered_issue", repoFullName: "acme/widgets", payload: { issueNumber: 2, rankScore: "high" } }); + ledger.appendEvent({ type: "pr_outcome", repoFullName: "acme/widgets", payload: { prNumber: 1, decision: "reopened", issueNumber: 2 } }); + expect(deriveTakenOpportunities([...ledger.readEvents(), null, 42, { type: "discovered_issue" }])).toEqual([]); + }); + + it("composes into the advisory backtest (backtestMinRankCandidate) end-to-end over a real ledger", () => { + const ledger = tempLedger(); + seedSkipFriendlyHistory(ledger); + const result = backtestMinRankCandidate(ledger.readEvents(), AMS_MIN_RANK_SHIPPED, 0.2); + expect(result).not.toBeNull(); + expect(result!.visible.verdict).toBe("improved"); + expect(backtestMinRankCandidate([], AMS_MIN_RANK_SHIPPED, 0.2)).toBeNull(); // empty ledger: floors unmet + }); +}); + +describe("backtest-run persistence + projections (#8184/#8185/#8186)", () => { + function persistedRun(ledger: EventLedger, candidate: number): PersistedAmsBacktestRun { + const result = backtestMinRankCandidate(ledger.readEvents(), AMS_MIN_RANK_SHIPPED, candidate); + expect(result).not.toBeNull(); + recordAmsThresholdBacktestRun(result!, { eventLedger: ledger }); + const runs = readAmsThresholdBacktestRuns(ledger); + return runs[runs.length - 1]!; + } + + it("round-trips a run event: full comparison metadata out equals what went in; corrupt/foreign rows are skipped", () => { + const ledger = tempLedger(); + seedSkipFriendlyHistory(ledger); + const run = persistedRun(ledger, 0.2); + expect(run.candidateThreshold).toBe(0.2); + expect(run.currentThreshold).toBe(AMS_MIN_RANK_SHIPPED); + expect(run.visible.verdict).toBe("improved"); + expect(run.heldOut.verdict).toBe("improved"); + expect(run.visibleCases).toBeGreaterThan(0); + // Corrupt rows: missing thresholds, malformed comparisons. + ledger.appendEvent({ type: MINER_AMS_THRESHOLD_BACKTEST_EVENT, payload: { candidateThreshold: 0.3 } }); + ledger.appendEvent({ type: MINER_AMS_THRESHOLD_BACKTEST_EVENT, payload: { currentThreshold: 0, candidateThreshold: 0.3, visible: { ruleId: "x", verdict: "weird" }, heldOut: run.heldOut } }); + expect(readAmsThresholdBacktestRuns(ledger)).toHaveLength(1); + expect(() => recordAmsThresholdBacktestRun(run as never, {})).toThrow("invalid_event_ledger"); + }); + + it("track record (#8185): counts both slices per run via the shared engine aggregation; empty history is zero-run", () => { + const ledger = tempLedger(); + seedSkipFriendlyHistory(ledger); + persistedRun(ledger, 0.2); + const record = computeAmsBacktestTrackRecord(readAmsThresholdBacktestRuns(ledger)); + expect(record.totalRuns).toBe(2); // visible + held-out comparisons + expect(record.regressedRuns).toBe(0); + expect(record.regressedRate).toBe(0); + expect(computeAmsBacktestTrackRecord([]).totalRuns).toBe(0); + expect(computeAmsBacktestTrackRecord([]).regressedRate).toBeNull(); + }); + + it("proposals (#8186): latest run per candidate, cleared-only, stale/undatable excluded, deterministic order", () => { + const ledger = tempLedger(); + seedSkipFriendlyHistory(ledger); + const cleared = persistedRun(ledger, 0.2); + const nowMs = Date.parse(cleared.createdAt ?? "") || Date.now(); + // A NOT-cleared run for another candidate (visible unchanged -- candidate equals current). + const flat = backtestMinRankCandidate(ledger.readEvents(), 0.15, 0.15); + if (flat) recordAmsThresholdBacktestRun(flat, { eventLedger: ledger }); + const runs = readAmsThresholdBacktestRuns(ledger); + const proposals = buildAmsBacktestProposals(runs, nowMs + 1000); + expect(proposals).toHaveLength(1); + expect(proposals[0]).toMatchObject({ candidateThreshold: 0.2, visibleVerdict: "improved" }); + // Stale: everything outside the lookback drops out. + expect(buildAmsBacktestProposals(runs, nowMs + 1000, 500)).toEqual([]); + // Undatable rows never propose. + expect(buildAmsBacktestProposals([{ ...runs[0]!, createdAt: null }], nowMs)).toEqual([]); + }); +}); + +describe("min-rank override: double-gated apply, gated read, revert (#8187)", () => { + it("bounds check: strictly above shipped, at/below the hard maximum", () => { + expect(isValidMinRankOverride(AMS_MIN_RANK_SHIPPED)).toBe(false); + expect(isValidMinRankOverride(0.2)).toBe(true); + expect(isValidMinRankOverride(AMS_MIN_RANK_HARD_MAXIMUM)).toBe(true); + expect(isValidMinRankOverride(AMS_MIN_RANK_HARD_MAXIMUM + 0.01)).toBe(false); + expect(isValidMinRankOverride("0.2")).toBe(false); + }); + + it("apply refuses in order -- flag_off, not_approved, out_of_bounds, no_supporting_run -- writing nothing", () => { + const ledger = tempLedger(); + expect(applyMinRankOverride(0.2, { eventLedger: ledger, enabled: false, approved: true })).toEqual({ applied: false, reason: "flag_off" }); + expect(applyMinRankOverride(0.2, { eventLedger: ledger, enabled: true, approved: false })).toEqual({ applied: false, reason: "not_approved" }); + expect(applyMinRankOverride(0.9, { eventLedger: ledger, enabled: true, approved: true })).toEqual({ applied: false, reason: "out_of_bounds" }); + expect(applyMinRankOverride(0.2, { eventLedger: ledger, enabled: true, approved: true })).toEqual({ applied: false, reason: "no_supporting_run" }); + expect(readMinRankOverride(ledger, { enabled: true })).toBeNull(); + expect(() => applyMinRankOverride(0.2, { eventLedger: undefined as never, enabled: true, approved: true })).toThrow("invalid_event_ledger"); + }); + + it("a cleared run + both gates applies; the read re-validates flag + bounds; revert restores shipped", () => { + const ledger = tempLedger(); + seedSkipFriendlyHistory(ledger); + const result = backtestMinRankCandidate(ledger.readEvents(), AMS_MIN_RANK_SHIPPED, 0.2)!; + recordAmsThresholdBacktestRun(result, { eventLedger: ledger }); + + const applied = applyMinRankOverride(0.2, { eventLedger: ledger, enabled: true, approved: true }); + expect(applied.applied).toBe(true); + expect(readMinRankOverride(ledger, { enabled: true })).toBe(0.2); + expect(readMinRankOverride(ledger, { enabled: false })).toBeNull(); // flag off => shipped, instantly + + // A hand-edited out-of-bounds applied row is ignored on read (bounds re-validate every read). + ledger.appendEvent({ type: MINER_AMS_MIN_RANK_APPLIED_EVENT, payload: { value: 0.9 } }); + expect(readMinRankOverride(ledger, { enabled: true })).toBe(0.2); + + expect(revertMinRankOverride({ eventLedger: ledger, approved: false })).toEqual({ reverted: false, reason: "not_approved" }); + expect(revertMinRankOverride({ eventLedger: ledger, approved: true })).toEqual({ reverted: true }); + expect(readMinRankOverride(ledger, { enabled: true })).toBeNull(); + const reverts = ledger.readEvents().filter((e) => e.type === MINER_AMS_MIN_RANK_REVERTED_EVENT); + expect(reverts).toHaveLength(1); + expect(() => revertMinRankOverride({ eventLedger: undefined as never, approved: true })).toThrow("invalid_event_ledger"); + }); +}); + +describe("readMinRankAutotuneEnabled (#8187 gate one)", () => { + it("reads the flag from .loopover-ams.yml; missing file, parse-safe defaults, and thrown reads are all OFF", () => { + const dir = mkdtempSync(join(tmpdir(), "miner-ams-policy-")); + tempDirs.push(dir); + const env = { LOOPOVER_MINER_CONFIG_DIR: dir }; + expect(readMinRankAutotuneEnabled(env)).toBe(false); // no file + const path = join(dir, ".loopover-ams.yml"); + writeFileSync(path, "minRankAutotuneEnabled: true\n"); + expect(readMinRankAutotuneEnabled(env)).toBe(true); + writeFileSync(path, "minRankAutotuneEnabled: banana\n"); + expect(readMinRankAutotuneEnabled(env)).toBe(false); // tolerant parser falls back to the OFF default + expect( + readMinRankAutotuneEnabled(env, { + readFileSync: (() => { + throw new Error("disk gone"); + }) as never, + }), + ).toBe(false); // fail CLOSED + }); +}); diff --git a/test/unit/miner-calibration-cli.test.ts b/test/unit/miner-calibration-cli.test.ts index 0bca476d98..b6cee9431f 100644 --- a/test/unit/miner-calibration-cli.test.ts +++ b/test/unit/miner-calibration-cli.test.ts @@ -1,4 +1,4 @@ -import { mkdtempSync, rmSync } from "node:fs"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -184,3 +184,146 @@ describe("loopover-miner calibration CLI (#4849)", () => { expect(err).not.toHaveBeenCalled(); }); }); + +// ── #8184/#8185/#8186/#8187: the calibration subcommands + report sections ───────────────────────────────── + +import { resolveAmsPolicyConfigPath } from "../../packages/loopover-miner/lib/ams-policy.js"; +import { + MINER_AMS_THRESHOLD_BACKTEST_EVENT, + readMinRankOverride, +} from "../../packages/loopover-miner/lib/ams-calibration.js"; + +function seedTakenHistory(env: Record): void { + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + for (let i = 1; i <= 60; i += 1) { + ledger.appendEvent({ type: "discovered_issue", repoFullName: "acme/widgets", payload: { issueNumber: i, rankScore: 0.15, title: "t", labels: [] } }); + ledger.appendEvent({ + type: "pr_outcome", + repoFullName: "acme/widgets", + payload: { prNumber: 1000 + i, decision: "closed", closedAt: "2026-07-10T00:00:00Z", reason: null, issueNumber: i }, + }); + } + for (const i of [101, 102]) { + ledger.appendEvent({ type: "discovered_issue", repoFullName: "acme/widgets", payload: { issueNumber: i, rankScore: 0.4, title: "t", labels: [] } }); + ledger.appendEvent({ type: "pr_outcome", repoFullName: "acme/widgets", payload: { prNumber: 1000 + i, decision: "merged", closedAt: null, reason: null, issueNumber: i } }); + } + ledger.close(); +} + +function enableAutotune(env: Record): void { + writeFileSync(resolveAmsPolicyConfigPath(env), "minRankAutotuneEnabled: true\n"); +} + +describe("calibration backtest-threshold (#8184)", () => { + it("replays, prints both split reports via the shared renderer, persists the run event, exits 0", () => { + const env = envForTempStores(); + seedTakenHistory(env); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(runCalibrationCli(["backtest-threshold", "--candidate", "0.2"], env)).toBe(0); + const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("current 0 -> candidate 0.2"); + expect(output).toContain("visible split"); + expect(output).toContain("held-out split"); + expect(output).toContain("advisory only"); + expect(output).toContain("Verdict"); + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + const runs = ledger.readEvents().filter((e) => e.type === MINER_AMS_THRESHOLD_BACKTEST_EVENT); + ledger.close(); + expect(runs).toHaveLength(1); + }); + + it("an under-floored corpus prints the explicit line, persists NOTHING, and still exits 0 (never on verdict)", () => { + const env = envForTempStores(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(runCalibrationCli(["backtest-threshold", "--candidate", "0.2"], env)).toBe(0); + expect(logSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain("not enough labeled taken opportunities"); + expect(runCalibrationCli(["backtest-threshold", "--candidate", "0.2", "--json"], env)).toBe(0); + const jsonLine = logSpy.mock.calls.map((call) => String(call[0])).find((line) => line.includes("insufficient_corpus")); + expect(jsonLine).toBeDefined(); + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + expect(ledger.readEvents().filter((e) => e.type === MINER_AMS_THRESHOLD_BACKTEST_EVENT)).toHaveLength(0); + ledger.close(); + }); + + it("JSON mode dumps the full result; a missing/invalid --candidate is a usage error (exit 1)", () => { + const env = envForTempStores(); + seedTakenHistory(env); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + vi.spyOn(console, "error").mockImplementation(() => undefined); + expect(runCalibrationCli(["backtest-threshold", "--candidate", "0.2", "--json"], env)).toBe(0); + const parsed = JSON.parse(logSpy.mock.calls.map((call) => String(call[0])).find((line) => line.trimStart().startsWith("{"))!) as { ran: boolean; visible: { verdict: string } }; + expect(parsed.ran).toBe(true); + expect(parsed.visible.verdict).toBe("improved"); + expect(runCalibrationCli(["backtest-threshold"], env)).toBe(1); + expect(runCalibrationCli(["backtest-threshold", "--candidate", "banana"], env)).toBe(1); + }); + + it("fails operationally (exit 1) when the ledger store cannot open", () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + const env = { LOOPOVER_MINER_EVENT_LEDGER_DB: "/dev/null/nope/ledger.sqlite" }; + expect(runCalibrationCli(["backtest-threshold", "--candidate", "0.2"], env)).toBe(2); + }); +}); + +describe("calibration apply-min-rank / revert-min-rank (#8187)", () => { + it("walks the full double-gated path: flag_off -> not_approved -> no_supporting_run -> applied -> reverted", () => { + const env = envForTempStores(); + seedTakenHistory(env); + vi.spyOn(console, "error").mockImplementation(() => undefined); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + + expect(runCalibrationCli(["apply-min-rank", "--candidate", "0.2", "--approve"], env)).toBe(1); // gate one off + enableAutotune(env); + expect(runCalibrationCli(["apply-min-rank", "--candidate", "0.2"], env)).toBe(1); // gate two missing + expect(runCalibrationCli(["apply-min-rank", "--candidate", "0.2", "--approve"], env)).toBe(1); // no evidence yet + expect(runCalibrationCli(["apply-min-rank", "--approve"], env)).toBe(1); // usage: no candidate + + expect(runCalibrationCli(["backtest-threshold", "--candidate", "0.2"], env)).toBe(0); // earn the evidence + expect(runCalibrationCli(["apply-min-rank", "--candidate", "0.2", "--approve"], env)).toBe(0); + expect(logSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain("min-rank override 0.2 applied"); + + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + expect(readMinRankOverride(ledger, { enabled: true })).toBe(0.2); + ledger.close(); + + expect(runCalibrationCli(["revert-min-rank"], env)).toBe(1); // revert also needs approval + expect(runCalibrationCli(["revert-min-rank", "--approve", "--json"], env)).toBe(0); + const after = initEventLedger(resolveEventLedgerDbPath(env)); + expect(readMinRankOverride(after, { enabled: true })).toBeNull(); + after.close(); + }); + + it("fails operationally (exit 1) when the ledger store cannot open", () => { + vi.spyOn(console, "error").mockImplementation(() => undefined); + const env = { LOOPOVER_MINER_EVENT_LEDGER_DB: "/dev/null/nope/ledger.sqlite" }; + expect(runCalibrationCli(["apply-min-rank", "--candidate", "0.2", "--approve"], env)).toBe(2); + expect(runCalibrationCli(["revert-min-rank", "--approve"], env)).toBe(2); + }); +}); + +describe("calibration report sections (#8185/#8186)", () => { + it("prints the explicit no-runs line on an empty history, and the track record + proposals + no-autonomy line once runs exist", () => { + const env = envForTempStores(); + seedTakenHistory(env); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => undefined); + expect(runCalibrationCli([], env)).toBe(0); + expect(logSpy.mock.calls.map((call) => String(call[0])).join("\n")).toContain("backtest track record: no backtest runs recorded."); + + expect(runCalibrationCli(["backtest-threshold", "--candidate", "0.2"], env)).toBe(0); + logSpy.mockClear(); + expect(runCalibrationCli([], env)).toBe(0); + const output = logSpy.mock.calls.map((call) => String(call[0])).join("\n"); + expect(output).toContain("backtest track record: 2 comparison(s) | REGRESSED 0 (rate 0.000)"); + expect(output).toContain("backtest-cleared proposal: min-rank 0 -> 0.2"); + expect(output).toContain("nothing applies automatically"); + + logSpy.mockClear(); + expect(runCalibrationCli(["--json"], env)).toBe(0); + const parsed = JSON.parse(logSpy.mock.calls.map((call) => String(call[0])).find((line) => line.trimStart().startsWith("{"))!) as { + backtestTrackRecord: { totalRuns: number; regressedRuns: number }; + backtestProposals: Array<{ candidateThreshold: number }>; + }; + expect(parsed.backtestTrackRecord).toMatchObject({ totalRuns: 2, regressedRuns: 0 }); + expect(parsed.backtestProposals[0]?.candidateThreshold).toBe(0.2); + }); +}); diff --git a/test/unit/miner-calibration-run.test.ts b/test/unit/miner-calibration-run.test.ts index 40f08a9994..e29e261eae 100644 --- a/test/unit/miner-calibration-run.test.ts +++ b/test/unit/miner-calibration-run.test.ts @@ -62,6 +62,7 @@ function validPayload(overrides: Record = {}): Record { replayRunId: "run-1", observedAt: "2026-07-09T00:00:00.000Z", replaySampleSize: 4, + backtestTrackRecord: null, }); }); @@ -396,3 +398,48 @@ describe("runHistoricalReplayCalibrationCycle (#4248)", () => { expect(noReplay.sampleSize).toBe(0); }); }); + +describe("backtestTrackRecord snapshot section (#8185)", () => { + const baseResult = { + enabled: true, + combinedAccuracy: 0.8, + baselineAccuracy: 0.62, + deltaFromBaseline: 0.18, + autonomyIncreasePermitted: false, + replayHarnessHold: false, + replayHarnessStatus: "healthy", + replayRunDue: false, + holdReasons: [], + audit: { contributingSources: ["pr_outcome"] }, + } as unknown as Parameters[0]; + + it("threads a valid track record through the payload and round-trips the normalizer; absence stays null", () => { + const record = { totalRuns: 4, regressedRuns: 1, regressedRate: 0.25 }; + const payload = snapshotPayloadFromResult(baseResult, { backtestTrackRecord: record }); + expect(payload.backtestTrackRecord).toEqual(record); + expect(normalizeCalibrationSnapshotPayload(payload)?.backtestTrackRecord).toEqual(record); + expect(snapshotPayloadFromResult(baseResult).backtestTrackRecord).toBeNull(); + // A zero-run record keeps its null rate. + const empty = snapshotPayloadFromResult(baseResult, { backtestTrackRecord: { totalRuns: 0, regressedRuns: 0, regressedRate: null } }); + expect(empty.backtestTrackRecord).toEqual({ totalRuns: 0, regressedRuns: 0, regressedRate: null }); + }); + + it("a malformed section degrades to null WITHOUT rejecting the snapshot (every invalid arm)", () => { + for (const bad of [ + "not-an-object", + { totalRuns: -1, regressedRuns: 0, regressedRate: null }, + { totalRuns: 1.5, regressedRuns: 0, regressedRate: null }, + { totalRuns: 2, regressedRuns: -2, regressedRate: null }, + { totalRuns: 2, regressedRuns: 1, regressedRate: "half" }, + { totalRuns: 2, regressedRuns: 1, regressedRate: Number.NaN }, + ]) { + const payload = snapshotPayloadFromResult(baseResult, { backtestTrackRecord: bad as never }); + expect(payload.backtestTrackRecord).toBeNull(); + expect(normalizeCalibrationSnapshotPayload(payload)).not.toBeNull(); + } + // Read-side tolerance: an old persisted row without the field normalizes with null. + const legacy = snapshotPayloadFromResult(baseResult) as unknown as Record; + delete legacy.backtestTrackRecord; + expect(normalizeCalibrationSnapshotPayload(legacy)?.backtestTrackRecord).toBeNull(); + }); +}); diff --git a/test/unit/miner-discover-cli.test.ts b/test/unit/miner-discover-cli.test.ts index ffdada3a22..9f9403607f 100644 --- a/test/unit/miner-discover-cli.test.ts +++ b/test/unit/miner-discover-cli.test.ts @@ -2124,3 +2124,57 @@ describe("discovery-index supplementation (#7168)", () => { expect(payload.fanOutCount).toBe(1); }); }); + +describe("min-rank override consumption at discover's enqueue (#8187)", () => { + it("threads shipped 0 by default, the earned override when the flag + applied event exist, and fails open on a broken ledger", async () => { + const { mkdtempSync, rmSync, writeFileSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { initEventLedger, resolveEventLedgerDbPath } = await import("../../packages/loopover-miner/lib/event-ledger.js"); + const { resolveAmsPolicyConfigPath } = await import("../../packages/loopover-miner/lib/ams-policy.js"); + const { MINER_AMS_MIN_RANK_APPLIED_EVENT } = await import("../../packages/loopover-miner/lib/ams-calibration.js"); + + const fetchCandidateIssuesWithSummary = vi.fn(async () => ({ + issues: [fanOutIssue({ issueNumber: 1, title: "candidate" })], + warnings: [], + rateLimitRemaining: 5000, + rateLimitResetAt: "2026-07-09T13:00:00.000Z", + })); + vi.spyOn(console, "log").mockImplementation(() => undefined); + + async function minRankSeenBy(env: Record): Promise { + const enqueueSpy = vi.fn(() => ({ enqueued: 1, skippedBelowMinRank: 0, skippedInvalid: 0, eventsAppended: 0 })); + const exitCode = await runDiscover(["acme/widgets", "--json"], { + nowMs: NOW, + env, + initPortfolioQueue: () => tempQueueStore(), + initPolicyDocCache: () => tempPolicyDocCacheStore(), + initPolicyVerdictCache: () => tempPolicyVerdictCacheStore(), + initRankedCandidatesStore: () => tempRankedCandidatesStore(), + fetchCandidateIssuesWithSummary, + searchCandidateIssuesWithSummary: vi.fn(), + enqueueRankedDiscovery: enqueueSpy as never, + }); + expect(exitCode).toBe(0); + return ((enqueueSpy.mock.calls[0] as unknown[] | undefined)?.[1] as { minRankScore?: number } | undefined)?.minRankScore; + } + + const dir = mkdtempSync(join(tmpdir(), "miner-discover-minrank-")); + try { + const env = { LOOPOVER_MINER_CONFIG_DIR: dir }; + expect(await minRankSeenBy(env)).toBe(0); // shipped default: no policy, no override + + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + ledger.appendEvent({ type: MINER_AMS_MIN_RANK_APPLIED_EVENT, payload: { value: 0.2 } }); + ledger.close(); + expect(await minRankSeenBy(env)).toBe(0); // applied row alone is inert: gate one still off + + writeFileSync(resolveAmsPolicyConfigPath(env), "minRankAutotuneEnabled: true\n"); + expect(await minRankSeenBy(env)).toBe(0.2); // both present: the earned override is consumed + + expect(await minRankSeenBy({ ...env, LOOPOVER_MINER_EVENT_LEDGER_DB: "/dev/null/nope/ledger.sqlite" })).toBe(0); // fail open + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/test/unit/miner-loop-cli.test.ts b/test/unit/miner-loop-cli.test.ts index a28cd431dc..b895f518e2 100644 --- a/test/unit/miner-loop-cli.test.ts +++ b/test/unit/miner-loop-cli.test.ts @@ -592,6 +592,8 @@ describe("runLoop (#5135)", () => { const prOutcomeEvents = after.eventLedger.readEvents({}).filter((e) => e.type === "pr_outcome"); expect(prOutcomeEvents).toHaveLength(1); expect(prOutcomeEvents[0]?.payload).toMatchObject({ prNumber: 123, decision: "merged" }); + // #8184: the outcome row carries the claimed issue so the AMS min-rank corpus can join it. + expect(prOutcomeEvents[0]?.payload).toMatchObject({ issueNumber: 7 }); // REGRESSION (#5394): the real CI-status observation was recorded in the loop's own event ledger. const ciStatusEvents = after.eventLedger.readEvents({}).filter((e) => e.type === "ci_status_observed"); diff --git a/test/unit/miner-orb-export.test.ts b/test/unit/miner-orb-export.test.ts index ab46428ce8..f41e708e16 100644 --- a/test/unit/miner-orb-export.test.ts +++ b/test/unit/miner-orb-export.test.ts @@ -92,8 +92,8 @@ describe("buildAnonymizedOrbBatch", () => { it("anonymizes a readPrOutcomes-shaped map, buckets a null reason to 'none', and sorts deterministically", () => { const outcomes = new Map([ - ["owner/repo:2", { repoFullName: "owner/repo", prNumber: 2, decision: "closed", closedAt: "2026-01-02T00:00:00Z", reason: "gate_close" }], - ["owner/repo:1", { repoFullName: "owner/repo", prNumber: 1, decision: "merged", closedAt: null, reason: null }], + ["owner/repo:2", { repoFullName: "owner/repo", prNumber: 2, decision: "closed", closedAt: "2026-01-02T00:00:00Z", reason: "gate_close", issueNumber: null }], + ["owner/repo:1", { repoFullName: "owner/repo", prNumber: 1, decision: "merged", closedAt: null, reason: null, issueNumber: null }], ]); const batch = buildAnonymizedOrbBatch(outcomes, anonKey); expect(batch).toHaveLength(2); @@ -127,7 +127,7 @@ describe("buildAnonymizedOrbBatch", () => { // A generator's returned iterator has [Symbol.iterator] but no .values() -- unlike arrays and Maps, both // of which have .values() too, so this is the only realistic way to exercise the non-Map iterable path. function* outcomes(): Generator { - yield { repoFullName: "owner/repo", prNumber: 3, decision: "merged", closedAt: "2026-01-03T00:00:00Z", reason: null }; + yield { repoFullName: "owner/repo", prNumber: 3, decision: "merged", closedAt: "2026-01-03T00:00:00Z", reason: null, issueNumber: null }; } const batch = buildAnonymizedOrbBatch(outcomes(), anonKey); expect(batch).toHaveLength(1); diff --git a/test/unit/miner-pr-outcome.test.ts b/test/unit/miner-pr-outcome.test.ts index fde991aa07..8f5533f6d3 100644 --- a/test/unit/miner-pr-outcome.test.ts +++ b/test/unit/miner-pr-outcome.test.ts @@ -38,6 +38,7 @@ describe("normalizePrOutcomePayload (#4274)", () => { decision: "closed", closedAt: "2026-07-09T00:00:00Z", reason: "gate_close", + issueNumber: null, }); // unrecognized reason → dropped expect(normalizePrOutcomePayload({ prNumber: 7, decision: "closed", reason: "because" })?.reason).toBeNull(); @@ -47,7 +48,7 @@ describe("normalizePrOutcomePayload (#4274)", () => { it("drops any reason on a merged decision (a merged PR has no rejection reason)", () => { const merged = normalizePrOutcomePayload({ prNumber: 9, decision: "merged", reason: "gate_close" }); - expect(merged).toEqual({ prNumber: 9, decision: "merged", closedAt: null, reason: null }); + expect(merged).toEqual({ prNumber: 9, decision: "merged", closedAt: null, reason: null, issueNumber: null }); }); it("coerces a null / non-string / whitespace closedAt to null", () => { @@ -76,7 +77,7 @@ describe("recordPrOutcomeSnapshot (#4274)", () => { const entry = recordPrOutcomeSnapshot({ repoFullName: " acme/widgets ", prNumber: 12, decision: "closed", reason: "superseded_by_duplicate", closedAt: "t" }, { eventLedger: ledger }) as Record; expect(entry.type).toBe(MINER_PR_OUTCOME_EVENT); expect(entry.repoFullName).toBe("acme/widgets"); - expect(entry.payload).toEqual({ prNumber: 12, decision: "closed", closedAt: "t", reason: "superseded_by_duplicate" }); + expect(entry.payload).toEqual({ prNumber: 12, decision: "closed", closedAt: "t", reason: "superseded_by_duplicate", issueNumber: null }); expect(MINER_PR_OUTCOME_DECISIONS).toEqual(["merged", "closed"]); }); @@ -128,7 +129,7 @@ describe("readPrOutcomes (#4274)", () => { recordPrOutcomeSnapshot({ repoFullName: "acme/widgets", prNumber: 1, decision: "merged" }, { eventLedger: ledger }); // supersedes recordPrOutcomeSnapshot({ repoFullName: "acme/other", prNumber: 2, decision: "closed" }, { eventLedger: ledger }); const latest = readPrOutcomes(ledger, { repoFullName: "acme/widgets" }); - expect(latest.get("acme/widgets:1")).toEqual({ repoFullName: "acme/widgets", prNumber: 1, decision: "merged", closedAt: null, reason: null }); + expect(latest.get("acme/widgets:1")).toEqual({ repoFullName: "acme/widgets", prNumber: 1, decision: "merged", closedAt: null, reason: null, issueNumber: null }); expect(latest.has("acme/other:2")).toBe(false); // filtered out by the repo filter }); @@ -157,3 +158,16 @@ describe("readPrOutcomes (#4274)", () => { expect([...readPrOutcomes(ledger).keys()]).toEqual(["acme/widgets:8"]); }); }); + +describe("pr_outcome issueNumber pairing (#8184)", () => { + it("carries a valid claimed-issue number through; malformed values degrade to null without rejecting the row", () => { + const base = { prNumber: 5, decision: "merged", closedAt: null, reason: null }; + expect(normalizePrOutcomePayload({ ...base, issueNumber: 7 })?.issueNumber).toBe(7); + expect(normalizePrOutcomePayload(base)?.issueNumber).toBeNull(); + for (const bad of [0, -3, 1.5, "7", null]) { + const normalized = normalizePrOutcomePayload({ ...base, issueNumber: bad }); + expect(normalized).not.toBeNull(); // the outcome itself is still real + expect(normalized?.issueNumber).toBeNull(); + } + }); +}); diff --git a/test/unit/miner-status.test.ts b/test/unit/miner-status.test.ts index 6eca51550f..2af943679d 100644 --- a/test/unit/miner-status.test.ts +++ b/test/unit/miner-status.test.ts @@ -128,6 +128,7 @@ describe("loopover-miner status/doctor (#2288)", () => { "github-token", "coding-agent-credential", "config-content", + "ams-backtest-proposals", "store-integrity:event-ledger", "store-integrity:governor-ledger", "store-integrity:prediction-ledger", @@ -501,3 +502,42 @@ describe("loopover-miner status/doctor (#2288)", () => { }); }); }); + +describe("checkAmsBacktestProposals (#8186)", () => { + it("is informational (always ok): explicit none-line empty, evidence + no-autonomy line with proposals, fail-open on a broken ledger", async () => { + const { mkdtempSync, rmSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const { initEventLedger, resolveEventLedgerDbPath } = await import("../../packages/loopover-miner/lib/event-ledger.js"); + const { backtestMinRankCandidate, recordAmsThresholdBacktestRun } = await import("../../packages/loopover-miner/lib/ams-calibration.js"); + const { checkAmsBacktestProposals } = await import("../../packages/loopover-miner/lib/status.js"); + + const dir = mkdtempSync(join(tmpdir(), "miner-status-ams-")); + try { + const env = { LOOPOVER_MINER_CONFIG_DIR: dir }; + const empty = checkAmsBacktestProposals(env); + expect(empty.ok).toBe(true); + expect(empty.detail).toContain("no backtest-cleared min-rank proposals"); + + const ledger = initEventLedger(resolveEventLedgerDbPath(env)); + for (let i = 1; i <= 60; i += 1) { + ledger.appendEvent({ type: "discovered_issue", repoFullName: "acme/widgets", payload: { issueNumber: i, rankScore: 0.15, title: "t", labels: [] } }); + ledger.appendEvent({ type: "pr_outcome", repoFullName: "acme/widgets", payload: { prNumber: 1000 + i, decision: "closed", closedAt: "2026-07-10T00:00:00Z", reason: null, issueNumber: i } }); + } + const result = backtestMinRankCandidate(ledger.readEvents(), 0, 0.2)!; + recordAmsThresholdBacktestRun(result, { eventLedger: ledger }); + ledger.close(); + + const withProposal = checkAmsBacktestProposals(env); + expect(withProposal.ok).toBe(true); + expect(withProposal.detail).toContain("min-rank 0 -> 0.2"); + expect(withProposal.detail).toContain("nothing applies automatically"); + + const broken = checkAmsBacktestProposals({ LOOPOVER_MINER_EVENT_LEDGER_DB: "/dev/null/nope/ledger.sqlite" }); + expect(broken.ok).toBe(true); + expect(broken.detail).toContain("unavailable"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +});