diff --git a/src/services/maintainer-recap-drift.ts b/src/services/maintainer-recap-drift.ts new file mode 100644 index 000000000..04a1f7586 --- /dev/null +++ b/src/services/maintainer-recap-drift.ts @@ -0,0 +1,102 @@ +// Maintainer-recap CONFIG-DRIFT section (#8214, epic #8211 track A). +// +// Pure section builder over a plain source struct, mirroring maintainer-recap-calibration.ts exactly: drift +// alerts are point-in-time, but the weekly recap is where a STANDING drift should be impossible to miss. The +// section renders each drifting knob's direction, live vs dominating value, corpus sizes, and how long the +// episode has stood — aggregate numbers + knob ids only, never corpus content (the same public-safe boundary +// as every other recap section). +// +// Ships independently of the sentinel runtime, the same way the calibration section shipped ahead of the full +// RecapReport (#2243's own header): this file only needs the per-knob {@link KnobDriftReport} projection plus +// the episode's first-fingerprinted timestamp, so a caller wires it the moment the sentinel persists episodes. +// Until then the flag-off arm renders the explicit disabled line — absence of data must be distinguishable +// from absence of drift. +import { PUBLIC_LOCAL_PATH_SCRUB_PATTERN } from "../signals/redaction"; +import type { KnobDriftReport } from "./loosening-knobs"; + +/** One standing drift episode: the sentinel's current report for a live knob, plus when the sentinel first + * fingerprinted the episode (its fingerprint timestamp — the "how long has this stood" anchor). */ +export type DriftRecapKnob = { + report: KnobDriftReport; + /** ISO timestamp of the episode's first sentinel fingerprint. */ + episodeSince: string; +}; + +/** Projection of the sentinel's state used by the drift section. Structurally compatible with what the + * sentinel evaluates per live knob ({@link KnobDriftReport} via evaluateKnobDrift, loosening-knobs.ts). */ +export type DriftRecapSource = { + /** Recap generation instant — episode ages are computed against this, never against a wall-clock read. */ + generatedAt: string; + /** False ⇒ the drift sentinel is not running; the section says so explicitly instead of looking clean. */ + sentinelEnabled: boolean; + /** Every live knob the sentinel currently reports as drifting. */ + drifting: DriftRecapKnob[]; + /** Count of evaluated live knobs with NO standing drift. */ + cleanKnobs: number; +}; + +/** One titled digest section: structured fields for consumers + ready-to-emit lines for the formatter — + * the CalibrationRecapSection shape verbatim, with drift counts in place of reversal counts. */ +export type DriftRecapSection = { + title: string; + drifting: number; + clean: number; + /** Plain-English status line (disabled / clean / drift-present). */ + note: string; + lines: string[]; +}; + +/** Public-safe scrub for free text pulled into the section (defense in depth — knob/rule ids and ISO + * timestamps are the only string inputs today). Mirrors maintainer-recap-calibration.ts. */ +function sanitizeRecapText(value: string): string { + return value.replace(PUBLIC_LOCAL_PATH_SCRUB_PATTERN, "").slice(0, 240); +} + +/** Whole days an episode has stood at `generatedAt`, floored; clock skew that puts the fingerprint in the + * future (or an unparseable timestamp) reads as 0 rather than a negative/NaN age. */ +function episodeStandingDays(episodeSince: string, generatedAt: string): number { + const elapsedMs = Date.parse(generatedAt) - Date.parse(episodeSince); + return Number.isFinite(elapsedMs) && elapsedMs > 0 ? Math.floor(elapsedMs / 86_400_000) : 0; +} + +/** + * Pure config-drift section over the sentinel projection, mirroring {@link buildCalibrationRecapSection}'s + * arms exactly: + * + * - sentinel off ⇒ the explicit disabled line (never a clean-looking silence); + * - no drifting knobs ⇒ one clean summary line over `cleanKnobs`; + * - drifting knobs ⇒ one line per knob (direction, live vs dominating value, corpus sizes, standing days), + * plus the clean-knob summary when the window is mixed. + */ +export function buildDriftRecapSection(source: DriftRecapSource): DriftRecapSection { + const title = "Config drift"; + const drifting = source.drifting.length; + + if (!source.sentinelEnabled) { + const note = "drift sentinel disabled — no drift evaluation ran this window."; + return { title, drifting: 0, clean: 0, note: sanitizeRecapText(note), lines: [sanitizeRecapText(note)] }; + } + + if (drifting === 0) { + const note = `Config drift clean: all ${source.cleanKnobs} evaluated knob(s) remain their best-supported live values.`; + return { title, drifting, clean: source.cleanKnobs, note: sanitizeRecapText(note), lines: [sanitizeRecapText(note)] }; + } + + const note = `config drift: ${drifting} live knob(s) are Pareto-dominated by another supported value; longest-standing episodes first below.`; + const knobLines = [...source.drifting] + .sort((left, right) => episodeStandingDays(right.episodeSince, source.generatedAt) - episodeStandingDays(left.episodeSince, source.generatedAt)) + .map(({ report, episodeSince }) => { + const days = episodeStandingDays(episodeSince, source.generatedAt); + return `${report.knobId} (${report.ruleId}): live ${report.liveValue} vs dominating ${report.dominatingValue} (${report.direction}) — visible n=${report.visibleCases}, held-out n=${report.heldOutCases}; standing ${days} day(s).`; + }); + const lines = [note, ...knobLines]; + if (source.cleanKnobs > 0) lines.push(`${source.cleanKnobs} other evaluated knob(s) are clean.`); + + return { + title, + drifting, + clean: source.cleanKnobs, + note: sanitizeRecapText(note), + lines: lines.map(sanitizeRecapText), + }; +} diff --git a/src/services/maintainer-recap.ts b/src/services/maintainer-recap.ts index c7ab21156..111c99c10 100644 --- a/src/services/maintainer-recap.ts +++ b/src/services/maintainer-recap.ts @@ -13,6 +13,7 @@ import { PUBLIC_LOCAL_PATH_SCRUB_PATTERN, PUBLIC_UNSAFE_PATTERN } from "../signals/redaction"; import { deliverRecapToDiscord, deliverRecapToSlack } from "./notify-discord"; import type { GatePrecisionReport } from "./gate-precision"; +import type { DriftRecapSection } from "./maintainer-recap-drift"; import type { OutcomeCalibration } from "./outcome-calibration"; import type { MaintainerRecapCohortCounts, MaintainerRecapRepo, RecapReport } from "../types"; import { nowIso } from "../utils/json"; @@ -161,7 +162,7 @@ function recapSectionLines(items: string[], fallback: string): string[] { * (Summary, Totals, Per-repo), mirroring formatWeeklyValueReportMarkdown at weekly-value-report.ts. PURE * string function — no delivery, no I/O. Every free-text value is routed through {@link redactRecapLine} so no * reward/trust/score/path term can leak into the digest even if the input report was hand-built. (#2240) */ -export function formatMaintainerRecap(report: RecapReport): string { +export function formatMaintainerRecap(report: RecapReport, options: { configDrift?: DriftRecapSection } = {}): string { const { totals } = report; const rate = totals.gateFalsePositiveRate !== null ? `${Math.round(totals.gateFalsePositiveRate * 100)}%` : "n/a"; const perRepoLines = report.repos.map( @@ -188,6 +189,11 @@ export function formatMaintainerRecap(report: RecapReport): string { "", "## Per-repo", ...recapSectionLines(perRepoLines, "_No repositories in this window._"), + // #8214: optional config-drift section (maintainer-recap-drift.ts) — appended only when the caller has a + // sentinel projection to render, so every existing digest stays byte-identical until the sentinel wires in. + ...(options.configDrift + ? ["", `## ${redactRecapLine(options.configDrift.title)}`, ...recapSectionLines(options.configDrift.lines, "_No drift lines for this window._")] + : []), ]; return `${lines.join("\n").replace(/\n{3,}/g, "\n\n").trimEnd()}\n`; } diff --git a/test/unit/maintainer-recap-drift.test.ts b/test/unit/maintainer-recap-drift.test.ts new file mode 100644 index 000000000..ad26c9835 --- /dev/null +++ b/test/unit/maintainer-recap-drift.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, it } from "vitest"; +import { + buildDriftRecapSection, + type DriftRecapKnob, + type DriftRecapSource, +} from "../../src/services/maintainer-recap-drift"; +import type { KnobDriftReport } from "../../src/services/loosening-knobs"; + +const GENERATED_AT = "2026-07-23T12:00:00.000Z"; + +function driftReport(overrides: Partial = {}): KnobDriftReport { + const comparison = { verdict: "improved", baseline: { precision: 0.7 }, proposed: { precision: 0.9 } } as unknown as KnobDriftReport["visible"]; + return { + knobId: "ai_consensus_defect.confidenceFloor", + ruleId: "ai_consensus_defect", + liveValue: 0.6, + dominatingValue: 0.8, + direction: "tighter", + visibleCases: 40, + heldOutCases: 12, + visible: comparison, + heldOut: comparison, + ...overrides, + }; +} + +function drifting(episodeSince: string, overrides: Partial = {}): DriftRecapKnob { + return { report: driftReport(overrides), episodeSince }; +} + +function source(overrides: Partial = {}): DriftRecapSource { + return { generatedAt: GENERATED_AT, sentinelEnabled: true, drifting: [], cleanKnobs: 0, ...overrides }; +} + +describe("buildDriftRecapSection (#8214)", () => { + it("renders the explicit disabled line when the sentinel flag is off — absence of data, not absence of drift", () => { + const section = buildDriftRecapSection(source({ sentinelEnabled: false, drifting: [drifting(GENERATED_AT)], cleanKnobs: 5 })); + expect(section.title).toBe("Config drift"); + expect(section.drifting).toBe(0); + expect(section.clean).toBe(0); + expect(section.note).toMatch(/drift sentinel disabled/); + expect(section.lines).toEqual([section.note]); + }); + + it("renders one clean summary line when every evaluated knob matches its best-supported value", () => { + const section = buildDriftRecapSection(source({ cleanKnobs: 6 })); + expect(section.drifting).toBe(0); + expect(section.clean).toBe(6); + expect(section.note).toMatch(/Config drift clean: all 6 evaluated knob\(s\)/); + expect(section.lines).toEqual([section.note]); + }); + + it("renders each drifting knob with direction, live vs dominating value, corpus sizes, and standing days", () => { + const section = buildDriftRecapSection( + source({ drifting: [drifting("2026-07-11T12:00:00.000Z")] }), + ); + expect(section.drifting).toBe(1); + expect(section.note).toMatch(/config drift: 1 live knob\(s\)/); + expect(section.lines[1]).toBe( + "ai_consensus_defect.confidenceFloor (ai_consensus_defect): live 0.6 vs dominating 0.8 (tighter) — visible n=40, held-out n=12; standing 12 day(s).", + ); + // All-drifting window: no clean-summary trailer. + expect(section.lines).toHaveLength(2); + }); + + it("orders a mixed window longest-standing first and appends the clean-knob summary", () => { + const section = buildDriftRecapSection( + source({ + drifting: [ + drifting("2026-07-22T12:00:00.000Z", { knobId: "young.knob", direction: "looser", liveValue: 0.9, dominatingValue: 0.5 }), + drifting("2026-07-01T12:00:00.000Z", { knobId: "old.knob", direction: "shipped" }), + ], + cleanKnobs: 4, + }), + ); + expect(section.drifting).toBe(2); + expect(section.clean).toBe(4); + expect(section.lines[1]).toContain("old.knob"); + expect(section.lines[1]).toContain("standing 22 day(s)"); + expect(section.lines[2]).toContain("young.knob"); + expect(section.lines[2]).toContain("(looser)"); + expect(section.lines[3]).toBe("4 other evaluated knob(s) are clean."); + }); + + it("clamps a future or unparseable episode timestamp to 0 standing days instead of a negative/NaN age", () => { + const section = buildDriftRecapSection( + source({ + drifting: [drifting("2026-08-01T12:00:00.000Z", { knobId: "future.knob" }), drifting("not-a-timestamp", { knobId: "garbled.knob" })], + }), + ); + for (const line of section.lines.slice(1)) expect(line).toContain("standing 0 day(s)"); + }); + + it("INVARIANT: only knob ids and aggregate numbers reach the section — never corpus/diff content, and local paths are scrubbed", () => { + // A hostile knob id smuggling an absolute path is scrubbed by the shared recap pattern; the diff-bearing + // comparison objects on the report never surface in any emitted line. + const hostile = drifting("2026-07-20T12:00:00.000Z", { + knobId: "/Users/operator/secret/corpus.knob", + visible: { verdict: "improved", corpusDiff: "diff --git a/leak b/leak" } as unknown as KnobDriftReport["visible"], + }); + const section = buildDriftRecapSection(source({ drifting: [hostile], cleanKnobs: 1 })); + const emitted = section.lines.join("\n"); + expect(emitted).toContain(""); + expect(emitted).not.toContain("/Users/operator"); + expect(emitted).not.toMatch(/diff --git|corpusDiff|verdict/); + }); +}); diff --git a/test/unit/maintainer-recap-format.test.ts b/test/unit/maintainer-recap-format.test.ts index a798a4c51..68800f993 100644 --- a/test/unit/maintainer-recap-format.test.ts +++ b/test/unit/maintainer-recap-format.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from "vitest"; import { formatMaintainerRecap } from "../../src/services/maintainer-recap"; +import { buildDriftRecapSection } from "../../src/services/maintainer-recap-drift"; import type { RecapReport } from "../../src/types"; const GEN = "2026-07-08T00:00:00.000Z"; @@ -32,6 +33,9 @@ describe("formatMaintainerRecap (#2240)", () => { expect(body).toContain("## Summary"); expect(body).toContain("## Totals"); expect(body).toContain("## Per-repo"); + // #8214: without a sentinel projection the drift section is entirely absent — the digest stays + // byte-identical to the pre-drift shape, not a dangling empty header. + expect(body).not.toContain("## Config drift"); // Empty sections show a single fallback line instead of dangling under the header. expect(body).toContain("_No summary lines for this window._"); expect(body).toContain("_No repositories in this window._"); @@ -43,6 +47,21 @@ describe("formatMaintainerRecap (#2240)", () => { expect(body).not.toMatch(/\n{3,}/); }); + it("appends the #8214 config-drift section as bullet lines when the caller supplies a sentinel projection", () => { + const configDrift = buildDriftRecapSection({ + generatedAt: GEN, + sentinelEnabled: false, + drifting: [], + cleanKnobs: 0, + }); + const body = formatMaintainerRecap(emptyReport(), { configDrift }); + expect(body).toContain("## Config drift"); + expect(body).toContain("- drift sentinel disabled — no drift evaluation ran this window."); + // The appended section keeps the digest's formatting invariants. + expect(body.endsWith("\n")).toBe(true); + expect(body).not.toMatch(/\n{3,}/); + }); + it("renders per-repo rows, a percent rate, and redacts both regex arms (path + economic term)", () => { const report: RecapReport = { generatedAt: GEN,