From 7d81d1af35e96d3c9f6ca15d359eb1811a42e0ec Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:20:49 -0700 Subject: [PATCH] =?UTF-8?q?feat(calibration):=20counterfactual=20prompt=20?= =?UTF-8?q?replay=20=E2=80=94=20offline=20harness=20+=20budget-gated=20adv?= =?UTF-8?q?isory=20CI=20check=20(#8221,=20#8222)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the #8218 sub-epic's phases 3+4 against the #8219 contract verbatim: - scripts/counterfactual-replay-core.ts (pure, unit-tested): abstention-safe variant-output parsing (never coerced), scoreBacktest mapping over the #8220 fixture plan, budget arithmetic + resume cursor, run-accounting invariant, artifact cache keys, Pareto baseline comparison - scripts/counterfactual-replay.ts (thin IO): ollama smoke path only (BYOK refused loudly), raw outputs cached under a local artifacts dir so re-scoring is free, --baseline prints the shared renderer's comparison; exit reflects operational success only, never verdict - src/services/ai-review.ts: REVIEW_PROMPT_VERSION + buildCanonicalJudgePrompt (the pure, input-free judge prompt the dual-checkout replay diffs) - scripts/print-review-prompt.ts: the #8139 dual-checkout extractor - .github/workflows/counterfactual-replay-check.yml: path-filtered on the judge-prompt surface, hard-gated on an explicitly configured replay budget (visible skipped-notice otherwise), byte-identical prompts short-circuit, fail-open on every step, update-in-place marker comment, persisted calibration.counterfactual_backtest_run events feeding the SAME REGRESSED-verdict track record as the threshold/logic backtests - docs: worked ollama example + the prompt-change workflow --- .../workflows/counterfactual-replay-check.yml | 188 +++++++++++++ .../content/docs/backtest-calibration.mdx | 35 +++ scripts/backtest-track-record.ts | 8 +- scripts/counterfactual-replay-core.ts | Bin 0 -> 10988 bytes scripts/counterfactual-replay.ts | 264 ++++++++++++++++++ scripts/print-review-prompt.ts | 41 +++ src/services/ai-review.ts | 14 + test/unit/counterfactual-replay-core.test.ts | 186 ++++++++++++ 8 files changed, 733 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/counterfactual-replay-check.yml create mode 100644 scripts/counterfactual-replay-core.ts create mode 100644 scripts/counterfactual-replay.ts create mode 100644 scripts/print-review-prompt.ts create mode 100644 test/unit/counterfactual-replay-core.test.ts diff --git a/.github/workflows/counterfactual-replay-check.yml b/.github/workflows/counterfactual-replay-check.yml new file mode 100644 index 000000000..e5139b470 --- /dev/null +++ b/.github/workflows/counterfactual-replay-check.yml @@ -0,0 +1,188 @@ +# Counterfactual prompt replay (#8222, sub-epic #8218, epic #8211 track C). When a PR touches the reviewer +# judge-prompt surface, this job replays the CANONICAL judge prompt from BOTH the PR's head and base +# checkouts (scripts/print-review-prompt.ts — the #8139 dual-checkout mechanism) against the recorded +# raw-context fixture corpus, and posts the Pareto-floored comparison as its own advisory comment. Mirrors +# backtest-logic-check.yml's posture end-to-end: separate workflow (untouched PRs pay nothing), advisory +# only (#8105 — never a required check, never blocks merge), every step fails OPEN (notice + green; the +# review engine auto-closes contributor PRs on ANY red check, so this job's own plumbing must never redden). +# +# SPEND GUARD (#8222's requirement): the replay costs real model inference, so the whole run is gated on +# the deployment explicitly configuring a budget — the COUNTERFACTUAL_REPLAY_BUDGET repo variable (neuron +# units, see COUNTERFACTUAL_DEFAULT_NEURON_BUDGET) plus the provider endpoint/model below. Unset (the +# default), the job posts a visible "skipped: no replay budget configured" notice and does nothing else. +name: counterfactual-replay + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + # Exactly the judge-prompt surface: REVIEW_PROMPT_VERSION + buildSystemPrompt/parseModelReview live + # here. Keep in sync with #8222's spec. + paths: + - "src/services/ai-review.ts" + +permissions: + contents: read + pull-requests: write + +concurrency: + group: counterfactual-replay-${{ github.ref }} + cancel-in-progress: true + +jobs: + replay: + name: counterfactual replay (advisory) + if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork != true }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + # The budget gate comes FIRST so an unconfigured deployment pays one echo, not an npm ci. + - name: Check replay budget configuration + id: budget + env: + REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }} + REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }} + REPLAY_PROVIDER_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} + run: | + if [ -n "$REPLAY_BUDGET" ] && [ -n "$REPLAY_MODEL" ] && [ -n "$REPLAY_PROVIDER_URL" ]; then + echo "configured=true" >> "$GITHUB_OUTPUT" + else + echo "configured=false" >> "$GITHUB_OUTPUT" + echo "::notice::Counterfactual replay skipped: no replay budget configured (set the COUNTERFACTUAL_REPLAY_BUDGET + COUNTERFACTUAL_REPLAY_MODEL repo variables and the COUNTERFACTUAL_OLLAMA_URL secret to enable). Advisory only, never fails the PR." + fi + + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: ${{ steps.budget.outputs.configured == 'true' }} + with: + persist-credentials: false + + # The PR's base commit inside the head workspace — base-side dynamic imports resolve bare specifiers + # by walking up into the head checkout's node_modules, so one npm ci serves both sides (#8139). + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + if: ${{ steps.budget.outputs.configured == 'true' }} + with: + ref: ${{ github.event.pull_request.base.sha }} + path: .replay-base + persist-credentials: false + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + if: ${{ steps.budget.outputs.configured == 'true' }} + with: + node-version-file: .nvmrc + cache: "npm" + + - name: Install deps + if: ${{ steps.budget.outputs.configured == 'true' }} + run: npm ci --ignore-scripts + + - name: Build engine package + if: ${{ steps.budget.outputs.configured == 'true' }} + run: npx turbo run build --filter=@loopover/engine + + # Everything below fails OPEN — notice + green, never a red check (#8105 held against our own infra). + - name: Extract base/head canonical prompts + if: ${{ steps.budget.outputs.configured == 'true' }} + id: prompts + run: | + if npx tsx scripts/print-review-prompt.ts --root . > head-prompt.txt \ + && npx tsx scripts/print-review-prompt.ts --root .replay-base > base-prompt.txt \ + && npx tsx scripts/print-review-prompt.ts --root . --version-only > head-version.txt \ + && npx tsx scripts/print-review-prompt.ts --root .replay-base --version-only > base-version.txt; then + if cmp -s head-prompt.txt base-prompt.txt; then + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "::notice::Judge prompt surface touched but the canonical prompt text is byte-identical across base/head — nothing to replay." + else + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + else + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "::notice::Prompt extraction failed — replay skipped. Advisory only, never fails the PR." + fi + + - name: Export corpus from D1 + if: ${{ steps.budget.outputs.configured == 'true' && steps.prompts.outputs.changed == 'true' }} + id: corpus + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + run: | + if npx tsx scripts/backtest-corpus-export.ts --rule-id ai_consensus_defect --output replay-corpus.json --remote; then + echo "available=true" >> "$GITHUB_OUTPUT" + else + echo "available=false" >> "$GITHUB_OUTPUT" + echo "::notice::Corpus export from D1 failed — replay skipped. Advisory only, never fails the PR." + fi + + # Two runs over the IDENTICAL deterministic sample (seed = head SHA, per the design contract's + # per-PR determinism requirement): base prompt first (the baseline artifact), then head with + # --baseline (comparison + comment + persisted run event). + - name: Replay base and head prompts + if: ${{ steps.corpus.outputs.available == 'true' }} + id: replay + env: + CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }} + CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }} + REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }} + REPLAY_PROVIDER_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + BASE_VERSION=$(cat base-version.txt) + HEAD_VERSION=$(cat head-version.txt) + if npx tsx scripts/counterfactual-replay.ts \ + --fixtures replay-corpus.json \ + --variant "${BASE_VERSION}@${REPLAY_MODEL}" \ + --prompt-file base-prompt.txt \ + --budget "$REPLAY_BUDGET" \ + --seed-suffix "$HEAD_SHA" \ + --ollama-url "$REPLAY_PROVIDER_URL" \ + --out base-scores.json \ + && npx tsx scripts/counterfactual-replay.ts \ + --fixtures replay-corpus.json \ + --variant "${HEAD_VERSION}@${REPLAY_MODEL}" \ + --prompt-file head-prompt.txt \ + --budget "$REPLAY_BUDGET" \ + --seed-suffix "$HEAD_SHA" \ + --ollama-url "$REPLAY_PROVIDER_URL" \ + --baseline base-scores.json \ + --base-variant-label "$BASE_VERSION" \ + --comment-out replay-comment.md \ + --head-sha "$HEAD_SHA" --base-sha "$BASE_SHA" \ + --persist --remote --db loopover \ + --repo "$GITHUB_REPOSITORY" --pr "$PR_NUMBER"; then + echo "ready=true" >> "$GITHUB_OUTPUT" + else + echo "ready=false" >> "$GITHUB_OUTPUT" + echo "::notice::Counterfactual replay failed — no comparison produced. Advisory only, never fails the PR." + fi + + - name: Post or update the PR comment + if: ${{ steps.replay.outputs.ready == 'true' }} + env: + GH_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + post_comment() { + marker="" + comment_id=$(gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" --paginate \ + --jq "[.[] | select(.body | contains(\"${marker}\")) | .id] | first // empty" | head -n 1) + if [ -n "$comment_id" ]; then + gh api "repos/${GITHUB_REPOSITORY}/issues/comments/${comment_id}" -X PATCH -F body=@replay-comment.md + else + gh api "repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/comments" -F body=@replay-comment.md + fi + } + if ! post_comment; then + echo "::notice::PR comment post failed — replay computed and persisted but not posted. Advisory only, never fails the PR." + fi + + # Fork half of the paired convention: fork runs get no secrets, so the replay cannot run — say so. + fork-notice: + name: counterfactual replay (skipped for fork PRs) + if: ${{ github.event.pull_request.draft != true && github.event.pull_request.head.repo.fork == true }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Explain the skip + run: echo "::notice::Fork PR — repo secrets are withheld, so the counterfactual replay is skipped. Advisory only; nothing blocks." diff --git a/apps/loopover-ui/content/docs/backtest-calibration.mdx b/apps/loopover-ui/content/docs/backtest-calibration.mdx index 6b6c0b125..3cee9b054 100644 --- a/apps/loopover-ui/content/docs/backtest-calibration.mdx +++ b/apps/loopover-ui/content/docs/backtest-calibration.mdx @@ -162,6 +162,41 @@ contract (`@loopover/engine`'s `counterfactual-contract` module). Three decision excluded from the confusion matrix and counted separately, never coerced to a verdict, so a degenerate prompt cannot farm precision by failing to answer on hard cases. +### The replay harness + +The contract is implemented by `scripts/counterfactual-replay.ts` (pure core in +`counterfactual-replay-core.ts`). A worked local example against an ollama model: + +```bash +# 1. Export the labeled corpus (raw-context diffs included) to a manifest: +tsx scripts/backtest-corpus-export.ts --rule-id ai_consensus_defect --output corpus.json --remote + +# 2. Score the current prompt as the baseline: +tsx scripts/counterfactual-replay.ts --fixtures corpus.json \ + --variant review-prompt-v1@llama3.1:8b --out baseline.json + +# 3. Score a candidate prompt against the SAME deterministic sample and print the Pareto comparison: +tsx scripts/counterfactual-replay.ts --fixtures corpus.json \ + --variant candidate@llama3.1:8b --prompt-file candidate-prompt.txt \ + --baseline baseline.json +``` + +Raw model outputs cache under `.counterfactual-artifacts/` (never committed), so re-scoring after a +parser or mapping change costs nothing. Exit codes reflect operational success only — never a verdict. + +### The prompt-change workflow + +A PR that touches the reviewer judge-prompt surface (`src/services/ai-review.ts` — bump +`REVIEW_PROMPT_VERSION` whenever a change shapes the judge's verdict) triggers the +`counterfactual-replay` advisory workflow: it extracts the **canonical judge prompt** from both the +base and head checkouts, replays both over the identical seeded sample (seed = head SHA, so re-runs +per push are deterministic), posts the comparison as an update-in-place PR comment, and persists a +`calibration.counterfactual_backtest_run` event feeding the same REGRESSED-verdict track record as the +threshold and logic backtests. The check spends nothing unless the deployment explicitly configures a +replay budget (repo variables `COUNTERFACTUAL_REPLAY_BUDGET`/`COUNTERFACTUAL_REPLAY_MODEL` plus the +provider endpoint secret) — otherwise it posts a visible "skipped: no replay budget configured" notice. +Advisory forever until a separate, explicit authority decision, exactly like every other backtest gate. + ## Self-hosting On a self-host deployment the same calibration system runs against your Postgres instead of D1 — diff --git a/scripts/backtest-track-record.ts b/scripts/backtest-track-record.ts index 6c4668348..94aea6a4a 100644 --- a/scripts/backtest-track-record.ts +++ b/scripts/backtest-track-record.ts @@ -1,7 +1,8 @@ #!/usr/bin/env node // Read-only D1 → REGRESSED-verdict track-record summary (#8140, epic #8082). Reads the BacktestComparison -// results the advisory backtests persist — #8138's ORB-native threshold runs AND #8139's CI-side logic -// runs, sibling event types with the same metadata.comparison shape — out of audit_events via `wrangler d1 +// results the advisory backtests persist — #8138's ORB-native threshold runs, #8139's CI-side logic +// runs, and #8222's CI-side counterfactual prompt replays, sibling event types with the same +// metadata.comparison shape — out of audit_events via `wrangler d1 // execute --json`, aggregates them with the pure computeRegressedVerdictTrackRecord (@loopover/engine), and // prints the summary #8105's Phase-2 merge-gating decision needs. The aggregation lives in the engine // (pure, unit-tested); this file is the thin IO wrapper — mirrors backtest-corpus-export.ts's identical split. @@ -14,6 +15,7 @@ import { spawnSync } from "node:child_process"; import { openPgDatabase, resolvePgConnection } from "./pg-cli.js"; import { computeRegressedVerdictTrackRecord, type BacktestComparison } from "@loopover/engine"; import { LOGIC_BACKTEST_EVENT_TYPE } from "./backtest-logic-check-core.js"; +import { COUNTERFACTUAL_BACKTEST_EVENT_TYPE } from "./counterfactual-replay-core.js"; // Mirrors THRESHOLD_BACKTEST_EVENT_TYPE in src/services/threshold-backtest-run.ts (#8138's writer) and must // be kept in sync with it by hand — that module is Worker-bound (D1 repositories import graph) and @@ -59,7 +61,7 @@ async function main() { console.error("Usage: tsx scripts/backtest-track-record.ts --db [--remote] | --pg "); process.exit(2); } - const sql = `SELECT metadata_json FROM audit_events WHERE event_type IN ('${THRESHOLD_BACKTEST_EVENT_TYPE}', '${LOGIC_BACKTEST_EVENT_TYPE}') ORDER BY created_at ASC`; + const sql = `SELECT metadata_json FROM audit_events WHERE event_type IN ('${THRESHOLD_BACKTEST_EVENT_TYPE}', '${LOGIC_BACKTEST_EVENT_TYPE}', '${COUNTERFACTUAL_BACKTEST_EVENT_TYPE}') ORDER BY created_at ASC`; const rows = pgConnection ? await pgQuery(pgConnection, sql) : d1Query(args.db!, args.remote, sql); const comparisons: BacktestComparison[] = []; for (const row of rows) { diff --git a/scripts/counterfactual-replay-core.ts b/scripts/counterfactual-replay-core.ts new file mode 100644 index 0000000000000000000000000000000000000000..e44d4a77dec3159d942c6849bf48f5ebb0f6ceae GIT binary patch literal 10988 zcma)CTXP%7u|CiG6=V2R2-vuga(rx4mK=(qBgPV`Lr_X3mJ1l{4uBCCyYS8~DMVpa z-twNxFZ3_T*WELF0l+veS;FpIx~DH+UuH)m^}5WBiZc8)&vI2P>7h&ulh1WjlsZwl zStojGBeO3KFQbn$#8+FkEgzf^9W*rKn98%T-><>u$o<|bBaT|j85KeKX6y@n_)PNL>{ zwqiOYs&63Qlq6Ir2xq1XyXvdd;5YUf*!0#w`0?VKivvcS)A5GbnOo(uGY8RP$)`*=On+KIJwk z45om)r20|_s2s!04r2{$m3|V%hQ*U!KqgGh2Z%oPn04Iveh&EQvE>fKg)n}te z-XUPQu283!Z!WISuD(Axy?%4@a&qkN9EKfYkUK+&YYn?j-o_JjO z@dKxFeW$XrSeJ#eX_2wZ1ZTl8xfhg9ZbO0}vS5Qi!1rohzyWR?pFg4(O`t0KRoR*1FV#YoPlV6)S08W&dJOV%w zrZ<6Uok9NUi({h3bI2T+ZELv*;#65azW6Q${N`ksV8Hz_QipxTvr4kU|dBhZ0FNLn8lJ?R~(ZfM687(8J{ zHfOti`V{sr>pSW@es!!mFnWSqS^^(~<6#Y!4lS_u!{xl&`Pk{v6gyX4KLPs;j(YG{ zKz6D+dWQEfj+nPX!1?>VF6kJu5uy7Dn3~fZG80b6Te2;Btg?N!TmCXxy zIR#V5R^+MiN}|44G6cpdj?EhG@>CrXccgQhj(JV(Qy$UfHF(RX zdZMiQR!*J(8;d2q`24BDIX`0mAfgD?fh8oNAMH<{5D5{DClY2WwTG%gbY!O?Sq5(F z74%m2Cq=z`*XcDjDe{sJMD(gXs@VsfgWlfrw@9|9%dx%!_99}Bw^}a7^ZE7hfhujWrx=Bzg?j+48Vn*_8Yovpu*_ZvY}X!dZrHH zYn>WgI|@$M23YnR+3p_51BYoH{y&kwX9I+m`Cs~*{@=IaULrjb4+L*h=D=*ElNSC7 zAj=VXRvi21dS#OhyDmqZ8O3x!KVTAwsJdaz1#jUHfXF_ODw=#S18BwslWK`k3i%O$ zil7*yE1U8g%1j4yN%x>vf}KnW72xJlmskb>M36utz>(fk3ZBj9$d^nr)K_2`c=Ac4 zXRYgrY(u4=g%p{?!X;w$5~=nnj^U0~TCQd$2ix+472(DCN*BvvWNgyy?pX?}^8|-8 z3J4}XPmw0LVQlAfZw);)QorqC=9fp0pFHv!`F%?|p&Ec4M1ap5vpEjwM@@kdlM<#u zULxvI+Q8G1vQ_yB$#39OH^3dMWu6;=FIPuUTp&e-hl?!bqXr@GdlVmTgZgGQaC&Z_ z%MW{{9+v-LHeH|MY>V0luzrxtdE5wK5(An7uFHsS2gS+bFaNL88;8D7O)8xtAbF_uuGt{7Nrf<6rGH6Wxhw`nUa_w9wH+J zLhy5>hKjsI9I2=NbbkHf^364T&Fid>5gd?pDk@x9lD8Pc<%7k?!7Tbky5u-cCSb-+BHGU;F5M26Z$o4+le$Kmj zvn6a5*m$Ma-%2d$Z(;IVnfr{^Ny_689nvBM_x>Xw2fIjcK|%{(3y~?tWTAliHZZ|a z1?FX@SIYAZ0wIP{bEkftr7qW0T&afY32UoTrZ$rRE$VNgaS(vw2?@A{5f*~wkc&ef z!vOUmrEW0_I-X@WL~0k(q_F55{R8YSsj;7^sC1;9W=>^Z*j1*~tnw`4jxy$8=Ah=e zsR_vbENNtxp{|h>mWrSz)kbw$)2%AbMgg*ayQXr6aFX5A_JcJ1vDv;w@k7MeWtVJ% z{-drKB{2JYOJrm8A4+Hl6{c{!d+sm?ahGB+>HvT}(u+^PToa&j~9nMy}z0cXhjLPQ1iz>-um-A0yun9NeUF{U|=AQmRM&Rp!Yrr_5 z8vWD6#)vtHj3J;B!sGPPl1IA&7Xg7)6a^J$`0d$HUA#^=@L42Rs3WMCyF!?!s|Tvj z&=8?^drZx$nysE%aoCPHd{vh%cS|(Ef;D4Hgi3_Zp}&?{8eF6hgLDrVji$VrU(bm=mFm^x`dvq5CW0B}CTk$Wf<5K=@e({t-^eO2o>;8DQrv zybMMA=knSMbC_lm)KwGG05#RNM%k21Jb_meunC1Ab~uuc{bE_9BLAXNTfM4QSG}%Q zPlc~&C>+8e_Od>JK3U_mt9OsU%;ybzsQGyCl!4JOp>a?j5X2JHZ+_F*_!);WZ`03# z1XNZ+Ds-nqmo>+1EU%G571>~p`v@|_fcP_0+)?IM+-2SsaR!`oL!CihI6grs65<_6 zfRal_L?w?_0le=B70t%>06&uONd5A!zvB;Ga6W!4l0AhN#xr{bw~bc@{BXH2yJG%2VJ(bEk3&yV=fES$zxHQAH0tx}J*t~|G1jZ&fY^SdD_U?Xq`RWyI`|9NChqEi% z_}fng14u>g=OF&()@ZHC^{*NXp3&Cf(ymhV>#Lw;03M8VAYBr5>(Ic#4UvBoZ;~>M zoB(+OY8srWvYz+ztA~xKd1w!DTiq z0}OCe52SHDw#VUw@@eb%r((ZUZ`8E+ITJ-xo6fTk_gk$Z0;7;l6FQ=J;SS(i@`)^c z0Wr0QtA0}ZHOX_l)a{of^PQK1`JvnpnicE|JJM#{8WL8TLmq=KvMYUeDsia2tq14J zvZ`)mPY1y_IYrU|7~1W<{m25@a)GvyO4ic(^wX^}%Zt$?wa-F3{@igMT5v?{uQaJWn9q4$GJXz3^e!6@_TSDaG8 zZekc4xRu{@D?Xfh6Y;u#=z9%(B@?oNo?kIJ2{DU1ff5m@p~2~;e0#z-GH7}lR9Mu- zK05reC*R9hTq4koA+iMOBXU=y`kuhwEpT@P1;m2#-r=NWrauke<2t$9d6&{%r_Wu+ z2y2NM{9I<#H5&i_%iuNs;Gy7T45hM^SE$X}VA_JNg(S&LXspQpC8f%)%!V1-=ZBFK`BlT`bXVi*Wz!6s3QP z8*hlR=a6aJ7x_zdf=vQbTC@Z zDZ&)-3!grSY$ZxHvjcFT5zi>R%EuQ&b&3=LZ(Qk{j5Z$0{R8S|G$WZ3x3fvmFhu$$ zW55?EiM=81&=*>0a{9cKvdz8mY*yMNZilmz67}7AN~O~H=LGFObU-Cy*%nq;#c=>^ zg7n&V_60zi;o+>g$b)9{RSOuO!{;9yB=~3!R#4b{xf|*#Y1Y&3+x8`iQP3uT)#C4V zB|h2t#&=(uhh}j#vO__T>d-RmA7!jMy^Q%s8QUWUnYA;pXTC^g&wLt?`3+&oyy}IF z2f4|hJkf|WW<)~ld=5FH1nC_YmbEtaV7fED71j47wgH9@LjHN+@o{G_p;cLCwU9JK zo#*($mD6@g+s+y!2*M8OsG@Xa$2*Z^z^0a{Wz_4Z;Y3c#&{Nnmj zX>!sDmo-3Liw~0c_k>CuM4hO}0PG@%32tf8lb5a@&>j_xP5KB_Qprr{lOYsuz#zzH XqKlq-d-C$l*;sWwu)6ymqv*c@g=L%q literal 0 HcmV?d00001 diff --git a/scripts/counterfactual-replay.ts b/scripts/counterfactual-replay.ts new file mode 100644 index 000000000..84d8d2265 --- /dev/null +++ b/scripts/counterfactual-replay.ts @@ -0,0 +1,264 @@ +#!/usr/bin/env node +// Counterfactual replay harness CLI (#8221, sub-epic #8218). Replays a judge variant (prompt version + +// model spec) against the replayable fixture set assembled from a backtest-corpus manifest +// (backtest-corpus-export.ts's output), scores it per the #8219 contract, and — with --baseline — +// prints the Pareto-floored comparison via the shared renderer. All mapping/aggregation logic lives in +// counterfactual-replay-core.ts (unit-tested); this wrapper owns provider IO, the artifacts cache, and +// file reads. OFFLINE-ONLY guarantees (#8219 contract point 2): never posts anywhere, never touches live +// reviews; local ollama is the smoke path and the ONLY provider wired here — BYOK providers are a later, +// explicitly-flagged addition, refused loudly today rather than silently attempted. +// +// tsx scripts/counterfactual-replay.ts --fixtures corpus.json --variant +// [--budget N] [--max-fixtures N] [--seed-suffix s] [--baseline scores.json] [--out scores.json] +// [--artifacts dir] [--ollama-url http://localhost:11434] [--prompt-file file.txt] +// +// Exit code reflects OPERATIONAL success only — never a verdict (the #8138 advisory guarantee). A run that +// exhausts its budget mid-set persists partial scores + a resume cursor and still exits 0. +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { + COUNTERFACTUAL_DEFAULT_NEURON_BUDGET, + COUNTERFACTUAL_SAMPLE_SEED_PREFIX, + renderBacktestScoreReport, + renderBacktestComparison, + type BacktestCase, + type BacktestScoreReport, + type CounterfactualVariant, + type CounterfactualVerdict, +} from "@loopover/engine"; +import { spawnSync } from "node:child_process"; +import { + artifactKey, + buildCounterfactualAuditInsertSql, + compareReplays, + estimateFixtureNeurons, + isRunAccountingValid, + parseVariantVerdict, + planReplay, + renderCounterfactualComment, +} from "./counterfactual-replay-core.js"; + +type Args = { + fixtures: string | undefined; + variant: string | undefined; + budget: number; + maxFixtures: number; + seedSuffix: string; + baseline: string | undefined; + out: string | undefined; + artifacts: string; + ollamaUrl: string; + promptFile: string | undefined; + // #8222 CI mode: emit the marker comment and/or persist the run event (wrangler d1, like #8139). + commentOut: string | undefined; + baseVariantLabel: string | undefined; + headSha: string; + baseSha: string; + persist: boolean; + repo: string | undefined; + pr: string | undefined; + db: string; + remote: boolean; +}; + +function parseArgs(argv: string[]): Args { + const args: Args = { + fixtures: undefined, + variant: undefined, + budget: COUNTERFACTUAL_DEFAULT_NEURON_BUDGET, + maxFixtures: 500, + seedSuffix: "default", + baseline: undefined, + out: undefined, + artifacts: ".counterfactual-artifacts", + ollamaUrl: "http://localhost:11434", + promptFile: undefined, + commentOut: undefined, + baseVariantLabel: undefined, + headSha: "", + baseSha: "", + persist: false, + repo: undefined, + pr: undefined, + db: "loopover", + remote: false, + }; + for (let i = 0; i < argv.length; i += 1) { + const flag = argv[i]; + if (flag === "--fixtures") args.fixtures = argv[++i]; + else if (flag === "--variant") args.variant = argv[++i]; + else if (flag === "--budget") args.budget = Number(argv[++i]); + else if (flag === "--max-fixtures") args.maxFixtures = Number(argv[++i]); + else if (flag === "--seed-suffix") args.seedSuffix = argv[++i] ?? "default"; + else if (flag === "--baseline") args.baseline = argv[++i]; + else if (flag === "--out") args.out = argv[++i]; + else if (flag === "--artifacts") args.artifacts = argv[++i] ?? args.artifacts; + else if (flag === "--ollama-url") args.ollamaUrl = argv[++i] ?? args.ollamaUrl; + else if (flag === "--prompt-file") args.promptFile = argv[++i]; + else if (flag === "--comment-out") args.commentOut = argv[++i]; + else if (flag === "--base-variant-label") args.baseVariantLabel = argv[++i]; + else if (flag === "--head-sha") args.headSha = argv[++i] ?? ""; + else if (flag === "--base-sha") args.baseSha = argv[++i] ?? ""; + else if (flag === "--persist") args.persist = true; + else if (flag === "--repo") args.repo = argv[++i]; + else if (flag === "--pr") args.pr = argv[++i]; + else if (flag === "--db") args.db = argv[++i] ?? "loopover"; + else if (flag === "--remote") args.remote = true; + } + return args; +} + +// The default judge prompt: the minimal would-flag question, JSON-only output so parseVariantVerdict's +// abstention accounting stays honest. A --prompt-file replaces it wholesale (that IS the variant surface +// #8222 will diff), and the prompt text is recorded in the artifacts dir alongside the outputs. +const DEFAULT_JUDGE_PROMPT = [ + "You are a strict senior code reviewer. Below is a unified diff from a pull request that a review gate", + "previously judged. Decide whether the diff contains a defect serious enough to block a merge.", + 'Respond with ONLY a JSON object: {"blockers": [..strings, empty if none..], "confidence": 0..1}.', +].join("\n"); + +async function judgeWithOllama(ollamaUrl: string, model: string, prompt: string, diff: string): Promise { + const response = await fetch(`${ollamaUrl}/api/generate`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ model, prompt: `${prompt}\n\n--- DIFF ---\n${diff}`, stream: false, options: { temperature: 0 } }), + }); + if (!response.ok) throw new Error(`ollama ${response.status}: ${(await response.text()).slice(0, 300)}`); + const body = (await response.json()) as { response?: unknown }; + return typeof body.response === "string" ? body.response : ""; +} + +async function main(): Promise { + const args = parseArgs(process.argv.slice(2)); + if (!args.fixtures || !args.variant || !args.variant.includes("@")) { + console.error("Usage: tsx scripts/counterfactual-replay.ts --fixtures --variant [...]"); + return 1; + } + if (!Number.isFinite(args.budget) || args.budget <= 0 || !Number.isInteger(args.maxFixtures) || args.maxFixtures <= 0) { + console.error("--budget and --max-fixtures must be positive."); + return 1; + } + const [promptVersion, ...modelParts] = args.variant.split("@"); + const variant: CounterfactualVariant = { promptVersion: promptVersion!, modelSpec: modelParts.join("@") }; + if (!variant.promptVersion || !variant.modelSpec) { + console.error("--variant must be ."); + return 1; + } + + const manifest = JSON.parse(readFileSync(args.fixtures, "utf8")) as { cases?: BacktestCase[] }; + if (!Array.isArray(manifest.cases)) { + console.error(`--fixtures file has no cases[] — expected a backtest-corpus-export manifest.`); + return 1; + } + const sampling = { seed: `${COUNTERFACTUAL_SAMPLE_SEED_PREFIX}:${args.seedSuffix}`, maxFixtures: args.maxFixtures }; + const plan = planReplay(manifest.cases, sampling); + if (plan.fixtures.length === 0) { + console.log("No replayable fixtures (every case lacked bounded raw context). Nothing to run."); + return 0; + } + + mkdirSync(args.artifacts, { recursive: true }); + const prompt = args.promptFile ? readFileSync(args.promptFile, "utf8") : DEFAULT_JUDGE_PROMPT; + writeFileSync(join(args.artifacts, `prompt-${variant.promptVersion}.txt`), prompt); + + const { scoreReplay } = await import("./counterfactual-replay-core.js"); + const verdicts = new Map(); + let neuronsSpent = 0; + for (const fixture of plan.fixtures) { + const cost = estimateFixtureNeurons(fixture, prompt.length); + if (neuronsSpent + cost > args.budget) break; // budget exhausted: partial run + cursor, by design + const cachePath = join(args.artifacts, `${artifactKey(variant, fixture.fixtureId)}.txt`); + let raw: string; + if (existsSync(cachePath)) { + raw = readFileSync(cachePath, "utf8"); // cached raw output: re-scoring is free + } else { + raw = await judgeWithOllama(args.ollamaUrl, variant.modelSpec, prompt, fixture.boundedInputs.diff); + writeFileSync(cachePath, raw); + neuronsSpent += cost; // cache hits spend nothing — only real provider calls count + } + verdicts.set(fixture.fixtureId, parseVariantVerdict(raw)); + } + + const scored = scoreReplay(plan, variant, sampling, verdicts, neuronsSpent); + if (!isRunAccountingValid(plan, scored.summary)) { + console.error("run accounting invariant violated — refusing to persist a summary that does not sum to the fixture universe."); + return 1; + } + + console.log(renderBacktestScoreReport(scored.report)); + console.log( + `scored ${scored.summary.scored} | abstained ${scored.summary.abstained} | skipped no_raw_context ${scored.summary.skipped.no_raw_context}, sampled_out ${scored.summary.skipped.sampled_out} | neurons ${scored.summary.neuronsSpent}/${args.budget}` + + (scored.summary.resumeFrom ? ` | resume from ${scored.summary.resumeFrom}` : ""), + ); + + if (args.out) writeFileSync(args.out, `${JSON.stringify({ report: scored.report, summary: scored.summary }, null, 2)}\n`); + + if (args.baseline) { + const baseline = JSON.parse(readFileSync(args.baseline, "utf8")) as { + report?: BacktestScoreReport; + summary?: { abstained?: number; variant?: { promptVersion?: string } }; + }; + if (!baseline.report) { + console.error("--baseline file has no report — expected a prior --out artifact."); + return 1; + } + const comparison = compareReplays(baseline.report, scored.report); + console.log(renderBacktestComparison(comparison)); + + // #8222 CI mode: the marker comment + the persisted run event, mirroring backtest-logic-check.ts. + if (args.commentOut) { + writeFileSync( + args.commentOut, + renderCounterfactualComment(comparison, { + promptVersionBase: args.baseVariantLabel ?? baseline.summary?.variant?.promptVersion ?? "base", + promptVersionHead: variant.promptVersion, + modelSpec: variant.modelSpec, + headSha: args.headSha, + baseSha: args.baseSha, + scored: scored.summary.scored, + abstainedBase: baseline.summary?.abstained ?? 0, + abstainedHead: scored.summary.abstained, + skippedNoRawContext: scored.summary.skipped.no_raw_context, + sampledOut: scored.summary.skipped.sampled_out, + seed: sampling.seed, + }), + ); + } + if (args.persist) { + if (!args.repo || !args.pr) { + console.error("--persist requires --repo and --pr."); + return 1; + } + const sql = buildCounterfactualAuditInsertSql({ + id: crypto.randomUUID(), + targetKey: `${args.repo}#${args.pr}`, + comparison, + headSha: args.headSha, + baseSha: args.baseSha, + promptVersionBase: args.baseVariantLabel ?? baseline.summary?.variant?.promptVersion ?? "base", + promptVersionHead: variant.promptVersion, + modelSpec: variant.modelSpec, + scored: scored.summary.scored, + createdAt: new Date().toISOString(), + }); + const result = spawnSync("npx", ["wrangler", "d1", "execute", args.db, args.remote ? "--remote" : "--local", "--json", "--command", sql], { + encoding: "utf8", + maxBuffer: 256 * 1024 * 1024, + }); + if (result.status !== 0) { + console.error(`persist failed (${result.status}): ${(result.stderr || result.stdout || "").slice(0, 400)}`); + return 1; + } + console.log("run persisted (calibration.counterfactual_backtest_run)."); + } + } + return 0; +} + +main().then( + (code) => process.exit(code), + (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }, +); diff --git a/scripts/print-review-prompt.ts b/scripts/print-review-prompt.ts new file mode 100644 index 000000000..2c8151b44 --- /dev/null +++ b/scripts/print-review-prompt.ts @@ -0,0 +1,41 @@ +#!/usr/bin/env node +// Canonical judge-prompt extractor (#8222) — the dual-checkout seam the counterfactual replay workflow +// uses: dynamically import src/services/ai-review.ts from a given checkout root (the PR's head or its +// base, exactly like backtest-logic-check.ts imports detection functions) and print either the canonical +// prompt text or its declared version. Read-only; no env, no network. +// +// tsx scripts/print-review-prompt.ts --root [--version-only] +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +async function main(): Promise { + const argv = process.argv.slice(2); + let root: string | undefined; + let versionOnly = false; + for (let i = 0; i < argv.length; i += 1) { + if (argv[i] === "--root") root = argv[++i]; + else if (argv[i] === "--version-only") versionOnly = true; + } + if (!root) { + console.error("Usage: tsx scripts/print-review-prompt.ts --root [--version-only]"); + return 1; + } + const moduleUrl = pathToFileURL(resolve(root, "src/services/ai-review.ts")).href; + const mod = (await import(moduleUrl)) as { REVIEW_PROMPT_VERSION?: string; buildCanonicalJudgePrompt?: () => string }; + if (typeof mod.REVIEW_PROMPT_VERSION !== "string" || typeof mod.buildCanonicalJudgePrompt !== "function") { + // A base checkout predating #8222 has neither export — print a sentinel so the workflow can treat the + // base as "unversioned" rather than failing (advisory discipline: never a red check over plumbing). + console.log(versionOnly ? "unversioned" : ""); + return 0; + } + console.log(versionOnly ? mod.REVIEW_PROMPT_VERSION : mod.buildCanonicalJudgePrompt()); + return 0; +} + +main().then( + (code) => process.exit(code), + (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }, +); diff --git a/src/services/ai-review.ts b/src/services/ai-review.ts index 768792b9b..1ea7c357c 100644 --- a/src/services/ai-review.ts +++ b/src/services/ai-review.ts @@ -987,6 +987,20 @@ const IMPROVEMENT_SIGNAL_SUFFIX = * prioritization suffix when on, then the inline-findings instruction when the caller asked for them, then the * improvement-signal instruction when the caller resolved that feature on; all absent (default) → the base * prompt, byte-identical to today. */ +/** #8222: the judge-prompt VERSION the counterfactual replay workflow keys on. Bump this on ANY change + * that shapes the judge's verdict surface — REVIEW_SYSTEM_PROMPT, buildSystemPrompt's suffix composition, + * or parseModelReview's accepted output shape. The CI replay compares base vs head canonical prompts and + * only spends when the version (or the canonical text) actually changed. */ +export const REVIEW_PROMPT_VERSION = "review-prompt-v1"; + +/** #8222: the CANONICAL judge prompt — buildSystemPrompt with every optional suffix absent, which is the + * exact base-model system prompt a default-configured repo's review runs under. The replay harness diffs + * and replays THIS text across a PR's base/head checkouts (the #8139 dual-checkout mechanism), so it must + * stay a pure function of the source alone: no inputs, no env, no clock. */ +export function buildCanonicalJudgePrompt(): string { + return buildSystemPrompt({ repoFullName: "canonical/fixture", title: "", body: "", diff: "" } as LoopOverAiReviewInput); +} + function buildSystemPrompt(input: LoopOverAiReviewInput): string { const groundingSuffix = input.grounding?.systemSuffix ?? ""; // Review-enrichment brief (#1472): the REES supplies a one-line discipline suffix ("treat a listed CVE/secret as diff --git a/test/unit/counterfactual-replay-core.test.ts b/test/unit/counterfactual-replay-core.test.ts new file mode 100644 index 000000000..562c92f7b --- /dev/null +++ b/test/unit/counterfactual-replay-core.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it } from "vitest"; +import type { BacktestCase, CounterfactualVerdict } from "@loopover/engine"; +import { REVIEW_PROMPT_VERSION, buildCanonicalJudgePrompt } from "../../src/services/ai-review"; +import { + artifactKey, + buildCounterfactualAuditInsertSql, + compareReplays, + COUNTERFACTUAL_BACKTEST_EVENT_TYPE, + COUNTERFACTUAL_COMMENT_MARKER, + COUNTERFACTUAL_RULE_ID, + renderCounterfactualComment, + estimateFixtureNeurons, + isRunAccountingValid, + parseVariantVerdict, + planReplay, + scoreReplay, +} from "../../scripts/counterfactual-replay-core.js"; + +// #8221: the harness's pure core. Provider IO lives in the wrapper; every mapping/aggregation arm is here. + +function replayCase(id: number, label: "confirmed" | "reversed", diff: string | null = "diff --git a/x b/x"): BacktestCase { + return { + ruleId: "ai_consensus_defect", + targetKey: `acme/widgets#${id}`, + outcome: "close", + label, + firedAt: "2026-06-01T00:00:00.000Z", + decidedAt: "2026-06-02T00:00:00.000Z", + ...(diff !== null ? { metadata: { diff } } : {}), + }; +} + +const SAMPLING = { seed: "counterfactual-replay-v1:test", maxFixtures: 100 }; +const VARIANT = { promptVersion: "v1", modelSpec: "llama3.1:8b" }; + +describe("parseVariantVerdict (#8219 scoring contract)", () => { + it("maps a parseable blockers array to would_flag/would_not_flag by emptiness", () => { + expect(parseVariantVerdict('{"blockers": ["race condition"], "confidence": 0.9}')).toBe("would_flag"); + expect(parseVariantVerdict('{"blockers": [], "confidence": 0.7}')).toBe("would_not_flag"); + }); + + it("tolerates fences and surrounding prose by extracting the first balanced object (string-aware)", () => { + expect(parseVariantVerdict('Sure! Here is my review:\n```json\n{"blockers": ["a \\"quoted\\" {brace} issue"]}\n```\nHope that helps.')).toBe( + "would_flag", + ); + expect(parseVariantVerdict('prefix {"blockers": []} suffix {"blockers": ["x"]}')).toBe("would_not_flag"); // FIRST object wins + }); + + it("ABSTAINS on everything else — never coerced: no JSON, unbalanced, invalid JSON, missing/non-array blockers, empty", () => { + for (const raw of ["I think it looks fine.", "{\"blockers\": [", '{"blockers": "none"}', '{"confidence": 1}', "", "{not json}"]) { + expect(parseVariantVerdict(raw)).toBe("abstained"); + } + expect(parseVariantVerdict(undefined as unknown as string)).toBe("abstained"); + }); +}); + +describe("planReplay + scoreReplay (#8221)", () => { + it("scores verdicts against labels with scoreBacktest unchanged: would_flag ⇒ predicted reversed", () => { + const plan = planReplay([replayCase(1, "reversed"), replayCase(2, "confirmed"), replayCase(3, "reversed")], SAMPLING); + const verdicts = new Map([ + ["acme/widgets#1", "would_flag"], // true positive + ["acme/widgets#2", "would_flag"], // false positive + ["acme/widgets#3", "would_not_flag"], // false negative + ]); + const { report, summary } = scoreReplay(plan, VARIANT, SAMPLING, verdicts, 1234); + expect(report.ruleId).toBe(COUNTERFACTUAL_RULE_ID); + expect(report.truePositive).toBe(1); + expect(report.falsePositive).toBe(1); + expect(report.falseNegative).toBe(1); + expect(report.precision).toBe(0.5); + expect(report.recall).toBe(0.5); + expect(summary).toMatchObject({ scored: 3, abstained: 0, neuronsSpent: 1234, resumeFrom: null, variant: VARIANT }); + expect(isRunAccountingValid(plan, summary)).toBe(true); + }); + + it("abstentions are counted and EXCLUDED from the matrix; a missing verdict sets the resume cursor at the first gap", () => { + const plan = planReplay([replayCase(1, "reversed"), replayCase(2, "confirmed"), replayCase(3, "confirmed")], SAMPLING); + const verdicts = new Map([ + ["acme/widgets#1", "abstained"], + // #2 unreached (budget), #3 unreached + ]); + const { report, summary } = scoreReplay(plan, VARIANT, SAMPLING, verdicts, 99); + expect(report.caseCount).toBe(0); // the abstention never entered the matrix + expect(summary.abstained).toBe(1); + expect(summary.resumeFrom).toBe("acme/widgets#2"); + expect(isRunAccountingValid(plan, summary)).toBe(true); + }); + + it("skip accounting flows from the assembler: non-replayable cases count no_raw_context and the invariant still holds", () => { + const plan = planReplay([replayCase(1, "reversed"), replayCase(2, "confirmed", ""), replayCase(3, "confirmed", null)], SAMPLING); + expect(plan.fixtures).toHaveLength(1); + expect(plan.skipped.no_raw_context).toBe(2); + const { summary } = scoreReplay(plan, VARIANT, SAMPLING, new Map([["acme/widgets#1", "would_flag" as const]]), 5); + expect(isRunAccountingValid(plan, summary)).toBe(true); + // A summary that lies about its universe fails the invariant. + expect(isRunAccountingValid(plan, { ...summary, scored: summary.scored + 1 })).toBe(false); + }); + + it("compareReplays is the unchanged Pareto floor over two runs of the same plan", () => { + const plan = planReplay([replayCase(1, "reversed"), replayCase(2, "confirmed")], SAMPLING); + const flagAll = scoreReplay(plan, VARIANT, SAMPLING, new Map([["acme/widgets#1", "would_flag" as const], ["acme/widgets#2", "would_flag" as const]]), 1); + const perfect = scoreReplay(plan, VARIANT, SAMPLING, new Map([["acme/widgets#1", "would_flag" as const], ["acme/widgets#2", "would_not_flag" as const]]), 1); + const comparison = compareReplays(flagAll.report, perfect.report); + expect(comparison.verdict).toBe("improved"); // precision up, recall held + }); +}); + +describe("budget + artifacts helpers (#8221)", () => { + it("estimateFixtureNeurons is deterministic, diff-proportional, and never zero", () => { + const small = planReplay([replayCase(1, "confirmed", "x")], SAMPLING).fixtures[0]!; + const large = planReplay([replayCase(2, "confirmed", "y".repeat(8000))], SAMPLING).fixtures[0]!; + expect(estimateFixtureNeurons(small, 100)).toBeGreaterThan(0); + expect(estimateFixtureNeurons(large, 100)).toBeGreaterThan(estimateFixtureNeurons(small, 100)); + expect(estimateFixtureNeurons(small, 100)).toBe(estimateFixtureNeurons(small, 100)); + }); + + it("artifactKey is stable per (variant, fixture) and distinct across either changing", () => { + const key = artifactKey(VARIANT, "acme/widgets#1"); + expect(key).toBe(artifactKey(VARIANT, "acme/widgets#1")); + expect(key).toMatch(/^[0-9a-f]{32}$/); + expect(artifactKey(VARIANT, "acme/widgets#2")).not.toBe(key); + expect(artifactKey({ ...VARIANT, promptVersion: "v2" }, "acme/widgets#1")).not.toBe(key); + }); +}); + +describe("#8222: prompt version + CI comment/persist helpers", () => { + it("the canonical judge prompt is a stable pure function of source and the version constant is pinned", () => { + expect(REVIEW_PROMPT_VERSION).toBe("review-prompt-v1"); + const prompt = buildCanonicalJudgePrompt(); + expect(prompt.length).toBeGreaterThan(100); + expect(prompt).toBe(buildCanonicalJudgePrompt()); // no clock, no env, no input + }); + + it("renders the marker comment with both prompt versions, the seed, and the advisory line", () => { + const plan = planReplay([replayCase(1, "reversed"), replayCase(2, "confirmed")], SAMPLING); + const base = scoreReplay(plan, { promptVersion: "review-prompt-v1", modelSpec: "m" }, SAMPLING, new Map([["acme/widgets#1", "would_flag" as const], ["acme/widgets#2", "would_flag" as const]]), 1); + const head = scoreReplay(plan, { promptVersion: "review-prompt-v2", modelSpec: "m" }, SAMPLING, new Map([["acme/widgets#1", "would_flag" as const], ["acme/widgets#2", "would_not_flag" as const]]), 1); + const comment = renderCounterfactualComment(compareReplays(base.report, head.report), { + promptVersionBase: "review-prompt-v1", + promptVersionHead: "review-prompt-v2", + modelSpec: "llama3.1:8b", + headSha: "abcdef1234567", + baseSha: "1234567abcdef", + scored: 2, + abstainedBase: 0, + abstainedHead: 1, + skippedNoRawContext: 3, + sampledOut: 0, + seed: SAMPLING.seed, + }); + expect(comment).toContain(COUNTERFACTUAL_COMMENT_MARKER); + expect(comment).toContain("review-prompt-v1"); + expect(comment).toContain("review-prompt-v2"); + expect(comment).toContain("abcdef1"); // short shas + expect(comment).toContain("Verdict"); + expect(comment).toContain("never blocks merge"); + expect(comment).toContain("3 case(s) lacked raw context"); + }); + + it("builds the audit INSERT with the shared metadata.comparison shape and escaped literals", () => { + const plan = planReplay([replayCase(1, "reversed")], SAMPLING); + const run = scoreReplay(plan, VARIANT, SAMPLING, new Map([["acme/widgets#1", "would_flag" as const]]), 1); + const sql = buildCounterfactualAuditInsertSql({ + id: "id-1", + targetKey: "acme/o'widgets#9", + comparison: compareReplays(run.report, run.report), + headSha: "h", + baseSha: "b", + promptVersionBase: "review-prompt-v1", + promptVersionHead: "review-prompt-v2", + modelSpec: "m", + scored: 1, + createdAt: "2026-07-23T00:00:00.000Z", + }); + expect(sql).toContain(`'${COUNTERFACTUAL_BACKTEST_EVENT_TYPE}'`); + expect(sql).toContain("INSERT INTO audit_events"); + expect(sql).toContain("acme/o''widgets#9"); // single-quote escaping + expect(sql).toContain('\"comparison\":'); // the field the shared track-record reader looks for + const metadataLiteral = sql.match(/'(\{.*\})'/)?.[1] ?? ""; + expect(JSON.parse(metadataLiteral.replace(/''/g, "'")) as Record).toMatchObject({ + promptVersionBase: "review-prompt-v1", + promptVersionHead: "review-prompt-v2", + scored: 1, + }); + }); +});