feat(calibration): Workers AI fallback provider for the replay harness; ollama stays the preferred path

.github/workflows/counterfactual-replay-check.yml

@@ -36,18 +36,22 @@ jobs:
     timeout-minutes: 20
     steps:
       # The budget gate comes FIRST so an unconfigured deployment pays one echo, not an npm ci.
+      # #8279: the default provider is Workers AI (the CLOUDFLARE_* secrets this workflow already holds for
+      # the corpus export double as its credentials — the token needs Workers AI run perms); an explicit
+      # COUNTERFACTUAL_OLLAMA_URL secret switches to a reachable ollama instead.
       - name: Check replay budget configuration
         id: budget
         env:
           REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }}
           REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }}
-          REPLAY_PROVIDER_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }}
+          REPLAY_OLLAMA_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }}
         run: |
-          if [ -n "$REPLAY_BUDGET" ] && [ -n "$REPLAY_MODEL" ] && [ -n "$REPLAY_PROVIDER_URL" ]; then
+          if [ -n "$REPLAY_BUDGET" ] && [ -n "$REPLAY_MODEL" ]; then
             echo "configured=true" >> "$GITHUB_OUTPUT"
+            if [ -n "$REPLAY_OLLAMA_URL" ]; then echo "provider=ollama" >> "$GITHUB_OUTPUT"; else echo "provider=workers-ai" >> "$GITHUB_OUTPUT"; fi
           else
             echo "configured=false" >> "$GITHUB_OUTPUT"
-            echo "::notice::Counterfactual replay skipped: no replay budget configured (set the COUNTERFACTUAL_REPLAY_BUDGET + COUNTERFACTUAL_REPLAY_MODEL repo variables and the COUNTERFACTUAL_OLLAMA_URL secret to enable). Advisory only, never fails the PR."
+            echo "::notice::Counterfactual replay skipped: no replay budget configured (set the COUNTERFACTUAL_REPLAY_BUDGET + COUNTERFACTUAL_REPLAY_MODEL repo variables; Workers AI runs on the existing CLOUDFLARE_* secrets, or set COUNTERFACTUAL_OLLAMA_URL to use a reachable ollama). Advisory only, never fails the PR."
           fi
 
       - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
@@ -123,7 +127,8 @@ jobs:
           CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
           REPLAY_BUDGET: ${{ vars.COUNTERFACTUAL_REPLAY_BUDGET }}
           REPLAY_MODEL: ${{ vars.COUNTERFACTUAL_REPLAY_MODEL }}
-          REPLAY_PROVIDER_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }}
+          REPLAY_OLLAMA_URL: ${{ secrets.COUNTERFACTUAL_OLLAMA_URL }}
+          REPLAY_PROVIDER: ${{ steps.budget.outputs.provider }}
           HEAD_SHA: ${{ github.event.pull_request.head.sha }}
           BASE_SHA: ${{ github.event.pull_request.base.sha }}
           PR_NUMBER: ${{ github.event.pull_request.number }}
@@ -136,15 +141,17 @@ jobs:
               --prompt-file base-prompt.txt \
               --budget "$REPLAY_BUDGET" \
               --seed-suffix "$HEAD_SHA" \
-              --ollama-url "$REPLAY_PROVIDER_URL" \
+              --provider "$REPLAY_PROVIDER" \
+              --ollama-url "${REPLAY_OLLAMA_URL:-http://localhost:11434}" \
               --out base-scores.json \
             && npx tsx scripts/counterfactual-replay.ts \
               --fixtures replay-corpus.json \
               --variant "${HEAD_VERSION}@${REPLAY_MODEL}" \
               --prompt-file head-prompt.txt \
               --budget "$REPLAY_BUDGET" \
               --seed-suffix "$HEAD_SHA" \
-              --ollama-url "$REPLAY_PROVIDER_URL" \
+              --provider "$REPLAY_PROVIDER" \
+              --ollama-url "${REPLAY_OLLAMA_URL:-http://localhost:11434}" \
               --baseline base-scores.json \
               --base-variant-label "$BASE_VERSION" \
               --comment-out replay-comment.md \

apps/loopover-ui/content/docs/backtest-calibration.mdx

@@ -193,8 +193,12 @@ base and head checkouts, replays both over the identical seeded sample (seed = h
 per push are deterministic), posts the comparison as an update-in-place PR comment, and persists a
 `calibration.counterfactual_backtest_run` event feeding the same REGRESSED-verdict track record as the
 threshold and logic backtests. The check spends nothing unless the deployment explicitly configures a
-replay budget (repo variables `COUNTERFACTUAL_REPLAY_BUDGET`/`COUNTERFACTUAL_REPLAY_MODEL` plus the
-provider endpoint secret) — otherwise it posts a visible "skipped: no replay budget configured" notice.
+replay budget (repo variables `COUNTERFACTUAL_REPLAY_BUDGET`/`COUNTERFACTUAL_REPLAY_MODEL`) — otherwise
+it posts a visible "skipped: no replay budget configured" notice. Provider selection: with a
+`COUNTERFACTUAL_OLLAMA_URL` secret set, the check uses that ollama endpoint (a self-hosted runner
+reaching its own local model is the zero-marginal-cost shape); without it, the default is the flag-gated
+Workers AI fallback on the deployment's existing Cloudflare credentials — OSS models, usage-priced,
+dormant until the budget variables exist.
 Advisory forever until a separate, explicit authority decision, exactly like every other backtest gate.
 
 ## Self-hosting

scripts/counterfactual-replay.ts

@@ -5,8 +5,9 @@
 // prints the Pareto-floored comparison via the shared renderer. All mapping/aggregation logic lives in
 // counterfactual-replay-core.ts (unit-tested); this wrapper owns provider IO, the artifacts cache, and
 // file reads. OFFLINE-ONLY guarantees (#8219 contract point 2): never posts anywhere, never touches live
-// reviews; local ollama is the smoke path and the ONLY provider wired here — BYOK providers are a later,
-// explicitly-flagged addition, refused loudly today rather than silently attempted.
+// reviews. Local ollama is the smoke path; `--provider workers-ai` (#8279) is the explicitly-flagged
+// hosted option for CI — credentials from CLOUDFLARE_ACCOUNT_ID/CLOUDFLARE_API_TOKEN, refused loudly when
+// absent. Every other provider remains unwired by design.
 //
 //   tsx scripts/counterfactual-replay.ts --fixtures corpus.json --variant <promptVersion@modelSpec>
 //        [--budget N] [--max-fixtures N] [--seed-suffix s] [--baseline scores.json] [--out scores.json]
@@ -30,6 +31,7 @@ import { spawnSync } from "node:child_process";
 import {
   artifactKey,
   buildCounterfactualAuditInsertSql,
+  extractWorkersAiResponse,
   compareReplays,
   estimateFixtureNeurons,
   isRunAccountingValid,
@@ -48,6 +50,7 @@ type Args = {
   out: string | undefined;
   artifacts: string;
   ollamaUrl: string;
+  provider: "ollama" | "workers-ai";
   promptFile: string | undefined;
   // #8222 CI mode: emit the marker comment and/or persist the run event (wrangler d1, like #8139).
   commentOut: string | undefined;
@@ -72,6 +75,7 @@ function parseArgs(argv: string[]): Args {
     out: undefined,
     artifacts: ".counterfactual-artifacts",
     ollamaUrl: "http://localhost:11434",
+    provider: "ollama",
     promptFile: undefined,
     commentOut: undefined,
     baseVariantLabel: undefined,
@@ -94,6 +98,7 @@ function parseArgs(argv: string[]): Args {
     else if (flag === "--out") args.out = argv[++i];
     else if (flag === "--artifacts") args.artifacts = argv[++i] ?? args.artifacts;
     else if (flag === "--ollama-url") args.ollamaUrl = argv[++i] ?? args.ollamaUrl;
+    else if (flag === "--provider") args.provider = (argv[++i] as "ollama" | "workers-ai") ?? "ollama";
     else if (flag === "--prompt-file") args.promptFile = argv[++i];
     else if (flag === "--comment-out") args.commentOut = argv[++i];
     else if (flag === "--base-variant-label") args.baseVariantLabel = argv[++i];
@@ -117,6 +122,18 @@ const DEFAULT_JUDGE_PROMPT = [
   'Respond with ONLY a JSON object: {"blockers": [..strings, empty if none..], "confidence": 0..1}.',
 ].join("\n");
 
+// Workers AI REST (#8279): the same account already running production reviews. Non-streaming text run;
+// a non-OK status throws (operational failure — the campaign stops loudly, never silently mis-scores).
+async function judgeWithWorkersAi(accountId: string, token: string, model: string, prompt: string, diff: string): Promise<string> {
+  const response = await fetch(`https://api.cloudflare.com/client/v4/accounts/${accountId}/ai/run/${model}`, {
+    method: "POST",
+    headers: { authorization: `Bearer ${token}`, "content-type": "application/json" },
+    body: JSON.stringify({ messages: [{ role: "user", content: `${prompt}\n\n--- DIFF ---\n${diff}` }], temperature: 0 }),
+  });
+  if (!response.ok) throw new Error(`workers-ai ${response.status}: ${(await response.text()).slice(0, 300)}`);
+  return extractWorkersAiResponse(await response.json());
+}
+
 async function judgeWithOllama(ollamaUrl: string, model: string, prompt: string, diff: string): Promise<string> {
   const response = await fetch(`${ollamaUrl}/api/generate`, {
     method: "POST",
@@ -157,6 +174,18 @@ async function main(): Promise<number> {
     return 0;
   }
 
+  // #8279: hosted-provider credentials resolve ONCE, loudly, before any spend.
+  let workersAi: { accountId: string; token: string } | null = null;
+  if (args.provider === "workers-ai") {
+    const accountId = process.env.CLOUDFLARE_ACCOUNT_ID?.trim();
+    const token = process.env.CLOUDFLARE_API_TOKEN?.trim();
+    if (!accountId || !token) {
+      console.error("--provider workers-ai requires CLOUDFLARE_ACCOUNT_ID and CLOUDFLARE_API_TOKEN (a token with Workers AI read/run).");
+      return 1;
+    }
+    workersAi = { accountId, token };
+  }
+
   mkdirSync(args.artifacts, { recursive: true });
   const prompt = args.promptFile ? readFileSync(args.promptFile, "utf8") : DEFAULT_JUDGE_PROMPT;
   writeFileSync(join(args.artifacts, `prompt-${variant.promptVersion}.txt`), prompt);
@@ -172,7 +201,9 @@ async function main(): Promise<number> {
     if (existsSync(cachePath)) {
       raw = readFileSync(cachePath, "utf8"); // cached raw output: re-scoring is free
     } else {
-      raw = await judgeWithOllama(args.ollamaUrl, variant.modelSpec, prompt, fixture.boundedInputs.diff);
+      raw = workersAi
+        ? await judgeWithWorkersAi(workersAi.accountId, workersAi.token, variant.modelSpec, prompt, fixture.boundedInputs.diff)
+        : await judgeWithOllama(args.ollamaUrl, variant.modelSpec, prompt, fixture.boundedInputs.diff);
       writeFileSync(cachePath, raw);
       neuronsSpent += cost; // cache hits spend nothing — only real provider calls count
     }

test/unit/counterfactual-replay-core.test.ts

@@ -3,6 +3,7 @@ import type { BacktestCase, CounterfactualVerdict } from "@loopover/engine";
 import { REVIEW_PROMPT_VERSION, buildCanonicalJudgePrompt } from "../../src/services/ai-review";
 import {
   artifactKey,
+  extractWorkersAiResponse,
   buildCounterfactualAuditInsertSql,
   compareReplays,
   COUNTERFACTUAL_BACKTEST_EVENT_TYPE,
@@ -184,3 +185,13 @@ describe("#8222: prompt version + CI comment/persist helpers", () => {
     });
   });
 });
+
+describe("extractWorkersAiResponse (#8279)", () => {
+  it("extracts result.response; every malformed envelope yields \"\" so the parser abstains rather than throwing", () => {
+    expect(extractWorkersAiResponse({ result: { response: '{"blockers": []}' } })).toBe('{"blockers": []}');
+    for (const bad of [null, undefined, {}, { result: {} }, { result: { response: 42 } }, "text"]) {
+      expect(extractWorkersAiResponse(bad)).toBe("");
+    }
+    expect(parseVariantVerdict(extractWorkersAiResponse({ result: {} }))).toBe("abstained"); // the composed fail-safe
+  });
+});