fix(evals): Include rubric judge cost

packages/junior-evals/evals/github-actions.md

@@ -94,7 +94,7 @@ Behavioral shard jobs keep running after individual case failures so every shard
 
 1. `behavioral / report` downloads all behavioral shard result files and publishes one combined `vitest-evals` summary (metric table, score distribution, quality misses)
 2. the same step publishes a `behavioral / score` Check Run with `min-pass-rate` (`EVAL_MIN_PASS_RATE`, currently `0.8`)
-3. `vitest-evals@0.16.1` attaches that Check Run to the PR head SHA and soft-fails the report step when the check publishes, so the Check Run title owns the pass-rate secondary line on the PR checks list
+3. `vitest-evals` attaches that Check Run to the PR head SHA and soft-fails the report step when the check publishes, so the Check Run title owns the pass-rate secondary line on the PR checks list
 
 If Check Run publishing is skipped or fails, the report step still fails on a rejected gate so status is not silently lost.
 

packages/junior-evals/package.json

@@ -27,7 +27,7 @@
     "typescript": "^6.0.3",
     "undici": "7.29.0",
     "vitest": "^4.1.7",
-    "vitest-evals": "0.16.1",
+    "vitest-evals": "0.17.0",
     "zod": "catalog:"
   }
 }

packages/junior-evals/src/helpers.ts

@@ -379,6 +379,37 @@ function usageTotal(usage: AgentTurnUsage | undefined): number | undefined {
     : undefined;
 }
 
+function normalizedTokenUsage(
+  usage: AgentTurnUsage | undefined,
+): HarnessRun["usage"] {
+  return {
+    ...(usage?.inputTokens !== undefined
+      ? { inputTokens: usage.inputTokens }
+      : {}),
+    ...(usage?.outputTokens !== undefined
+      ? { outputTokens: usage.outputTokens }
+      : {}),
+    ...(usage?.reasoningTokens !== undefined
+      ? { reasoningTokens: usage.reasoningTokens }
+      : {}),
+    ...(usageTotal(usage) !== undefined
+      ? { totalTokens: usageTotal(usage) }
+      : {}),
+  };
+}
+
+function toJudgeUsage(
+  usage: AgentTurnUsage | undefined,
+  model: string,
+): HarnessRun["usage"] {
+  return {
+    provider: GEN_AI_PROVIDER_NAME,
+    model,
+    ...normalizedTokenUsage(usage),
+    ...(usage?.cost?.total !== undefined ? { costUsd: usage.cost.total } : {}),
+  };
+}
+
 function toHarnessUsage(result: EvalResult): HarnessRun["usage"] {
   const usage = result.usage;
   const metadata = toJsonRecord({
@@ -392,28 +423,15 @@ function toHarnessUsage(result: EvalResult): HarnessRun["usage"] {
       ? {
           currency: "USD",
           cost: usage.cost,
-          ...(usage.cost.total !== undefined
-            ? { costUsd: usage.cost.total }
-            : {}),
         }
       : {}),
     ...(result.modelIds.length > 1 ? { modelIds: result.modelIds } : {}),
   });
   return {
     provider: GEN_AI_PROVIDER_NAME,
     ...(result.modelIds.length === 1 ? { model: result.modelIds[0] } : {}),
-    ...(usage?.inputTokens !== undefined
-      ? { inputTokens: usage.inputTokens }
-      : {}),
-    ...(usage?.outputTokens !== undefined
-      ? { outputTokens: usage.outputTokens }
-      : {}),
-    ...(usage?.reasoningTokens !== undefined
-      ? { reasoningTokens: usage.reasoningTokens }
-      : {}),
-    ...(usageTotal(usage) !== undefined
-      ? { totalTokens: usageTotal(usage) }
-      : {}),
+    ...normalizedTokenUsage(usage),
+    ...(usage?.cost?.total !== undefined ? { costUsd: usage.cost.total } : {}),
     toolCalls: result.toolInvocations.length,
     ...(Object.keys(metadata).length > 0 ? { metadata } : {}),
   };
@@ -631,7 +649,7 @@ const EVAL_JUDGE_MODEL_ID = resolveGatewayModel("openai/gpt-5.4").id;
 const judgeHarness = createJudgeHarness({
   name: "slack-rubric-judge-model",
   run: async ({ prompt, system }) => {
-    const { text } = await completeText({
+    const { message, text } = await completeText({
       modelId: EVAL_JUDGE_MODEL_ID,
       system,
       messages: [
@@ -643,7 +661,26 @@ const judgeHarness = createJudgeHarness({
       ],
       temperature: 0,
     });
-    return text;
+    return {
+      output: text,
+      session: {
+        events: [
+          ...(system
+            ? [
+                {
+                  type: "message" as const,
+                  role: "system" as const,
+                  content: system,
+                },
+              ]
+            : []),
+          { type: "message", role: "user", content: prompt },
+          { type: "message", role: "assistant", content: text },
+        ],
+      },
+      usage: toJudgeUsage(message.usage, message.model ?? EVAL_JUDGE_MODEL_ID),
+      errors: [],
+    };
   },
 });
 

packages/junior-evals/tests/unit/harness/helpers.test.ts

@@ -1,19 +1,27 @@
 import { expect, it, vi } from "vitest";
 
-const { runError, runEvalScenarioMock } = vi.hoisted(() => ({
+const { completeTextMock, runError, runEvalScenarioMock } = vi.hoisted(() => ({
+  completeTextMock: vi.fn(),
   runError: new Error("stop after capturing harness options"),
   runEvalScenarioMock: vi.fn(async () => {
     throw new Error("uninitialized run error");
   }),
 }));
 
+vi.mock("@/chat/pi/client", () => ({
+  completeText: completeTextMock,
+  GEN_AI_PROVIDER_NAME: "vercel-ai-gateway",
+  resolveGatewayModel: vi.fn((modelId: string) => ({ id: modelId })),
+}));
+
 vi.mock("../../../src/behavior-harness", () => ({
   runEvalScenario: runEvalScenarioMock,
 }));
 
 import {
   hasImageAttachment,
   serializeVisibleTranscript,
+  slackEvals,
   slackHarness,
   visibleAssistantText,
   visibleThreadReplies,
@@ -143,6 +151,52 @@ it("includes captured Slack posts in the rubric-visible transcript", async () =>
   ).not.toHaveProperty("rubric_visible", false);
 });
 
+it("reports rubric judge usage through the judge harness", async () => {
+  completeTextMock.mockResolvedValueOnce({
+    message: {
+      model: "openai/gpt-5.4",
+      usage: {
+        inputTokens: 120,
+        outputTokens: 20,
+        totalTokens: 140,
+        cost: { total: 0.031 },
+      },
+    },
+    text: '{"answer":"A","rationale":"The response meets the rubric."}',
+  });
+
+  const judgeRun = await slackEvals.judgeHarness.run(
+    { prompt: "Grade this.", system: "Return JSON." },
+    { artifacts: {}, setArtifact: vi.fn() },
+  );
+  expect(judgeRun.usage).toEqual({
+    provider: "vercel-ai-gateway",
+    model: "openai/gpt-5.4",
+    inputTokens: 120,
+    outputTokens: 20,
+    totalTokens: 140,
+    costUsd: 0.031,
+  });
+});
+
+it("omits unknown rubric judge cost", async () => {
+  completeTextMock.mockResolvedValueOnce({
+    message: {
+      usage: {},
+    },
+    text: '{"answer":"A","rationale":"The response meets the rubric."}',
+  });
+
+  const judgeRun = await slackEvals.judgeHarness.run(
+    { prompt: "Grade this.", system: "Return JSON." },
+    { artifacts: {}, setArtifact: vi.fn() },
+  );
+  expect(judgeRun.usage).toEqual({
+    provider: "vercel-ai-gateway",
+    model: "openai/gpt-5.4",
+  });
+});
+
 it("forwards the Vitest abort signal to the eval scenario", async () => {
   runEvalScenarioMock.mockRejectedValueOnce(runError);
   const controller = new AbortController();

pnpm-workspace.yaml

@@ -34,9 +34,9 @@ syncInjectedDepsAfterScripts:
 minimumReleaseAge: 1440
 minimumReleaseAgeExclude:
   - "@sentry/starlight-theme"
-  - "@vitest-evals/core@0.16.1"
-  - "@vitest-evals/report-ui@0.16.1"
-  - "vitest-evals@0.16.1"
+  - "@vitest-evals/core@0.17.0"
+  - "@vitest-evals/report-ui@0.17.0"
+  - "vitest-evals@0.17.0"
   - "@sentry/core@10.65.0"
   - "@sentry/node-core@10.65.0"
   - "@sentry/node@10.65.0"