From 3b926585731221d89ef7e8d8a6e4f0fca300cae0 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 02:36:27 +0000 Subject: [PATCH 1/8] fix(evals): Record rubric judge costs Co-Authored-By: David Cramer --- packages/junior-evals/src/helpers.ts | 64 +++++++++++++++---- .../tests/unit/harness/helpers.test.ts | 61 +++++++++++++++++- 2 files changed, 112 insertions(+), 13 deletions(-) diff --git a/packages/junior-evals/src/helpers.ts b/packages/junior-evals/src/helpers.ts index f4668161d0..f099c3c1fe 100644 --- a/packages/junior-evals/src/helpers.ts +++ b/packages/junior-evals/src/helpers.ts @@ -616,6 +616,11 @@ interface JudgeResultPayload { rationale: string; } +interface JudgeHarnessPayload extends Record { + text: string; + usage: Record; +} + const CHOICE_SCORES: Record = { A: 1, B: 0.75, @@ -631,7 +636,7 @@ const EVAL_JUDGE_MODEL_ID = resolveGatewayModel("openai/gpt-5.4").id; const judgeHarness = createJudgeHarness({ name: "slack-rubric-judge-model", run: async ({ prompt, system }) => { - const { text } = await completeText({ + const { message, text } = await completeText({ modelId: EVAL_JUDGE_MODEL_ID, system, messages: [ @@ -643,7 +648,18 @@ const judgeHarness = createJudgeHarness({ ], temperature: 0, }); - return text; + const usage = toJsonRecord({ + inputTokens: message.usage.input, + outputTokens: message.usage.output, + cachedInputTokens: message.usage.cacheRead, + cacheCreationTokens: message.usage.cacheWrite, + reasoningTokens: message.usage.reasoning, + totalTokens: message.usage.totalTokens, + currency: "USD", + cost: message.usage.cost, + costUsd: message.usage.cost.total, + }); + return { text, usage } satisfies JudgeHarnessPayload; }, }); @@ -675,6 +691,27 @@ function isJudgeAnswer(value: unknown): value is JudgeAnswer { ); } +function parseJudgeHarnessPayload(result: JsonValue | undefined): { + text: string; + usage?: Record; +} { + if (typeof result === "string") { + return { text: result }; + } + if ( + result && + typeof result === "object" && + !Array.isArray(result) && + typeof result.text === "string" && + result.usage && + typeof result.usage === "object" && + !Array.isArray(result.usage) + ) { + return { text: result.text, usage: result.usage }; + } + throw new Error("Rubric judge returned an invalid harness payload."); +} + function parseJudgeResult(text: string): JudgeResultPayload { const parsed = JSON.parse(text) as unknown; if ( @@ -759,17 +796,16 @@ export const RubricJudge = createJudge( if (!runJudge) { throw new Error("RubricJudge requires a configured judgeHarness."); } - const object = parseJudgeResult( - String( - await runJudge({ - prompt: formatJudgePrompt( - serializeVisibleTranscript(session), - formatRubric(input.criteria), - ), - system: EVAL_SYSTEM, - }), - ), + const judgeResult = parseJudgeHarnessPayload( + await runJudge({ + prompt: formatJudgePrompt( + serializeVisibleTranscript(session), + formatRubric(input.criteria), + ), + system: EVAL_SYSTEM, + }), ); + const object = parseJudgeResult(judgeResult.text); const answer = object.answer as keyof typeof CHOICE_SCORES; return { @@ -777,6 +813,10 @@ export const RubricJudge = createJudge( metadata: { answer, rationale: object.rationale, + ...(judgeResult.usage ? { usage: judgeResult.usage } : {}), + ...(typeof judgeResult.usage?.costUsd === "number" + ? { costUsd: judgeResult.usage.costUsd } + : {}), }, }; }, diff --git a/packages/junior-evals/tests/unit/harness/helpers.test.ts b/packages/junior-evals/tests/unit/harness/helpers.test.ts index c7fce35de4..305222bfd4 100644 --- a/packages/junior-evals/tests/unit/harness/helpers.test.ts +++ b/packages/junior-evals/tests/unit/harness/helpers.test.ts @@ -1,19 +1,28 @@ import { expect, it, vi } from "vitest"; -const { runError, runEvalScenarioMock } = vi.hoisted(() => ({ +const { completeTextMock, runError, runEvalScenarioMock } = vi.hoisted(() => ({ + completeTextMock: vi.fn(), runError: new Error("stop after capturing harness options"), runEvalScenarioMock: vi.fn(async () => { throw new Error("uninitialized run error"); }), })); +vi.mock("@/chat/pi/client", () => ({ + completeText: completeTextMock, + GEN_AI_PROVIDER_NAME: "vercel-ai-gateway", + resolveGatewayModel: vi.fn((modelId: string) => ({ id: modelId })), +})); + vi.mock("../../../src/behavior-harness", () => ({ runEvalScenario: runEvalScenarioMock, })); import { hasImageAttachment, + RubricJudge, serializeVisibleTranscript, + slackEvals, slackHarness, visibleAssistantText, visibleThreadReplies, @@ -143,6 +152,56 @@ it("includes captured Slack posts in the rubric-visible transcript", async () => ).not.toHaveProperty("rubric_visible", false); }); +it("records rubric judge usage and cost in score metadata", async () => { + completeTextMock.mockResolvedValueOnce({ + message: { + usage: { + input: 100, + output: 20, + cacheRead: 10, + cacheWrite: 0, + reasoning: 5, + totalTokens: 130, + cost: { + input: 0.01, + output: 0.02, + cacheRead: 0.001, + cacheWrite: 0, + total: 0.031, + }, + }, + }, + text: '{"answer":"A","rationale":"The response meets the rubric."}', + }); + + const judgeRun = await slackEvals.judgeHarness.run( + { prompt: "Grade this.", system: "Return JSON." }, + { artifacts: {}, setArtifact: vi.fn() }, + ); + const result = await RubricJudge.assess({ + harness: slackHarness, + input: { criteria: { pass: ["Answers correctly"] }, initialEvents: [] }, + output: undefined, + run: {} as never, + runJudge: async () => judgeRun.output, + session: { events: [] }, + toolCalls: [], + }); + + expect(result.metadata).toMatchObject({ + answer: "A", + costUsd: 0.031, + usage: { + inputTokens: 100, + outputTokens: 20, + cachedInputTokens: 10, + reasoningTokens: 5, + totalTokens: 130, + costUsd: 0.031, + }, + }); +}); + it("forwards the Vitest abort signal to the eval scenario", async () => { runEvalScenarioMock.mockRejectedValueOnce(runError); const controller = new AbortController(); From a7750bffb22883ab1f8180812c3f290f29de41ae Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 02:48:37 +0000 Subject: [PATCH 2/8] refactor(evals): Simplify judge cost capture Co-Authored-By: David Cramer --- packages/junior-evals/src/helpers.ts | 69 +++++-------------- .../tests/unit/harness/helpers.test.ts | 31 +-------- 2 files changed, 21 insertions(+), 79 deletions(-) diff --git a/packages/junior-evals/src/helpers.ts b/packages/junior-evals/src/helpers.ts index f099c3c1fe..34bf88b518 100644 --- a/packages/junior-evals/src/helpers.ts +++ b/packages/junior-evals/src/helpers.ts @@ -616,11 +616,6 @@ interface JudgeResultPayload { rationale: string; } -interface JudgeHarnessPayload extends Record { - text: string; - usage: Record; -} - const CHOICE_SCORES: Record = { A: 1, B: 0.75, @@ -648,18 +643,7 @@ const judgeHarness = createJudgeHarness({ ], temperature: 0, }); - const usage = toJsonRecord({ - inputTokens: message.usage.input, - outputTokens: message.usage.output, - cachedInputTokens: message.usage.cacheRead, - cacheCreationTokens: message.usage.cacheWrite, - reasoningTokens: message.usage.reasoning, - totalTokens: message.usage.totalTokens, - currency: "USD", - cost: message.usage.cost, - costUsd: message.usage.cost.total, - }); - return { text, usage } satisfies JudgeHarnessPayload; + return { costUsd: message.usage.cost.total, text }; }, }); @@ -691,27 +675,6 @@ function isJudgeAnswer(value: unknown): value is JudgeAnswer { ); } -function parseJudgeHarnessPayload(result: JsonValue | undefined): { - text: string; - usage?: Record; -} { - if (typeof result === "string") { - return { text: result }; - } - if ( - result && - typeof result === "object" && - !Array.isArray(result) && - typeof result.text === "string" && - result.usage && - typeof result.usage === "object" && - !Array.isArray(result.usage) - ) { - return { text: result.text, usage: result.usage }; - } - throw new Error("Rubric judge returned an invalid harness payload."); -} - function parseJudgeResult(text: string): JudgeResultPayload { const parsed = JSON.parse(text) as unknown; if ( @@ -796,15 +759,22 @@ export const RubricJudge = createJudge( if (!runJudge) { throw new Error("RubricJudge requires a configured judgeHarness."); } - const judgeResult = parseJudgeHarnessPayload( - await runJudge({ - prompt: formatJudgePrompt( - serializeVisibleTranscript(session), - formatRubric(input.criteria), - ), - system: EVAL_SYSTEM, - }), - ); + const judgeResult = await runJudge({ + prompt: formatJudgePrompt( + serializeVisibleTranscript(session), + formatRubric(input.criteria), + ), + system: EVAL_SYSTEM, + }); + if ( + !judgeResult || + typeof judgeResult !== "object" || + Array.isArray(judgeResult) || + typeof judgeResult.text !== "string" || + typeof judgeResult.costUsd !== "number" + ) { + throw new Error("Rubric judge returned an invalid result."); + } const object = parseJudgeResult(judgeResult.text); const answer = object.answer as keyof typeof CHOICE_SCORES; @@ -813,10 +783,7 @@ export const RubricJudge = createJudge( metadata: { answer, rationale: object.rationale, - ...(judgeResult.usage ? { usage: judgeResult.usage } : {}), - ...(typeof judgeResult.usage?.costUsd === "number" - ? { costUsd: judgeResult.usage.costUsd } - : {}), + costUsd: judgeResult.costUsd, }, }; }, diff --git a/packages/junior-evals/tests/unit/harness/helpers.test.ts b/packages/junior-evals/tests/unit/harness/helpers.test.ts index 305222bfd4..f32146e09f 100644 --- a/packages/junior-evals/tests/unit/harness/helpers.test.ts +++ b/packages/junior-evals/tests/unit/harness/helpers.test.ts @@ -152,24 +152,10 @@ it("includes captured Slack posts in the rubric-visible transcript", async () => ).not.toHaveProperty("rubric_visible", false); }); -it("records rubric judge usage and cost in score metadata", async () => { +it("records rubric judge cost in score metadata", async () => { completeTextMock.mockResolvedValueOnce({ message: { - usage: { - input: 100, - output: 20, - cacheRead: 10, - cacheWrite: 0, - reasoning: 5, - totalTokens: 130, - cost: { - input: 0.01, - output: 0.02, - cacheRead: 0.001, - cacheWrite: 0, - total: 0.031, - }, - }, + usage: { cost: { total: 0.031 } }, }, text: '{"answer":"A","rationale":"The response meets the rubric."}', }); @@ -188,18 +174,7 @@ it("records rubric judge usage and cost in score metadata", async () => { toolCalls: [], }); - expect(result.metadata).toMatchObject({ - answer: "A", - costUsd: 0.031, - usage: { - inputTokens: 100, - outputTokens: 20, - cachedInputTokens: 10, - reasoningTokens: 5, - totalTokens: 130, - costUsd: 0.031, - }, - }); + expect(result.metadata).toMatchObject({ answer: "A", costUsd: 0.031 }); }); it("forwards the Vitest abort signal to the eval scenario", async () => { From 15162371b82a107e81756928030a3b434a38bd49 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 02:58:53 +0000 Subject: [PATCH 3/8] fix(evals): Treat missing judge cost as optional Bugbot flagged that message.usage.cost.total can be undefined, which made costUsd validation throw and fail otherwise-valid rubric verdicts. Cost is now optional end to end: the judge harness run, the RubricJudge validation, and the score metadata. --- packages/junior-evals/src/helpers.ts | 9 ++++--- .../tests/unit/harness/helpers.test.ts | 26 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/packages/junior-evals/src/helpers.ts b/packages/junior-evals/src/helpers.ts index 34bf88b518..b85fc67817 100644 --- a/packages/junior-evals/src/helpers.ts +++ b/packages/junior-evals/src/helpers.ts @@ -643,7 +643,7 @@ const judgeHarness = createJudgeHarness({ ], temperature: 0, }); - return { costUsd: message.usage.cost.total, text }; + return { costUsd: message.usage.cost?.total, text }; }, }); @@ -771,7 +771,8 @@ export const RubricJudge = createJudge( typeof judgeResult !== "object" || Array.isArray(judgeResult) || typeof judgeResult.text !== "string" || - typeof judgeResult.costUsd !== "number" + (judgeResult.costUsd !== undefined && + typeof judgeResult.costUsd !== "number") ) { throw new Error("Rubric judge returned an invalid result."); } @@ -783,7 +784,9 @@ export const RubricJudge = createJudge( metadata: { answer, rationale: object.rationale, - costUsd: judgeResult.costUsd, + ...(judgeResult.costUsd !== undefined + ? { costUsd: judgeResult.costUsd } + : undefined), }, }; }, diff --git a/packages/junior-evals/tests/unit/harness/helpers.test.ts b/packages/junior-evals/tests/unit/harness/helpers.test.ts index f32146e09f..aaaf0a0240 100644 --- a/packages/junior-evals/tests/unit/harness/helpers.test.ts +++ b/packages/junior-evals/tests/unit/harness/helpers.test.ts @@ -177,6 +177,32 @@ it("records rubric judge cost in score metadata", async () => { expect(result.metadata).toMatchObject({ answer: "A", costUsd: 0.031 }); }); +it("scores the rubric judge without failing when cost is missing", async () => { + completeTextMock.mockResolvedValueOnce({ + message: { + usage: {}, + }, + text: '{"answer":"A","rationale":"The response meets the rubric."}', + }); + + const judgeRun = await slackEvals.judgeHarness.run( + { prompt: "Grade this.", system: "Return JSON." }, + { artifacts: {}, setArtifact: vi.fn() }, + ); + const result = await RubricJudge.assess({ + harness: slackHarness, + input: { criteria: { pass: ["Answers correctly"] }, initialEvents: [] }, + output: undefined, + run: {} as never, + runJudge: async () => judgeRun.output, + session: { events: [] }, + toolCalls: [], + }); + + expect(result.metadata).toMatchObject({ answer: "A" }); + expect(result.metadata).not.toHaveProperty("costUsd"); +}); + it("forwards the Vitest abort signal to the eval scenario", async () => { runEvalScenarioMock.mockRejectedValueOnce(runError); const controller = new AbortController(); From 6a56bf6fa81e9c79d5a7e44c1df86750b78356db Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:12:06 +0000 Subject: [PATCH 4/8] feat(evals): Record rubric judge usage --- packages/junior-evals/src/helpers.ts | 46 ++++++++++++++++--- .../tests/unit/harness/helpers.test.ts | 32 +++++++++++-- 2 files changed, 67 insertions(+), 11 deletions(-) diff --git a/packages/junior-evals/src/helpers.ts b/packages/junior-evals/src/helpers.ts index b85fc67817..1bdf93099a 100644 --- a/packages/junior-evals/src/helpers.ts +++ b/packages/junior-evals/src/helpers.ts @@ -379,6 +379,38 @@ function usageTotal(usage: AgentTurnUsage | undefined): number | undefined { : undefined; } +function toJudgeUsage( + usage: AgentTurnUsage | undefined, + model: string, +): HarnessRun["usage"] { + const metadata = toJsonRecord({ + ...(usage?.cachedInputTokens !== undefined + ? { cachedInputTokens: usage.cachedInputTokens } + : {}), + ...(usage?.cacheCreationTokens !== undefined + ? { cacheCreationTokens: usage.cacheCreationTokens } + : {}), + ...(usage?.cost?.total !== undefined ? { costUsd: usage.cost.total } : {}), + }); + return { + provider: GEN_AI_PROVIDER_NAME, + model, + ...(usage?.inputTokens !== undefined + ? { inputTokens: usage.inputTokens } + : {}), + ...(usage?.outputTokens !== undefined + ? { outputTokens: usage.outputTokens } + : {}), + ...(usage?.reasoningTokens !== undefined + ? { reasoningTokens: usage.reasoningTokens } + : {}), + ...(usageTotal(usage) !== undefined + ? { totalTokens: usageTotal(usage) } + : {}), + ...(Object.keys(metadata).length > 0 ? { metadata } : {}), + }; +} + function toHarnessUsage(result: EvalResult): HarnessRun["usage"] { const usage = result.usage; const metadata = toJsonRecord({ @@ -643,7 +675,10 @@ const judgeHarness = createJudgeHarness({ ], temperature: 0, }); - return { costUsd: message.usage.cost?.total, text }; + return { + text, + usage: toJudgeUsage(message.usage, message.model ?? EVAL_JUDGE_MODEL_ID), + }; }, }); @@ -771,8 +806,9 @@ export const RubricJudge = createJudge( typeof judgeResult !== "object" || Array.isArray(judgeResult) || typeof judgeResult.text !== "string" || - (judgeResult.costUsd !== undefined && - typeof judgeResult.costUsd !== "number") + !judgeResult.usage || + typeof judgeResult.usage !== "object" || + Array.isArray(judgeResult.usage) ) { throw new Error("Rubric judge returned an invalid result."); } @@ -784,9 +820,7 @@ export const RubricJudge = createJudge( metadata: { answer, rationale: object.rationale, - ...(judgeResult.costUsd !== undefined - ? { costUsd: judgeResult.costUsd } - : undefined), + usage: judgeResult.usage, }, }; }, diff --git a/packages/junior-evals/tests/unit/harness/helpers.test.ts b/packages/junior-evals/tests/unit/harness/helpers.test.ts index aaaf0a0240..8679c799e8 100644 --- a/packages/junior-evals/tests/unit/harness/helpers.test.ts +++ b/packages/junior-evals/tests/unit/harness/helpers.test.ts @@ -152,10 +152,16 @@ it("includes captured Slack posts in the rubric-visible transcript", async () => ).not.toHaveProperty("rubric_visible", false); }); -it("records rubric judge cost in score metadata", async () => { +it("records rubric judge usage in score metadata", async () => { completeTextMock.mockResolvedValueOnce({ message: { - usage: { cost: { total: 0.031 } }, + model: "openai/gpt-5.4", + usage: { + inputTokens: 120, + outputTokens: 20, + totalTokens: 140, + cost: { total: 0.031 }, + }, }, text: '{"answer":"A","rationale":"The response meets the rubric."}', }); @@ -174,7 +180,17 @@ it("records rubric judge cost in score metadata", async () => { toolCalls: [], }); - expect(result.metadata).toMatchObject({ answer: "A", costUsd: 0.031 }); + expect(result.metadata).toMatchObject({ + answer: "A", + usage: { + provider: "vercel-ai-gateway", + model: "openai/gpt-5.4", + inputTokens: 120, + outputTokens: 20, + totalTokens: 140, + metadata: { costUsd: 0.031 }, + }, + }); }); it("scores the rubric judge without failing when cost is missing", async () => { @@ -199,8 +215,14 @@ it("scores the rubric judge without failing when cost is missing", async () => { toolCalls: [], }); - expect(result.metadata).toMatchObject({ answer: "A" }); - expect(result.metadata).not.toHaveProperty("costUsd"); + expect(result.metadata).toMatchObject({ + answer: "A", + usage: { + provider: "vercel-ai-gateway", + model: "openai/gpt-5.4", + }, + }); + expect(result.metadata).not.toHaveProperty("usage.metadata.costUsd"); }); it("forwards the Vitest abort signal to the eval scenario", async () => { From 95eeddff20c9e545a4f1412303ef1da94034c2de Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 03:19:59 +0000 Subject: [PATCH 5/8] refactor(evals): Reuse token normalization Co-Authored-By: David Cramer --- packages/junior-evals/src/helpers.ts | 42 +++++++++++----------------- 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/packages/junior-evals/src/helpers.ts b/packages/junior-evals/src/helpers.ts index 1bdf93099a..3c1bf65e9f 100644 --- a/packages/junior-evals/src/helpers.ts +++ b/packages/junior-evals/src/helpers.ts @@ -379,22 +379,10 @@ function usageTotal(usage: AgentTurnUsage | undefined): number | undefined { : undefined; } -function toJudgeUsage( +function normalizedTokenUsage( usage: AgentTurnUsage | undefined, - model: string, ): HarnessRun["usage"] { - const metadata = toJsonRecord({ - ...(usage?.cachedInputTokens !== undefined - ? { cachedInputTokens: usage.cachedInputTokens } - : {}), - ...(usage?.cacheCreationTokens !== undefined - ? { cacheCreationTokens: usage.cacheCreationTokens } - : {}), - ...(usage?.cost?.total !== undefined ? { costUsd: usage.cost.total } : {}), - }); return { - provider: GEN_AI_PROVIDER_NAME, - model, ...(usage?.inputTokens !== undefined ? { inputTokens: usage.inputTokens } : {}), @@ -407,7 +395,20 @@ function toJudgeUsage( ...(usageTotal(usage) !== undefined ? { totalTokens: usageTotal(usage) } : {}), - ...(Object.keys(metadata).length > 0 ? { metadata } : {}), + }; +} + +function toJudgeUsage( + usage: AgentTurnUsage | undefined, + model: string, +): HarnessRun["usage"] { + return { + provider: GEN_AI_PROVIDER_NAME, + model, + ...normalizedTokenUsage(usage), + ...(usage?.cost?.total !== undefined + ? { metadata: { costUsd: usage.cost.total } } + : {}), }; } @@ -434,18 +435,7 @@ function toHarnessUsage(result: EvalResult): HarnessRun["usage"] { return { provider: GEN_AI_PROVIDER_NAME, ...(result.modelIds.length === 1 ? { model: result.modelIds[0] } : {}), - ...(usage?.inputTokens !== undefined - ? { inputTokens: usage.inputTokens } - : {}), - ...(usage?.outputTokens !== undefined - ? { outputTokens: usage.outputTokens } - : {}), - ...(usage?.reasoningTokens !== undefined - ? { reasoningTokens: usage.reasoningTokens } - : {}), - ...(usageTotal(usage) !== undefined - ? { totalTokens: usageTotal(usage) } - : {}), + ...normalizedTokenUsage(usage), toolCalls: result.toolInvocations.length, ...(Object.keys(metadata).length > 0 ? { metadata } : {}), }; From 02820e2dd949c5c58dfd1e42c852b727e4e6ede9 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 04:21:08 +0000 Subject: [PATCH 6/8] fix(evals): Report total scenario cost Co-Authored-By: David Cramer --- packages/junior-evals/src/helpers.ts | 33 +++++++++++++++++-- .../tests/unit/harness/helpers.test.ts | 16 ++++++--- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/packages/junior-evals/src/helpers.ts b/packages/junior-evals/src/helpers.ts index 3c1bf65e9f..6fd9abbac8 100644 --- a/packages/junior-evals/src/helpers.ts +++ b/packages/junior-evals/src/helpers.ts @@ -398,6 +398,18 @@ function normalizedTokenUsage( }; } +function usageCostUsd(usage: unknown): number | undefined { + if (!usage || typeof usage !== "object" || Array.isArray(usage)) { + return undefined; + } + const metadata = (usage as Record).metadata; + if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) { + return undefined; + } + const value = (metadata as Record).costUsd; + return typeof value === "number" ? value : undefined; +} + function toJudgeUsage( usage: AgentTurnUsage | undefined, model: string, @@ -768,6 +780,7 @@ export const RubricJudge = createJudge( "RubricJudge", async ({ input, + run, session, runJudge, }: JudgeContext< @@ -775,10 +788,16 @@ export const RubricJudge = createJudge( JsonValue | undefined, typeof slackHarness >) => { + const applicationCostUsd = usageCostUsd(run.usage); if (!input.criteria) { return { score: 1, - metadata: { skipped: "deterministic-only" }, + metadata: { + skipped: "deterministic-only", + ...(typeof applicationCostUsd === "number" + ? { costUsd: applicationCostUsd, applicationCostUsd } + : {}), + }, }; } if (!runJudge) { @@ -804,13 +823,23 @@ export const RubricJudge = createJudge( } const object = parseJudgeResult(judgeResult.text); const answer = object.answer as keyof typeof CHOICE_SCORES; + const judgeCostUsd = usageCostUsd(judgeResult.usage); + const costUsd = + typeof applicationCostUsd === "number" && typeof judgeCostUsd === "number" + ? applicationCostUsd + judgeCostUsd + : undefined; return { score: CHOICE_SCORES[answer], metadata: { answer, rationale: object.rationale, - usage: judgeResult.usage, + judgeUsage: judgeResult.usage, + ...(costUsd !== undefined ? { costUsd } : {}), + ...(typeof applicationCostUsd === "number" + ? { applicationCostUsd } + : {}), + ...(typeof judgeCostUsd === "number" ? { judgeCostUsd } : {}), }, }; }, diff --git a/packages/junior-evals/tests/unit/harness/helpers.test.ts b/packages/junior-evals/tests/unit/harness/helpers.test.ts index 8679c799e8..249fc10bdc 100644 --- a/packages/junior-evals/tests/unit/harness/helpers.test.ts +++ b/packages/junior-evals/tests/unit/harness/helpers.test.ts @@ -174,7 +174,9 @@ it("records rubric judge usage in score metadata", async () => { harness: slackHarness, input: { criteria: { pass: ["Answers correctly"] }, initialEvents: [] }, output: undefined, - run: {} as never, + run: { + usage: { metadata: { costUsd: 0.5 } }, + } as never, runJudge: async () => judgeRun.output, session: { events: [] }, toolCalls: [], @@ -182,7 +184,10 @@ it("records rubric judge usage in score metadata", async () => { expect(result.metadata).toMatchObject({ answer: "A", - usage: { + costUsd: 0.531, + applicationCostUsd: 0.5, + judgeCostUsd: 0.031, + judgeUsage: { provider: "vercel-ai-gateway", model: "openai/gpt-5.4", inputTokens: 120, @@ -209,7 +214,7 @@ it("scores the rubric judge without failing when cost is missing", async () => { harness: slackHarness, input: { criteria: { pass: ["Answers correctly"] }, initialEvents: [] }, output: undefined, - run: {} as never, + run: { usage: {} } as never, runJudge: async () => judgeRun.output, session: { events: [] }, toolCalls: [], @@ -217,12 +222,13 @@ it("scores the rubric judge without failing when cost is missing", async () => { expect(result.metadata).toMatchObject({ answer: "A", - usage: { + judgeUsage: { provider: "vercel-ai-gateway", model: "openai/gpt-5.4", }, }); - expect(result.metadata).not.toHaveProperty("usage.metadata.costUsd"); + expect(result.metadata).not.toHaveProperty("costUsd"); + expect(result.metadata).not.toHaveProperty("judgeUsage.metadata.costUsd"); }); it("forwards the Vitest abort signal to the eval scenario", async () => { From 28316cc70fbec9c83e049b5eae8543ab2622b844 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:27:07 +0000 Subject: [PATCH 7/8] fix(evals): Use formal cost reporting Co-Authored-By: David Cramer --- packages/junior-evals/package.json | 2 +- packages/junior-evals/src/helpers.ts | 86 +++++++------------ .../tests/unit/harness/helpers.test.ts | 39 +++------ pnpm-lock.yaml | 28 +++--- pnpm-workspace.yaml | 6 +- 5 files changed, 62 insertions(+), 99 deletions(-) diff --git a/packages/junior-evals/package.json b/packages/junior-evals/package.json index 129bebcf99..1900194e01 100644 --- a/packages/junior-evals/package.json +++ b/packages/junior-evals/package.json @@ -27,7 +27,7 @@ "typescript": "^6.0.3", "undici": "7.29.0", "vitest": "^4.1.7", - "vitest-evals": "0.16.1", + "vitest-evals": "0.17.0", "zod": "catalog:" } } diff --git a/packages/junior-evals/src/helpers.ts b/packages/junior-evals/src/helpers.ts index 6fd9abbac8..c71359e500 100644 --- a/packages/junior-evals/src/helpers.ts +++ b/packages/junior-evals/src/helpers.ts @@ -398,18 +398,6 @@ function normalizedTokenUsage( }; } -function usageCostUsd(usage: unknown): number | undefined { - if (!usage || typeof usage !== "object" || Array.isArray(usage)) { - return undefined; - } - const metadata = (usage as Record).metadata; - if (!metadata || typeof metadata !== "object" || Array.isArray(metadata)) { - return undefined; - } - const value = (metadata as Record).costUsd; - return typeof value === "number" ? value : undefined; -} - function toJudgeUsage( usage: AgentTurnUsage | undefined, model: string, @@ -418,9 +406,7 @@ function toJudgeUsage( provider: GEN_AI_PROVIDER_NAME, model, ...normalizedTokenUsage(usage), - ...(usage?.cost?.total !== undefined - ? { metadata: { costUsd: usage.cost.total } } - : {}), + ...(usage?.cost?.total !== undefined ? { costUsd: usage.cost.total } : {}), }; } @@ -437,9 +423,6 @@ function toHarnessUsage(result: EvalResult): HarnessRun["usage"] { ? { currency: "USD", cost: usage.cost, - ...(usage.cost.total !== undefined - ? { costUsd: usage.cost.total } - : {}), } : {}), ...(result.modelIds.length > 1 ? { modelIds: result.modelIds } : {}), @@ -448,6 +431,7 @@ function toHarnessUsage(result: EvalResult): HarnessRun["usage"] { provider: GEN_AI_PROVIDER_NAME, ...(result.modelIds.length === 1 ? { model: result.modelIds[0] } : {}), ...normalizedTokenUsage(usage), + ...(usage?.cost?.total !== undefined ? { costUsd: usage.cost.total } : {}), toolCalls: result.toolInvocations.length, ...(Object.keys(metadata).length > 0 ? { metadata } : {}), }; @@ -678,8 +662,24 @@ const judgeHarness = createJudgeHarness({ temperature: 0, }); return { - text, + output: text, + session: { + events: [ + ...(system + ? [ + { + type: "message" as const, + role: "system" as const, + content: system, + }, + ] + : []), + { type: "message", role: "user", content: prompt }, + { type: "message", role: "assistant", content: text }, + ], + }, usage: toJudgeUsage(message.usage, message.model ?? EVAL_JUDGE_MODEL_ID), + errors: [], }; }, }); @@ -780,7 +780,6 @@ export const RubricJudge = createJudge( "RubricJudge", async ({ input, - run, session, runJudge, }: JudgeContext< @@ -788,58 +787,33 @@ export const RubricJudge = createJudge( JsonValue | undefined, typeof slackHarness >) => { - const applicationCostUsd = usageCostUsd(run.usage); if (!input.criteria) { return { score: 1, - metadata: { - skipped: "deterministic-only", - ...(typeof applicationCostUsd === "number" - ? { costUsd: applicationCostUsd, applicationCostUsd } - : {}), - }, + metadata: { skipped: "deterministic-only" }, }; } if (!runJudge) { throw new Error("RubricJudge requires a configured judgeHarness."); } - const judgeResult = await runJudge({ - prompt: formatJudgePrompt( - serializeVisibleTranscript(session), - formatRubric(input.criteria), + const object = parseJudgeResult( + String( + await runJudge({ + prompt: formatJudgePrompt( + serializeVisibleTranscript(session), + formatRubric(input.criteria), + ), + system: EVAL_SYSTEM, + }), ), - system: EVAL_SYSTEM, - }); - if ( - !judgeResult || - typeof judgeResult !== "object" || - Array.isArray(judgeResult) || - typeof judgeResult.text !== "string" || - !judgeResult.usage || - typeof judgeResult.usage !== "object" || - Array.isArray(judgeResult.usage) - ) { - throw new Error("Rubric judge returned an invalid result."); - } - const object = parseJudgeResult(judgeResult.text); + ); const answer = object.answer as keyof typeof CHOICE_SCORES; - const judgeCostUsd = usageCostUsd(judgeResult.usage); - const costUsd = - typeof applicationCostUsd === "number" && typeof judgeCostUsd === "number" - ? applicationCostUsd + judgeCostUsd - : undefined; return { score: CHOICE_SCORES[answer], metadata: { answer, rationale: object.rationale, - judgeUsage: judgeResult.usage, - ...(costUsd !== undefined ? { costUsd } : {}), - ...(typeof applicationCostUsd === "number" - ? { applicationCostUsd } - : {}), - ...(typeof judgeCostUsd === "number" ? { judgeCostUsd } : {}), }, }; }, diff --git a/packages/junior-evals/tests/unit/harness/helpers.test.ts b/packages/junior-evals/tests/unit/harness/helpers.test.ts index 249fc10bdc..9423dde195 100644 --- a/packages/junior-evals/tests/unit/harness/helpers.test.ts +++ b/packages/junior-evals/tests/unit/harness/helpers.test.ts @@ -152,7 +152,7 @@ it("includes captured Slack posts in the rubric-visible transcript", async () => ).not.toHaveProperty("rubric_visible", false); }); -it("records rubric judge usage in score metadata", async () => { +it("reports rubric judge usage through the judge harness", async () => { completeTextMock.mockResolvedValueOnce({ message: { model: "openai/gpt-5.4", @@ -174,28 +174,21 @@ it("records rubric judge usage in score metadata", async () => { harness: slackHarness, input: { criteria: { pass: ["Answers correctly"] }, initialEvents: [] }, output: undefined, - run: { - usage: { metadata: { costUsd: 0.5 } }, - } as never, + run: { usage: {} } as never, runJudge: async () => judgeRun.output, session: { events: [] }, toolCalls: [], }); - expect(result.metadata).toMatchObject({ - answer: "A", - costUsd: 0.531, - applicationCostUsd: 0.5, - judgeCostUsd: 0.031, - judgeUsage: { - provider: "vercel-ai-gateway", - model: "openai/gpt-5.4", - inputTokens: 120, - outputTokens: 20, - totalTokens: 140, - metadata: { costUsd: 0.031 }, - }, + expect(judgeRun.usage).toEqual({ + provider: "vercel-ai-gateway", + model: "openai/gpt-5.4", + inputTokens: 120, + outputTokens: 20, + totalTokens: 140, + costUsd: 0.031, }); + expect(result.metadata).toMatchObject({ answer: "A" }); }); it("scores the rubric judge without failing when cost is missing", async () => { @@ -220,15 +213,11 @@ it("scores the rubric judge without failing when cost is missing", async () => { toolCalls: [], }); - expect(result.metadata).toMatchObject({ - answer: "A", - judgeUsage: { - provider: "vercel-ai-gateway", - model: "openai/gpt-5.4", - }, + expect(judgeRun.usage).toEqual({ + provider: "vercel-ai-gateway", + model: "openai/gpt-5.4", }); - expect(result.metadata).not.toHaveProperty("costUsd"); - expect(result.metadata).not.toHaveProperty("judgeUsage.metadata.costUsd"); + expect(result.metadata).toMatchObject({ answer: "A" }); }); it("forwards the Vitest abort signal to the eval scenario", async () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a2803e76a1..b8006a5ed8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -474,8 +474,8 @@ importers: specifier: ^4.1.7 version: 4.1.7(tsx@4.22.3) vitest-evals: - specifier: 0.16.1 - version: 0.16.1(ai@6.0.190(zod@4.5.4))(tinyrainbow@3.1.0)(vitest@4.1.7(tsx@4.22.3))(zod@4.5.4) + specifier: 0.17.0 + version: 0.17.0(ai@6.0.190(zod@4.5.4))(tinyrainbow@3.1.0)(vitest@4.1.7(tsx@4.22.3))(zod@4.5.4) zod: specifier: 'catalog:' version: 4.5.4 @@ -4370,11 +4370,11 @@ packages: '@vercel/static-config@3.3.0': resolution: {integrity: sha512-GpS3tPwUeDJCkrKbMNtS2XLRFgfxTlN7YNUL+Bo23+fGolrDw6Oq79R3yvxTYgqRaJMGSEqC7iMw6mj6I5loxg==} - '@vitest-evals/core@0.16.1': - resolution: {integrity: sha512-e4BfGirm4HOP2HfoYvE61iWp7K389tYwUOdPj+CS8lCI1D2efIlLptClU1knhDocLVaU6F/9ty2+p7u1tDkaIg==} + '@vitest-evals/core@0.17.0': + resolution: {integrity: sha512-X1OBxLYWQnB+siIhI/BcDySMfpZ3AfeLiPRoZUi0iWMXeELwxxzOuzfsd2UZbJjeqyDbixRspdCfIzXYqzGxAw==} - '@vitest-evals/report-ui@0.16.1': - resolution: {integrity: sha512-3oHbMntbksRWQB1Nu1j8UXIy8+we6iUng24wAKxhSli7vjIgkaX02JjjfXgK4CWG4DhgOQkwJqw+wNaOMduY5A==} + '@vitest-evals/report-ui@0.17.0': + resolution: {integrity: sha512-7IWu7QoJpAomDsKO/RTKl8eichmJ9qQEmbQTJU5tao2gwQVzGvHx9bt79dB3b1MWuG7ZKMohv1h4FLGkZHUMqQ==} '@vitest/coverage-v8@4.1.7': resolution: {integrity: sha512-qsYPeXc5Q9dFLd1i8Ap+Bx8sQgcp+rFVQo4R0dDsWNBzl26ldVF1qOO+RL24K7FDrR6pA+50XedRLSoSG24bVQ==} @@ -8650,8 +8650,8 @@ packages: vite: optional: true - vitest-evals@0.16.1: - resolution: {integrity: sha512-MuPVetClAOn55ewgajxGxZcQDFWUftrQ19/rZQn8zXvaxHri+ElgEb5W0OIMRQwSvF5TBYIsE5rWdQB/qrWJkg==} + vitest-evals@0.17.0: + resolution: {integrity: sha512-GCQEveS5tVhCNBjAW77usjeWxPi+TqI+wJEfF0BqpD6bq5Qw/9JhAmnbrnzbOvIn8MWzUA0Wc1aonCgWxRKawg==} hasBin: true peerDependencies: ai: 6.0.190 @@ -12994,13 +12994,13 @@ snapshots: json-schema-to-ts: 1.6.4 ts-morph: 12.0.0 - '@vitest-evals/core@0.16.1': + '@vitest-evals/core@0.17.0': dependencies: zod: 4.5.4 - '@vitest-evals/report-ui@0.16.1': + '@vitest-evals/report-ui@0.17.0': dependencies: - '@vitest-evals/core': 0.16.1 + '@vitest-evals/core': 0.17.0 '@vitest/coverage-v8@4.1.7(vitest@4.1.7)': dependencies: @@ -18023,10 +18023,10 @@ snapshots: optionalDependencies: vite: 8.0.14(esbuild@0.28.1)(tsx@4.22.3) - vitest-evals@0.16.1(ai@6.0.190(zod@4.5.4))(tinyrainbow@3.1.0)(vitest@4.1.7(tsx@4.22.3))(zod@4.5.4): + vitest-evals@0.17.0(ai@6.0.190(zod@4.5.4))(tinyrainbow@3.1.0)(vitest@4.1.7(tsx@4.22.3))(zod@4.5.4): dependencies: - '@vitest-evals/core': 0.16.1 - '@vitest-evals/report-ui': 0.16.1 + '@vitest-evals/core': 0.17.0 + '@vitest-evals/report-ui': 0.17.0 tinyrainbow: 3.1.0 vitest: 4.1.7(tsx@4.22.3) optionalDependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index da88f460b1..f081bea2f6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -34,9 +34,9 @@ syncInjectedDepsAfterScripts: minimumReleaseAge: 1440 minimumReleaseAgeExclude: - "@sentry/starlight-theme" - - "@vitest-evals/core@0.16.1" - - "@vitest-evals/report-ui@0.16.1" - - "vitest-evals@0.16.1" + - "@vitest-evals/core@0.17.0" + - "@vitest-evals/report-ui@0.17.0" + - "vitest-evals@0.17.0" - "@sentry/core@10.65.0" - "@sentry/node-core@10.65.0" - "@sentry/node@10.65.0" From 688c1d4f8ddc76ff6263e446d5f8c3ef998f55d3 Mon Sep 17 00:00:00 2001 From: "sentry-junior[bot]" <264270552+sentry-junior[bot]@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:33:52 +0000 Subject: [PATCH 8/8] refactor(evals): Simplify judge cost coverage Co-Authored-By: David Cramer --- packages/junior-evals/evals/github-actions.md | 2 +- .../tests/unit/harness/helpers.test.ts | 25 +------------------ 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/packages/junior-evals/evals/github-actions.md b/packages/junior-evals/evals/github-actions.md index 25954ee30a..eaeb1f901c 100644 --- a/packages/junior-evals/evals/github-actions.md +++ b/packages/junior-evals/evals/github-actions.md @@ -94,7 +94,7 @@ Behavioral shard jobs keep running after individual case failures so every shard 1. `behavioral / report` downloads all behavioral shard result files and publishes one combined `vitest-evals` summary (metric table, score distribution, quality misses) 2. the same step publishes a `behavioral / score` Check Run with `min-pass-rate` (`EVAL_MIN_PASS_RATE`, currently `0.8`) -3. `vitest-evals@0.16.1` attaches that Check Run to the PR head SHA and soft-fails the report step when the check publishes, so the Check Run title owns the pass-rate secondary line on the PR checks list +3. `vitest-evals` attaches that Check Run to the PR head SHA and soft-fails the report step when the check publishes, so the Check Run title owns the pass-rate secondary line on the PR checks list If Check Run publishing is skipped or fails, the report step still fails on a rejected gate so status is not silently lost. diff --git a/packages/junior-evals/tests/unit/harness/helpers.test.ts b/packages/junior-evals/tests/unit/harness/helpers.test.ts index 9423dde195..46b5d9ab94 100644 --- a/packages/junior-evals/tests/unit/harness/helpers.test.ts +++ b/packages/junior-evals/tests/unit/harness/helpers.test.ts @@ -20,7 +20,6 @@ vi.mock("../../../src/behavior-harness", () => ({ import { hasImageAttachment, - RubricJudge, serializeVisibleTranscript, slackEvals, slackHarness, @@ -170,16 +169,6 @@ it("reports rubric judge usage through the judge harness", async () => { { prompt: "Grade this.", system: "Return JSON." }, { artifacts: {}, setArtifact: vi.fn() }, ); - const result = await RubricJudge.assess({ - harness: slackHarness, - input: { criteria: { pass: ["Answers correctly"] }, initialEvents: [] }, - output: undefined, - run: { usage: {} } as never, - runJudge: async () => judgeRun.output, - session: { events: [] }, - toolCalls: [], - }); - expect(judgeRun.usage).toEqual({ provider: "vercel-ai-gateway", model: "openai/gpt-5.4", @@ -188,10 +177,9 @@ it("reports rubric judge usage through the judge harness", async () => { totalTokens: 140, costUsd: 0.031, }); - expect(result.metadata).toMatchObject({ answer: "A" }); }); -it("scores the rubric judge without failing when cost is missing", async () => { +it("omits unknown rubric judge cost", async () => { completeTextMock.mockResolvedValueOnce({ message: { usage: {}, @@ -203,21 +191,10 @@ it("scores the rubric judge without failing when cost is missing", async () => { { prompt: "Grade this.", system: "Return JSON." }, { artifacts: {}, setArtifact: vi.fn() }, ); - const result = await RubricJudge.assess({ - harness: slackHarness, - input: { criteria: { pass: ["Answers correctly"] }, initialEvents: [] }, - output: undefined, - run: { usage: {} } as never, - runJudge: async () => judgeRun.output, - session: { events: [] }, - toolCalls: [], - }); - expect(judgeRun.usage).toEqual({ provider: "vercel-ai-gateway", model: "openai/gpt-5.4", }); - expect(result.metadata).toMatchObject({ answer: "A" }); }); it("forwards the Vitest abort signal to the eval scenario", async () => {