diff --git a/packages/junior-evals/evals/github-actions.md b/packages/junior-evals/evals/github-actions.md index 25954ee30a..eaeb1f901c 100644 --- a/packages/junior-evals/evals/github-actions.md +++ b/packages/junior-evals/evals/github-actions.md @@ -94,7 +94,7 @@ Behavioral shard jobs keep running after individual case failures so every shard 1. `behavioral / report` downloads all behavioral shard result files and publishes one combined `vitest-evals` summary (metric table, score distribution, quality misses) 2. the same step publishes a `behavioral / score` Check Run with `min-pass-rate` (`EVAL_MIN_PASS_RATE`, currently `0.8`) -3. `vitest-evals@0.16.1` attaches that Check Run to the PR head SHA and soft-fails the report step when the check publishes, so the Check Run title owns the pass-rate secondary line on the PR checks list +3. `vitest-evals` attaches that Check Run to the PR head SHA and soft-fails the report step when the check publishes, so the Check Run title owns the pass-rate secondary line on the PR checks list If Check Run publishing is skipped or fails, the report step still fails on a rejected gate so status is not silently lost. diff --git a/packages/junior-evals/package.json b/packages/junior-evals/package.json index 129bebcf99..1900194e01 100644 --- a/packages/junior-evals/package.json +++ b/packages/junior-evals/package.json @@ -27,7 +27,7 @@ "typescript": "^6.0.3", "undici": "7.29.0", "vitest": "^4.1.7", - "vitest-evals": "0.16.1", + "vitest-evals": "0.17.0", "zod": "catalog:" } } diff --git a/packages/junior-evals/src/helpers.ts b/packages/junior-evals/src/helpers.ts index f4668161d0..c71359e500 100644 --- a/packages/junior-evals/src/helpers.ts +++ b/packages/junior-evals/src/helpers.ts @@ -379,6 +379,37 @@ function usageTotal(usage: AgentTurnUsage | undefined): number | undefined { : undefined; } +function normalizedTokenUsage( + usage: AgentTurnUsage | undefined, +): HarnessRun["usage"] { + return { + ...(usage?.inputTokens !== undefined + ? { inputTokens: usage.inputTokens } + : {}), + ...(usage?.outputTokens !== undefined + ? { outputTokens: usage.outputTokens } + : {}), + ...(usage?.reasoningTokens !== undefined + ? { reasoningTokens: usage.reasoningTokens } + : {}), + ...(usageTotal(usage) !== undefined + ? { totalTokens: usageTotal(usage) } + : {}), + }; +} + +function toJudgeUsage( + usage: AgentTurnUsage | undefined, + model: string, +): HarnessRun["usage"] { + return { + provider: GEN_AI_PROVIDER_NAME, + model, + ...normalizedTokenUsage(usage), + ...(usage?.cost?.total !== undefined ? { costUsd: usage.cost.total } : {}), + }; +} + function toHarnessUsage(result: EvalResult): HarnessRun["usage"] { const usage = result.usage; const metadata = toJsonRecord({ @@ -392,9 +423,6 @@ function toHarnessUsage(result: EvalResult): HarnessRun["usage"] { ? { currency: "USD", cost: usage.cost, - ...(usage.cost.total !== undefined - ? { costUsd: usage.cost.total } - : {}), } : {}), ...(result.modelIds.length > 1 ? { modelIds: result.modelIds } : {}), @@ -402,18 +430,8 @@ function toHarnessUsage(result: EvalResult): HarnessRun["usage"] { return { provider: GEN_AI_PROVIDER_NAME, ...(result.modelIds.length === 1 ? { model: result.modelIds[0] } : {}), - ...(usage?.inputTokens !== undefined - ? { inputTokens: usage.inputTokens } - : {}), - ...(usage?.outputTokens !== undefined - ? { outputTokens: usage.outputTokens } - : {}), - ...(usage?.reasoningTokens !== undefined - ? { reasoningTokens: usage.reasoningTokens } - : {}), - ...(usageTotal(usage) !== undefined - ? { totalTokens: usageTotal(usage) } - : {}), + ...normalizedTokenUsage(usage), + ...(usage?.cost?.total !== undefined ? { costUsd: usage.cost.total } : {}), toolCalls: result.toolInvocations.length, ...(Object.keys(metadata).length > 0 ? { metadata } : {}), }; @@ -631,7 +649,7 @@ const EVAL_JUDGE_MODEL_ID = resolveGatewayModel("openai/gpt-5.4").id; const judgeHarness = createJudgeHarness({ name: "slack-rubric-judge-model", run: async ({ prompt, system }) => { - const { text } = await completeText({ + const { message, text } = await completeText({ modelId: EVAL_JUDGE_MODEL_ID, system, messages: [ @@ -643,7 +661,26 @@ const judgeHarness = createJudgeHarness({ ], temperature: 0, }); - return text; + return { + output: text, + session: { + events: [ + ...(system + ? [ + { + type: "message" as const, + role: "system" as const, + content: system, + }, + ] + : []), + { type: "message", role: "user", content: prompt }, + { type: "message", role: "assistant", content: text }, + ], + }, + usage: toJudgeUsage(message.usage, message.model ?? EVAL_JUDGE_MODEL_ID), + errors: [], + }; }, }); diff --git a/packages/junior-evals/tests/unit/harness/helpers.test.ts b/packages/junior-evals/tests/unit/harness/helpers.test.ts index c7fce35de4..46b5d9ab94 100644 --- a/packages/junior-evals/tests/unit/harness/helpers.test.ts +++ b/packages/junior-evals/tests/unit/harness/helpers.test.ts @@ -1,12 +1,19 @@ import { expect, it, vi } from "vitest"; -const { runError, runEvalScenarioMock } = vi.hoisted(() => ({ +const { completeTextMock, runError, runEvalScenarioMock } = vi.hoisted(() => ({ + completeTextMock: vi.fn(), runError: new Error("stop after capturing harness options"), runEvalScenarioMock: vi.fn(async () => { throw new Error("uninitialized run error"); }), })); +vi.mock("@/chat/pi/client", () => ({ + completeText: completeTextMock, + GEN_AI_PROVIDER_NAME: "vercel-ai-gateway", + resolveGatewayModel: vi.fn((modelId: string) => ({ id: modelId })), +})); + vi.mock("../../../src/behavior-harness", () => ({ runEvalScenario: runEvalScenarioMock, })); @@ -14,6 +21,7 @@ vi.mock("../../../src/behavior-harness", () => ({ import { hasImageAttachment, serializeVisibleTranscript, + slackEvals, slackHarness, visibleAssistantText, visibleThreadReplies, @@ -143,6 +151,52 @@ it("includes captured Slack posts in the rubric-visible transcript", async () => ).not.toHaveProperty("rubric_visible", false); }); +it("reports rubric judge usage through the judge harness", async () => { + completeTextMock.mockResolvedValueOnce({ + message: { + model: "openai/gpt-5.4", + usage: { + inputTokens: 120, + outputTokens: 20, + totalTokens: 140, + cost: { total: 0.031 }, + }, + }, + text: '{"answer":"A","rationale":"The response meets the rubric."}', + }); + + const judgeRun = await slackEvals.judgeHarness.run( + { prompt: "Grade this.", system: "Return JSON." }, + { artifacts: {}, setArtifact: vi.fn() }, + ); + expect(judgeRun.usage).toEqual({ + provider: "vercel-ai-gateway", + model: "openai/gpt-5.4", + inputTokens: 120, + outputTokens: 20, + totalTokens: 140, + costUsd: 0.031, + }); +}); + +it("omits unknown rubric judge cost", async () => { + completeTextMock.mockResolvedValueOnce({ + message: { + usage: {}, + }, + text: '{"answer":"A","rationale":"The response meets the rubric."}', + }); + + const judgeRun = await slackEvals.judgeHarness.run( + { prompt: "Grade this.", system: "Return JSON." }, + { artifacts: {}, setArtifact: vi.fn() }, + ); + expect(judgeRun.usage).toEqual({ + provider: "vercel-ai-gateway", + model: "openai/gpt-5.4", + }); +}); + it("forwards the Vitest abort signal to the eval scenario", async () => { runEvalScenarioMock.mockRejectedValueOnce(runError); const controller = new AbortController(); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a2803e76a1..b8006a5ed8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -474,8 +474,8 @@ importers: specifier: ^4.1.7 version: 4.1.7(tsx@4.22.3) vitest-evals: - specifier: 0.16.1 - version: 0.16.1(ai@6.0.190(zod@4.5.4))(tinyrainbow@3.1.0)(vitest@4.1.7(tsx@4.22.3))(zod@4.5.4) + specifier: 0.17.0 + version: 0.17.0(ai@6.0.190(zod@4.5.4))(tinyrainbow@3.1.0)(vitest@4.1.7(tsx@4.22.3))(zod@4.5.4) zod: specifier: 'catalog:' version: 4.5.4 @@ -4370,11 +4370,11 @@ packages: '@vercel/static-config@3.3.0': resolution: {integrity: sha512-GpS3tPwUeDJCkrKbMNtS2XLRFgfxTlN7YNUL+Bo23+fGolrDw6Oq79R3yvxTYgqRaJMGSEqC7iMw6mj6I5loxg==} - '@vitest-evals/core@0.16.1': - resolution: {integrity: sha512-e4BfGirm4HOP2HfoYvE61iWp7K389tYwUOdPj+CS8lCI1D2efIlLptClU1knhDocLVaU6F/9ty2+p7u1tDkaIg==} + '@vitest-evals/core@0.17.0': + resolution: {integrity: sha512-X1OBxLYWQnB+siIhI/BcDySMfpZ3AfeLiPRoZUi0iWMXeELwxxzOuzfsd2UZbJjeqyDbixRspdCfIzXYqzGxAw==} - '@vitest-evals/report-ui@0.16.1': - resolution: {integrity: sha512-3oHbMntbksRWQB1Nu1j8UXIy8+we6iUng24wAKxhSli7vjIgkaX02JjjfXgK4CWG4DhgOQkwJqw+wNaOMduY5A==} + '@vitest-evals/report-ui@0.17.0': + resolution: {integrity: sha512-7IWu7QoJpAomDsKO/RTKl8eichmJ9qQEmbQTJU5tao2gwQVzGvHx9bt79dB3b1MWuG7ZKMohv1h4FLGkZHUMqQ==} '@vitest/coverage-v8@4.1.7': resolution: {integrity: sha512-qsYPeXc5Q9dFLd1i8Ap+Bx8sQgcp+rFVQo4R0dDsWNBzl26ldVF1qOO+RL24K7FDrR6pA+50XedRLSoSG24bVQ==} @@ -8650,8 +8650,8 @@ packages: vite: optional: true - vitest-evals@0.16.1: - resolution: {integrity: sha512-MuPVetClAOn55ewgajxGxZcQDFWUftrQ19/rZQn8zXvaxHri+ElgEb5W0OIMRQwSvF5TBYIsE5rWdQB/qrWJkg==} + vitest-evals@0.17.0: + resolution: {integrity: sha512-GCQEveS5tVhCNBjAW77usjeWxPi+TqI+wJEfF0BqpD6bq5Qw/9JhAmnbrnzbOvIn8MWzUA0Wc1aonCgWxRKawg==} hasBin: true peerDependencies: ai: 6.0.190 @@ -12994,13 +12994,13 @@ snapshots: json-schema-to-ts: 1.6.4 ts-morph: 12.0.0 - '@vitest-evals/core@0.16.1': + '@vitest-evals/core@0.17.0': dependencies: zod: 4.5.4 - '@vitest-evals/report-ui@0.16.1': + '@vitest-evals/report-ui@0.17.0': dependencies: - '@vitest-evals/core': 0.16.1 + '@vitest-evals/core': 0.17.0 '@vitest/coverage-v8@4.1.7(vitest@4.1.7)': dependencies: @@ -18023,10 +18023,10 @@ snapshots: optionalDependencies: vite: 8.0.14(esbuild@0.28.1)(tsx@4.22.3) - vitest-evals@0.16.1(ai@6.0.190(zod@4.5.4))(tinyrainbow@3.1.0)(vitest@4.1.7(tsx@4.22.3))(zod@4.5.4): + vitest-evals@0.17.0(ai@6.0.190(zod@4.5.4))(tinyrainbow@3.1.0)(vitest@4.1.7(tsx@4.22.3))(zod@4.5.4): dependencies: - '@vitest-evals/core': 0.16.1 - '@vitest-evals/report-ui': 0.16.1 + '@vitest-evals/core': 0.17.0 + '@vitest-evals/report-ui': 0.17.0 tinyrainbow: 3.1.0 vitest: 4.1.7(tsx@4.22.3) optionalDependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index da88f460b1..f081bea2f6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -34,9 +34,9 @@ syncInjectedDepsAfterScripts: minimumReleaseAge: 1440 minimumReleaseAgeExclude: - "@sentry/starlight-theme" - - "@vitest-evals/core@0.16.1" - - "@vitest-evals/report-ui@0.16.1" - - "vitest-evals@0.16.1" + - "@vitest-evals/core@0.17.0" + - "@vitest-evals/report-ui@0.17.0" + - "vitest-evals@0.17.0" - "@sentry/core@10.65.0" - "@sentry/node-core@10.65.0" - "@sentry/node@10.65.0"