diff --git a/packages/core/src/report/metadata.ts b/packages/core/src/report/metadata.ts index 09fdc03..d559490 100644 --- a/packages/core/src/report/metadata.ts +++ b/packages/core/src/report/metadata.ts @@ -1,7 +1,7 @@ import { z } from "zod"; import { HarnessRunSchema, ToolCallSchema } from "../harness"; import { JsonObjectSchema, JsonValueSchema } from "../json"; -import { isJsonObject, NullableFiniteNumberSchema } from "../schema-utils"; +import { NullableFiniteNumberSchema, isJsonObject } from "../schema-utils"; /** Harness metadata stored by vitest-evals on Vitest task metadata. */ export const HarnessMetaSchema = z @@ -31,6 +31,8 @@ export const EvalMetaSchema = z .object({ scores: z.array(EvalScoreSchema).optional(), avgScore: NullableFiniteNumberSchema, + input: JsonValueSchema.optional(), + expected: JsonValueSchema.optional(), output: JsonValueSchema.optional(), thresholdFailed: z.boolean().optional(), toolCalls: z.array(ToolCallSchema).optional(), diff --git a/packages/report-ui/package.json b/packages/report-ui/package.json index facaac0..449ee0c 100644 --- a/packages/report-ui/package.json +++ b/packages/report-ui/package.json @@ -18,9 +18,7 @@ "type": "module", "main": "./dist/index.js", "module": "./dist/index.js", - "files": [ - "dist" - ], + "files": ["dist"], "publishConfig": { "access": "public" }, @@ -41,13 +39,15 @@ }, "devDependencies": { "@tailwindcss/vite": "^4.3.0", + "@tanstack/react-router": "^1.170.32", "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", - "@vitejs/plugin-react": "^5.1.1", + "@vitejs/plugin-react": "^6.1.1", "react": "^19.2.3", "react-dom": "^19.2.3", + "react18-json-view": "^0.2.10", "tailwindcss": "^4.3.0", - "vite": "^7.3.0", + "vite": "^8.2.2", "vite-tsconfig-paths": "^6.1.1" } } diff --git a/packages/report-ui/src/app/App.test.ts b/packages/report-ui/src/app/App.test.ts index df8a5e6..7699dbb 100644 --- a/packages/report-ui/src/app/App.test.ts +++ b/packages/report-ui/src/app/App.test.ts @@ -1,7 +1,9 @@ -import { afterEach, describe, expect, test, vi } from "vitest"; import type { ReportWorkspace } from "@vitest-evals/core"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { + adjacentVisibleCase, loadWorkspace, + nextSortSearch, resolveSelectedCase, resolveSelectedCaseId, summarizeVisibleWorkspace, @@ -40,6 +42,46 @@ afterEach(() => { vi.unstubAllGlobals(); }); +describe("nextSortSearch", () => { + test("starts a new column with its default direction and toggles the active one", () => { + expect(nextSortSearch(undefined, "asc", "score")).toEqual({ + dir: "desc", + sort: "score", + }); + expect(nextSortSearch("score", "desc", "score")).toEqual({ + dir: "asc", + sort: "score", + }); + expect(nextSortSearch("score", "asc", "case")).toEqual({ + dir: "asc", + sort: "case", + }); + expect(nextSortSearch(undefined, "asc", "model")).toEqual({ + dir: "asc", + sort: "model", + }); + }); +}); + +describe("adjacentVisibleCase", () => { + test("opens the first or last case when nothing is selected", () => { + expect(adjacentVisibleCase(cases, undefined, 1)?.id).toBe("failed-case"); + expect(adjacentVisibleCase(cases, undefined, -1)?.id).toBe("passed-case"); + }); + + test("moves to the next visible case and stays at the ends", () => { + expect(adjacentVisibleCase(cases, "failed-case", 1)?.id).toBe( + "passed-case", + ); + expect(adjacentVisibleCase(cases, "passed-case", 1)?.id).toBe( + "passed-case", + ); + expect(adjacentVisibleCase(cases, "passed-case", -1)?.id).toBe( + "failed-case", + ); + }); +}); + describe("case selection", () => { test("keeps selection scoped to visible filtered cases", () => { const visibleCases = [cases[1]!]; @@ -109,6 +151,7 @@ describe("visible summary", () => { }, { query: "", + model: "all", runId: "run-2", status: "passed", }, @@ -134,6 +177,7 @@ describe("visible summary", () => { runs, { query: "", + model: "all", runId: "run-2", status: "passed", }, @@ -151,6 +195,7 @@ describe("visible summary", () => { runs, { query: "", + model: "all", runId: "all", status: "all", }, @@ -183,6 +228,7 @@ describe("visible summary", () => { }, { query: "", + model: "all", runId: "all", status: "all", }, diff --git a/packages/report-ui/src/app/App.tsx b/packages/report-ui/src/app/App.tsx index 6f6e0b4..15c6d7c 100644 --- a/packages/report-ui/src/app/App.tsx +++ b/packages/report-ui/src/app/App.tsx @@ -1,22 +1,54 @@ -import { useEffect, useMemo, useState } from "react"; import { - ReportWorkspaceSchema, + type ReportCase, type ReportWorkspace, + ReportWorkspaceSchema, } from "@vitest-evals/core"; +import { useCallback, useEffect, useMemo, useState } from "react"; +import { caseToMarkdown } from "./case-markdown"; +import { formatJunitXml, formatPullRequestComment } from "./ci-artifacts"; +import { + type CaseDelta, + caseDelta, + resolveBaselineRun, + workspaceDelta, +} from "./compare"; import { CaseDrawer } from "./components/CaseDrawer"; import { CaseWorkbench } from "./components/CaseWorkbench"; +import { + CommandPalette, + type PaletteCommand, + downloadTextFile, +} from "./components/CommandPalette"; import { ReportHeader, RunStrip, SummaryBar } from "./components/ReportChrome"; import { + RerunSessionProvider, + useRerunSession, +} from "./components/RerunSession"; +import { relativeDisplayPath, resolveOpenPath } from "./display-path"; +import { editorTargets } from "./file-open"; +import { + type CaseFilters, + type CaseSortColumn, filterReportCases, + formatJson, + hasActiveCaseFilters, + sortReportCases, summarizeWorkspace, - type CaseFilters, + uniqueCaseModels, } from "./model"; -import type { DetailTab } from "./types"; +import { estimateWorkspaceUsageCost } from "./pricing"; +import { + DEFAULT_REPORT_META, + type ReportMeta, + ReportMetaContext, + readReportMeta, +} from "./report-meta"; +import { useReportSearch } from "./report-state"; type LoadState = | { status: "loading" } | { status: "error"; message: string } - | { status: "ready"; workspace: ReportWorkspace }; + | { status: "ready"; workspace: ReportWorkspace; meta: ReportMeta }; export function App() { const [loadState, setLoadState] = useState({ status: "loading" }); @@ -31,6 +63,14 @@ export function App() { return () => abortController.abort(); }, []); + const reloadReadyWorkspace = useCallback(() => { + void loadWorkspace(new AbortController().signal).then((nextState) => { + if (nextState?.status === "ready") { + setLoadState(nextState); + } + }); + }, []); + if (loadState.status === "loading") { return ; } @@ -41,96 +81,477 @@ export function App() { ); } - return ; + return ( + + + + ); } -function ReportApp({ workspace }: { workspace: ReportWorkspace }) { - const [filters, setFilters] = useState({ - query: "", - status: "all", - runId: "all", - }); - const [selectedCaseId, setSelectedCaseId] = useState( - () => workspace.cases.find((testCase) => testCase.status === "failed")?.id, +function ReportApp({ + meta, + workspace, +}: { + meta: ReportMeta; + workspace: ReportWorkspace; +}) { + const { requestRerun } = useRerunSession(); + const { search, setSearch } = useReportSearch(); + const filters = useMemo( + () => ({ + query: search.q, + status: search.status, + runId: search.run, + model: search.model, + }), + [search.model, search.q, search.run, search.status], ); - const [isDrawerOpen, setIsDrawerOpen] = useState(false); - const [detailTab, setDetailTab] = useState("overview"); const filteredCases = useMemo( () => filterReportCases(workspace.cases, filters), [workspace.cases, filters], ); - const visibleRuns = useMemo( - () => visibleWorkspaceRuns(workspace.runs, filters, filteredCases), - [workspace.runs, filters, filteredCases], + const baselineRun = useMemo( + () => resolveBaselineRun(workspace.runs, search.vs), + [search.vs, workspace.runs], + ); + const baselineCases = useMemo( + () => + baselineRun + ? workspace.cases.filter( + (testCase) => testCase.runId === baselineRun.id, + ) + : [], + [baselineRun, workspace.cases], + ); + const deltas = useMemo(() => { + const next = new Map(); + if (!baselineRun) { + return next; + } + for (const testCase of workspace.cases) { + if (testCase.runId === baselineRun.id) { + continue; + } + const delta = caseDelta(testCase, baselineCases, meta.pricing); + if (delta) { + next.set(testCase.id, delta); + } + } + return next; + }, [baselineCases, baselineRun, meta.pricing, workspace.cases]); + const scoreDeltas = useMemo(() => { + const next = new Map(); + for (const [id, delta] of deltas) { + if (delta.score !== undefined) { + next.set(id, delta.score); + } + } + return next; + }, [deltas]); + const visibleCases = useMemo( + () => + sortReportCases( + filteredCases, + search.sort, + search.dir, + meta.pricing, + scoreDeltas, + ), + [filteredCases, meta.pricing, scoreDeltas, search.sort, search.dir], + ); + const runDelta = useMemo( + () => workspaceDelta(workspace, meta.pricing), + [meta.pricing, workspace], ); - const summary = useMemo( + const workspaceSummary = useMemo( + () => summarizeWorkspace(workspace), + [workspace], + ); + const workspaceCost = useMemo( () => - summarizeWorkspace({ - ...workspace, - cases: filteredCases, - runs: visibleRuns, - }), - [workspace, filteredCases, visibleRuns], + estimateWorkspaceUsageCost( + workspace.cases.map((testCase) => testCase.harness?.run?.usage ?? {}), + meta.pricing, + ), + [meta.pricing, workspace.cases], ); - const selectedCase = resolveSelectedCase(selectedCaseId, filteredCases); + const sourceLabel = + relativeDisplayPath( + workspace.runs[0]?.source ?? workspace.runs[0]?.id, + meta.workspaceRoot, + ) || "Eval report"; + const selectedCase = resolveSelectedCase(search.case, workspace.cases); + const isDrawerOpen = Boolean(search.case && selectedCase); + const [paletteOpen, setPaletteOpen] = useState(false); - useEffect(() => { - const nextSelectedCaseId = resolveSelectedCaseId( - selectedCaseId, - filteredCases, + const clearFilters = () => + setSearch({ + q: "", + status: "all", + run: "all", + model: "all", + }); + + const openCase = (testCase: ReportCase | undefined, tab = search.tab) => { + if (!testCase) { + return; + } + setSearch({ case: testCase.id, tab }, { replace: false }); + }; + + const moveVisibleCase = (delta: 1 | -1) => { + openCase(adjacentVisibleCase(visibleCases, selectedCase?.id, delta)); + }; + + const paletteCommands: PaletteCommand[] = (() => { + const selectedFile = + resolveOpenPath(selectedCase?.file, meta.workspaceRoot) ?? + selectedCase?.file; + const selectedRun = workspace.runs.find( + (run) => run.id === selectedCase?.runId, ); - if (nextSelectedCaseId !== selectedCaseId) { - setSelectedCaseId(nextSelectedCaseId); + const commands: PaletteCommand[] = [ + { + id: "filter-failed", + group: "Filters", + label: "Show failed cases", + run: () => setSearch({ status: "failed" }), + }, + { + id: "filter-passed", + group: "Filters", + label: "Show passed cases", + run: () => setSearch({ status: "passed" }), + }, + { + id: "filter-all", + group: "Filters", + label: "Show all cases", + run: () => setSearch({ status: "all" }), + }, + { + id: "clear-filters", + group: "Filters", + label: "Reset filters", + run: clearFilters, + }, + { + id: "focus-search", + group: "Navigation", + label: "Focus search", + hint: "/", + run: () => document.getElementById("case-search")?.focus(), + }, + { + id: "next-case", + group: "Navigation", + label: "Next case", + hint: "j", + run: () => moveVisibleCase(1), + }, + { + id: "prev-case", + group: "Navigation", + label: "Previous case", + hint: "k", + run: () => moveVisibleCase(-1), + }, + { + id: "download-junit", + group: "Export", + label: "Download JUnit XML", + run: () => + downloadTextFile( + "vitest-evals.junit.xml", + formatJunitXml(workspace), + "application/xml", + ), + }, + { + id: "download-comment", + group: "Export", + label: "Download PR comment", + run: () => + downloadTextFile( + "vitest-evals.comment.md", + formatPullRequestComment(workspace, meta.pricing), + "text/markdown", + ), + }, + ]; + if (selectedCase) { + commands.push({ + id: "rerun-selected", + group: "Actions", + label: "Re-run selected case", + run: () => requestRerun(selectedCase), + }); + commands.push({ + id: "open-selected", + group: "Navigation", + label: "Open selected case", + run: () => openCase(selectedCase, "overview"), + }); + commands.push({ + id: "compare-tab", + group: "Navigation", + label: "Open Compare tab", + run: () => openCase(selectedCase, "compare"), + }); + commands.push({ + id: "copy-markdown", + group: "Copy", + label: "Copy as Markdown", + run: () => { + void navigator.clipboard.writeText( + caseToMarkdown(selectedCase, selectedRun), + ); + }, + }); + commands.push({ + id: "copy-json", + group: "Copy", + label: "Copy JSON", + run: () => { + void navigator.clipboard.writeText(formatJson(selectedCase)); + }, + }); + if (selectedCase.failureMessages.length > 0) { + commands.push({ + id: "copy-failure", + group: "Copy", + label: "Copy failure", + run: () => { + void navigator.clipboard.writeText( + selectedCase.failureMessages.join("\n\n"), + ); + }, + }); + } + if (selectedFile) { + for (const target of editorTargets({ file: selectedFile })) { + commands.push({ + id: `open-${target.id}`, + group: "Editor", + label: target.label, + hint: selectedCase.displayName, + run: () => { + window.location.href = target.href; + }, + }); + } + } } - }, [filteredCases, selectedCaseId]); + for (const testCase of visibleCases) { + commands.push({ + id: `case-${testCase.id}`, + group: "Cases", + label: testCase.displayName, + hint: testCase.status, + run: () => openCase(testCase), + }); + } + return commands; + })(); + + useEffect(() => { + const onKeyDown = (event: KeyboardEvent) => { + if ((event.metaKey || event.ctrlKey) && event.key.toLowerCase() === "k") { + event.preventDefault(); + setPaletteOpen((open) => !open); + return; + } + if (paletteOpen) { + return; + } + if (event.key === "Escape") { + if (isDrawerOpen) { + return; + } + if (hasActiveCaseFilters(filters)) { + event.preventDefault(); + setSearch({ + q: "", + status: "all", + run: "all", + model: "all", + }); + } + return; + } + if (isEditableTarget(event.target)) { + return; + } + if (event.key === "/" && !event.metaKey && !event.ctrlKey) { + event.preventDefault(); + document.getElementById("case-search")?.focus(); + return; + } + if (event.key === "j" || event.key === "k") { + event.preventDefault(); + const next = adjacentVisibleCase( + visibleCases, + selectedCase?.id, + event.key === "j" ? 1 : -1, + ); + if (next) { + setSearch({ case: next.id }, { replace: false }); + } + } + }; + window.addEventListener("keydown", onKeyDown); + return () => window.removeEventListener("keydown", onKeyDown); + }, [ + filters, + isDrawerOpen, + paletteOpen, + selectedCase, + setSearch, + visibleCases, + ]); return ( -
-
- -
- - - +
+
+ setPaletteOpen(true)} + /> +
+ + {workspace.runs.length > 1 ? ( + + ) : null} + + setSearch({ + q: nextFilters.query, + model: nextFilters.model, + run: nextFilters.runId, + status: nextFilters.status, + }) + } + onSelectCase={(testCase) => + setSearch( + { case: testCase.id, tab: "overview" }, + { replace: false }, + ) + } + onSortChange={(column) => + setSearch(nextSortSearch(search.sort, search.dir, column)) + } + /> +
+ + { - setSelectedCaseId(testCase.id); - setDetailTab("overview"); - setIsDrawerOpen(true); - }} + testCase={selectedCase} + onClose={() => + setSearch( + { case: undefined, tab: "overview" }, + { replace: false }, + ) + } + onNext={() => moveVisibleCase(1)} + onPrev={() => moveVisibleCase(-1)} + onResetFilters={clearFilters} + onTabChange={(tab) => setSearch({ tab })} /> -
- - setIsDrawerOpen(false)} - onTabChange={setDetailTab} - /> -
-
+ setPaletteOpen(false)} + /> + + + ); } +export function nextSortSearch( + currentColumn: CaseSortColumn | undefined, + currentDirection: "asc" | "desc", + column: CaseSortColumn, +) { + if (currentColumn !== column) { + return { + sort: column, + dir: defaultSortDirection(column), + }; + } + + return { + sort: column, + dir: currentDirection === "asc" ? ("desc" as const) : ("asc" as const), + }; +} + +function defaultSortDirection(column: CaseSortColumn): "asc" | "desc" { + return column === "case" || + column === "model" || + column === "status" || + column === "expected" + ? "asc" + : "desc"; +} + export function resolveSelectedCase( selectedCaseId: string | undefined, - filteredCases: ReportWorkspace["cases"], + cases: ReportWorkspace["cases"], +) { + return cases.find((testCase) => testCase.id === selectedCaseId); +} + +export function adjacentVisibleCase( + cases: ReportCase[], + currentId: string | undefined, + delta: 1 | -1, ) { - return filteredCases.find((testCase) => testCase.id === selectedCaseId); + if (cases.length === 0) { + return undefined; + } + const index = cases.findIndex((testCase) => testCase.id === currentId); + if (index < 0) { + return cases[delta === 1 ? 0 : cases.length - 1]; + } + const next = index + delta; + if (next < 0 || next >= cases.length) { + return cases[index]; + } + return cases[next]; } export function resolveSelectedCaseId( @@ -171,14 +592,6 @@ export function visibleWorkspaceRuns( return visibleRunsForCases(runs, filteredCases); } -function hasActiveCaseFilters(filters: CaseFilters) { - return ( - filters.query.trim().length > 0 || - filters.status !== "all" || - filters.runId !== "all" - ); -} - function visibleRunsForCases( runs: ReportWorkspace["runs"], filteredCases: ReportWorkspace["cases"], @@ -193,12 +606,20 @@ export async function loadWorkspace( signal: AbortSignal, ): Promise { try { - const response = await fetch("/data/workspace.json", { signal }); - if (!response.ok) { - throw new Error(`HTTP ${response.status}`); + const [workspaceResponse, metaResponse] = await Promise.all([ + fetch("/data/workspace.json", { signal }), + fetch("/data/meta.json", { signal }), + ]); + if (!workspaceResponse.ok) { + throw new Error(`HTTP ${workspaceResponse.status}`); } - const workspace = ReportWorkspaceSchema.parse(await response.json()); - return { status: "ready", workspace }; + const workspace = ReportWorkspaceSchema.parse( + await workspaceResponse.json(), + ); + const meta = metaResponse.ok + ? readReportMeta(await metaResponse.json()) + : DEFAULT_REPORT_META; + return { status: "ready", meta, workspace }; } catch (error) { if (signal.aborted) { return undefined; @@ -210,6 +631,17 @@ export async function loadWorkspace( } } +function isEditableTarget(target: EventTarget | null) { + if (!(target instanceof HTMLElement)) { + return false; + } + if (target.isContentEditable) { + return true; + } + const tag = target.tagName; + return tag === "INPUT" || tag === "TEXTAREA" || tag === "SELECT"; +} + function CenteredState({ title, detail }: { title: string; detail: string }) { return (
diff --git a/packages/report-ui/src/app/case-markdown.test.ts b/packages/report-ui/src/app/case-markdown.test.ts new file mode 100644 index 0000000..6c659ee --- /dev/null +++ b/packages/report-ui/src/app/case-markdown.test.ts @@ -0,0 +1,49 @@ +import type { ReportCase } from "@vitest-evals/core"; +import { describe, expect, test } from "vitest"; +import { caseToMarkdown } from "./case-markdown"; + +const testCase: ReportCase = { + ancestorTitles: ["refund"], + displayFile: "refund.eval.ts", + displayName: "refund agent > rejects fraud", + failureMessages: ["Score: 0.20 below threshold: 1.00"], + file: "/repo/refund.eval.ts", + fullName: "refund agent rejects fraud", + id: "failed-case", + runId: "run-1", + status: "failed", + title: "rejects fraud", + eval: { + avgScore: 0.2, + output: { status: "denied" }, + scores: [{ name: "StructuredOutputJudge", score: 0.2 }], + }, +}; + +describe("caseToMarkdown", () => { + test("includes agent instructions and the failing case evidence", () => { + const markdown = caseToMarkdown(testCase, { + id: "run-1", + source: "eval-results/refund.json", + status: "failed", + totals: { + evalFailed: 1, + evalPassed: 0, + evalTotal: 1, + failed: 1, + passed: 0, + skipped: 0, + total: 1, + }, + }); + + expect(markdown).toContain("# Eval case: refund agent > rejects fraud"); + expect(markdown).toContain("Help fix or improve this vitest-evals case."); + expect(markdown).toContain("**Result:** failed"); + expect(markdown).toContain("**File:** `refund.eval.ts`"); + expect(markdown).toContain("Score: 0.20 below threshold: 1.00"); + expect(markdown).toContain("| StructuredOutputJudge | 20% |"); + expect(markdown).toContain('"status": "denied"'); + expect(markdown).toContain("## Raw case JSON"); + }); +}); diff --git a/packages/report-ui/src/app/case-markdown.ts b/packages/report-ui/src/app/case-markdown.ts new file mode 100644 index 0000000..a070d9c --- /dev/null +++ b/packages/report-ui/src/app/case-markdown.ts @@ -0,0 +1,54 @@ +import type { ReportCase, ReportRun } from "@vitest-evals/core"; +import { formatDuration, formatJson, formatScore } from "./model"; + +/** Builds a paste-ready Markdown brief for an agent working on one eval case. */ +export function caseToMarkdown( + testCase: ReportCase, + run: ReportRun | undefined, +): string { + const lines = [ + `# Eval case: ${testCase.displayName}`, + "", + "Help fix or improve this vitest-evals case. Keep the change minimal, preserve the eval intent, and say what you would change.", + "", + "## Status", + "", + `- **Result:** ${testCase.status}`, + `- **Score:** ${formatScore(testCase.eval?.avgScore)}`, + `- **Duration:** ${formatDuration(testCase.durationMs)}`, + `- **File:** \`${testCase.displayFile}\``, + `- **Run:** ${run?.source ?? testCase.runId}`, + ]; + + if (testCase.failureMessages.length > 0) { + lines.push("", "## Failures", ""); + for (const message of testCase.failureMessages) { + lines.push("```", message, "```", ""); + } + } + + const output = testCase.eval?.output ?? testCase.harness?.run?.output; + if (output !== undefined) { + lines.push("## Output", "", "```json", formatJson(output), "```", ""); + } + + const scores = testCase.eval?.scores ?? []; + if (scores.length > 0) { + lines.push("## Judge evidence", "", "| Judge | Score |", "| --- | --- |"); + for (const score of scores) { + lines.push(`| ${score.name ?? "Score"} | ${formatScore(score.score)} |`); + } + lines.push(""); + } + + lines.push( + "## Raw case JSON", + "", + "```json", + formatJson(testCase), + "```", + "", + ); + + return lines.join("\n"); +} diff --git a/packages/report-ui/src/app/ci-artifacts.test.ts b/packages/report-ui/src/app/ci-artifacts.test.ts new file mode 100644 index 0000000..e1cbf29 --- /dev/null +++ b/packages/report-ui/src/app/ci-artifacts.test.ts @@ -0,0 +1,75 @@ +import type { ReportWorkspace } from "@vitest-evals/core"; +import { describe, expect, test } from "vitest"; +import { formatJunitXml, formatPullRequestComment } from "./ci-artifacts"; +import { FALLBACK_PRICING } from "./pricing"; + +const workspace: ReportWorkspace = { + schemaVersion: 1, + runs: [ + { + id: "run-1", + source: "eval-results/a.json", + status: "failed", + durationMs: 1500, + totals: { + total: 2, + passed: 1, + failed: 1, + skipped: 0, + evalTotal: 2, + evalPassed: 1, + evalFailed: 1, + }, + }, + ], + cases: [ + { + id: "ok", + ancestorTitles: ["suite"], + displayFile: "suite.eval.ts", + displayName: "suite > passes", + failureMessages: [], + file: "/repo/suite.eval.ts", + fullName: "suite passes", + runId: "run-1", + status: "passed", + title: "passes", + durationMs: 200, + eval: { avgScore: 1, scores: [] }, + }, + { + id: "bad", + ancestorTitles: ["suite"], + displayFile: "suite.eval.ts", + displayName: "suite > fails", + failureMessages: ["Score: 0.20 below threshold: 1.00"], + file: "/repo/suite.eval.ts", + fullName: "suite fails", + runId: "run-1", + status: "failed", + title: "fails", + durationMs: 400, + eval: { avgScore: 0.2, scores: [] }, + }, + ], +}; + +describe("formatJunitXml", () => { + test("emits one failing testcase with the failure message", () => { + const xml = formatJunitXml(workspace); + expect(xml).toContain('tests="2"'); + expect(xml).toContain('failures="1"'); + expect(xml).toContain('name="suite > fails"'); + expect(xml).toContain("Score: 0.20 below threshold: 1.00"); + expect(xml).toContain('name="suite > passes"'); + }); +}); + +describe("formatPullRequestComment", () => { + test("summarizes pass rate, cost, and failures", () => { + const markdown = formatPullRequestComment(workspace, FALLBACK_PRICING); + expect(markdown).toContain("50%"); + expect(markdown).toContain("1 passed, 1 failed"); + expect(markdown).toContain("suite > fails"); + }); +}); diff --git a/packages/report-ui/src/app/ci-artifacts.ts b/packages/report-ui/src/app/ci-artifacts.ts new file mode 100644 index 0000000..bd1359e --- /dev/null +++ b/packages/report-ui/src/app/ci-artifacts.ts @@ -0,0 +1,116 @@ +import type { ReportCase, ReportWorkspace } from "@vitest-evals/core"; +import { formatSignedScore, formatSignedUsd, workspaceDelta } from "./compare"; +import { + formatDuration, + formatNumber, + formatScore, + summarizeWorkspace, +} from "./model"; +import { type PricingTable, estimateWorkspaceCost, formatUsd } from "./pricing"; + +/** Writes a JUnit XML document for CI systems that consume xUnit reports. */ +export function formatJunitXml(workspace: ReportWorkspace): string { + const summary = summarizeWorkspace(workspace); + const suites = workspace.runs.map((run) => { + const cases = workspace.cases.filter( + (testCase) => testCase.runId === run.id, + ); + const failures = cases.filter((testCase) => testCase.status === "failed"); + const body = cases.map((testCase) => formatJunitCase(testCase)).join("\n"); + return [ + ` `, + body, + " ", + ].join("\n"); + }); + + return [ + ``, + ``, + ...suites, + "", + "", + ].join("\n"); +} + +/** Writes a pull-request comment body from the collected workspace. */ +export function formatPullRequestComment( + workspace: ReportWorkspace, + pricing?: PricingTable, +): string { + const summary = summarizeWorkspace(workspace); + const cost = estimateWorkspaceCost( + workspace.cases.map((testCase) => testCase.harness?.run?.usage ?? {}), + pricing ?? { source: "none", sources: [], models: [] }, + ); + const delta = workspaceDelta(workspace, pricing); + const executed = summary.passed + summary.failed; + const passRate = + executed === 0 + ? "n/a" + : `${Math.round((summary.passed / executed) * 100)}%`; + const failures = workspace.cases.filter( + (testCase) => testCase.status === "failed", + ); + const lines = [ + "## vitest-evals", + "", + "| | |", + "| --- | --- |", + `| Pass rate | ${passRate} |`, + `| Cases | ${summary.passed} passed, ${summary.failed} failed, ${summary.skipped} skipped |`, + `| Avg score | ${formatScore(summary.averageScore)} |`, + `| Runtime | ${formatDuration(summary.durationMs)} |`, + `| Tokens | ${formatNumber(summary.totalTokens)} |`, + `| Est. cost | ${formatUsd(cost)} |`, + ]; + if (delta) { + lines.push( + `| vs previous | ${formatSignedScore(delta.passRate)} pass · ${formatSignedScore(delta.averageScore)} score · ${formatSignedUsd(delta.costUsd)} (${delta.matchedCases} matched) |`, + ); + } + lines.push("", "### Failures", ""); + if (failures.length === 0) { + lines.push("No eval failures.", ""); + } else { + for (const testCase of failures.slice(0, 20)) { + const score = formatScore(testCase.eval?.avgScore); + const firstFailure = testCase.failureMessages[0]?.split("\n")[0] ?? ""; + lines.push( + `- **${testCase.displayName}** (${score}) — \`${testCase.displayFile}\`${firstFailure ? ` — ${firstFailure}` : ""}`, + ); + } + if (failures.length > 20) { + lines.push("", `${failures.length - 20} more failure(s) omitted.`); + } + lines.push(""); + } + return `${lines.join("\n")}\n`; +} + +function formatJunitCase(testCase: ReportCase) { + const time = seconds(testCase.durationMs); + const open = ` `; + } + const message = testCase.failureMessages[0] ?? "eval failed"; + return [ + `${open}>`, + ` ${escapeXml(testCase.failureMessages.join("\n\n"))}`, + " ", + ].join("\n"); +} + +function seconds(durationMs: number | undefined) { + return ((durationMs ?? 0) / 1000).toFixed(3); +} + +function escapeXml(value: string) { + return value + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """) + .replace(/'/g, "'"); +} diff --git a/packages/report-ui/src/app/compare.test.ts b/packages/report-ui/src/app/compare.test.ts new file mode 100644 index 0000000..cbd40f7 --- /dev/null +++ b/packages/report-ui/src/app/compare.test.ts @@ -0,0 +1,196 @@ +import type { ReportCase, ReportWorkspace } from "@vitest-evals/core"; +import { describe, expect, test } from "vitest"; +import { + caseDelta, + caseHasCompare, + caseKey, + formatSignedScore, + formatSignedUsd, + judgeSpread, + trialStats, + workspaceDelta, +} from "./compare"; +import { FALLBACK_PRICING } from "./pricing"; + +function testCase( + overrides: Partial & Pick, +): ReportCase { + return { + ancestorTitles: ["refund"], + displayFile: "refund.eval.ts", + displayName: "refund > fraud", + failureMessages: [], + file: "/repo/refund.eval.ts", + fullName: "refund fraud", + runId: "run-new", + status: "passed", + title: "fraud", + ...overrides, + }; +} + +describe("case identity", () => { + test("keys a case by file and full name", () => { + expect( + caseKey( + testCase({ + id: "a", + file: "/repo/a.ts", + fullName: "suite case", + }), + ), + ).toBe("/repo/a.ts::suite case"); + }); +}); + +describe("caseDelta", () => { + test("subtracts baseline score and cost", () => { + const current = testCase({ + id: "now", + eval: { avgScore: 1, scores: [] }, + harness: { + run: { + output: {}, + session: { events: [] }, + usage: { + model: "gpt-4o-mini", + inputTokens: 1_000_000, + outputTokens: 0, + }, + errors: [], + }, + }, + }); + const baseline = testCase({ + id: "then", + runId: "run-old", + status: "failed", + eval: { avgScore: 0.5, scores: [] }, + harness: { + run: { + output: {}, + session: { events: [] }, + usage: { + model: "gpt-4o-mini", + inputTokens: 2_000_000, + outputTokens: 0, + }, + errors: [], + }, + }, + }); + + expect(caseDelta(current, [baseline], FALLBACK_PRICING)).toMatchObject({ + score: 0.5, + costUsd: -0.15, + statusChanged: true, + }); + }); +}); + +describe("trialStats and judgeSpread", () => { + test("computes trial count and judge standard deviation", () => { + const first = testCase({ + id: "t1", + eval: { + avgScore: 0.5, + scores: [ + { name: "a", score: 0 }, + { name: "b", score: 1 }, + ], + }, + }); + const second = testCase({ + id: "t2", + eval: { avgScore: 1, scores: [] }, + }); + + expect(trialStats(first, [first, second])).toMatchObject({ + count: 2, + meanScore: 0.75, + }); + expect(judgeSpread(first).stdevScore).toBeCloseTo(Math.SQRT1_2); + }); +}); + +describe("workspaceDelta", () => { + test("compares the oldest run to the newest", () => { + const workspace: ReportWorkspace = { + schemaVersion: 1, + runs: [ + { + id: "old", + startedAt: 1, + status: "failed", + totals: { + total: 1, + passed: 0, + failed: 1, + skipped: 0, + evalTotal: 1, + evalPassed: 0, + evalFailed: 1, + }, + }, + { + id: "new", + startedAt: 2, + status: "passed", + totals: { + total: 1, + passed: 1, + failed: 0, + skipped: 0, + evalTotal: 1, + evalPassed: 1, + evalFailed: 0, + }, + }, + ], + cases: [ + testCase({ + id: "old-case", + runId: "old", + status: "failed", + eval: { avgScore: 0.5, scores: [] }, + }), + testCase({ + id: "new-case", + runId: "new", + status: "passed", + eval: { avgScore: 1, scores: [] }, + }), + ], + }; + + expect(workspaceDelta(workspace)).toMatchObject({ + baseline: { id: "old" }, + current: { id: "new" }, + passRate: 1, + averageScore: 0.5, + matchedCases: 1, + }); + }); +}); + +describe("caseHasCompare", () => { + test("is true when a baseline sibling exists", () => { + const current = testCase({ id: "now" }); + const baseline = testCase({ id: "then", runId: "run-old" }); + expect(caseHasCompare(current, [current, baseline], [baseline])).toBe(true); + }); + + test("is false for a lone case", () => { + const current = testCase({ id: "only" }); + expect(caseHasCompare(current, [current], [])).toBe(false); + }); +}); + +describe("signed formatters", () => { + test("formats score points and usd deltas", () => { + expect(formatSignedScore(0.12)).toBe("+12pp"); + expect(formatSignedScore(-0.04)).toBe("-4pp"); + expect(formatSignedUsd(0.02)).toBe("+$0.020"); + expect(formatSignedUsd(-0.15)).toBe("-$0.150"); + }); +}); diff --git a/packages/report-ui/src/app/compare.ts b/packages/report-ui/src/app/compare.ts new file mode 100644 index 0000000..56cc46a --- /dev/null +++ b/packages/report-ui/src/app/compare.ts @@ -0,0 +1,297 @@ +import type { + ReportCase, + ReportRun, + ReportWorkspace, +} from "@vitest-evals/core"; +import { caseModel } from "./model"; +import { type PricingTable, estimateUsageCost } from "./pricing"; + +/** Stable identity for the same eval assertion across runs. */ +export function caseKey(testCase: ReportCase): string { + return `${testCase.file}::${testCase.fullName}`; +} + +/** Same leaf title in a file — used to find model variants. */ +export function caseFamilyKey(testCase: ReportCase): string { + return `${testCase.file}::${testCase.title}`; +} + +/** Picks the older run as the default baseline when two or more exist. */ +export function defaultBaselineRun(runs: ReportRun[]): ReportRun | undefined { + if (runs.length < 2) { + return undefined; + } + return [...runs].sort(compareRuns)[0]; +} + +/** Resolves the URL baseline, falling back to the oldest run. */ +export function resolveBaselineRun( + runs: ReportRun[], + baselineRunId: string | undefined, +): ReportRun | undefined { + if (baselineRunId && baselineRunId !== "auto") { + return runs.find((run) => run.id === baselineRunId); + } + return defaultBaselineRun(runs); +} + +export type CaseDelta = { + baseline: ReportCase; + score?: number; + costUsd?: number; + statusChanged: boolean; +}; + +/** Finds the baseline sibling and score/cost deltas for one case. */ +export function caseDelta( + testCase: ReportCase, + baselineCases: ReportCase[], + pricing?: PricingTable, +): CaseDelta | undefined { + const baseline = baselineCases.find( + (candidate) => + candidate.runId !== testCase.runId && + caseKey(candidate) === caseKey(testCase), + ); + if (!baseline) { + return undefined; + } + return { + baseline, + score: subtractNullable(testCase.eval?.avgScore, baseline.eval?.avgScore), + costUsd: subtractNullable( + pricing + ? estimateUsageCost(testCase.harness?.run?.usage ?? {}, pricing) + ?.totalUsd + : undefined, + pricing + ? estimateUsageCost(baseline.harness?.run?.usage ?? {}, pricing) + ?.totalUsd + : undefined, + ), + statusChanged: baseline.status !== testCase.status, + }; +} + +/** Other recordings of the same assertion (other runs or retries). */ +export function caseSiblings( + testCase: ReportCase, + cases: ReportCase[], +): ReportCase[] { + const key = caseKey(testCase); + return cases.filter( + (candidate) => candidate.id !== testCase.id && caseKey(candidate) === key, + ); +} + +/** True when another run, retry, or model can be compared to this case. */ +export function caseHasCompare( + testCase: ReportCase, + cases: ReportCase[], + baselineCases: ReportCase[], +) { + return ( + caseSiblings(testCase, cases).length > 0 || + caseModelVariants(testCase, cases).length > 0 || + caseDelta(testCase, baselineCases) !== undefined + ); +} + +/** Same title in the same file, different recorded model. */ +export function caseModelVariants( + testCase: ReportCase, + cases: ReportCase[], +): ReportCase[] { + const key = caseFamilyKey(testCase); + const model = caseModel(testCase); + return cases.filter( + (candidate) => + candidate.id !== testCase.id && + caseFamilyKey(candidate) === key && + caseModel(candidate) !== model, + ); +} + +export type TrialStats = { + count: number; + meanScore?: number; + stdevScore?: number; +}; + +/** Groups same-key cases in one run (retries / trialCount). */ +export function trialStats( + testCase: ReportCase, + cases: ReportCase[], +): TrialStats { + const key = caseKey(testCase); + const trials = cases.filter( + (candidate) => + candidate.runId === testCase.runId && caseKey(candidate) === key, + ); + const scores = trials + .map((trial) => trial.eval?.avgScore) + .filter((score): score is number => typeof score === "number"); + return { + count: trials.length, + meanScore: mean(scores), + stdevScore: stdev(scores), + }; +} + +/** Spread of judge scores on one case. */ +export function judgeSpread(testCase: ReportCase): TrialStats { + const scores = (testCase.eval?.scores ?? []) + .map((score) => score.score) + .filter((score): score is number => typeof score === "number"); + return { + count: scores.length, + meanScore: mean(scores), + stdevScore: stdev(scores), + }; +} + +export type WorkspaceDelta = { + baseline: ReportRun; + current: ReportRun; + passRate?: number; + averageScore?: number; + costUsd?: number; + matchedCases: number; +}; + +/** Workspace-level change from the baseline run to the newest run. */ +export function workspaceDelta( + workspace: ReportWorkspace, + pricing?: PricingTable, +): WorkspaceDelta | undefined { + if (workspace.runs.length < 2) { + return undefined; + } + const ordered = [...workspace.runs].sort(compareRuns); + const baseline = ordered[0]; + const current = ordered[ordered.length - 1]; + if (!baseline || !current || baseline.id === current.id) { + return undefined; + } + const baselineCases = workspace.cases.filter( + (testCase) => testCase.runId === baseline.id, + ); + const currentCases = workspace.cases.filter( + (testCase) => testCase.runId === current.id, + ); + const pairs = currentCases + .map((testCase) => caseDelta(testCase, baselineCases, pricing)) + .filter((delta): delta is CaseDelta => Boolean(delta)); + return { + baseline, + current, + passRate: subtractNullable( + executedPassRate(currentCases), + executedPassRate(baselineCases), + ), + averageScore: mean( + pairs + .map((pair) => pair.score) + .filter((score): score is number => typeof score === "number"), + ), + costUsd: subtractNullable( + runCostUsd(currentCases, pricing), + runCostUsd(baselineCases, pricing), + ), + matchedCases: pairs.length, + }; +} + +export function formatSignedScore(delta: number | undefined) { + if (delta === undefined) { + return "n/a"; + } + const points = Math.round(delta * 100); + if (points === 0) { + return "0pp"; + } + return `${points > 0 ? "+" : ""}${points}pp`; +} + +export function formatSignedUsd(delta: number | undefined) { + if (delta === undefined) { + return "n/a"; + } + if (Math.abs(delta) < 0.00005) { + return "$0.00"; + } + const digits = Math.abs(delta) < 0.01 ? 4 : 3; + const body = Math.abs(delta).toFixed(digits); + if (delta > 0) { + return `+$${body}`; + } + return `-$${body}`; +} + +function compareRuns(left: ReportRun, right: ReportRun) { + return ( + (left.startedAt ?? 0) - (right.startedAt ?? 0) || + left.id.localeCompare(right.id) + ); +} + +function executedPassRate(cases: ReportCase[]) { + const executed = cases.filter( + (testCase) => testCase.status === "passed" || testCase.status === "failed", + ); + if (executed.length === 0) { + return undefined; + } + return ( + executed.filter((testCase) => testCase.status === "passed").length / + executed.length + ); +} + +function runCostUsd(cases: ReportCase[], pricing?: PricingTable) { + if (!pricing) { + return undefined; + } + let total = 0; + let matched = false; + for (const testCase of cases) { + const cost = estimateUsageCost(testCase.harness?.run?.usage ?? {}, pricing); + if (!cost) { + continue; + } + matched = true; + total += cost.totalUsd; + } + return matched ? total : undefined; +} + +function subtractNullable( + left: number | null | undefined, + right: number | null | undefined, +) { + if (left == null || right == null) { + return undefined; + } + return left - right; +} + +function mean(values: number[]) { + if (values.length === 0) { + return undefined; + } + return values.reduce((total, value) => total + value, 0) / values.length; +} + +function stdev(values: number[]) { + if (values.length < 2) { + return undefined; + } + const average = mean(values); + if (average === undefined) { + return undefined; + } + const variance = + values.reduce((total, value) => total + (value - average) ** 2, 0) / + (values.length - 1); + return Math.sqrt(variance); +} diff --git a/packages/report-ui/src/app/components/CaseDrawer.tsx b/packages/report-ui/src/app/components/CaseDrawer.tsx index 82e2ab5..89392c7 100644 --- a/packages/report-ui/src/app/components/CaseDrawer.tsx +++ b/packages/report-ui/src/app/components/CaseDrawer.tsx @@ -1,32 +1,58 @@ -import { useEffect, useRef } from "react"; import type { ReportCase, ReportRun } from "@vitest-evals/core"; -import { formatDuration } from "../model"; +import { useEffect, useRef } from "react"; +import { caseToMarkdown } from "../case-markdown"; +import { caseHasCompare } from "../compare"; +import { relativeDisplayPath } from "../display-path"; +import { judgeTally } from "../judge-score"; +import { caseModel, caseScoreTone, formatDuration, formatJson } from "../model"; +import { estimateUsageCost, formatUsd } from "../pricing"; +import { useReportMeta } from "../report-meta"; +import { suggestBetterModel } from "../suggest-model"; import type { DetailTab } from "../types"; import { TabButton } from "../ui"; +import { CompareTab } from "./CompareTab"; +import { CopyButton } from "./CopyButton"; +import { CostHelp } from "./CostHelp"; +import { ModelHint } from "./ModelHint"; import { OverviewTab } from "./OverviewTab"; +import { PathLabel } from "./PathLabel"; import { RawTab } from "./RawTab"; -import { Fact, FactsGrid, ScoreValue, StatusMark } from "./ReportPrimitives"; +import { ScoreValue, StatusMark } from "./ReportPrimitives"; +import { RerunButton } from "./RerunButton"; import { TranscriptTab } from "./TranscriptTab"; const DETAIL_TABS: Array<{ id: DetailTab; label: string }> = [ { id: "overview", label: "Overview" }, { id: "transcript", label: "Transcript" }, + { id: "compare", label: "Compare" }, { id: "raw", label: "Raw" }, ]; export function CaseDrawer({ + baselineCases, + cases, detailTab, + filtersActive, open, runs, testCase, onClose, + onNext, + onPrev, + onResetFilters, onTabChange, }: { + baselineCases: ReportCase[]; + cases: ReportCase[]; detailTab: DetailTab; + filtersActive: boolean; open: boolean; runs: ReportRun[]; testCase: ReportCase | undefined; onClose: () => void; + onNext: () => void; + onPrev: () => void; + onResetFilters: () => void; onTabChange: (tab: DetailTab) => void; }) { const dialogRef = useRef(null); @@ -75,8 +101,16 @@ export function CaseDrawer({ return null; } + const { pricing, workspaceRoot } = useReportMeta(); const run = runs.find((candidate) => candidate.id === testCase.runId); const harnessRun = testCase.harness?.run; + const model = caseModel(testCase); + const usageCost = estimateUsageCost(harnessRun?.usage ?? {}, pricing); + const failureText = testCase.failureMessages.join("\n\n"); + const suggestion = suggestBetterModel(model, pricing); + const canCompare = caseHasCompare(testCase, cases, baselineCases); + const activeTab = + detailTab === "compare" && !canCompare ? "overview" : detailTab; return ( -
+
-
-
+
+

{testCase.displayName}

-

- {testCase.displayFile} -

-
- - Score - - +
+
-
-
- +
+ {model ? ( +
+
+ + Model + + + {model} + +
+ {suggestion ? : null} +
+ ) : null} + {usageCost ? ( +
+ + Cost + + + {formatUsd(usageCost.totalUsd)} + + +
+ ) : null} +
+ Score - +
- + + +
+
+
+ {failureText ? ( + + ) : ( + -
+
+ {model ? ( + + {model} + + ) : null} +
- - - - -
- {detailTab === "overview" ? ( + {activeTab === "overview" ? ( ) : null} - {detailTab === "transcript" ? ( + {activeTab === "transcript" ? ( ) : null} - {detailTab === "raw" ? : null} + {activeTab === "compare" && canCompare ? ( + + ) : null} + {activeTab === "raw" ? : null}
diff --git a/packages/report-ui/src/app/components/CaseWorkbench.tsx b/packages/report-ui/src/app/components/CaseWorkbench.tsx index 6dea4c9..a3c9e63 100644 --- a/packages/report-ui/src/app/components/CaseWorkbench.tsx +++ b/packages/report-ui/src/app/components/CaseWorkbench.tsx @@ -1,22 +1,54 @@ -import type { ReactNode } from "react"; import type { ReportCase, ReportRun } from "@vitest-evals/core"; +import type { ReactNode } from "react"; +import { + type CaseDelta, + formatSignedScore, + formatSignedUsd, + judgeSpread, + trialStats, +} from "../compare"; +import { relativeDisplayPath, resolveOpenPath } from "../display-path"; +import { judgeTally, judgeTallyLabel } from "../judge-score"; import { + type CaseFilters, + type CaseSortColumn, + type CaseSortDirection, + type CaseStatusFilter, + caseExpected, + caseModel, + caseScoreTone, caseToolCallCount, caseTotalTokens, + compactValue, formatDuration, formatNumber, - type CaseFilters, - type CaseStatusFilter, + hasActiveCaseFilters, } from "../model"; -import { EmptyState, Field, Input, Select, cx } from "../ui"; +import { estimateUsageCost, formatUsd } from "../pricing"; +import { useReportMeta } from "../report-meta"; +import { formatElapsed, jobElapsedMs, runningJobForCase } from "../rerun"; +import { suggestBetterModel } from "../suggest-model"; +import { EmptyState, Input, Select, cx } from "../ui"; +import { FileOpenMenu } from "./FileOpenMenu"; +import { InstantTooltip } from "./InstantTooltip"; +import { ModelHint } from "./ModelHint"; import { ScoreValue, StatusMark } from "./ReportPrimitives"; +import { RerunButton } from "./RerunButton"; +import { TaskSpinner, useRerunSession } from "./RerunSession"; type CaseColumn = { - id: string; + id: CaseSortColumn; header: string; className: string; }; +const EMPTY_FILTERS: CaseFilters = { + query: "", + status: "all", + runId: "all", + model: "all", +}; + const STATUS_OPTIONS: Array<{ value: CaseStatusFilter; label: string }> = [ { value: "all", label: "All" }, { value: "failed", label: "Failed" }, @@ -31,18 +63,33 @@ const CASE_COLUMNS: CaseColumn[] = [ { id: "status", header: "Status", - className: "w-[96px]", + className: "w-[44px]", }, { id: "case", header: "Case", className: "min-w-[220px]", }, + { + id: "model", + header: "Model", + className: "w-[160px]", + }, + { + id: "expected", + header: "Expected", + className: "w-[140px]", + }, { id: "score", header: "Score", className: "w-[82px] text-right", }, + { + id: "delta", + header: "Δ", + className: "w-[72px] text-right", + }, { id: "duration", header: "Duration", @@ -51,7 +98,12 @@ const CASE_COLUMNS: CaseColumn[] = [ { id: "tokens", header: "Tokens", - className: "w-[92px] text-right", + className: "w-[88px] text-right", + }, + { + id: "cost", + header: "Cost", + className: "w-[88px] text-right", }, { id: "tools", @@ -61,45 +113,91 @@ const CASE_COLUMNS: CaseColumn[] = [ ]; export function CaseWorkbench({ + allCases, cases, + deltas, filters, + modelOptions, runs, selectedCaseId, + showDelta, + sortColumn, + sortDirection, totalCases, onFiltersChange, onSelectCase, + onSortChange, }: { + allCases: ReportCase[]; cases: ReportCase[]; + deltas: Map; filters: CaseFilters; + modelOptions: string[]; runs: ReportRun[]; selectedCaseId: string | undefined; + showDelta: boolean; + sortColumn: CaseSortColumn | undefined; + sortDirection: CaseSortDirection; totalCases: number; onFiltersChange: (filters: CaseFilters) => void; onSelectCase: (testCase: ReportCase) => void; + onSortChange: (column: CaseSortColumn) => void; }) { + const showExpected = allCases.some( + (testCase) => caseExpected(testCase) !== undefined, + ); + const columns = CASE_COLUMNS.filter((column) => { + if (column.id === "delta") { + return showDelta; + } + if (column.id === "expected") { + return showExpected; + } + return true; + }); return ( -
-
-
-
-

- Case ledger -

-

- {cases.length} of {totalCases} case(s) -

-
+
+
+
+

+ Ledger + + {cases.length}/{totalCases} + +

+ +
- +
); @@ -107,45 +205,70 @@ export function CaseWorkbench({ function CaseFilterControls({ filters, + modelOptions, runs, onFiltersChange, }: { filters: CaseFilters; + modelOptions: string[]; runs: ReportRun[]; onFiltersChange: (filters: CaseFilters) => void; }) { + const { workspaceRoot } = useReportMeta(); return ( -
- - - onFiltersChange({ ...filters, query: event.target.value }) - } - placeholder="Case, file, judge" - /> - - +
+ + onFiltersChange({ ...filters, query: event.target.value }) + } + placeholder="Case, file, judge, model — /" + /> + + {modelOptions.length > 1 ? ( - - + ) : null} + {runs.length > 1 ? ( - + ) : null}
); } function CaseTable({ + allCases, cases, + columns, + deltas, + query, selectedCaseId, + sortColumn, + sortDirection, onSelectCase, + onSortChange, }: { + allCases: ReportCase[]; cases: ReportCase[]; + columns: CaseColumn[]; + deltas: Map; + query: string; selectedCaseId: string | undefined; + sortColumn: CaseSortColumn | undefined; + sortDirection: CaseSortDirection; onSelectCase: (testCase: ReportCase) => void; + onSortChange: (column: CaseSortColumn) => void; }) { if (cases.length === 0) { return No matching eval cases; } return ( -
- +
+
- {CASE_COLUMNS.map((column) => ( - - ))} + {columns.map((column) => { + const active = sortColumn === column.id; + return ( + + ); + })} {cases.map((testCase) => ( column.id === "delta")} + showExpected={columns.some((column) => column.id === "expected")} testCase={testCase} onSelectCase={onSelectCase} /> @@ -208,15 +390,52 @@ function CaseTable({ } function CaseRow({ + allCases, + delta, + query, selected, + showDelta, + showExpected, testCase, onSelectCase, }: { + allCases: ReportCase[]; + delta: CaseDelta | undefined; + query: string; selected: boolean; + showDelta: boolean; + showExpected: boolean; testCase: ReportCase; onSelectCase: (testCase: ReportCase) => void; }) { + const { jobs, now } = useRerunSession(); + const runningJob = runningJobForCase(jobs, testCase); + const { pricing, workspaceRoot } = useReportMeta(); + const model = caseModel(testCase); + const suggestion = suggestBetterModel(model, pricing); + const expected = compactValue(caseExpected(testCase)); + const displayFile = + relativeDisplayPath(testCase.displayFile, workspaceRoot) || + testCase.displayFile; + const usageCost = estimateUsageCost( + testCase.harness?.run?.usage ?? {}, + pricing, + ); + const spread = judgeSpread(testCase); + const trials = trialStats(testCase, allCases); + const tally = judgeTally(testCase); + const scoreHint = [ + judgeTallyLabel(tally), + spread.stdevScore !== undefined + ? `σ ${Math.round(spread.stdevScore * 100)}pp` + : undefined, + trials.count > 1 ? `${trials.count} trials` : undefined, + ] + .filter(Boolean) + .join(" · "); const selectCase = () => onSelectCase(testCase); + const fileHit = includesIgnoreCase(displayFile, query); + const cellSelected = selected; return ( @@ -227,49 +446,139 @@ function CaseRow({ caseRailClass(testCase.status, selected), )} label={`Open ${testCase.displayName}`} + selected={cellSelected} onClick={selectCase} > - + {runningJob ? ( + + ) : ( + + )} + + {showExpected ? ( + + ) : null} + {showDelta ? ( + + ) : null} +
- {column.header} - +
+ +
+
+
+ + + + + {fileHit ? ( + + + + ) : null} + +
+ + +
+
+ 		- - {testCase.displayName} - - - {testCase.displayFile} + + + + + {suggestion ? : null} + + + {expected || "n/a"} + + + - + + + + + {delta ? ( + + {formatSignedScore(delta.score)} + + ) : ( + "—" + )} + + - {formatDuration(testCase.durationMs)} + {runningJob ? ( + + {formatElapsed(jobElapsedMs(runningJob, now))} + + ) : ( + formatDuration(testCase.durationMs) + )} {formatNumber(caseTotalTokens(testCase))} @@ -279,6 +588,17 @@ function CaseRow({ + {usageCost ? formatUsd(usageCost.totalUsd) : "n/a"} + + + {formatNumber(caseToolCallCount(testCase))} @@ -306,7 +626,10 @@ function CaseCellButton({ return ( + ))} + + ); +} + +function HighlightQuery({ text, query }: { text: string; query: string }) { + const needle = query.trim(); + if (!needle) { + return text; + } + const index = text.toLowerCase().indexOf(needle.toLowerCase()); + if (index < 0) { + return text; + } + return ( + <> + {text.slice(0, index)} + + {text.slice(index, index + needle.length)} + + {text.slice(index + needle.length)} + + ); +} + +function includesIgnoreCase(text: string, query: string) { + const needle = query.trim(); + return needle.length > 0 && text.toLowerCase().includes(needle.toLowerCase()); +} + +function ledgerMinWidth(columns: CaseColumn[]) { + return columns.reduce((total, column) => { + if (column.id === "case") { + return total + 220; + } + if (column.id === "model") { + return total + 160; + } + if (column.id === "expected") { + return total + 140; + } + return total + 90; + }, 0); +} diff --git a/packages/report-ui/src/app/components/CommandPalette.tsx b/packages/report-ui/src/app/components/CommandPalette.tsx new file mode 100644 index 0000000..aace6fc --- /dev/null +++ b/packages/report-ui/src/app/components/CommandPalette.tsx @@ -0,0 +1,168 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import { cx } from "../ui"; + +export type PaletteCommand = { + id: string; + group: string; + label: string; + hint?: string; + run: () => void; +}; + +export function CommandPalette({ + commands, + open, + onClose, +}: { + commands: PaletteCommand[]; + open: boolean; + onClose: () => void; +}) { + const [query, setQuery] = useState(""); + const [active, setActive] = useState(0); + const inputRef = useRef(null); + const matches = useMemo(() => { + const needle = query.trim().toLowerCase(); + if (!needle) { + return commands; + } + return commands.filter((command) => + `${command.group} ${command.label} ${command.hint ?? ""}` + .toLowerCase() + .includes(needle), + ); + }, [commands, query]); + + useEffect(() => { + if (!open) { + setQuery(""); + setActive(0); + return; + } + const frame = window.requestAnimationFrame(() => { + inputRef.current?.focus(); + }); + return () => window.cancelAnimationFrame(frame); + }, [open]); + + if (!open) { + return null; + } + + const runActive = () => { + const command = matches[active]; + if (!command) { + return; + } + command.run(); + onClose(); + }; + + return ( +
+ + + )) + )} + + +
+ ); +} + +/** Triggers a browser download for a generated text artifact. */ +export function downloadTextFile( + filename: string, + text: string, + mime = "text/plain", +) { + const blob = new Blob([text], { type: mime }); + const url = URL.createObjectURL(blob); + const link = document.createElement("a"); + link.href = url; + link.download = filename; + document.body.append(link); + link.click(); + link.remove(); + URL.revokeObjectURL(url); +} diff --git a/packages/report-ui/src/app/components/CompareTab.tsx b/packages/report-ui/src/app/components/CompareTab.tsx new file mode 100644 index 0000000..c6887c8 --- /dev/null +++ b/packages/report-ui/src/app/components/CompareTab.tsx @@ -0,0 +1,164 @@ +import type { ReportCase } from "@vitest-evals/core"; +import { + caseDelta, + caseModelVariants, + caseSiblings, + formatSignedScore, + formatSignedUsd, + judgeSpread, + trialStats, +} from "../compare"; +import { + caseModel, + compactValue, + formatDuration, + formatNumber, +} from "../model"; +import { estimateUsageCost, formatUsd } from "../pricing"; +import { useReportMeta } from "../report-meta"; +import { EmptyState } from "../ui"; +import { DetailContent, DetailSection } from "./DetailLayout"; +import { ScoreValue, StatusMark } from "./ReportPrimitives"; + +export function CompareTab({ + baselineCases, + cases, + testCase, +}: { + baselineCases: ReportCase[]; + cases: ReportCase[]; + testCase: ReportCase; +}) { + const { pricing } = useReportMeta(); + const siblings = caseSiblings(testCase, cases); + const variants = caseModelVariants(testCase, cases); + const delta = caseDelta(testCase, baselineCases, pricing); + const trials = trialStats(testCase, cases); + const spread = judgeSpread(testCase); + const rows = uniqueCases([testCase, ...siblings, ...variants]); + + return ( + + +
+ + vs baseline{" "} + + {delta ? formatSignedScore(delta.score) : "n/a"} + + {delta?.costUsd !== undefined ? ( + + {" "} + · {formatSignedUsd(delta.costUsd)} + + ) : null} + + {trials.count > 1 ? ( + + {trials.count} trials + {trials.stdevScore !== undefined + ? ` · σ ${formatSignedScore(trials.stdevScore).replace("+", "")}` + : ""} + + ) : null} + {spread.count > 1 && spread.stdevScore !== undefined ? ( + judge σ {Math.round(spread.stdevScore * 100)}pp + ) : null} +
+ + + {rows.length <= 1 ? ( + + Load another run or model to compare this case. + + ) : ( +
+ + + + + + + + + + + + + {rows.map((row) => { + const rowDelta = caseDelta(row, baselineCases, pricing); + const cost = estimateUsageCost( + row.harness?.run?.usage ?? {}, + pricing, + ); + return ( + + + + + + + + + ); + })} + +
Run + Model + + Score + + Δ + + Cost + + Duration +
+
+ + + {compactValue(row.source ?? row.runId, 36) || + row.runId} + {row.id === testCase.id ? " · this" : ""} + +
+ 		+ {caseModel(row) ?? "n/a"} + + + + {formatSignedScore(rowDelta?.score)} + + {cost ? formatUsd(cost.totalUsd) : "n/a"} + + {formatDuration(row.durationMs)} +
+
+ )} + + +

+ This case {formatNumber(testCase.harness?.run?.usage?.totalTokens)}{" "} + tokens + {delta?.baseline + ? ` · baseline ${formatNumber(delta.baseline.harness?.run?.usage?.totalTokens)}` + : ""} + . +

+ + + ); +} + +function uniqueCases(cases: ReportCase[]) { + const seen = new Set(); + const unique: ReportCase[] = []; + for (const testCase of cases) { + if (seen.has(testCase.id)) { + continue; + } + seen.add(testCase.id); + unique.push(testCase); + } + return unique; +} diff --git a/packages/report-ui/src/app/components/CopyButton.tsx b/packages/report-ui/src/app/components/CopyButton.tsx new file mode 100644 index 0000000..f113fd6 --- /dev/null +++ b/packages/report-ui/src/app/components/CopyButton.tsx @@ -0,0 +1,117 @@ +import { useEffect, useState } from "react"; +import { cx } from "../ui"; + +export function CopyButton({ + emphasis = "secondary", + icon = false, + label = "Copy", + size = "md", + text, +}: { + emphasis?: "primary" | "secondary"; + icon?: boolean; + label?: string; + size?: "md" | "compact"; + text: string; +}) { + const [copied, setCopied] = useState(false); + + useEffect(() => { + if (!copied) { + return; + } + const timeout = window.setTimeout(() => setCopied(false), 1500); + return () => window.clearTimeout(timeout); + }, [copied]); + + return ( + + ); +} + +function ClipboardIcon() { + return ( + + ); +} + +function CheckIcon() { + return ( + + ); +} diff --git a/packages/report-ui/src/app/components/CostHelp.tsx b/packages/report-ui/src/app/components/CostHelp.tsx new file mode 100644 index 0000000..016a9e6 --- /dev/null +++ b/packages/report-ui/src/app/components/CostHelp.tsx @@ -0,0 +1,198 @@ +import { useEffect, useId, useRef, useState } from "react"; +import { formatNumber } from "../model"; +import { type PricingTable, type UsageCost, formatUsd } from "../pricing"; +import { cx } from "../ui"; + +export function CostHelp({ + cost, + pricing, + align = "left", +}: { + cost?: UsageCost; + pricing: PricingTable; + align?: "left" | "right"; +}) { + const [open, setOpen] = useState(false); + const menuId = useId(); + const rootRef = useRef(null); + + useEffect(() => { + if (!open) { + return; + } + const onPointerDown = (event: PointerEvent) => { + if ( + event.target instanceof Node && + !rootRef.current?.contains(event.target) + ) { + setOpen(false); + } + }; + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") { + setOpen(false); + } + }; + document.addEventListener("pointerdown", onPointerDown); + document.addEventListener("keydown", onKeyDown); + return () => { + document.removeEventListener("pointerdown", onPointerDown); + document.removeEventListener("keydown", onKeyDown); + }; + }, [open]); + + return ( +
+ + {open ? ( +
+

Estimated cost

+ {cost ? ( + + ) : ( +

{FORMULA}

+ )} + {cost ? ( +

+ {FORMULA} +

+ ) : null} + {pricing.source === "fallback" ? ( +

+ Live catalog unavailable — using bundled fallback rates. +

+ ) : null} + {pricing.fetchedAt ? ( +

+ Rates fetched {formatFetchedAt(pricing.fetchedAt)} +

+ ) : null} + +
+ ) : null} +
+ ); +} + +const FORMULA = + "Uncached input, cache reads, cache writes, and output, each × its USD / 1M-token rate. models.dev first, then LiteLLM. Unknown models are skipped."; + +function CostBreakdown({ cost }: { cost: UsageCost }) { + return ( +
+ + {cost.cachedReadTokens > 0 ? ( + + ) : null} + {cost.cacheWriteTokens > 0 ? ( + + ) : null} + + +

+ {cost.matchedId ?? "unmatched"} + {cost.usedTotalTokensFallback ? " · total tokens billed as input" : ""} + {cost.cachedReadTokens > 0 && !cost.pricedCachedReads + ? " · cache hits at the input rate" + : ""} +

+
+ ); +} + +function tokenRowLabel(kind: string, tokens: number, perMillionUsd?: number) { + if (perMillionUsd === undefined) { + return `${kind} ${formatNumber(tokens)}`; + } + return `${kind} ${formatNumber(tokens)} × ${formatUsd(perMillionUsd)}/M`; +} + +function BreakdownRow({ label, value }: { label: string; value: string }) { + return ( +
+
{label}
+
{value}
+
+ ); +} + +function formatFetchedAt(value: string) { + const timestamp = Date.parse(value); + if (!Number.isFinite(timestamp)) { + return value; + } + return new Intl.DateTimeFormat(undefined, { + dateStyle: "medium", + timeStyle: "short", + }).format(timestamp); +} diff --git a/packages/report-ui/src/app/components/FailureList.tsx b/packages/report-ui/src/app/components/FailureList.tsx new file mode 100644 index 0000000..fc9b7c5 --- /dev/null +++ b/packages/report-ui/src/app/components/FailureList.tsx @@ -0,0 +1,73 @@ +import { parseFailureMessage } from "../failure"; +import { EmptyState, cx } from "../ui"; +import { CopyButton } from "./CopyButton"; + +export function FailureList({ messages }: { messages: string[] }) { + if (messages.length === 0) { + return No failure messages; + } + + return ( +
    + {messages.map((message) => ( + + ))} +
+ ); +} + +function FailureCard({ message }: { message: string }) { + const failure = parseFailureMessage(message); + + return ( +
  • +
    +
    +

    + {failure.name} +

    +

    + {failure.headline} +

    +
    + +
    + {failure.diffLines.length > 0 ? ( + 
    +          {failure.diffLines.map((line, index) => (
+            
+              {line.type === "add"
+                ? `+ ${line.text}`
+                : line.type === "remove"
+                  ? `- ${line.text}`
+                  : line.text}
+            
+          ))}
+
    + ) : null} + {failure.body ? ( + 
    +          {failure.body}
+
    + ) : null} + {failure.stack ? ( +
    + + Stack + + 
    +            {failure.stack}
+
    +
    + ) : null} +
    • + ); +} diff --git a/packages/report-ui/src/app/components/FileOpenMenu.tsx b/packages/report-ui/src/app/components/FileOpenMenu.tsx new file mode 100644 index 0000000..d1bfd8c --- /dev/null +++ b/packages/report-ui/src/app/components/FileOpenMenu.tsx @@ -0,0 +1,131 @@ +import { useEffect, useId, useRef, useState } from "react"; +import { type OpenFileTarget, editorTargets } from "../file-open"; +import { cx } from "../ui"; + +export function FileOpenMenu({ file, line, column }: OpenFileTarget) { + const [open, setOpen] = useState(false); + const menuId = useId(); + const rootRef = useRef(null); + + useEffect(() => { + if (!open) { + return; + } + const onPointerDown = (event: PointerEvent) => { + if ( + event.target instanceof Node && + !rootRef.current?.contains(event.target) + ) { + setOpen(false); + } + }; + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") { + setOpen(false); + } + }; + document.addEventListener("pointerdown", onPointerDown); + document.addEventListener("keydown", onKeyDown); + return () => { + document.removeEventListener("pointerdown", onPointerDown); + document.removeEventListener("keydown", onKeyDown); + }; + }, [open]); + + return ( +
    + + {open ? ( + + ) : null} +
    + ); +} + +function CopyAbsPathButton({ file }: { file: string }) { + const [copied, setCopied] = useState(false); + + useEffect(() => { + if (!copied) { + return; + } + const timeout = window.setTimeout(() => setCopied(false), 1500); + return () => window.clearTimeout(timeout); + }, [copied]); + + return ( + + ); +} + +function ExternalLinkIcon() { + return ( + + ); +} diff --git a/packages/report-ui/src/app/components/InstantTooltip.tsx b/packages/report-ui/src/app/components/InstantTooltip.tsx new file mode 100644 index 0000000..fa72342 --- /dev/null +++ b/packages/report-ui/src/app/components/InstantTooltip.tsx @@ -0,0 +1,62 @@ +import { type ReactNode, useState } from "react"; + +const TOOLTIP_PAD = 8; +const TOOLTIP_MAX_WIDTH = 360; +const TOOLTIP_ESTIMATED_HEIGHT = 40; + +export function InstantTooltip({ + content, + children, +}: { + content: ReactNode; + children: ReactNode; +}) { + const [anchor, setAnchor] = useState(); + + return ( + + setAnchor(event.currentTarget.getBoundingClientRect()) + } + onMouseLeave={() => setAnchor(undefined)} + > + {children} + {anchor ? : null} + + ); +} + +function TooltipBubble({ + anchor, + content, +}: { + anchor: DOMRect; + content: ReactNode; +}) { + const maxWidth = Math.min( + TOOLTIP_MAX_WIDTH, + window.innerWidth - TOOLTIP_PAD * 2, + ); + const center = anchor.left + anchor.width / 2; + const left = Math.min( + Math.max(center, TOOLTIP_PAD + maxWidth / 2), + window.innerWidth - TOOLTIP_PAD - maxWidth / 2, + ); + const fitsAbove = anchor.top - TOOLTIP_ESTIMATED_HEIGHT - TOOLTIP_PAD >= 0; + const top = fitsAbove + ? anchor.top - TOOLTIP_PAD + : anchor.bottom + TOOLTIP_PAD; + return ( + + {content} + + ); +} diff --git a/packages/report-ui/src/app/components/JsonInspector.tsx b/packages/report-ui/src/app/components/JsonInspector.tsx new file mode 100644 index 0000000..201f978 --- /dev/null +++ b/packages/report-ui/src/app/components/JsonInspector.tsx @@ -0,0 +1,100 @@ +import { useState } from "react"; +import JsonView from "react18-json-view"; +import "react18-json-view/src/style.css"; +import { formatJson } from "../model"; +import { CodeBlock, EmptyState, cx } from "../ui"; +import { CopyButton } from "./CopyButton"; + +export function JsonInspector({ + value, + mode, + onModeChange, +}: { + value: unknown; + mode?: "tree" | "text"; + onModeChange?: (mode: "tree" | "text") => void; +}) { + const [localMode, setLocalMode] = useState<"tree" | "text">("tree"); + const currentMode = mode ?? localMode; + const setMode = onModeChange ?? setLocalMode; + if (value === undefined || value === "") { + return n/a; + } + + if (isShortScalar(value)) { + const text = typeof value === "string" ? value : formatJson(value); + return ( +
    + + {typeof value === "string" ? value : String(value)} + + +
    + ); + } + + const text = formatJson(value); + + return ( +
    +
    +
    + setMode("tree")} + > + Inspector + + setMode("text")} + > + Text + +
    + +
    + {currentMode === "tree" ? ( +
    + +
    + ) : ( + + )} +
    + ); +} + +function isShortScalar(value: unknown) { + if (typeof value === "number" || typeof value === "boolean") { + return true; + } + return ( + typeof value === "string" && value.length <= 80 && !value.includes("\n") + ); +} + +function ModeButton({ + children, + selected, + onClick, +}: { + children: string; + selected: boolean; + onClick: () => void; +}) { + return ( + + ); +} diff --git a/packages/report-ui/src/app/components/ModelHint.tsx b/packages/report-ui/src/app/components/ModelHint.tsx new file mode 100644 index 0000000..58e5fcd --- /dev/null +++ b/packages/report-ui/src/app/components/ModelHint.tsx @@ -0,0 +1,16 @@ +import { type ModelSuggestion, suggestionLabel } from "../suggest-model"; +import { InstantTooltip } from "./InstantTooltip"; + +export function ModelHint({ suggestion }: { suggestion: ModelSuggestion }) { + return ( + + + ! + + + ); +} diff --git a/packages/report-ui/src/app/components/OverviewTab.tsx b/packages/report-ui/src/app/components/OverviewTab.tsx index c8585f6..9c6d172 100644 --- a/packages/report-ui/src/app/components/OverviewTab.tsx +++ b/packages/report-ui/src/app/components/OverviewTab.tsx @@ -1,8 +1,18 @@ import type { HarnessRun, ReportCase } from "@vitest-evals/core"; -import { formatDuration, formatNumber } from "../model"; -import { EmptyState } from "../ui"; +import { + caseExpected, + caseInput, + formatDuration, + formatNumber, +} from "../model"; +import { estimateUsageCost, formatUsd } from "../pricing"; +import { useReportMeta } from "../report-meta"; +import { EmptyState, cx } from "../ui"; +import { CostHelp } from "./CostHelp"; import { DetailContent, DetailSection } from "./DetailLayout"; -import { Fact, FactsGrid, JsonBlock, ScoreValue } from "./ReportPrimitives"; +import { FailureList } from "./FailureList"; +import { JsonInspector } from "./JsonInspector"; +import { Fact, FactsGrid, ScoreValue } from "./ReportPrimitives"; export function OverviewTab({ testCase, @@ -11,30 +21,66 @@ export function OverviewTab({ testCase: ReportCase; run: HarnessRun | undefined; }) { + const input = caseInput(testCase); + const expected = caseExpected(testCase); + const showInput = input !== undefined && input !== ""; + const showExpected = expected !== undefined && expected !== ""; + + const datasetTitle = + showInput && showExpected ? "Dataset" : showInput ? "Input" : "Expected"; + return ( + {showInput || showExpected ? ( + +
    + {showInput ? ( +
    + {showInput && showExpected ? ( +

    + Input +

    + ) : null} + +
    + ) : null} + {showExpected ? ( +
    + {showInput && showExpected ? ( +

    + Expected +

    + ) : null} + +
    + ) : null} +
    +     + ) : null} - + - - - - - {testCase.failureMessages.length > 0 ? ( -
      - {testCase.failureMessages.map((message) => ( -
    • - {message} -
      • - ))} -
    - ) : ( - No failure messages - )} -     +
    + + Usage + +
    + +
    +
    + {testCase.failureMessages.length > 0 ? ( + + + + ) : null}     ); } @@ -140,7 +186,9 @@ function EvidenceLine({ label, value }: { label: string; value: string }) { } function UsageGrid({ run }: { run: HarnessRun | undefined }) { + const { pricing } = useReportMeta(); const usage = run?.usage; + const cost = estimateUsageCost(usage ?? {}, pricing); return ( @@ -149,6 +197,19 @@ function UsageGrid({ run }: { run: HarnessRun | undefined }) { + + {formatUsd(cost.totalUsd)} + + + ) : ( + "n/a" + ) + } + /> diff --git a/packages/report-ui/src/app/components/PathLabel.tsx b/packages/report-ui/src/app/components/PathLabel.tsx new file mode 100644 index 0000000..30df82a --- /dev/null +++ b/packages/report-ui/src/app/components/PathLabel.tsx @@ -0,0 +1,34 @@ +import { relativeDisplayPath, resolveOpenPath } from "../display-path"; +import { useReportMeta } from "../report-meta"; +import { FileOpenMenu } from "./FileOpenMenu"; + +export function PathLabel({ + path, + file, + line, + column, + className, +}: { + path: string | undefined; + file?: string; + line?: number; + column?: number; + className?: string; +}) { + const { workspaceRoot } = useReportMeta(); + const display = relativeDisplayPath(path, workspaceRoot); + const openFile = resolveOpenPath(file ?? path, workspaceRoot); + + if (!display) { + return n/a; + } + + return ( + + {display} + {openFile ? ( + + ) : null} + + ); +} diff --git a/packages/report-ui/src/app/components/RawTab.tsx b/packages/report-ui/src/app/components/RawTab.tsx index e93f48a..4afb1eb 100644 --- a/packages/report-ui/src/app/components/RawTab.tsx +++ b/packages/report-ui/src/app/components/RawTab.tsx @@ -1,21 +1,32 @@ import type { ReportCase } from "@vitest-evals/core"; +import { useState } from "react"; import { DetailContent, DetailSection } from "./DetailLayout"; -import { JsonBlock } from "./ReportPrimitives"; +import { JsonInspector } from "./JsonInspector"; export function RawTab({ testCase }: { testCase: ReportCase }) { + const [mode, setMode] = useState<"tree" | "text">("tree"); + return ( - + {testCase.harness?.run?.artifacts ? ( - + ) : null} {testCase.harness?.run?.errors?.length ? ( - + ) : null} diff --git a/packages/report-ui/src/app/components/ReportChrome.tsx b/packages/report-ui/src/app/components/ReportChrome.tsx index 8749caa..23391f2 100644 --- a/packages/report-ui/src/app/components/ReportChrome.tsx +++ b/packages/report-ui/src/app/components/ReportChrome.tsx @@ -1,11 +1,24 @@ +import { Link } from "@tanstack/react-router"; import type { ReportRun } from "@vitest-evals/core"; +import type { ReactNode } from "react"; import { + type WorkspaceDelta, + formatSignedScore, + formatSignedUsd, +} from "../compare"; +import { + type CaseStatusFilter, formatDuration, formatNumber, formatScore, type summarizeWorkspace, } from "../model"; -import { cx, toneTextClass, type Tone } from "../ui"; +import { type UsageCost, formatUsd } from "../pricing"; +import { useReportMeta } from "../report-meta"; +import { toReportSearch } from "../search"; +import { type Tone, cx, toneTextClass } from "../ui"; +import { CostHelp } from "./CostHelp"; +import { PathLabel } from "./PathLabel"; import { executedCaseCount, passRate, @@ -15,124 +28,147 @@ import { export function ReportHeader({ caseCount, runCount, + sourceLabel, + visibleCaseCount, + onOpenPalette, }: { caseCount: number; runCount: number; + sourceLabel: string; + visibleCaseCount: number; + onOpenPalette: () => void; }) { + const shortcut = commandShortcut(); return ( -
    +
    - vitest-evals + toReportSearch({})} + to="/" + > + vitest-evals +
    -

    - Run inspection +

    + {sourceLabel}

    -
    - - - {formatNumber(runCount)} - {" "} - runs - - - - {formatNumber(caseCount)} - {" "} - cases - -
    +

    + {formatNumber(runCount)} run{runCount === 1 ? "" : "s"} + {visibleCaseCount !== caseCount + ? ` · showing ${formatNumber(visibleCaseCount)} of ${formatNumber(caseCount)}` + : ` · ${formatNumber(caseCount)} cases`} +

    +
    ); } export function SummaryBar({ + currentStatus, + runDelta, summary, + workspaceCost, }: { + currentStatus: CaseStatusFilter; + runDelta?: WorkspaceDelta; summary: ReturnType; + workspaceCost?: UsageCost; }) { + const { pricing } = useReportMeta(); const verdictTone = passRateTone(summary); return (
    -
    -
    - - Verdict - -
    - - {passRate(summary)} - - - pass rate - -
    -
    - - - {summary.failed} - {" "} - failed - - +
    +
    + + {passRate(summary)} + + + + + {summary.failed} + {" "} + failed + + + + {summary.passed} + {" "} + passed + + {summary.skipped > 0 ? ( + - {summary.caseCount} + {summary.skipped} {" "} - cases + skipped - - avg{" "} + ) : null} + {runDelta ? ( + + vs previous{" "} - {formatScore(summary.averageScore)} + {formatSignedScore(runDelta.passRate)} + · {formatSignedUsd(runDelta.costUsd)} -
    + ) : null}
    - -
    -
    - - Outcome mix - - - {formatNumber(summary.caseCount)} cases - -
    - -
    - - - -
    -
    - -
    +
    + } + label="Cost" + value={`${formatUsd(workspaceCost?.totalUsd)} · ${formatNumber(summary.totalTokens)}`} /> + {summary.toolCallCount > 0 ? ( + + ) : null} -
    @@ -142,9 +178,11 @@ export function SummaryBar({ } export function RunStrip({ + currentStatus, runs, selectedRunId, }: { + currentStatus: CaseStatusFilter; runs: ReportRun[]; selectedRunId: string; }) { @@ -171,8 +209,8 @@ export function RunStrip({ aria-hidden="true" />
    - - {run.source ?? run.id} + + {formatDuration(run.durationMs)} @@ -180,18 +218,26 @@ export function RunStrip({
    - + {run.totals.evalPassed} passed - - + + {run.totals.evalFailed} failed - +
    ))} @@ -200,13 +246,22 @@ export function RunStrip({ ); } -function SummaryCounter({ label, value }: { label: string; value: string }) { +function SummaryCounter({ + hint, + label, + value, +}: { + hint?: ReactNode; + label: string; + value: string; +}) { return ( -
    -
    +
    +
    {label} + {hint ? {hint} : null}
    -
    +
    {value}
    @@ -225,11 +280,16 @@ function OutcomeBar({ ].filter((segment) => segment.value > 0); if (summary.caseCount === 0) { - return
    ; + return ( +