From 87c318d00401ed013fc04bb9b6ec2e2e63086fa1 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:09:58 +0200 Subject: [PATCH 01/47] feat(report-ui): persist inspection state in the URL Keep the selected case, drawer tab, and ledger filters shareable so a report link opens the same view. --- packages/report-ui/package.json | 6 +- packages/report-ui/src/app/App.test.ts | 20 +++- packages/report-ui/src/app/App.tsx | 120 +++++++++++++-------- packages/report-ui/src/app/main.tsx | 5 +- packages/report-ui/src/app/report-state.ts | 26 +++++ packages/report-ui/src/app/router.tsx | 45 ++++++++ packages/report-ui/src/app/search.test.ts | 119 ++++++++++++++++++++ packages/report-ui/src/app/search.ts | 117 ++++++++++++++++++++ pnpm-lock.yaml | 115 ++++++++++++++++++++ 9 files changed, 525 insertions(+), 48 deletions(-) create mode 100644 packages/report-ui/src/app/report-state.ts create mode 100644 packages/report-ui/src/app/router.tsx create mode 100644 packages/report-ui/src/app/search.test.ts create mode 100644 packages/report-ui/src/app/search.ts diff --git a/packages/report-ui/package.json b/packages/report-ui/package.json index facaac0..4be9e0a 100644 --- a/packages/report-ui/package.json +++ b/packages/report-ui/package.json @@ -18,9 +18,7 @@ "type": "module", "main": "./dist/index.js", "module": "./dist/index.js", - "files": [ - "dist" - ], + "files": ["dist"], "publishConfig": { "access": "public" }, @@ -41,11 +39,13 @@ }, "devDependencies": { "@tailwindcss/vite": "^4.3.0", + "@tanstack/react-router": "^1.170.32", "@types/react": "^19.2.7", "@types/react-dom": "^19.2.3", "@vitejs/plugin-react": "^5.1.1", "react": "^19.2.3", "react-dom": "^19.2.3", + "react18-json-view": "^0.2.10", "tailwindcss": "^4.3.0", "vite": "^7.3.0", "vite-tsconfig-paths": "^6.1.1" diff --git a/packages/report-ui/src/app/App.test.ts b/packages/report-ui/src/app/App.test.ts index df8a5e6..a43dd9c 100644 --- a/packages/report-ui/src/app/App.test.ts +++ b/packages/report-ui/src/app/App.test.ts @@ -1,7 +1,8 @@ -import { afterEach, describe, expect, test, vi } from "vitest"; import type { ReportWorkspace } from "@vitest-evals/core"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { loadWorkspace, + nextSortSearch, resolveSelectedCase, resolveSelectedCaseId, summarizeVisibleWorkspace, @@ -40,6 +41,23 @@ afterEach(() => { vi.unstubAllGlobals(); }); +describe("nextSortSearch", () => { + test("starts a new column with its default direction and toggles the active one", () => { + expect(nextSortSearch(undefined, "asc", "score")).toEqual({ + dir: "desc", + sort: "score", + }); + expect(nextSortSearch("score", "desc", "score")).toEqual({ + dir: "asc", + sort: "score", + }); + expect(nextSortSearch("score", "asc", "case")).toEqual({ + dir: "asc", + sort: "case", + }); + }); +}); + describe("case selection", () => { test("keeps selection scoped to visible filtered cases", () => { const visibleCases = [cases[1]!]; diff --git a/packages/report-ui/src/app/App.tsx b/packages/report-ui/src/app/App.tsx index 6f6e0b4..95eec01 100644 --- a/packages/report-ui/src/app/App.tsx +++ b/packages/report-ui/src/app/App.tsx @@ -1,17 +1,19 @@ -import { useEffect, useMemo, useState } from "react"; import { - ReportWorkspaceSchema, type ReportWorkspace, + ReportWorkspaceSchema, } from "@vitest-evals/core"; +import { useEffect, useMemo, useState } from "react"; import { CaseDrawer } from "./components/CaseDrawer"; import { CaseWorkbench } from "./components/CaseWorkbench"; import { ReportHeader, RunStrip, SummaryBar } from "./components/ReportChrome"; import { + type CaseFilters, + type CaseSortColumn, filterReportCases, + sortReportCases, summarizeWorkspace, - type CaseFilters, } from "./model"; -import type { DetailTab } from "./types"; +import { useReportSearch } from "./report-state"; type LoadState = | { status: "loading" } @@ -45,45 +47,39 @@ export function App() { } function ReportApp({ workspace }: { workspace: ReportWorkspace }) { - const [filters, setFilters] = useState({ - query: "", - status: "all", - runId: "all", - }); - const [selectedCaseId, setSelectedCaseId] = useState( - () => workspace.cases.find((testCase) => testCase.status === "failed")?.id, + const { search, setSearch } = useReportSearch(); + const filters = useMemo( + () => ({ + query: search.q, + status: search.status, + runId: search.run, + }), + [search.q, search.run, search.status], ); - const [isDrawerOpen, setIsDrawerOpen] = useState(false); - const [detailTab, setDetailTab] = useState("overview"); const filteredCases = useMemo( () => filterReportCases(workspace.cases, filters), [workspace.cases, filters], ); + const visibleCases = useMemo( + () => sortReportCases(filteredCases, search.sort, search.dir), + [filteredCases, search.sort, search.dir], + ); const visibleRuns = useMemo( - () => visibleWorkspaceRuns(workspace.runs, filters, filteredCases), - [workspace.runs, filters, filteredCases], + () => visibleWorkspaceRuns(workspace.runs, filters, visibleCases), + [workspace.runs, filters, visibleCases], ); const summary = useMemo( () => summarizeWorkspace({ ...workspace, - cases: filteredCases, + cases: visibleCases, runs: visibleRuns, }), - [workspace, filteredCases, visibleRuns], + [workspace, visibleCases, visibleRuns], ); - const selectedCase = resolveSelectedCase(selectedCaseId, filteredCases); - - useEffect(() => { - const nextSelectedCaseId = resolveSelectedCaseId( - selectedCaseId, - filteredCases, - ); - if (nextSelectedCaseId !== selectedCaseId) { - setSelectedCaseId(nextSelectedCaseId); - } - }, [filteredCases, selectedCaseId]); + const selectedCase = resolveSelectedCase(search.case, workspace.cases); + const isDrawerOpen = Boolean(search.case && selectedCase); return (
@@ -96,41 +92,81 @@ function ReportApp({ workspace }: { workspace: ReportWorkspace }) { className="overflow-hidden rounded-lg border border-line-subtle bg-panel shadow-[0_14px_34px_rgba(23,32,28,0.06)]" aria-label="Report workspace" > - - + + { - setSelectedCaseId(testCase.id); - setDetailTab("overview"); - setIsDrawerOpen(true); - }} + onFiltersChange={(nextFilters) => + setSearch({ + q: nextFilters.query, + run: nextFilters.runId, + status: nextFilters.status, + }) + } + onSelectCase={(testCase) => + setSearch( + { case: testCase.id, tab: "overview" }, + { replace: false }, + ) + } + onSortChange={(column) => + setSearch(nextSortSearch(search.sort, search.dir, column)) + } /> setIsDrawerOpen(false)} - onTabChange={setDetailTab} + onClose={() => + setSearch({ case: undefined, tab: "overview" }, { replace: false }) + } + onTabChange={(tab) => setSearch({ tab })} />
); } +export function nextSortSearch( + currentColumn: CaseSortColumn | undefined, + currentDirection: "asc" | "desc", + column: CaseSortColumn, +) { + if (currentColumn !== column) { + return { + sort: column, + dir: defaultSortDirection(column), + }; + } + + return { + sort: column, + dir: currentDirection === "asc" ? ("desc" as const) : ("asc" as const), + }; +} + +function defaultSortDirection(column: CaseSortColumn): "asc" | "desc" { + return column === "case" || column === "status" ? "asc" : "desc"; +} + export function resolveSelectedCase( selectedCaseId: string | undefined, - filteredCases: ReportWorkspace["cases"], + cases: ReportWorkspace["cases"], ) { - return filteredCases.find((testCase) => testCase.id === selectedCaseId); + return cases.find((testCase) => testCase.id === selectedCaseId); } export function resolveSelectedCaseId( diff --git a/packages/report-ui/src/app/main.tsx b/packages/report-ui/src/app/main.tsx index 3d84a8c..93e85b7 100644 --- a/packages/report-ui/src/app/main.tsx +++ b/packages/report-ui/src/app/main.tsx @@ -1,10 +1,11 @@ +import { RouterProvider } from "@tanstack/react-router"; import { StrictMode } from "react"; import { createRoot } from "react-dom/client"; -import { App } from "./App"; +import { router } from "./router"; import "./styles.css"; createRoot(document.getElementById("root") as HTMLElement).render( - + , ); diff --git a/packages/report-ui/src/app/report-state.ts b/packages/report-ui/src/app/report-state.ts new file mode 100644 index 0000000..c4a7e72 --- /dev/null +++ b/packages/report-ui/src/app/report-state.ts @@ -0,0 +1,26 @@ +import { getRouteApi } from "@tanstack/react-router"; +import { type ReportSearch, toReportSearch } from "./search"; + +const reportRoute = getRouteApi("/"); + +/** Reads and writes report inspection state through the URL search. */ +export function useReportSearch() { + const search = reportRoute.useSearch(); + const navigate = reportRoute.useNavigate(); + + const setSearch = ( + patch: Partial, + options?: { replace?: boolean }, + ) => { + void navigate({ + replace: options?.replace ?? true, + search: (previous) => + toReportSearch({ + ...previous, + ...patch, + }), + }); + }; + + return { search, setSearch }; +} diff --git a/packages/report-ui/src/app/router.tsx b/packages/report-ui/src/app/router.tsx new file mode 100644 index 0000000..2250ed4 --- /dev/null +++ b/packages/report-ui/src/app/router.tsx @@ -0,0 +1,45 @@ +import { + Outlet, + createRootRoute, + createRoute, + createRouter, + stripSearchParams, +} from "@tanstack/react-router"; +import { App } from "./App"; +import { validateReportSearch } from "./search"; + +const rootRoute = createRootRoute({ + component: Outlet, +}); + +export const indexRoute = createRoute({ + getParentRoute: () => rootRoute, + path: "/", + validateSearch: (search: Record) => + validateReportSearch(search), + search: { + middlewares: [ + stripSearchParams({ + q: "", + status: "all", + run: "all", + dir: "asc", + tab: "overview", + }), + ], + }, + component: App, +}); + +const routeTree = rootRoute.addChildren([indexRoute]); + +export const router = createRouter({ + routeTree, + trailingSlash: "never", +}); + +declare module "@tanstack/react-router" { + interface Register { + router: typeof router; + } +} diff --git a/packages/report-ui/src/app/search.test.ts b/packages/report-ui/src/app/search.test.ts new file mode 100644 index 0000000..6288fa4 --- /dev/null +++ b/packages/report-ui/src/app/search.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, test } from "vitest"; +import { + compactReportSearch, + toReportSearch, + validateReportSearch, +} from "./search"; + +describe("validateReportSearch", () => { + test("fills defaults for an empty query string", () => { + expect(validateReportSearch({})).toEqual({ + q: "", + status: "all", + run: "all", + sort: undefined, + dir: "asc", + case: undefined, + tab: "overview", + }); + }); + + test("keeps recognized values and ignores unknown ones", () => { + expect( + validateReportSearch({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "score", + dir: "desc", + case: "case-1", + tab: "raw", + extra: "nope", + }), + ).toEqual({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "score", + dir: "desc", + case: "case-1", + tab: "raw", + }); + + expect( + validateReportSearch({ + status: "bogus", + sort: "nope", + dir: "sideways", + tab: "secret", + }), + ).toMatchObject({ + status: "all", + sort: undefined, + dir: "asc", + tab: "overview", + }); + }); +}); + +describe("toReportSearch", () => { + test("fills defaults for a partial search patch", () => { + expect(toReportSearch({ status: "failed" })).toEqual({ + q: "", + status: "failed", + run: "all", + sort: undefined, + dir: "asc", + case: undefined, + tab: "overview", + }); + }); +}); + +describe("compactReportSearch", () => { + test("omits default values from the serialized search", () => { + expect( + compactReportSearch({ + q: "", + status: "all", + run: "all", + dir: "asc", + tab: "overview", + }), + ).toEqual({}); + }); + + test("keeps only the active inspection state", () => { + expect( + compactReportSearch({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "score", + dir: "desc", + case: "case-1", + tab: "raw", + }), + ).toEqual({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "score", + dir: "desc", + case: "case-1", + tab: "raw", + }); + }); + + test("drops the drawer tab when no case is open", () => { + expect( + compactReportSearch({ + q: "", + status: "all", + run: "all", + dir: "asc", + tab: "raw", + }), + ).toEqual({}); + }); +}); diff --git a/packages/report-ui/src/app/search.ts b/packages/report-ui/src/app/search.ts new file mode 100644 index 0000000..0a216a1 --- /dev/null +++ b/packages/report-ui/src/app/search.ts @@ -0,0 +1,117 @@ +import type { CaseSortColumn, CaseStatusFilter } from "./model"; +import type { DetailTab } from "./types"; + +export type ReportSearch = { + q: string; + status: CaseStatusFilter; + run: string; + sort?: CaseSortColumn; + dir: "asc" | "desc"; + case?: string; + tab: DetailTab; +}; + +const STATUS_VALUES = new Set([ + "all", + "passed", + "failed", + "skipped", + "pending", + "todo", + "disabled", +]); + +const SORT_VALUES = new Set([ + "status", + "case", + "score", + "duration", + "tokens", + "tools", +]); + +const TAB_VALUES = new Set(["overview", "transcript", "raw"]); + +/** Parses report UI search params and fills defaults for missing keys. */ +export function validateReportSearch( + search: Record, +): ReportSearch { + return { + q: readString(search.q) ?? "", + status: readStatus(search.status), + run: readString(search.run) ?? "all", + sort: readSort(search.sort), + dir: search.dir === "desc" ? "desc" : "asc", + case: readString(search.case), + tab: readTab(search.tab), + }; +} + +/** Fills missing search keys so router navigations stay fully typed. */ +export function toReportSearch(search: Partial): ReportSearch { + return { + q: search.q ?? "", + status: search.status ?? "all", + run: search.run ?? "all", + sort: search.sort, + dir: search.dir ?? "asc", + case: search.case, + tab: search.tab ?? "overview", + }; +} + +/** Drops default search values so shared URLs stay short. */ +export function compactReportSearch( + search: Partial, +): Record { + return compactFilledSearch(toReportSearch(search)); +} + +function compactFilledSearch(search: ReportSearch): Record { + const next: Record = {}; + if (search.q.trim().length > 0) { + next.q = search.q; + } + if (search.status !== "all") { + next.status = search.status; + } + if (search.run !== "all") { + next.run = search.run; + } + if (search.sort) { + next.sort = search.sort; + if (search.dir !== "asc") { + next.dir = search.dir; + } + } + if (search.case) { + next.case = search.case; + if (search.tab !== "overview") { + next.tab = search.tab; + } + } + return next; +} + +function readString(value: unknown) { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function readStatus(value: unknown): CaseStatusFilter { + return typeof value === "string" && + STATUS_VALUES.has(value as CaseStatusFilter) + ? (value as CaseStatusFilter) + : "all"; +} + +function readSort(value: unknown): CaseSortColumn | undefined { + return typeof value === "string" && SORT_VALUES.has(value as CaseSortColumn) + ? (value as CaseSortColumn) + : undefined; +} + +function readTab(value: unknown): DetailTab { + return typeof value === "string" && TAB_VALUES.has(value as DetailTab) + ? (value as DetailTab) + : "overview"; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index afb2b0d..8d494f7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -191,6 +191,9 @@ importers: '@tailwindcss/vite': specifier: ^4.3.0 version: 4.3.0(vite@7.3.3(@types/node@25.5.0)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0)) + '@tanstack/react-router': + specifier: ^1.170.32 + version: 1.170.32(react-dom@19.2.7(react@19.2.7))(react@19.2.7) '@types/react': specifier: ^19.2.7 version: 19.2.16 @@ -206,6 +209,9 @@ importers: react-dom: specifier: ^19.2.3 version: 19.2.7(react@19.2.7) + react18-json-view: + specifier: ^0.2.10 + version: 0.2.10(react@19.2.7) tailwindcss: specifier: ^4.3.0 version: 4.3.0 @@ -1746,6 +1752,30 @@ packages: peerDependencies: vite: ^5.2.0 || ^6 || ^7 || ^8 + '@tanstack/history@1.162.1': + resolution: {integrity: sha512-DR9t6lfLVdrjgCwpglrR9DR7Ok8/HlXjcOE+goWXF3zyuLUO/ug7vMbSFxTqrQTtbRghJfyhmIZ0S6LhPIy44w==} + engines: {node: '>=20.19'} + + '@tanstack/react-router@1.170.32': + resolution: {integrity: sha512-SIpxvaTKco100a5ZR3ePmArbhtm3XOx+w1dpGYY9gxHDta4iXSKDdQuhLonwJbIMkVJsU1rwXf0UDHMrF/1snw==} + engines: {node: '>=20.19'} + peerDependencies: + react: '>=18.0.0 || >=19.0.0' + react-dom: '>=18.0.0 || >=19.0.0' + + '@tanstack/react-store@0.9.3': + resolution: {integrity: sha512-y2iHd/N9OkoQbFJLUX1T9vbc2O9tjH0pQRgTcx1/Nz4IlwLvkgpuglXUx+mXt0g5ZDFrEeDnONPqkbfxXJKwRg==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + react-dom: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + + '@tanstack/router-core@1.171.27': + resolution: {integrity: sha512-wDwSLvoLwIaNcnx9UNcN9Mb7Y8QwCYq1U1RQZwyN186gnkIoIYI2SOxy8VqH1vFigbkHkk4FmwMAQlghPgDK2g==} + engines: {node: '>=20.19'} + + '@tanstack/store@0.9.3': + resolution: {integrity: sha512-8reSzl/qGWGGVKhBoxXPMWzATSbZLZFWhwBAFO9NAyp0TxzfBP0mIrGb8CP8KrQTmvzXlR/vFPPUrHTLBGyFyw==} + '@tootallnate/quickjs-emscripten@0.23.0': resolution: {integrity: sha512-C5Mc6rdnsaJDjO3UpGW/CQTHtCKaYlScZTly4JIu97Jxo/odCiH0ITnDXSJPTOrEKk/ycSZ0AOgTmkDtkOsvIA==} @@ -2199,6 +2229,9 @@ packages: cookie-es@1.2.3: resolution: {integrity: sha512-lXVyvUvrNXblMqzIRrxHb57UUVmqsSWlxqt3XIjCkUP0wDAf6uicO6KMbEgYrMNtEvWgWHwe42CKxPu9MYAnWw==} + cookie-es@3.1.1: + resolution: {integrity: sha512-UaXxwISYJPTr9hwQxMFYZ7kNhSXboMXP+Z3TRX6f1/NyaGPfuNUZOWP1pUEb75B2HjfklIYLVRfWiFZJyC6Npg==} + cookie-signature@1.2.2: resolution: {integrity: sha512-D76uU73ulSXrD1UXF4KE2TMxVVwhsnCgfAyTg9k8P6KGZjlXKrOLe4dJQKI3Bxi5wjesZoFXJWElNWBjPZMbhg==} engines: {node: '>=6.6.0'} @@ -2211,6 +2244,9 @@ packages: resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} engines: {node: '>=18'} + copy-to-clipboard@3.3.3: + resolution: {integrity: sha512-2KV8NhB5JqC3ky0r9PMCAZKbUHSwtEo4CwCs0KXgruG43gX5PMqDEBbVU4OUzw2MuAWUfsuFmWvEKG5QRfSnJA==} + cors@2.8.6: resolution: {integrity: sha512-tJtZBBHA6vjIAaF6EnIaq6laBBP9aq/Y3ouVJjEfoHbRBcHBAHYcMh/w8LDrk2PvIMMq8gmopa5D4V8RmbrxGw==} engines: {node: '>= 0.10'} @@ -2875,6 +2911,10 @@ packages: resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} engines: {node: '>=16'} + isbot@5.2.2: + resolution: {integrity: sha512-iQcBXcd+Rv/pkubRyGh2utW2j1oPG5hZY6TUhVPpqK4G+o3IbxpJNx04hgksjc/N7GK5pEorUxDeg31cFgEk/w==} + engines: {node: '>=18'} + isexe@2.0.0: resolution: {integrity: sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==} @@ -3675,6 +3715,11 @@ packages: resolution: {integrity: sha512-QgT5//D3jfjJb6Gsjxv0Slpj23ip+HtOpnNgnb2S5zU3CB26G/IDPGoy4RJB42wzFE46DRsstbW6tKHoKbhAxw==} engines: {node: '>=0.10.0'} + react18-json-view@0.2.10: + resolution: {integrity: sha512-rYEbaCG/U4THY1qp1xY14/Kbnp9yY3W6Qm3Rmu+jlCIdxzMS5EcD+wI97kCKRoN3CuJyJU8hqkax5xWfl8A4EA==} + peerDependencies: + react: '>=16.8.0' + react@19.2.7: resolution: {integrity: sha512-HNe9WslTbXmFK8o8cmwgAeJFSBvt1bPdHCVKtaaV+WlAN36mpT4hcRpwbf3fY56ar2oIXzsBpOAiIRHAdY0OlQ==} engines: {node: '>=0.10.0'} @@ -3827,6 +3872,16 @@ packages: resolution: {integrity: sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==} engines: {node: '>= 18'} + seroval-plugins@1.6.4: + resolution: {integrity: sha512-R0f1U9hmn38+dFMz6b6ab8lwucmw4AtiY7St+JPWudy1dm+Bs3g884nyrsH9Cy6rKpZKLYayXuMda9GZ/fl8JQ==} + engines: {node: '>=10'} + peerDependencies: + seroval: ^1.0 + + seroval@1.6.4: + resolution: {integrity: sha512-LErWMNS2RRFdu2RMA5u/PA59/IWs0XsikyEXGQ2/36iEWFrdG0ABmg17E17cikrv76891kOAMq3TkTFXpwAHXw==} + engines: {node: '>=10'} + serve-static@2.2.1: resolution: {integrity: sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==} engines: {node: '>= 18'} @@ -4051,6 +4106,9 @@ packages: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} + toggle-selection@1.0.6: + resolution: {integrity: sha512-BiZS+C1OS8g/q2RRbJmy59xpyghNBqrr6k5L/uKBGRsTfxmu3ffiRnd8mlGPUVayg8pvfi5urfnu8TU7DVOkLQ==} + toidentifier@1.0.1: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} @@ -4262,6 +4320,11 @@ packages: peerDependencies: browserslist: '>= 4.21.0' + use-sync-external-store@1.6.0: + resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -6233,6 +6296,33 @@ snapshots: tailwindcss: 4.3.0 vite: 7.3.3(@types/node@25.5.0)(jiti@2.7.0)(lightningcss@1.32.0)(yaml@2.9.0) + '@tanstack/history@1.162.1': {} + + '@tanstack/react-router@1.170.32(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': + dependencies: + '@tanstack/history': 1.162.1 + '@tanstack/react-store': 0.9.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7) + '@tanstack/router-core': 1.171.27 + isbot: 5.2.2 + react: 19.2.7 + react-dom: 19.2.7(react@19.2.7) + + '@tanstack/react-store@0.9.3(react-dom@19.2.7(react@19.2.7))(react@19.2.7)': + dependencies: + '@tanstack/store': 0.9.3 + react: 19.2.7 + react-dom: 19.2.7(react@19.2.7) + use-sync-external-store: 1.6.0(react@19.2.7) + + '@tanstack/router-core@1.171.27': + dependencies: + '@tanstack/history': 1.162.1 + cookie-es: 3.1.1 + seroval: 1.6.4 + seroval-plugins: 1.6.4(seroval@1.6.4) + + '@tanstack/store@0.9.3': {} + '@tootallnate/quickjs-emscripten@0.23.0': {} '@types/babel__core@7.20.5': @@ -6771,6 +6861,8 @@ snapshots: cookie-es@1.2.3: {} + cookie-es@3.1.1: {} + cookie-signature@1.2.2: optional: true @@ -6779,6 +6871,10 @@ snapshots: cookie@1.1.1: {} + copy-to-clipboard@3.3.3: + dependencies: + toggle-selection: 1.0.6 + cors@2.8.6: dependencies: object-assign: 4.1.1 @@ -7650,6 +7746,8 @@ snapshots: dependencies: is-inside-container: 1.0.0 + isbot@5.2.2: {} + isexe@2.0.0: {} istanbul-lib-coverage@3.2.2: {} @@ -8727,6 +8825,11 @@ snapshots: react-refresh@0.18.0: {} + react18-json-view@0.2.10(react@19.2.7): + dependencies: + copy-to-clipboard: 3.3.3 + react: 19.2.7 + react@19.2.7: {} readdirp@4.1.2: {} @@ -8980,6 +9083,12 @@ snapshots: - supports-color optional: true + seroval-plugins@1.6.4(seroval@1.6.4): + dependencies: + seroval: 1.6.4 + + seroval@1.6.4: {} + serve-static@2.2.1: dependencies: encodeurl: 2.0.0 @@ -9252,6 +9361,8 @@ snapshots: dependencies: is-number: 7.0.0 + toggle-selection@1.0.6: {} + toidentifier@1.0.1: optional: true @@ -9425,6 +9536,10 @@ snapshots: escalade: 3.2.0 picocolors: 1.1.1 + use-sync-external-store@1.6.0(react@19.2.7): + dependencies: + react: 19.2.7 + util-deprecate@1.0.2: {} validate.io-array@1.0.6: {} From 99e7010bda41649109f3c0d061676049e1706acf Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:10:10 +0200 Subject: [PATCH 02/47] feat(report-ui): sort the case ledger by column Let inspectors order cases by status, name, score, duration, tokens, or tools without losing the original report order by default. --- .../src/app/components/CaseWorkbench.tsx | 69 +++++++++++++--- packages/report-ui/src/app/model.test.ts | 21 ++++- packages/report-ui/src/app/model.ts | 81 ++++++++++++++++++- 3 files changed, 157 insertions(+), 14 deletions(-) diff --git a/packages/report-ui/src/app/components/CaseWorkbench.tsx b/packages/report-ui/src/app/components/CaseWorkbench.tsx index 6dea4c9..9aaf2ab 100644 --- a/packages/report-ui/src/app/components/CaseWorkbench.tsx +++ b/packages/report-ui/src/app/components/CaseWorkbench.tsx @@ -1,18 +1,20 @@ -import type { ReactNode } from "react"; import type { ReportCase, ReportRun } from "@vitest-evals/core"; +import type { ReactNode } from "react"; import { + type CaseFilters, + type CaseSortColumn, + type CaseSortDirection, + type CaseStatusFilter, caseToolCallCount, caseTotalTokens, formatDuration, formatNumber, - type CaseFilters, - type CaseStatusFilter, } from "../model"; import { EmptyState, Field, Input, Select, cx } from "../ui"; import { ScoreValue, StatusMark } from "./ReportPrimitives"; type CaseColumn = { - id: string; + id: CaseSortColumn; header: string; className: string; }; @@ -65,17 +67,23 @@ export function CaseWorkbench({ filters, runs, selectedCaseId, + sortColumn, + sortDirection, totalCases, onFiltersChange, onSelectCase, + onSortChange, }: { cases: ReportCase[]; filters: CaseFilters; runs: ReportRun[]; selectedCaseId: string | undefined; + sortColumn: CaseSortColumn | undefined; + sortDirection: CaseSortDirection; totalCases: number; onFiltersChange: (filters: CaseFilters) => void; onSelectCase: (testCase: ReportCase) => void; + onSortChange: (column: CaseSortColumn) => void; }) { return (
@@ -99,7 +107,10 @@ export function CaseWorkbench({
); @@ -167,11 +178,17 @@ function CaseFilterControls({ function CaseTable({ cases, selectedCaseId, + sortColumn, + sortDirection, onSelectCase, + onSortChange, }: { cases: ReportCase[]; selectedCaseId: string | undefined; + sortColumn: CaseSortColumn | undefined; + sortDirection: CaseSortDirection; onSelectCase: (testCase: ReportCase) => void; + onSortChange: (column: CaseSortColumn) => void; }) { if (cases.length === 0) { return No matching eval cases; @@ -182,14 +199,42 @@ function CaseTable({ - {CASE_COLUMNS.map((column) => ( - - ))} + {CASE_COLUMNS.map((column) => { + const active = sortColumn === column.id; + return ( + + ); + })} diff --git a/packages/report-ui/src/app/model.test.ts b/packages/report-ui/src/app/model.test.ts index d9adf73..fae0574 100644 --- a/packages/report-ui/src/app/model.test.ts +++ b/packages/report-ui/src/app/model.test.ts @@ -1,6 +1,6 @@ -import { describe, expect, test } from "vitest"; import { messagesToTranscriptEvents } from "@vitest-evals/core"; import type { ReportWorkspace } from "@vitest-evals/core"; +import { describe, expect, test } from "vitest"; import { buildSpanTree, buildTranscript, @@ -10,6 +10,7 @@ import { filterReportCases, formatScore, scoreTone, + sortReportCases, summarizeWorkspace, } from "./model"; @@ -233,6 +234,24 @@ describe("summarizeWorkspace", () => { }); }); +describe("sortReportCases", () => { + test("orders failed cases first when sorting by status", () => { + expect( + sortReportCases(workspace.cases, "status", "asc").map( + (testCase) => testCase.status, + ), + ).toEqual(["failed", "passed"]); + }); + + test("orders higher scores first when sorting score descending", () => { + expect( + sortReportCases(workspace.cases, "score", "desc").map( + (testCase) => testCase.eval?.avgScore, + ), + ).toEqual([1, 0.2]); + }); +}); + describe("filterReportCases", () => { test("filters by status, run, and search query", () => { expect( diff --git a/packages/report-ui/src/app/model.ts b/packages/report-ui/src/app/model.ts index eadf324..7c59403 100644 --- a/packages/report-ui/src/app/model.ts +++ b/packages/report-ui/src/app/model.ts @@ -1,5 +1,4 @@ import { - toolCalls, type HarnessRun, type JsonValue, type NormalizedError, @@ -9,6 +8,7 @@ import { type TranscriptMessageEvent, type TranscriptToolCallEvent, type TranscriptToolResultEvent, + toolCalls, } from "@vitest-evals/core"; export type CaseStatusFilter = "all" | ReportCase["status"]; @@ -19,6 +19,25 @@ export type CaseFilters = { runId: string; }; +export type CaseSortColumn = + | "status" + | "case" + | "score" + | "duration" + | "tokens" + | "tools"; + +export type CaseSortDirection = "asc" | "desc"; + +const STATUS_RANK: Record = { + failed: 0, + passed: 1, + pending: 2, + todo: 3, + skipped: 4, + disabled: 5, +}; + export type WorkspaceSummary = { runCount: number; caseCount: number; @@ -95,6 +114,66 @@ export function summarizeWorkspace( }; } +/** Sorts filtered cases for the report ledger. */ +export function sortReportCases( + cases: ReportCase[], + column: CaseSortColumn | undefined, + direction: CaseSortDirection, +) { + if (!column) { + return cases; + } + + const ranked = [...cases].sort((left, right) => { + const comparison = compareCaseColumn(left, right, column); + return direction === "desc" ? -comparison : comparison; + }); + return ranked; +} + +function compareCaseColumn( + left: ReportCase, + right: ReportCase, + column: CaseSortColumn, +) { + switch (column) { + case "status": + return STATUS_RANK[left.status] - STATUS_RANK[right.status]; + case "case": + return left.displayName.localeCompare(right.displayName); + case "score": + return compareNullableNumber(left.eval?.avgScore, right.eval?.avgScore); + case "duration": + return compareNullableNumber(left.durationMs, right.durationMs); + case "tokens": + return compareNullableNumber( + caseTotalTokens(left), + caseTotalTokens(right), + ); + case "tools": + return compareNullableNumber( + caseToolCallCount(left), + caseToolCallCount(right), + ); + } +} + +function compareNullableNumber( + left: number | null | undefined, + right: number | null | undefined, +) { + if (left == null && right == null) { + return 0; + } + if (left == null) { + return 1; + } + if (right == null) { + return -1; + } + return left - right; +} + /** Filters cases for the report explorer. */ export function filterReportCases(cases: ReportCase[], filters: CaseFilters) { const query = filters.query.trim().toLowerCase(); From c33ee9db215a09e25ca60a06002fe2d21acbf6f8 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:10:11 +0200 Subject: [PATCH 03/47] feat(report-ui): filter the ledger from outcome count links Turn passed/failed/skipped counts into real links so a click jumps straight to that slice of the report. --- .../src/app/components/ReportChrome.tsx | 133 ++++++++++++++++-- 1 file changed, 121 insertions(+), 12 deletions(-) diff --git a/packages/report-ui/src/app/components/ReportChrome.tsx b/packages/report-ui/src/app/components/ReportChrome.tsx index 8749caa..ae14193 100644 --- a/packages/report-ui/src/app/components/ReportChrome.tsx +++ b/packages/report-ui/src/app/components/ReportChrome.tsx @@ -1,11 +1,15 @@ +import { Link } from "@tanstack/react-router"; import type { ReportRun } from "@vitest-evals/core"; +import type { ReactNode } from "react"; import { + type CaseStatusFilter, formatDuration, formatNumber, formatScore, type summarizeWorkspace, } from "../model"; -import { cx, toneTextClass, type Tone } from "../ui"; +import { toReportSearch } from "../search"; +import { type Tone, cx, toneTextClass } from "../ui"; import { executedCaseCount, passRate, @@ -50,8 +54,10 @@ export function ReportHeader({ } export function SummaryBar({ + currentStatus, summary, }: { + currentStatus: CaseStatusFilter; summary: ReturnType; }) { const verdictTone = passRateTone(summary); @@ -80,12 +86,15 @@ export function SummaryBar({
- + {summary.failed} {" "} failed - + {summary.caseCount} @@ -112,9 +121,27 @@ export function SummaryBar({
- - - + + +
@@ -142,9 +169,11 @@ export function SummaryBar({ } export function RunStrip({ + currentStatus, runs, selectedRunId, }: { + currentStatus: CaseStatusFilter; runs: ReportRun[]; selectedRunId: string; }) { @@ -180,18 +209,26 @@ export function RunStrip({
- + {run.totals.evalPassed} passed - - + + {run.totals.evalFailed} failed - +
))} @@ -245,11 +282,15 @@ function OutcomeBar({ } function OutcomeStat({ + active, label, + status, tone, value, }: { + active: boolean; label: string; + status: CaseStatusFilter; tone: Tone; value: number; }) { @@ -259,16 +300,84 @@ function OutcomeStat({ className={cx("size-2 shrink-0 rounded-[2px]", statusFillClass(tone))} aria-hidden="true" /> - + {value} {" "} {label.toLowerCase()} - + ); } +function StatusFilterLink({ + active, + children, + className, + status, +}: { + active: boolean; + children: ReactNode; + className?: string; + status: CaseStatusFilter; +}) { + return ( + + toReportSearch({ + ...previous, + status: active ? "all" : status, + }) + } + to="/" + > + {children} + + ); +} + +function RunStatusLink({ + active, + children, + runId, + status, +}: { + active: boolean; + children: ReactNode; + runId: string; + status: CaseStatusFilter; +}) { + return ( + + toReportSearch({ + ...previous, + run: active ? "all" : runId, + status: active ? "all" : status, + }) + } + to="/" + > + {children} + + ); +} + function passRateTone(summary: ReturnType): Tone { const executedCases = executedCaseCount(summary); if (executedCases === 0) { From 90e62474db70c4d04b6f8fea4096f42985ad97a6 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:10:11 +0200 Subject: [PATCH 04/47] feat(report-ui): inspect raw case JSON in a collapsible tree Give the Raw tab a Chrome-style inspector plus a text toggle so large case payloads are readable. --- .../src/app/components/JsonInspector.tsx | 80 +++++++++++++++++++ .../report-ui/src/app/components/RawTab.tsx | 19 ++++- packages/report-ui/src/app/styles.css | 10 +++ 3 files changed, 105 insertions(+), 4 deletions(-) create mode 100644 packages/report-ui/src/app/components/JsonInspector.tsx diff --git a/packages/report-ui/src/app/components/JsonInspector.tsx b/packages/report-ui/src/app/components/JsonInspector.tsx new file mode 100644 index 0000000..03ff012 --- /dev/null +++ b/packages/report-ui/src/app/components/JsonInspector.tsx @@ -0,0 +1,80 @@ +import JsonView from "react18-json-view"; +import "react18-json-view/src/style.css"; +import { formatJson } from "../model"; +import { CodeBlock, EmptyState, cx } from "../ui"; +import { CopyButton } from "./CopyButton"; + +export function JsonInspector({ + value, + mode, + onModeChange, +}: { + value: unknown; + mode: "tree" | "text"; + onModeChange: (mode: "tree" | "text") => void; +}) { + if (value === undefined || value === "") { + return n/a; + } + + const text = formatJson(value); + + return ( +
+
+
+ onModeChange("tree")} + > + Inspector + + onModeChange("text")} + > + Text + +
+ +
+ {mode === "tree" ? ( +
+ +
+ ) : ( + + )} +
+ ); +} + +function ModeButton({ + children, + selected, + onClick, +}: { + children: string; + selected: boolean; + onClick: () => void; +}) { + return ( + + ); +} diff --git a/packages/report-ui/src/app/components/RawTab.tsx b/packages/report-ui/src/app/components/RawTab.tsx index e93f48a..4afb1eb 100644 --- a/packages/report-ui/src/app/components/RawTab.tsx +++ b/packages/report-ui/src/app/components/RawTab.tsx @@ -1,21 +1,32 @@ import type { ReportCase } from "@vitest-evals/core"; +import { useState } from "react"; import { DetailContent, DetailSection } from "./DetailLayout"; -import { JsonBlock } from "./ReportPrimitives"; +import { JsonInspector } from "./JsonInspector"; export function RawTab({ testCase }: { testCase: ReportCase }) { + const [mode, setMode] = useState<"tree" | "text">("tree"); + return ( - + {testCase.harness?.run?.artifacts ? ( - + ) : null} {testCase.harness?.run?.errors?.length ? ( - + ) : null} diff --git a/packages/report-ui/src/app/styles.css b/packages/report-ui/src/app/styles.css index 2deb1f2..8017625 100644 --- a/packages/report-ui/src/app/styles.css +++ b/packages/report-ui/src/app/styles.css @@ -48,4 +48,14 @@ button { color: inherit; } + + .json-view { + color: var(--color-code); + --json-property: var(--color-pass); + --json-index: var(--color-trace); + --json-number: var(--color-trace); + --json-string: var(--color-warn); + --json-boolean: var(--color-fail); + --json-null: var(--color-fail); + } } From dc2d3ef107103b6b78724b6b8b68115e22131623 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:10:11 +0200 Subject: [PATCH 05/47] feat(report-ui): copy JSON with temporary Copied! feedback Make it obvious that the clipboard write succeeded without leaving the report. --- .../src/app/components/CopyButton.tsx | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 packages/report-ui/src/app/components/CopyButton.tsx diff --git a/packages/report-ui/src/app/components/CopyButton.tsx b/packages/report-ui/src/app/components/CopyButton.tsx new file mode 100644 index 0000000..349ddbd --- /dev/null +++ b/packages/report-ui/src/app/components/CopyButton.tsx @@ -0,0 +1,39 @@ +import { useEffect, useState } from "react"; +import { cx } from "../ui"; + +export function CopyButton({ + label = "Copy", + text, +}: { + label?: string; + text: string; +}) { + const [copied, setCopied] = useState(false); + + useEffect(() => { + if (!copied) { + return; + } + const timeout = window.setTimeout(() => setCopied(false), 1500); + return () => window.clearTimeout(timeout); + }, [copied]); + + return ( + + ); +} From fc0131346a09c03d1e28f184511eef15d1d5e29f Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:10:17 +0200 Subject: [PATCH 06/47] feat(report-ui): make failure messages easier to scan Parse assertion headlines, diffs, and stacks so a failed eval is readable instead of one dumped string. --- .../src/app/components/FailureList.tsx | 71 +++++++++++++++++ .../src/app/components/OverviewTab.tsx | 13 +--- packages/report-ui/src/app/failure.test.ts | 54 +++++++++++++ packages/report-ui/src/app/failure.ts | 76 +++++++++++++++++++ 4 files changed, 203 insertions(+), 11 deletions(-) create mode 100644 packages/report-ui/src/app/components/FailureList.tsx create mode 100644 packages/report-ui/src/app/failure.test.ts create mode 100644 packages/report-ui/src/app/failure.ts diff --git a/packages/report-ui/src/app/components/FailureList.tsx b/packages/report-ui/src/app/components/FailureList.tsx new file mode 100644 index 0000000..476cde1 --- /dev/null +++ b/packages/report-ui/src/app/components/FailureList.tsx @@ -0,0 +1,71 @@ +import { parseFailureMessage } from "../failure"; +import { EmptyState, cx } from "../ui"; + +export function FailureList({ messages }: { messages: string[] }) { + if (messages.length === 0) { + return No failure messages; + } + + return ( +
    + {messages.map((message) => ( + + ))} +
+ ); +} + +function FailureCard({ message }: { message: string }) { + const failure = parseFailureMessage(message); + + return ( +
  • +
    +
    +

    + {failure.name} +

    +

    + {failure.headline} +

    +
    +
    + {failure.diffLines.length > 0 ? ( + 
    +          {failure.diffLines.map((line, index) => (
+            
+              {line.type === "add"
+                ? `+ ${line.text}`
+                : line.type === "remove"
+                  ? `- ${line.text}`
+                  : line.text}
+            
+          ))}
+
    + ) : null} + {failure.body ? ( + 
    +          {failure.body}
+
    + ) : null} + {failure.stack ? ( +
    + + Stack + + 
    +            {failure.stack}
+
    +
    + ) : null} +
    • + ); +} diff --git a/packages/report-ui/src/app/components/OverviewTab.tsx b/packages/report-ui/src/app/components/OverviewTab.tsx index c8585f6..d7f23cb 100644 --- a/packages/report-ui/src/app/components/OverviewTab.tsx +++ b/packages/report-ui/src/app/components/OverviewTab.tsx @@ -2,6 +2,7 @@ import type { HarnessRun, ReportCase } from "@vitest-evals/core"; import { formatDuration, formatNumber } from "../model"; import { EmptyState } from "../ui"; import { DetailContent, DetailSection } from "./DetailLayout"; +import { FailureList } from "./FailureList"; import { Fact, FactsGrid, JsonBlock, ScoreValue } from "./ReportPrimitives"; export function OverviewTab({ @@ -23,17 +24,7 @@ export function OverviewTab({ - {testCase.failureMessages.length > 0 ? ( -
      - {testCase.failureMessages.map((message) => ( -
    • - {message} -
      • - ))} -
    - ) : ( - No failure messages - )} +     ); diff --git a/packages/report-ui/src/app/failure.test.ts b/packages/report-ui/src/app/failure.test.ts new file mode 100644 index 0000000..0da9a29 --- /dev/null +++ b/packages/report-ui/src/app/failure.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, test } from "vitest"; +import { parseFailureMessage } from "./failure"; + +describe("parseFailureMessage", () => { + test("splits an assertion error into headline, diff, and stack", () => { + expect( + parseFailureMessage( + [ + "AssertionError: Score: 0.75 below threshold: 0.80", + "", + "- Expected", + "+ Received", + "", + "- 0.80", + "+ 0.75", + " at Module.scoreThreshold (score.ts:10:5)", + " at Object. (eval.ts:22:3)", + ].join("\n"), + ), + ).toEqual({ + name: "AssertionError", + headline: "Score: 0.75 below threshold: 0.80", + body: "", + stack: [ + "at Module.scoreThreshold (score.ts:10:5)", + " at Object. (eval.ts:22:3)", + ].join("\n"), + diffLines: [ + { type: "context", text: "- Expected" }, + { type: "context", text: "+ Received" }, + { type: "remove", text: "0.80" }, + { type: "add", text: "0.75" }, + ], + }); + }); + + test("keeps unnamed messages and leftover body text", () => { + expect(parseFailureMessage("expected approved, received denied")).toEqual({ + name: "Error", + headline: "expected approved, received denied", + body: "", + stack: undefined, + diffLines: [], + }); + + expect( + parseFailureMessage("Error: boom\nmodel returned unstructured output"), + ).toMatchObject({ + name: "Error", + headline: "boom", + body: "model returned unstructured output", + }); + }); +}); diff --git a/packages/report-ui/src/app/failure.ts b/packages/report-ui/src/app/failure.ts new file mode 100644 index 0000000..b4febbb --- /dev/null +++ b/packages/report-ui/src/app/failure.ts @@ -0,0 +1,76 @@ +export type FailureDiffLine = { + type: "add" | "remove" | "context"; + text: string; +}; + +export type ParsedFailure = { + name: string; + headline: string; + body: string; + stack?: string; + diffLines: FailureDiffLine[]; +}; + +const STACK_LINE = /^\s+at\s+/; +const DIFF_LINE = /^([+-])\s.*$/; +const DIFF_HEADER = /^(Expected|Received|Actual|- Expected|\+ Received)\b/; + +/** Splits a raw Vitest failure string into headline, diff, body, and stack. */ +export function parseFailureMessage(message: string): ParsedFailure { + const lines = message.replace(/\r\n/g, "\n").split("\n"); + const stackStart = lines.findIndex((line) => STACK_LINE.test(line)); + const contentLines = stackStart === -1 ? lines : lines.slice(0, stackStart); + const stack = + stackStart === -1 ? undefined : lines.slice(stackStart).join("\n").trim(); + + const firstLine = contentLines[0] ?? "Failure"; + const named = firstLine.match(/^([A-Za-z_$][\w$]*(?:Error)?):\s*(.*)$/); + const name = named?.[1] ?? "Error"; + const headline = (named?.[2] ?? firstLine).trim() || name; + const rest = contentLines.slice(1); + + const diffLines = collectDiffLines(rest); + const body = rest + .filter((line) => !isDiffNoise(line)) + .join("\n") + .trim(); + + return { + name, + headline, + body, + stack: stack && stack.length > 0 ? stack : undefined, + diffLines, + }; +} + +function collectDiffLines(lines: string[]): FailureDiffLine[] { + const collected: FailureDiffLine[] = []; + for (const line of lines) { + if (DIFF_HEADER.test(line.trim())) { + collected.push({ type: "context", text: line.trim() }); + continue; + } + const diff = line.match(DIFF_LINE); + if (!diff) { + continue; + } + if (line.startsWith("---") || line.startsWith("+++")) { + continue; + } + collected.push({ + type: diff[1] === "+" ? "add" : "remove", + text: line.slice(1).trimStart(), + }); + } + return collected; +} + +function isDiffNoise(line: string) { + return ( + DIFF_HEADER.test(line.trim()) || + DIFF_LINE.test(line) || + line.startsWith("---") || + line.startsWith("+++") + ); +} From 41515a5cae7e9d01c0e5d974246b89bf8a9fce5d Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:10:17 +0200 Subject: [PATCH 07/47] feat(report-ui): open the case file in a local editor Jump from the drawer file path into VS Code, Cursor, or Zed at the failing location. --- .../src/app/components/FileOpenMenu.tsx | 101 ++++++++++++++++++ packages/report-ui/src/app/file-open.test.ts | 27 +++++ packages/report-ui/src/app/file-open.ts | 38 +++++++ 3 files changed, 166 insertions(+) create mode 100644 packages/report-ui/src/app/components/FileOpenMenu.tsx create mode 100644 packages/report-ui/src/app/file-open.test.ts create mode 100644 packages/report-ui/src/app/file-open.ts diff --git a/packages/report-ui/src/app/components/FileOpenMenu.tsx b/packages/report-ui/src/app/components/FileOpenMenu.tsx new file mode 100644 index 0000000..7ae20d6 --- /dev/null +++ b/packages/report-ui/src/app/components/FileOpenMenu.tsx @@ -0,0 +1,101 @@ +import type { ReportCase } from "@vitest-evals/core"; +import { useEffect, useId, useRef, useState } from "react"; +import { editorTargets } from "../file-open"; +import { cx } from "../ui"; + +export function FileOpenMenu({ testCase }: { testCase: ReportCase }) { + const [open, setOpen] = useState(false); + const menuId = useId(); + const rootRef = useRef(null); + + useEffect(() => { + if (!open) { + return; + } + const onPointerDown = (event: PointerEvent) => { + if ( + event.target instanceof Node && + !rootRef.current?.contains(event.target) + ) { + setOpen(false); + } + }; + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") { + setOpen(false); + } + }; + document.addEventListener("pointerdown", onPointerDown); + document.addEventListener("keydown", onKeyDown); + return () => { + document.removeEventListener("pointerdown", onPointerDown); + document.removeEventListener("keydown", onKeyDown); + }; + }, [open]); + + return ( +
    + + {open ? ( + + ) : null} +
    + ); +} + +function ExternalLinkIcon() { + return ( + + ); +} diff --git a/packages/report-ui/src/app/file-open.test.ts b/packages/report-ui/src/app/file-open.test.ts new file mode 100644 index 0000000..2320852 --- /dev/null +++ b/packages/report-ui/src/app/file-open.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from "vitest"; +import { editorTargets } from "./file-open"; + +describe("editorTargets", () => { + test("includes line and column when the case has a location", () => { + const targets = editorTargets({ + ancestorTitles: [], + displayFile: "refund.eval.ts", + displayName: "refund", + failureMessages: [], + file: "/repo/refund.eval.ts", + fullName: "refund", + id: "case-1", + location: { column: 4, line: 12 }, + runId: "run-1", + status: "failed", + title: "refund", + }); + + expect(targets.map((target) => target.href)).toEqual([ + "vscode://file/repo/refund.eval.ts:12:4", + "vscode-insiders://file/repo/refund.eval.ts:12:4", + "cursor://file/repo/refund.eval.ts:12:4", + "zed://file/repo/refund.eval.ts:12:4", + ]); + }); +}); diff --git a/packages/report-ui/src/app/file-open.ts b/packages/report-ui/src/app/file-open.ts new file mode 100644 index 0000000..9ec87ac --- /dev/null +++ b/packages/report-ui/src/app/file-open.ts @@ -0,0 +1,38 @@ +import type { ReportCase } from "@vitest-evals/core"; + +export type EditorTarget = { + id: string; + label: string; + href: string; +}; + +/** Builds editor deep links for the case source file. */ +export function editorTargets(testCase: ReportCase): EditorTarget[] { + const location = fileLocation(testCase); + return [ + { + id: "vscode", + label: "Open in VS Code", + href: `vscode://file${location}`, + }, + { + id: "vscode-insiders", + label: "Open in VS Code Insiders", + href: `vscode-insiders://file${location}`, + }, + { id: "cursor", label: "Open in Cursor", href: `cursor://file${location}` }, + { id: "zed", label: "Open in Zed", href: `zed://file${location}` }, + ]; +} + +function fileLocation(testCase: ReportCase) { + const line = testCase.location?.line; + const column = testCase.location?.column; + if (line === undefined) { + return testCase.file; + } + if (column === undefined) { + return `${testCase.file}:${line}`; + } + return `${testCase.file}:${line}:${column}`; +} From 6513ac3f9b9ed97504757afc24ff262fd7bf4ef3 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:10:17 +0200 Subject: [PATCH 08/47] feat(report-ui): copy a case as Markdown for an agent Bundle the failing case, judge evidence, and raw JSON into a paste-ready brief with short fix instructions. --- .../report-ui/src/app/case-markdown.test.ts | 49 +++++++++++++++++ packages/report-ui/src/app/case-markdown.ts | 54 +++++++++++++++++++ .../src/app/components/CaseDrawer.tsx | 33 ++++++++---- 3 files changed, 126 insertions(+), 10 deletions(-) create mode 100644 packages/report-ui/src/app/case-markdown.test.ts create mode 100644 packages/report-ui/src/app/case-markdown.ts diff --git a/packages/report-ui/src/app/case-markdown.test.ts b/packages/report-ui/src/app/case-markdown.test.ts new file mode 100644 index 0000000..6c659ee --- /dev/null +++ b/packages/report-ui/src/app/case-markdown.test.ts @@ -0,0 +1,49 @@ +import type { ReportCase } from "@vitest-evals/core"; +import { describe, expect, test } from "vitest"; +import { caseToMarkdown } from "./case-markdown"; + +const testCase: ReportCase = { + ancestorTitles: ["refund"], + displayFile: "refund.eval.ts", + displayName: "refund agent > rejects fraud", + failureMessages: ["Score: 0.20 below threshold: 1.00"], + file: "/repo/refund.eval.ts", + fullName: "refund agent rejects fraud", + id: "failed-case", + runId: "run-1", + status: "failed", + title: "rejects fraud", + eval: { + avgScore: 0.2, + output: { status: "denied" }, + scores: [{ name: "StructuredOutputJudge", score: 0.2 }], + }, +}; + +describe("caseToMarkdown", () => { + test("includes agent instructions and the failing case evidence", () => { + const markdown = caseToMarkdown(testCase, { + id: "run-1", + source: "eval-results/refund.json", + status: "failed", + totals: { + evalFailed: 1, + evalPassed: 0, + evalTotal: 1, + failed: 1, + passed: 0, + skipped: 0, + total: 1, + }, + }); + + expect(markdown).toContain("# Eval case: refund agent > rejects fraud"); + expect(markdown).toContain("Help fix or improve this vitest-evals case."); + expect(markdown).toContain("**Result:** failed"); + expect(markdown).toContain("**File:** `refund.eval.ts`"); + expect(markdown).toContain("Score: 0.20 below threshold: 1.00"); + expect(markdown).toContain("| StructuredOutputJudge | 20% |"); + expect(markdown).toContain('"status": "denied"'); + expect(markdown).toContain("## Raw case JSON"); + }); +}); diff --git a/packages/report-ui/src/app/case-markdown.ts b/packages/report-ui/src/app/case-markdown.ts new file mode 100644 index 0000000..a070d9c --- /dev/null +++ b/packages/report-ui/src/app/case-markdown.ts @@ -0,0 +1,54 @@ +import type { ReportCase, ReportRun } from "@vitest-evals/core"; +import { formatDuration, formatJson, formatScore } from "./model"; + +/** Builds a paste-ready Markdown brief for an agent working on one eval case. */ +export function caseToMarkdown( + testCase: ReportCase, + run: ReportRun | undefined, +): string { + const lines = [ + `# Eval case: ${testCase.displayName}`, + "", + "Help fix or improve this vitest-evals case. Keep the change minimal, preserve the eval intent, and say what you would change.", + "", + "## Status", + "", + `- **Result:** ${testCase.status}`, + `- **Score:** ${formatScore(testCase.eval?.avgScore)}`, + `- **Duration:** ${formatDuration(testCase.durationMs)}`, + `- **File:** \`${testCase.displayFile}\``, + `- **Run:** ${run?.source ?? testCase.runId}`, + ]; + + if (testCase.failureMessages.length > 0) { + lines.push("", "## Failures", ""); + for (const message of testCase.failureMessages) { + lines.push("```", message, "```", ""); + } + } + + const output = testCase.eval?.output ?? testCase.harness?.run?.output; + if (output !== undefined) { + lines.push("## Output", "", "```json", formatJson(output), "```", ""); + } + + const scores = testCase.eval?.scores ?? []; + if (scores.length > 0) { + lines.push("## Judge evidence", "", "| Judge | Score |", "| --- | --- |"); + for (const score of scores) { + lines.push(`| ${score.name ?? "Score"} | ${formatScore(score.score)} |`); + } + lines.push(""); + } + + lines.push( + "## Raw case JSON", + "", + "```json", + formatJson(testCase), + "```", + "", + ); + + return lines.join("\n"); +} diff --git a/packages/report-ui/src/app/components/CaseDrawer.tsx b/packages/report-ui/src/app/components/CaseDrawer.tsx index 82e2ab5..8b659b1 100644 --- a/packages/report-ui/src/app/components/CaseDrawer.tsx +++ b/packages/report-ui/src/app/components/CaseDrawer.tsx @@ -1,8 +1,11 @@ -import { useEffect, useRef } from "react"; import type { ReportCase, ReportRun } from "@vitest-evals/core"; -import { formatDuration } from "../model"; +import { useEffect, useRef } from "react"; +import { caseToMarkdown } from "../case-markdown"; +import { formatDuration, formatJson } from "../model"; import type { DetailTab } from "../types"; import { TabButton } from "../ui"; +import { CopyButton } from "./CopyButton"; +import { FileOpenMenu } from "./FileOpenMenu"; import { OverviewTab } from "./OverviewTab"; import { RawTab } from "./RawTab"; import { Fact, FactsGrid, ScoreValue, StatusMark } from "./ReportPrimitives"; @@ -109,14 +112,24 @@ export function CaseDrawer({ {testCase.displayName} -

    - {testCase.displayFile} -

    -
    - - Score - - +
    +

    + {testCase.displayFile} +

    + +
    +
    + + +
    + + Score + + +
    From f046b374ec2d51d0e3a8e1a5eaff81d1ebb91e45 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:41:55 +0200 Subject: [PATCH 09/47] feat(report-ui): keep header totals on the full workspace --- packages/report-ui/src/app/App.test.ts | 24 +- packages/report-ui/src/app/App.tsx | 226 ++++++++++++------ .../src/app/components/ReportChrome.tsx | 166 +++++++++++-- 3 files changed, 320 insertions(+), 96 deletions(-) diff --git a/packages/report-ui/src/app/App.test.ts b/packages/report-ui/src/app/App.test.ts index df8a5e6..a199d26 100644 --- a/packages/report-ui/src/app/App.test.ts +++ b/packages/report-ui/src/app/App.test.ts @@ -1,7 +1,8 @@ -import { afterEach, describe, expect, test, vi } from "vitest"; import type { ReportWorkspace } from "@vitest-evals/core"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { loadWorkspace, + nextSortSearch, resolveSelectedCase, resolveSelectedCaseId, summarizeVisibleWorkspace, @@ -40,6 +41,27 @@ afterEach(() => { vi.unstubAllGlobals(); }); +describe("nextSortSearch", () => { + test("starts a new column with its default direction and toggles the active one", () => { + expect(nextSortSearch(undefined, "asc", "score")).toEqual({ + dir: "desc", + sort: "score", + }); + expect(nextSortSearch("score", "desc", "score")).toEqual({ + dir: "asc", + sort: "score", + }); + expect(nextSortSearch("score", "asc", "case")).toEqual({ + dir: "asc", + sort: "case", + }); + expect(nextSortSearch(undefined, "asc", "model")).toEqual({ + dir: "asc", + sort: "model", + }); + }); +}); + describe("case selection", () => { test("keeps selection scoped to visible filtered cases", () => { const visibleCases = [cases[1]!]; diff --git a/packages/report-ui/src/app/App.tsx b/packages/report-ui/src/app/App.tsx index 6f6e0b4..4289a93 100644 --- a/packages/report-ui/src/app/App.tsx +++ b/packages/report-ui/src/app/App.tsx @@ -1,22 +1,31 @@ -import { useEffect, useMemo, useState } from "react"; import { - ReportWorkspaceSchema, type ReportWorkspace, + ReportWorkspaceSchema, } from "@vitest-evals/core"; +import { useEffect, useMemo, useState } from "react"; import { CaseDrawer } from "./components/CaseDrawer"; import { CaseWorkbench } from "./components/CaseWorkbench"; import { ReportHeader, RunStrip, SummaryBar } from "./components/ReportChrome"; import { + type CaseFilters, + type CaseSortColumn, filterReportCases, + sortReportCases, summarizeWorkspace, - type CaseFilters, } from "./model"; -import type { DetailTab } from "./types"; +import { estimateWorkspaceCost } from "./pricing"; +import { + DEFAULT_REPORT_META, + type ReportMeta, + ReportMetaContext, + readReportMeta, +} from "./report-meta"; +import { useReportSearch } from "./report-state"; type LoadState = | { status: "loading" } | { status: "error"; message: string } - | { status: "ready"; workspace: ReportWorkspace }; + | { status: "ready"; workspace: ReportWorkspace; meta: ReportMeta }; export function App() { const [loadState, setLoadState] = useState({ status: "loading" }); @@ -41,96 +50,147 @@ export function App() { ); } - return ; + return ; } -function ReportApp({ workspace }: { workspace: ReportWorkspace }) { - const [filters, setFilters] = useState({ - query: "", - status: "all", - runId: "all", - }); - const [selectedCaseId, setSelectedCaseId] = useState( - () => workspace.cases.find((testCase) => testCase.status === "failed")?.id, +function ReportApp({ + meta, + workspace, +}: { + meta: ReportMeta; + workspace: ReportWorkspace; +}) { + const { search, setSearch } = useReportSearch(); + const filters = useMemo( + () => ({ + query: search.q, + status: search.status, + runId: search.run, + }), + [search.q, search.run, search.status], ); - const [isDrawerOpen, setIsDrawerOpen] = useState(false); - const [detailTab, setDetailTab] = useState("overview"); const filteredCases = useMemo( () => filterReportCases(workspace.cases, filters), [workspace.cases, filters], ); - const visibleRuns = useMemo( - () => visibleWorkspaceRuns(workspace.runs, filters, filteredCases), - [workspace.runs, filters, filteredCases], + const visibleCases = useMemo( + () => sortReportCases(filteredCases, search.sort, search.dir), + [filteredCases, search.sort, search.dir], + ); + const workspaceSummary = useMemo( + () => summarizeWorkspace(workspace), + [workspace], ); - const summary = useMemo( + const estimatedCostUsd = useMemo( () => - summarizeWorkspace({ - ...workspace, - cases: filteredCases, - runs: visibleRuns, - }), - [workspace, filteredCases, visibleRuns], + estimateWorkspaceCost( + workspace.cases.map((testCase) => testCase.harness?.run?.usage ?? {}), + meta.pricing, + ), + [meta.pricing, workspace.cases], ); - const selectedCase = resolveSelectedCase(selectedCaseId, filteredCases); - - useEffect(() => { - const nextSelectedCaseId = resolveSelectedCaseId( - selectedCaseId, - filteredCases, - ); - if (nextSelectedCaseId !== selectedCaseId) { - setSelectedCaseId(nextSelectedCaseId); - } - }, [filteredCases, selectedCaseId]); + const selectedCase = resolveSelectedCase(search.case, workspace.cases); + const isDrawerOpen = Boolean(search.case && selectedCase); return ( -
    -
    - -
    - - - +
    +
    + +
    + + + + setSearch({ + q: nextFilters.query, + run: nextFilters.runId, + status: nextFilters.status, + }) + } + onSelectCase={(testCase) => + setSearch( + { case: testCase.id, tab: "overview" }, + { replace: false }, + ) + } + onSortChange={(column) => + setSearch(nextSortSearch(search.sort, search.dir, column)) + } + /> +
    + + { - setSelectedCaseId(testCase.id); - setDetailTab("overview"); - setIsDrawerOpen(true); - }} + testCase={selectedCase} + onClose={() => + setSearch( + { case: undefined, tab: "overview" }, + { replace: false }, + ) + } + onTabChange={(tab) => setSearch({ tab })} /> -
    - - setIsDrawerOpen(false)} - onTabChange={setDetailTab} - /> -
    -
    +
    + + ); } +export function nextSortSearch( + currentColumn: CaseSortColumn | undefined, + currentDirection: "asc" | "desc", + column: CaseSortColumn, +) { + if (currentColumn !== column) { + return { + sort: column, + dir: defaultSortDirection(column), + }; + } + + return { + sort: column, + dir: currentDirection === "asc" ? ("desc" as const) : ("asc" as const), + }; +} + +function defaultSortDirection(column: CaseSortColumn): "asc" | "desc" { + return column === "case" || column === "model" || column === "status" + ? "asc" + : "desc"; +} + export function resolveSelectedCase( selectedCaseId: string | undefined, - filteredCases: ReportWorkspace["cases"], + cases: ReportWorkspace["cases"], ) { - return filteredCases.find((testCase) => testCase.id === selectedCaseId); + return cases.find((testCase) => testCase.id === selectedCaseId); } export function resolveSelectedCaseId( @@ -193,12 +253,20 @@ export async function loadWorkspace( signal: AbortSignal, ): Promise { try { - const response = await fetch("/data/workspace.json", { signal }); - if (!response.ok) { - throw new Error(`HTTP ${response.status}`); + const [workspaceResponse, metaResponse] = await Promise.all([ + fetch("/data/workspace.json", { signal }), + fetch("/data/meta.json", { signal }), + ]); + if (!workspaceResponse.ok) { + throw new Error(`HTTP ${workspaceResponse.status}`); } - const workspace = ReportWorkspaceSchema.parse(await response.json()); - return { status: "ready", workspace }; + const workspace = ReportWorkspaceSchema.parse( + await workspaceResponse.json(), + ); + const meta = metaResponse.ok + ? readReportMeta(await metaResponse.json()) + : DEFAULT_REPORT_META; + return { status: "ready", meta, workspace }; } catch (error) { if (signal.aborted) { return undefined; diff --git a/packages/report-ui/src/app/components/ReportChrome.tsx b/packages/report-ui/src/app/components/ReportChrome.tsx index 8749caa..0ea3663 100644 --- a/packages/report-ui/src/app/components/ReportChrome.tsx +++ b/packages/report-ui/src/app/components/ReportChrome.tsx @@ -1,11 +1,17 @@ +import { Link } from "@tanstack/react-router"; import type { ReportRun } from "@vitest-evals/core"; +import type { ReactNode } from "react"; import { + type CaseStatusFilter, formatDuration, formatNumber, formatScore, type summarizeWorkspace, } from "../model"; -import { cx, toneTextClass, type Tone } from "../ui"; +import { formatUsd } from "../pricing"; +import { toReportSearch } from "../search"; +import { type Tone, cx, toneTextClass } from "../ui"; +import { PathLabel } from "./PathLabel"; import { executedCaseCount, passRate, @@ -15,15 +21,23 @@ import { export function ReportHeader({ caseCount, runCount, + visibleCaseCount, }: { caseCount: number; runCount: number; + visibleCaseCount: number; }) { return (
    - vitest-evals + toReportSearch({})} + to="/" + > + vitest-evals +
    @@ -43,6 +57,15 @@ export function ReportHeader({     {" "} cases + {visibleCaseCount !== caseCount ? ( + + showing{" "} + + {formatNumber(visibleCaseCount)} + {" "} + of {formatNumber(caseCount)} + + ) : null}
    @@ -50,8 +73,12 @@ export function ReportHeader({ } export function SummaryBar({ + currentStatus, + estimatedCostUsd, summary, }: { + currentStatus: CaseStatusFilter; + estimatedCostUsd?: number; summary: ReturnType; }) { const verdictTone = passRateTone(summary); @@ -80,12 +107,15 @@ export function SummaryBar({
    - + {summary.failed} {" "} failed - + {summary.caseCount} @@ -112,13 +142,31 @@ export function SummaryBar({
    - - - + + +
    -
    +
    +
    - {column.header} - + +
    +
    - {CASE_COLUMNS.map((column) => ( - - ))} + {CASE_COLUMNS.map((column) => { + const active = sortColumn === column.id; + return ( + + ); + })} @@ -216,6 +275,15 @@ function CaseRow({ testCase: ReportCase; onSelectCase: (testCase: ReportCase) => void; }) { + const { pricing, workspaceRoot } = useReportMeta(); + const model = caseModel(testCase); + const displayFile = + relativeDisplayPath(testCase.displayFile, workspaceRoot) || + testCase.displayFile; + const usageCost = estimateUsageCost( + testCase.harness?.run?.usage ?? {}, + pricing, + ); const selectCase = () => onSelectCase(testCase); return ( @@ -232,19 +300,39 @@ function CaseRow({ + @@ -254,7 +342,10 @@ function CaseRow({ label={`Open ${testCase.displayName}`} onClick={selectCase} > - +
    - {column.header} - + +
    +
    + + + {testCase.displayName} + + + {displayFile} + + +
    + +
    +
    +     		- - {testCase.displayName} - - - {testCase.displayFile} + + {model ?? "n/a"} @@ -272,7 +363,13 @@ function CaseRow({ label={`Open ${testCase.displayName}`} onClick={selectCase} > - {formatNumber(caseTotalTokens(testCase))} + {usageCost ? ( + + {formatNumber(caseTotalTokens(testCase))} + + ) : ( + formatNumber(caseTotalTokens(testCase)) + )} diff --git a/packages/report-ui/src/app/model.test.ts b/packages/report-ui/src/app/model.test.ts index d9adf73..826c370 100644 --- a/packages/report-ui/src/app/model.test.ts +++ b/packages/report-ui/src/app/model.test.ts @@ -1,6 +1,6 @@ -import { describe, expect, test } from "vitest"; import { messagesToTranscriptEvents } from "@vitest-evals/core"; import type { ReportWorkspace } from "@vitest-evals/core"; +import { describe, expect, test } from "vitest"; import { buildSpanTree, buildTranscript, @@ -10,6 +10,7 @@ import { filterReportCases, formatScore, scoreTone, + sortReportCases, summarizeWorkspace, } from "./model"; @@ -233,6 +234,59 @@ describe("summarizeWorkspace", () => { }); }); +describe("sortReportCases", () => { + test("orders failed cases first when sorting by status", () => { + expect( + sortReportCases(workspace.cases, "status", "asc").map( + (testCase) => testCase.status, + ), + ).toEqual(["failed", "passed"]); + }); + + test("orders higher scores first when sorting score descending", () => { + expect( + sortReportCases(workspace.cases, "score", "desc").map( + (testCase) => testCase.eval?.avgScore, + ), + ).toEqual([1, 0.2]); + }); + + test("orders models alphabetically", () => { + expect( + sortReportCases( + [ + { + ...workspace.cases[0]!, + harness: { + name: "pi-ai", + run: { + errors: [], + output: {}, + session: { events: [] }, + usage: { model: "gpt-4o" }, + }, + }, + }, + { + ...workspace.cases[1]!, + harness: { + name: "pi-ai", + run: { + errors: [], + output: {}, + session: { events: [] }, + usage: { model: "gemini-2.5-flash" }, + }, + }, + }, + ], + "model", + "asc", + ).map((testCase) => testCase.harness?.run?.usage?.model), + ).toEqual(["gemini-2.5-flash", "gpt-4o"]); + }); +}); + describe("filterReportCases", () => { test("filters by status, run, and search query", () => { expect( @@ -261,6 +315,32 @@ describe("filterReportCases", () => { }), ).toEqual([]); }); + + test("searches recorded model ids", () => { + expect( + filterReportCases( + [ + { + ...workspace.cases[0]!, + harness: { + name: "pi-ai", + run: { + errors: [], + output: {}, + session: { events: [] }, + usage: { model: "gemini-2.5-flash" }, + }, + }, + }, + ], + { + status: "all", + runId: "all", + query: "gemini-2.5", + }, + ), + ).toHaveLength(1); + }); }); describe("case helpers", () => { diff --git a/packages/report-ui/src/app/model.ts b/packages/report-ui/src/app/model.ts index eadf324..4cf98a0 100644 --- a/packages/report-ui/src/app/model.ts +++ b/packages/report-ui/src/app/model.ts @@ -1,5 +1,4 @@ import { - toolCalls, type HarnessRun, type JsonValue, type NormalizedError, @@ -9,6 +8,7 @@ import { type TranscriptMessageEvent, type TranscriptToolCallEvent, type TranscriptToolResultEvent, + toolCalls, } from "@vitest-evals/core"; export type CaseStatusFilter = "all" | ReportCase["status"]; @@ -19,6 +19,26 @@ export type CaseFilters = { runId: string; }; +export type CaseSortColumn = + | "status" + | "case" + | "model" + | "score" + | "duration" + | "tokens" + | "tools"; + +export type CaseSortDirection = "asc" | "desc"; + +const STATUS_RANK: Record = { + failed: 0, + passed: 1, + pending: 2, + todo: 3, + skipped: 4, + disabled: 5, +}; + export type WorkspaceSummary = { runCount: number; caseCount: number; @@ -95,6 +115,68 @@ export function summarizeWorkspace( }; } +/** Sorts filtered cases for the report ledger. */ +export function sortReportCases( + cases: ReportCase[], + column: CaseSortColumn | undefined, + direction: CaseSortDirection, +) { + if (!column) { + return cases; + } + + const ranked = [...cases].sort((left, right) => { + const comparison = compareCaseColumn(left, right, column); + return direction === "desc" ? -comparison : comparison; + }); + return ranked; +} + +function compareCaseColumn( + left: ReportCase, + right: ReportCase, + column: CaseSortColumn, +) { + switch (column) { + case "status": + return STATUS_RANK[left.status] - STATUS_RANK[right.status]; + case "case": + return left.displayName.localeCompare(right.displayName); + case "model": + return (caseModel(left) ?? "").localeCompare(caseModel(right) ?? ""); + case "score": + return compareNullableNumber(left.eval?.avgScore, right.eval?.avgScore); + case "duration": + return compareNullableNumber(left.durationMs, right.durationMs); + case "tokens": + return compareNullableNumber( + caseTotalTokens(left), + caseTotalTokens(right), + ); + case "tools": + return compareNullableNumber( + caseToolCallCount(left), + caseToolCallCount(right), + ); + } +} + +function compareNullableNumber( + left: number | null | undefined, + right: number | null | undefined, +) { + if (left == null && right == null) { + return 0; + } + if (left == null) { + return 1; + } + if (right == null) { + return -1; + } + return left - right; +} + /** Filters cases for the report explorer. */ export function filterReportCases(cases: ReportCase[], filters: CaseFilters) { const query = filters.query.trim().toLowerCase(); @@ -118,6 +200,11 @@ export function caseToolCalls(testCase: ReportCase) { return toolCallsForCase(testCase); } +/** Returns the recorded application model for a report case. */ +export function caseModel(testCase: ReportCase) { + return testCase.harness?.run?.usage?.model; +} + /** Returns the best available token total for a report case. */ export function caseTotalTokens(testCase: ReportCase) { const run = testCase.harness?.run; @@ -390,6 +477,7 @@ function searchableCaseText(testCase: ReportCase) { testCase.fullName, testCase.displayFile, testCase.source, + caseModel(testCase), ...(testCase.eval?.scores ?? []).map((score) => score.name ?? ""), ] .filter(Boolean) diff --git a/packages/report-ui/src/app/search.test.ts b/packages/report-ui/src/app/search.test.ts new file mode 100644 index 0000000..748c8e3 --- /dev/null +++ b/packages/report-ui/src/app/search.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, test } from "vitest"; +import { + compactReportSearch, + toReportSearch, + validateReportSearch, +} from "./search"; + +describe("validateReportSearch", () => { + test("fills defaults for an empty query string", () => { + expect(validateReportSearch({})).toEqual({ + q: "", + status: "all", + run: "all", + sort: undefined, + dir: "asc", + case: undefined, + tab: "overview", + }); + }); + + test("keeps recognized values and ignores unknown ones", () => { + expect( + validateReportSearch({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "model", + dir: "desc", + case: "case-1", + tab: "raw", + extra: "nope", + }), + ).toEqual({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "model", + dir: "desc", + case: "case-1", + tab: "raw", + }); + + expect( + validateReportSearch({ + status: "bogus", + sort: "nope", + dir: "sideways", + tab: "secret", + }), + ).toMatchObject({ + status: "all", + sort: undefined, + dir: "asc", + tab: "overview", + }); + }); +}); + +describe("toReportSearch", () => { + test("fills defaults for a partial search patch", () => { + expect(toReportSearch({ status: "failed" })).toEqual({ + q: "", + status: "failed", + run: "all", + sort: undefined, + dir: "asc", + case: undefined, + tab: "overview", + }); + }); +}); + +describe("compactReportSearch", () => { + test("omits default values from the serialized search", () => { + expect( + compactReportSearch({ + q: "", + status: "all", + run: "all", + dir: "asc", + tab: "overview", + }), + ).toEqual({}); + }); + + test("keeps only the active inspection state", () => { + expect( + compactReportSearch({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "model", + dir: "desc", + case: "case-1", + tab: "raw", + }), + ).toEqual({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "model", + dir: "desc", + case: "case-1", + tab: "raw", + }); + }); + + test("drops the drawer tab when no case is open", () => { + expect( + compactReportSearch({ + q: "", + status: "all", + run: "all", + dir: "asc", + tab: "raw", + }), + ).toEqual({}); + }); +}); diff --git a/packages/report-ui/src/app/search.ts b/packages/report-ui/src/app/search.ts new file mode 100644 index 0000000..244bfe6 --- /dev/null +++ b/packages/report-ui/src/app/search.ts @@ -0,0 +1,118 @@ +import type { CaseSortColumn, CaseStatusFilter } from "./model"; +import type { DetailTab } from "./types"; + +export type ReportSearch = { + q: string; + status: CaseStatusFilter; + run: string; + sort?: CaseSortColumn; + dir: "asc" | "desc"; + case?: string; + tab: DetailTab; +}; + +const STATUS_VALUES = new Set([ + "all", + "passed", + "failed", + "skipped", + "pending", + "todo", + "disabled", +]); + +const SORT_VALUES = new Set([ + "status", + "case", + "model", + "score", + "duration", + "tokens", + "tools", +]); + +const TAB_VALUES = new Set(["overview", "transcript", "raw"]); + +/** Parses report UI search params and fills defaults for missing keys. */ +export function validateReportSearch( + search: Record, +): ReportSearch { + return { + q: readString(search.q) ?? "", + status: readStatus(search.status), + run: readString(search.run) ?? "all", + sort: readSort(search.sort), + dir: search.dir === "desc" ? "desc" : "asc", + case: readString(search.case), + tab: readTab(search.tab), + }; +} + +/** Fills missing search keys so router navigations stay fully typed. */ +export function toReportSearch(search: Partial): ReportSearch { + return { + q: search.q ?? "", + status: search.status ?? "all", + run: search.run ?? "all", + sort: search.sort, + dir: search.dir ?? "asc", + case: search.case, + tab: search.tab ?? "overview", + }; +} + +/** Drops default search values so shared URLs stay short. */ +export function compactReportSearch( + search: Partial, +): Record { + return compactFilledSearch(toReportSearch(search)); +} + +function compactFilledSearch(search: ReportSearch): Record { + const next: Record = {}; + if (search.q.trim().length > 0) { + next.q = search.q; + } + if (search.status !== "all") { + next.status = search.status; + } + if (search.run !== "all") { + next.run = search.run; + } + if (search.sort) { + next.sort = search.sort; + if (search.dir !== "asc") { + next.dir = search.dir; + } + } + if (search.case) { + next.case = search.case; + if (search.tab !== "overview") { + next.tab = search.tab; + } + } + return next; +} + +function readString(value: unknown) { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function readStatus(value: unknown): CaseStatusFilter { + return typeof value === "string" && + STATUS_VALUES.has(value as CaseStatusFilter) + ? (value as CaseStatusFilter) + : "all"; +} + +function readSort(value: unknown): CaseSortColumn | undefined { + return typeof value === "string" && SORT_VALUES.has(value as CaseSortColumn) + ? (value as CaseSortColumn) + : undefined; +} + +function readTab(value: unknown): DetailTab { + return typeof value === "string" && TAB_VALUES.has(value as DetailTab) + ? (value as DetailTab) + : "overview"; +} From 86f3dc20fbcaca80b3440c058e3df965d01c4c0b Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:42:02 +0200 Subject: [PATCH 11/47] feat(report-ui): show passed judges on score hover --- .../src/app/components/InstantTooltip.tsx | 35 +++++++++++++++++++ .../src/app/components/ReportPrimitives.tsx | 34 +++++++++++++++--- .../report-ui/src/app/judge-score.test.ts | 32 +++++++++++++++++ packages/report-ui/src/app/judge-score.ts | 25 +++++++++++++ 4 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 packages/report-ui/src/app/components/InstantTooltip.tsx create mode 100644 packages/report-ui/src/app/judge-score.test.ts create mode 100644 packages/report-ui/src/app/judge-score.ts diff --git a/packages/report-ui/src/app/components/InstantTooltip.tsx b/packages/report-ui/src/app/components/InstantTooltip.tsx new file mode 100644 index 0000000..98c1f5c --- /dev/null +++ b/packages/report-ui/src/app/components/InstantTooltip.tsx @@ -0,0 +1,35 @@ +import { type ReactNode, useState } from "react"; + +export function InstantTooltip({ + content, + children, +}: { + content: ReactNode; + children: ReactNode; +}) { + const [anchor, setAnchor] = useState(); + + return ( + + setAnchor(event.currentTarget.getBoundingClientRect()) + } + onMouseLeave={() => setAnchor(undefined)} + > + {children} + {anchor ? ( + + {content} + + ) : null} + + ); +} diff --git a/packages/report-ui/src/app/components/ReportPrimitives.tsx b/packages/report-ui/src/app/components/ReportPrimitives.tsx index 688f056..2e5414d 100644 --- a/packages/report-ui/src/app/components/ReportPrimitives.tsx +++ b/packages/report-ui/src/app/components/ReportPrimitives.tsx @@ -1,12 +1,14 @@ -import type { ReactNode } from "react"; import type { ReportCase } from "@vitest-evals/core"; +import type { ReactNode } from "react"; +import { type JudgeTally, judgeTallyLabel } from "../judge-score"; import { formatJson, formatScore, scoreTone, type summarizeWorkspace, } from "../model"; -import { CodeBlock, EmptyState, cx, toneTextClass, type Tone } from "../ui"; +import { CodeBlock, EmptyState, type Tone, cx, toneTextClass } from "../ui"; +import { InstantTooltip } from "./InstantTooltip"; export function JsonBlock({ value }: { value: unknown }) { if (value === undefined || value === "") { @@ -42,13 +44,25 @@ export function FactsGrid({ ); } -export function Fact({ label, value }: { label: string; value: string }) { +export function Fact({ + label, + value, +}: { + label: string; + value: ReactNode; +}) { return (
    {label}
    -
    {value}
    +
    + {typeof value === "string" ? ( + {value} + ) : ( + value + )} +
    ); } @@ -80,12 +94,14 @@ export function StatusMark({ export function ScoreValue({ score, size = "md", + tally, }: { score: number | null | undefined; size?: "md" | "lg"; + tally?: JudgeTally; }) { const tone = scoreTone(score) as Tone; - return ( + const value = ( ); + + if (!tally || tally.total === 0) { + return value; + } + + return ( + {value} + ); } export function passRate(summary: ReturnType) { diff --git a/packages/report-ui/src/app/judge-score.test.ts b/packages/report-ui/src/app/judge-score.test.ts new file mode 100644 index 0000000..f3a2da0 --- /dev/null +++ b/packages/report-ui/src/app/judge-score.test.ts @@ -0,0 +1,32 @@ +import type { ReportCase } from "@vitest-evals/core"; +import { describe, expect, test } from "vitest"; +import { judgeTally, judgeTallyLabel } from "./judge-score"; + +const testCase: ReportCase = { + ancestorTitles: [], + displayFile: "refund.eval.ts", + displayName: "refund", + failureMessages: [], + file: "/repo/refund.eval.ts", + fullName: "refund", + id: "case-1", + runId: "run-1", + status: "failed", + title: "refund", + eval: { + avgScore: 0.75, + scores: [ + { name: "range", score: 0 }, + { name: "domain", score: 1 }, + { name: "present", score: 1 }, + { name: "relevance", score: 1 }, + ], + }, +}; + +describe("judgeTally", () => { + test("counts full-mark judges against the recorded set", () => { + expect(judgeTally(testCase)).toEqual({ passed: 3, total: 4 }); + expect(judgeTallyLabel(judgeTally(testCase))).toBe("3/4 judges passed"); + }); +}); diff --git a/packages/report-ui/src/app/judge-score.ts b/packages/report-ui/src/app/judge-score.ts new file mode 100644 index 0000000..ef2a357 --- /dev/null +++ b/packages/report-ui/src/app/judge-score.ts @@ -0,0 +1,25 @@ +import type { ReportCase } from "@vitest-evals/core"; + +export type JudgeTally = { + passed: number; + total: number; +}; + +/** Counts judges that fully passed (score >= 1) against all recorded judges. */ +export function judgeTally(testCase: ReportCase): JudgeTally { + const scores = testCase.eval?.scores ?? []; + return { + passed: scores.filter( + (score) => typeof score.score === "number" && score.score >= 1, + ).length, + total: scores.length, + }; +} + +/** Builds the ledger tooltip for a case average score. */ +export function judgeTallyLabel(tally: JudgeTally): string { + if (tally.total === 0) { + return "No judges"; + } + return `${tally.passed}/${tally.total} judges passed`; +} From 7866f8b725865456c601b65465bdb1a9f73fbb85 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:42:02 +0200 Subject: [PATCH 12/47] feat(report-ui): open relative paths from the CLI workspace --- .../src/app/components/FileOpenMenu.tsx | 103 ++++++++++++++++++ .../src/app/components/PathLabel.tsx | 34 ++++++ .../report-ui/src/app/display-path.test.ts | 33 ++++++ packages/report-ui/src/app/display-path.ts | 39 +++++++ packages/report-ui/src/app/file-open.test.ts | 19 ++++ packages/report-ui/src/app/file-open.ts | 40 +++++++ 6 files changed, 268 insertions(+) create mode 100644 packages/report-ui/src/app/components/FileOpenMenu.tsx create mode 100644 packages/report-ui/src/app/components/PathLabel.tsx create mode 100644 packages/report-ui/src/app/display-path.test.ts create mode 100644 packages/report-ui/src/app/display-path.ts create mode 100644 packages/report-ui/src/app/file-open.test.ts create mode 100644 packages/report-ui/src/app/file-open.ts diff --git a/packages/report-ui/src/app/components/FileOpenMenu.tsx b/packages/report-ui/src/app/components/FileOpenMenu.tsx new file mode 100644 index 0000000..1552135 --- /dev/null +++ b/packages/report-ui/src/app/components/FileOpenMenu.tsx @@ -0,0 +1,103 @@ +import { useEffect, useId, useRef, useState } from "react"; +import { type OpenFileTarget, editorTargets } from "../file-open"; +import { cx } from "../ui"; + +export function FileOpenMenu({ file, line, column }: OpenFileTarget) { + const [open, setOpen] = useState(false); + const menuId = useId(); + const rootRef = useRef(null); + + useEffect(() => { + if (!open) { + return; + } + const onPointerDown = (event: PointerEvent) => { + if ( + event.target instanceof Node && + !rootRef.current?.contains(event.target) + ) { + setOpen(false); + } + }; + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") { + setOpen(false); + } + }; + document.addEventListener("pointerdown", onPointerDown); + document.addEventListener("keydown", onKeyDown); + return () => { + document.removeEventListener("pointerdown", onPointerDown); + document.removeEventListener("keydown", onKeyDown); + }; + }, [open]); + + return ( +
    + + {open ? ( + + ) : null} +
    + ); +} + +function ExternalLinkIcon() { + return ( + + ); +} diff --git a/packages/report-ui/src/app/components/PathLabel.tsx b/packages/report-ui/src/app/components/PathLabel.tsx new file mode 100644 index 0000000..30df82a --- /dev/null +++ b/packages/report-ui/src/app/components/PathLabel.tsx @@ -0,0 +1,34 @@ +import { relativeDisplayPath, resolveOpenPath } from "../display-path"; +import { useReportMeta } from "../report-meta"; +import { FileOpenMenu } from "./FileOpenMenu"; + +export function PathLabel({ + path, + file, + line, + column, + className, +}: { + path: string | undefined; + file?: string; + line?: number; + column?: number; + className?: string; +}) { + const { workspaceRoot } = useReportMeta(); + const display = relativeDisplayPath(path, workspaceRoot); + const openFile = resolveOpenPath(file ?? path, workspaceRoot); + + if (!display) { + return n/a; + } + + return ( + + {display} + {openFile ? ( + + ) : null} + + ); +} diff --git a/packages/report-ui/src/app/display-path.test.ts b/packages/report-ui/src/app/display-path.test.ts new file mode 100644 index 0000000..94b7c63 --- /dev/null +++ b/packages/report-ui/src/app/display-path.test.ts @@ -0,0 +1,33 @@ +import { describe, expect, test } from "vitest"; +import { relativeDisplayPath, resolveOpenPath } from "./display-path"; + +describe("relativeDisplayPath", () => { + test("strips the CLI workspace prefix", () => { + expect( + relativeDisplayPath( + "/Users/me/app/eval-results/vitest-evals.json", + "/Users/me/app", + ), + ).toBe("eval-results/vitest-evals.json"); + }); + + test("keeps paths that are already relative", () => { + expect(relativeDisplayPath("src/ai/prompt.eval.ts", "/Users/me/app")).toBe( + "src/ai/prompt.eval.ts", + ); + }); +}); + +describe("resolveOpenPath", () => { + test("joins a relative path onto the workspace root", () => { + expect(resolveOpenPath("src/ai/prompt.eval.ts", "/Users/me/app")).toBe( + "/Users/me/app/src/ai/prompt.eval.ts", + ); + }); + + test("keeps an absolute path", () => { + expect(resolveOpenPath("/repo/file.ts", "/Users/me/app")).toBe( + "/repo/file.ts", + ); + }); +}); diff --git a/packages/report-ui/src/app/display-path.ts b/packages/report-ui/src/app/display-path.ts new file mode 100644 index 0000000..fec2603 --- /dev/null +++ b/packages/report-ui/src/app/display-path.ts @@ -0,0 +1,39 @@ +/** Turns an absolute path into the path relative to the CLI workspace root. */ +export function relativeDisplayPath( + path: string | undefined, + workspaceRoot: string | undefined, +): string { + if (!path) { + return ""; + } + if (!workspaceRoot) { + return path; + } + + const root = workspaceRoot.replace(/\/+$/, ""); + if (path === root) { + return path.split("/").pop() ?? path; + } + const prefix = `${root}/`; + if (path.startsWith(prefix)) { + return path.slice(prefix.length); + } + return path; +} + +/** Resolves a displayed path back to an editor-openable absolute file. */ +export function resolveOpenPath( + path: string | undefined, + workspaceRoot: string | undefined, +): string | undefined { + if (!path) { + return undefined; + } + if (path.startsWith("/") || /^[A-Za-z]:[\\/]/.test(path)) { + return path; + } + if (!workspaceRoot) { + return undefined; + } + return `${workspaceRoot.replace(/\/+$/, "")}/${path}`; +} diff --git a/packages/report-ui/src/app/file-open.test.ts b/packages/report-ui/src/app/file-open.test.ts new file mode 100644 index 0000000..004fe21 --- /dev/null +++ b/packages/report-ui/src/app/file-open.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, test } from "vitest"; +import { editorTargets } from "./file-open"; + +describe("editorTargets", () => { + test("includes line and column when the file has a location", () => { + expect( + editorTargets({ + column: 4, + file: "/repo/refund.eval.ts", + line: 12, + }).map((target) => target.href), + ).toEqual([ + "vscode://file/repo/refund.eval.ts:12:4", + "vscode-insiders://file/repo/refund.eval.ts:12:4", + "cursor://file/repo/refund.eval.ts:12:4", + "zed://file/repo/refund.eval.ts:12:4", + ]); + }); +}); diff --git a/packages/report-ui/src/app/file-open.ts b/packages/report-ui/src/app/file-open.ts new file mode 100644 index 0000000..e81b727 --- /dev/null +++ b/packages/report-ui/src/app/file-open.ts @@ -0,0 +1,40 @@ +export type EditorTarget = { + id: string; + label: string; + href: string; +}; + +export type OpenFileTarget = { + file: string; + line?: number; + column?: number; +}; + +/** Builds editor deep links for a source file. */ +export function editorTargets(target: OpenFileTarget): EditorTarget[] { + const location = fileLocation(target); + return [ + { + id: "vscode", + label: "Open in VS Code", + href: `vscode://file${location}`, + }, + { + id: "vscode-insiders", + label: "Open in VS Code Insiders", + href: `vscode-insiders://file${location}`, + }, + { id: "cursor", label: "Open in Cursor", href: `cursor://file${location}` }, + { id: "zed", label: "Open in Zed", href: `zed://file${location}` }, + ]; +} + +function fileLocation(target: OpenFileTarget) { + if (target.line === undefined) { + return target.file; + } + if (target.column === undefined) { + return `${target.file}:${target.line}`; + } + return `${target.file}:${target.line}:${target.column}`; +} From e42abe26da0c14632fa24fccee845c6ab4cc97b7 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:42:07 +0200 Subject: [PATCH 13/47] feat(report-ui): inspect case output like the Raw tab --- .../src/app/components/JsonInspector.tsx | 84 +++++++++++++++++++ .../src/app/components/OverviewTab.tsx | 23 +++-- 2 files changed, 94 insertions(+), 13 deletions(-) create mode 100644 packages/report-ui/src/app/components/JsonInspector.tsx diff --git a/packages/report-ui/src/app/components/JsonInspector.tsx b/packages/report-ui/src/app/components/JsonInspector.tsx new file mode 100644 index 0000000..43ff845 --- /dev/null +++ b/packages/report-ui/src/app/components/JsonInspector.tsx @@ -0,0 +1,84 @@ +import { useState } from "react"; +import JsonView from "react18-json-view"; +import "react18-json-view/src/style.css"; +import { formatJson } from "../model"; +import { CodeBlock, EmptyState, cx } from "../ui"; +import { CopyButton } from "./CopyButton"; + +export function JsonInspector({ + value, + mode, + onModeChange, +}: { + value: unknown; + mode?: "tree" | "text"; + onModeChange?: (mode: "tree" | "text") => void; +}) { + const [localMode, setLocalMode] = useState<"tree" | "text">("tree"); + const currentMode = mode ?? localMode; + const setMode = onModeChange ?? setLocalMode; + if (value === undefined || value === "") { + return n/a; + } + + const text = formatJson(value); + + return ( +
    +
    +
    + setMode("tree")} + > + Inspector + + setMode("text")} + > + Text + +
    + +
    + {currentMode === "tree" ? ( +
    + +
    + ) : ( + + )} +
    + ); +} + +function ModeButton({ + children, + selected, + onClick, +}: { + children: string; + selected: boolean; + onClick: () => void; +}) { + return ( + + ); +} diff --git a/packages/report-ui/src/app/components/OverviewTab.tsx b/packages/report-ui/src/app/components/OverviewTab.tsx index c8585f6..5aa4865 100644 --- a/packages/report-ui/src/app/components/OverviewTab.tsx +++ b/packages/report-ui/src/app/components/OverviewTab.tsx @@ -1,8 +1,12 @@ import type { HarnessRun, ReportCase } from "@vitest-evals/core"; import { formatDuration, formatNumber } from "../model"; +import { estimateUsageCost, formatUsd } from "../pricing"; +import { useReportMeta } from "../report-meta"; import { EmptyState } from "../ui"; import { DetailContent, DetailSection } from "./DetailLayout"; -import { Fact, FactsGrid, JsonBlock, ScoreValue } from "./ReportPrimitives"; +import { FailureList } from "./FailureList"; +import { JsonInspector } from "./JsonInspector"; +import { Fact, FactsGrid, ScoreValue } from "./ReportPrimitives"; export function OverviewTab({ testCase, @@ -14,7 +18,7 @@ export function OverviewTab({ return ( - + @@ -23,17 +27,7 @@ export function OverviewTab({ - {testCase.failureMessages.length > 0 ? ( -
      - {testCase.failureMessages.map((message) => ( -
    • - {message} -
      • - ))} -
    - ) : ( - No failure messages - )} +     ); @@ -140,7 +134,9 @@ function EvidenceLine({ label, value }: { label: string; value: string }) { } function UsageGrid({ run }: { run: HarnessRun | undefined }) { + const { pricing } = useReportMeta(); const usage = run?.usage; + const cost = estimateUsageCost(usage ?? {}, pricing); return ( @@ -149,6 +145,7 @@ function UsageGrid({ run }: { run: HarnessRun | undefined }) { + From 91fbeb23a23b8a54916b95add50886b77f1c6200 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:42:08 +0200 Subject: [PATCH 14/47] feat(report-ui): estimate eval cost from models.dev rates --- packages/report-ui/src/app/pricing.test.ts | 67 ++++++++ packages/report-ui/src/app/pricing.ts | 147 ++++++++++++++++ .../report-ui/src/app/report-meta.test.ts | 41 +++++ packages/report-ui/src/app/report-meta.ts | 51 ++++++ .../report-ui/src/pricing-catalog.test.ts | 84 +++++++++ packages/report-ui/src/pricing-catalog.ts | 160 ++++++++++++++++++ packages/report-ui/src/server.test.ts | 9 +- packages/report-ui/src/server.ts | 25 ++- 8 files changed, 580 insertions(+), 4 deletions(-) create mode 100644 packages/report-ui/src/app/pricing.test.ts create mode 100644 packages/report-ui/src/app/pricing.ts create mode 100644 packages/report-ui/src/app/report-meta.test.ts create mode 100644 packages/report-ui/src/app/report-meta.ts create mode 100644 packages/report-ui/src/pricing-catalog.test.ts create mode 100644 packages/report-ui/src/pricing-catalog.ts diff --git a/packages/report-ui/src/app/pricing.test.ts b/packages/report-ui/src/app/pricing.test.ts new file mode 100644 index 0000000..9286e7b --- /dev/null +++ b/packages/report-ui/src/app/pricing.test.ts @@ -0,0 +1,67 @@ +import { describe, expect, test } from "vitest"; +import { + FALLBACK_PRICING, + estimateUsageCost, + estimateWorkspaceCost, + formatUsd, + matchModelRate, +} from "./pricing"; + +describe("matchModelRate", () => { + test("matches a bare model id and a provider-prefixed id", () => { + expect(matchModelRate("gemini-2.5-flash", FALLBACK_PRICING)?.id).toBe( + "gemini-2.5-flash", + ); + expect( + matchModelRate("google/gemini-2.5-flash", FALLBACK_PRICING)?.id, + ).toBe("gemini-2.5-flash"); + }); +}); + +describe("estimateUsageCost", () => { + test("prices one million input and output tokens at the catalog rate", () => { + expect( + estimateUsageCost( + { model: "gpt-4o", inputTokens: 1_000_000, outputTokens: 1_000_000 }, + FALLBACK_PRICING, + ), + ).toMatchObject({ + matchedId: "gpt-4o", + inputUsd: 2.5, + outputUsd: 10, + totalUsd: 12.5, + }); + }); +}); + +describe("estimateWorkspaceCost", () => { + test("sums matched cases and treats totalTokens as input when split is missing", () => { + expect( + estimateUsageCost( + { model: "gpt-4o-mini", totalTokens: 1_000_000 }, + FALLBACK_PRICING, + ), + ).toMatchObject({ + inputUsd: 0.15, + outputUsd: 0, + totalUsd: 0.15, + }); + + expect( + estimateWorkspaceCost( + [ + { model: "gpt-4o-mini", inputTokens: 1_000_000, outputTokens: 0 }, + { model: "unknown-model", inputTokens: 1_000_000 }, + ], + FALLBACK_PRICING, + ), + ).toBe(0.15); + }); +}); + +describe("formatUsd", () => { + test("uses four digits for sub-cent eval spends", () => { + expect(formatUsd(0.009)).toBe("$0.0090"); + expect(formatUsd(0.12)).toBe("$0.120"); + }); +}); diff --git a/packages/report-ui/src/app/pricing.ts b/packages/report-ui/src/app/pricing.ts new file mode 100644 index 0000000..4832fd9 --- /dev/null +++ b/packages/report-ui/src/app/pricing.ts @@ -0,0 +1,147 @@ +export type ModelRate = { + id: string; + inputPerMillionUsd: number; + outputPerMillionUsd: number; +}; + +export type PricingTable = { + fetchedAt?: string; + source: string; + models: ModelRate[]; +}; + +export type UsageCost = { + model?: string; + matchedId?: string; + inputUsd: number; + outputUsd: number; + totalUsd: number; +}; + +export type ReportUiMeta = { + workspaceRoot?: string; + pricing: PricingTable; +}; + +/** Bundled first-party rates used when the live catalog is unavailable. */ +export const FALLBACK_PRICING: PricingTable = { + source: "fallback", + models: [ + { + id: "gemini-1.5-flash", + inputPerMillionUsd: 0.075, + outputPerMillionUsd: 0.3, + }, + { id: "gemini-1.5-pro", inputPerMillionUsd: 1.25, outputPerMillionUsd: 5 }, + { + id: "gemini-2.0-flash", + inputPerMillionUsd: 0.1, + outputPerMillionUsd: 0.4, + }, + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + { + id: "gemini-2.5-flash-lite", + inputPerMillionUsd: 0.1, + outputPerMillionUsd: 0.4, + }, + { id: "gpt-4o", inputPerMillionUsd: 2.5, outputPerMillionUsd: 10 }, + { id: "gpt-4o-mini", inputPerMillionUsd: 0.15, outputPerMillionUsd: 0.6 }, + { id: "gpt-5.6-terra", inputPerMillionUsd: 1.25, outputPerMillionUsd: 10 }, + { id: "claude-sonnet-4.5", inputPerMillionUsd: 3, outputPerMillionUsd: 15 }, + { id: "claude-sonnet-4", inputPerMillionUsd: 3, outputPerMillionUsd: 15 }, + { id: "claude-3-5-sonnet", inputPerMillionUsd: 3, outputPerMillionUsd: 15 }, + ], +}; + +/** Finds the best catalog row for a recorded model id. */ +export function matchModelRate( + model: string | undefined, + table: PricingTable, +): ModelRate | undefined { + if (!model) { + return undefined; + } + const exact = table.models.find((row) => row.id === model); + if (exact) { + return exact; + } + const bare = model.split("/").pop() ?? model; + return ( + table.models.find((row) => row.id === bare) ?? + table.models.find( + (row) => row.id.endsWith(`/${bare}`) || row.id.endsWith(`/${model}`), + ) + ); +} + +/** Estimates input/output USD from token counts and a matched model rate. */ +export function estimateUsageCost( + usage: { + model?: string; + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + }, + table: PricingTable, +): UsageCost | undefined { + const rate = matchModelRate(usage.model, table); + if (!rate) { + return undefined; + } + const inputTokens = usage.inputTokens ?? 0; + const outputTokens = usage.outputTokens ?? 0; + const pricedInputTokens = + inputTokens === 0 && outputTokens === 0 + ? (usage.totalTokens ?? 0) + : inputTokens; + const inputUsd = (pricedInputTokens / 1_000_000) * rate.inputPerMillionUsd; + const outputUsd = (outputTokens / 1_000_000) * rate.outputPerMillionUsd; + return { + model: usage.model, + matchedId: rate.id, + inputUsd, + outputUsd, + totalUsd: inputUsd + outputUsd, + }; +} + +/** Sums priced cases. Unmatched models are skipped. */ +export function estimateWorkspaceCost( + usages: Array<{ + model?: string; + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + }>, + table: PricingTable, +): number | undefined { + let total = 0; + let matched = false; + for (const usage of usages) { + const cost = estimateUsageCost(usage, table); + if (!cost) { + continue; + } + matched = true; + total += cost.totalUsd; + } + return matched ? total : undefined; +} + +/** Formats a USD estimate with enough digits for eval-sized spends. */ +export function formatUsd(amount: number | undefined): string { + if (amount === undefined) { + return "n/a"; + } + if (amount === 0) { + return "$0.00"; + } + if (amount < 0.01) { + return `$${amount.toFixed(4)}`; + } + return `$${amount.toFixed(3)}`; +} diff --git a/packages/report-ui/src/app/report-meta.test.ts b/packages/report-ui/src/app/report-meta.test.ts new file mode 100644 index 0000000..86f749a --- /dev/null +++ b/packages/report-ui/src/app/report-meta.test.ts @@ -0,0 +1,41 @@ +import { describe, expect, test } from "vitest"; +import { FALLBACK_PRICING } from "./pricing"; +import { readReportMeta } from "./report-meta"; + +describe("readReportMeta", () => { + test("keeps a valid workspace root and pricing table", () => { + expect( + readReportMeta({ + workspaceRoot: "/repo", + pricing: { + source: "https://models.dev/api.json", + models: [ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + ], + }, + }), + ).toEqual({ + workspaceRoot: "/repo", + pricing: { + source: "https://models.dev/api.json", + models: [ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + ], + }, + }); + }); + + test("falls back when the payload is empty", () => { + expect(readReportMeta({})).toEqual({ + pricing: FALLBACK_PRICING, + }); + }); +}); diff --git a/packages/report-ui/src/app/report-meta.ts b/packages/report-ui/src/app/report-meta.ts new file mode 100644 index 0000000..d3dc872 --- /dev/null +++ b/packages/report-ui/src/app/report-meta.ts @@ -0,0 +1,51 @@ +import { createContext, useContext } from "react"; +import { FALLBACK_PRICING, type ReportUiMeta } from "./pricing"; + +export type ReportMeta = ReportUiMeta; + +export const DEFAULT_REPORT_META: ReportMeta = { + pricing: FALLBACK_PRICING, +}; + +export const ReportMetaContext = createContext(DEFAULT_REPORT_META); + +/** Reads the CLI workspace root and live/fallback pricing for the report UI. */ +export function useReportMeta(): ReportMeta { + return useContext(ReportMetaContext); +} + +/** Accepts /data/meta.json or falls back to bundled pricing. */ +export function readReportMeta(value: unknown): ReportMeta { + if (!isRecord(value)) { + return DEFAULT_REPORT_META; + } + + const workspaceRoot = + typeof value.workspaceRoot === "string" && value.workspaceRoot.length > 0 + ? value.workspaceRoot + : undefined; + const pricing = isPricingTable(value.pricing) + ? value.pricing + : FALLBACK_PRICING; + return { workspaceRoot, pricing }; +} + +function isPricingTable(value: unknown): value is ReportMeta["pricing"] { + if (!isRecord(value) || typeof value.source !== "string") { + return false; + } + if (!Array.isArray(value.models)) { + return false; + } + return value.models.every( + (row) => + isRecord(row) && + typeof row.id === "string" && + typeof row.inputPerMillionUsd === "number" && + typeof row.outputPerMillionUsd === "number", + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/report-ui/src/pricing-catalog.test.ts b/packages/report-ui/src/pricing-catalog.test.ts new file mode 100644 index 0000000..6c0e644 --- /dev/null +++ b/packages/report-ui/src/pricing-catalog.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, test } from "vitest"; +import { + flattenLiteLlm, + flattenModelsDev, + mergeModelRates, +} from "./pricing-catalog"; + +describe("flattenModelsDev", () => { + test("prefers first-party google rates over reseller copies", () => { + expect( + flattenModelsDev({ + google: { + models: { + "gemini-2.5-flash": { cost: { input: 0.3, output: 2.5 } }, + }, + }, + openrouter: { + models: { + "gemini-2.5-flash": { cost: { input: 9, output: 9 } }, + }, + }, + }), + ).toEqual([ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + ]); + }); +}); + +describe("flattenLiteLlm", () => { + test("converts per-token prices into per-million USD", () => { + const [rate] = flattenLiteLlm({ + "gemini/gemini-2.0-flash": { + input_cost_per_token: 1e-7, + output_cost_per_token: 4e-7, + }, + }); + expect(rate?.id).toBe("gemini-2.0-flash"); + expect(rate?.inputPerMillionUsd).toBeCloseTo(0.1); + expect(rate?.outputPerMillionUsd).toBeCloseTo(0.4); + }); +}); + +describe("mergeModelRates", () => { + test("keeps primary rows and fills missing LiteLLM models", () => { + expect( + mergeModelRates( + [ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + ], + [ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 9, + outputPerMillionUsd: 9, + }, + { + id: "gpt-4o-mini", + inputPerMillionUsd: 0.15, + outputPerMillionUsd: 0.6, + }, + ], + ), + ).toEqual([ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + { + id: "gpt-4o-mini", + inputPerMillionUsd: 0.15, + outputPerMillionUsd: 0.6, + }, + ]); + }); +}); diff --git a/packages/report-ui/src/pricing-catalog.ts b/packages/report-ui/src/pricing-catalog.ts new file mode 100644 index 0000000..5b2edf1 --- /dev/null +++ b/packages/report-ui/src/pricing-catalog.ts @@ -0,0 +1,160 @@ +import { + FALLBACK_PRICING, + type ModelRate, + type PricingTable, +} from "./app/pricing"; + +const MODELS_DEV_URL = "https://models.dev/api.json"; +const LITELLM_URL = + "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; +const FIRST_PARTY = new Set(["google", "openai", "anthropic"]); + +/** Loads models.dev + LiteLLM pricing, falling back to the bundled table. */ +export async function loadPricingTable(): Promise { + const [modelsDev, liteLlm] = await Promise.all([ + fetchJson(MODELS_DEV_URL), + fetchJson(LITELLM_URL), + ]); + const models = mergeModelRates( + modelsDev ? flattenModelsDev(modelsDev) : [], + liteLlm ? flattenLiteLlm(liteLlm) : [], + ); + if (models.length === 0) { + return FALLBACK_PRICING; + } + + const sources = [ + modelsDev ? MODELS_DEV_URL : undefined, + liteLlm ? LITELLM_URL : undefined, + ].filter((source): source is string => Boolean(source)); + + return { + fetchedAt: new Date().toISOString(), + source: sources.join(" + "), + models, + }; +} + +/** Flattens models.dev provider catalogs into bare-id first-party rates. */ +export function flattenModelsDev(catalog: unknown): ModelRate[] { + if (!isRecord(catalog)) { + return []; + } + + const ranked: Array = []; + for (const [providerId, provider] of Object.entries(catalog)) { + if (!isRecord(provider) || !isRecord(provider.models)) { + continue; + } + const priority = FIRST_PARTY.has(providerId) ? 0 : 1; + for (const [modelId, model] of Object.entries(provider.models)) { + if (!isRecord(model) || !isRecord(model.cost)) { + continue; + } + const input = asFiniteNumber(model.cost.input); + const output = asFiniteNumber(model.cost.output); + if (input === undefined || output === undefined) { + continue; + } + ranked.push({ + id: modelId.includes("/") ? modelId : `${providerId}/${modelId}`, + inputPerMillionUsd: input, + outputPerMillionUsd: output, + priority, + }); + } + } + + ranked.sort((left, right) => left.priority - right.priority); + const seen = new Set(); + const models: ModelRate[] = []; + for (const row of ranked) { + const bare = row.id.split("/").pop() ?? row.id; + if (seen.has(row.id) || seen.has(bare)) { + continue; + } + seen.add(row.id); + seen.add(bare); + models.push({ + id: bare, + inputPerMillionUsd: row.inputPerMillionUsd, + outputPerMillionUsd: row.outputPerMillionUsd, + }); + } + return models; +} + +/** Flattens LiteLLM per-token prices into per-million USD rates. */ +export function flattenLiteLlm(catalog: unknown): ModelRate[] { + if (!isRecord(catalog)) { + return []; + } + + const models: ModelRate[] = []; + const seen = new Set(); + for (const [id, row] of Object.entries(catalog)) { + if (!isRecord(row)) { + continue; + } + const input = asFiniteNumber(row.input_cost_per_token); + const output = asFiniteNumber(row.output_cost_per_token); + if (input === undefined || output === undefined) { + continue; + } + const bare = id.split("/").pop() ?? id; + if (seen.has(bare)) { + continue; + } + seen.add(bare); + models.push({ + id: bare, + inputPerMillionUsd: input * 1_000_000, + outputPerMillionUsd: output * 1_000_000, + }); + } + return models; +} + +/** Prefers models.dev rows, then fills gaps from LiteLLM. */ +export function mergeModelRates( + primary: ModelRate[], + secondary: ModelRate[], +): ModelRate[] { + const seen = new Set(); + const models: ModelRate[] = []; + for (const row of [...primary, ...secondary]) { + const bare = row.id.split("/").pop() ?? row.id; + if (seen.has(row.id) || seen.has(bare)) { + continue; + } + seen.add(row.id); + seen.add(bare); + models.push(row); + } + return models; +} + +async function fetchJson(url: string): Promise { + try { + const response = await fetch(url, { + headers: { Accept: "application/json" }, + signal: AbortSignal.timeout(8000), + }); + if (!response.ok) { + return undefined; + } + return await response.json(); + } catch { + return undefined; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function asFiniteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} diff --git a/packages/report-ui/src/server.test.ts b/packages/report-ui/src/server.test.ts index aadcc7e..afd0570 100644 --- a/packages/report-ui/src/server.test.ts +++ b/packages/report-ui/src/server.test.ts @@ -1,8 +1,8 @@ import { chmod, mkdir, mkdtemp, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { describe, expect, test } from "vitest"; import type { ReportWorkspace } from "@vitest-evals/core"; +import { describe, expect, test } from "vitest"; import { parseCliArgs } from "./cli-options"; import { serveReportWorkspace } from "./server"; @@ -88,6 +88,7 @@ describe("serveReportWorkspace", () => { assetsDir, host: "127.0.0.1", port: 0, + workspaceRoot: "/repo", }); try { @@ -97,6 +98,12 @@ describe("serveReportWorkspace", () => { cases: [{ id: "case-1" }], }); + const metaResponse = await fetch(`${server.url}/data/meta.json`); + await expect(metaResponse.json()).resolves.toMatchObject({ + workspaceRoot: "/repo", + pricing: { source: "fallback" }, + }); + const htmlResponse = await fetch(server.url); await expect(htmlResponse.text()).resolves.toContain("report ui"); diff --git a/packages/report-ui/src/server.ts b/packages/report-ui/src/server.ts index 724712d..3ad70e6 100644 --- a/packages/report-ui/src/server.ts +++ b/packages/report-ui/src/server.ts @@ -1,15 +1,17 @@ import { readFile, stat } from "node:fs/promises"; import { - createServer, type IncomingMessage, type Server, type ServerResponse, + createServer, } from "node:http"; import { dirname, extname, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import type { ReportWorkspace } from "@vitest-evals/core"; import { readReportWorkspace } from "@vitest-evals/core/node"; +import { FALLBACK_PRICING, type ReportUiMeta } from "./app/pricing.js"; import { currentModuleUrl } from "./esm-runtime.js"; +import { loadPricingTable } from "./pricing-catalog.js"; /** Options for serving a report UI from one or more JSON result inputs. */ export type ServeReportUiOptions = { @@ -26,6 +28,8 @@ export type ServeReportWorkspaceOptions = { host?: string; port?: number; assetsDir?: string; + workspaceRoot?: string; + pricing?: ReportUiMeta["pricing"]; }; /** Handle returned by the local report UI server. */ @@ -53,7 +57,9 @@ export async function serveReportUi( assetsDir: options.assetsDir, host: options.host, port: options.port, + pricing: await loadPricingTable(), resultFiles, + workspaceRoot: resolve(options.workspace ?? options.cwd ?? process.cwd()), }); } @@ -65,7 +71,11 @@ export async function serveReportWorkspace( const host = options.host ?? DEFAULT_HOST; const port = options.port ?? DEFAULT_PORT; const assetsDir = resolve(options.assetsDir ?? defaultAssetsDir()); - const server = createServer(createRequestHandler(workspace, assetsDir)); + const meta: ReportUiMeta = { + pricing: options.pricing ?? FALLBACK_PRICING, + workspaceRoot: options.workspaceRoot, + }; + const server = createServer(createRequestHandler(workspace, assetsDir, meta)); await listen(server, port, host); @@ -78,7 +88,11 @@ export async function serveReportWorkspace( }; } -function createRequestHandler(workspace: ReportWorkspace, assetsDir: string) { +function createRequestHandler( + workspace: ReportWorkspace, + assetsDir: string, + meta: ReportUiMeta, +) { return async (request: IncomingMessage, response: ServerResponse) => { try { const requestUrl = new URL( @@ -91,6 +105,11 @@ function createRequestHandler(workspace: ReportWorkspace, assetsDir: string) { return; } + if (requestUrl.pathname === "/data/meta.json") { + sendJson(response, meta); + return; + } + if (requestUrl.pathname === "/healthz") { sendText(response, 200, "ok\n", "text/plain; charset=utf-8"); return; From f312ddc2797e55ad28784ebcfb797a48bce357c8 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 17:42:08 +0200 Subject: [PATCH 15/47] feat(report-ui): put the recorded model in the drawer header --- .../src/app/components/CaseDrawer.tsx | 69 +++++++++++++++---- 1 file changed, 57 insertions(+), 12 deletions(-) diff --git a/packages/report-ui/src/app/components/CaseDrawer.tsx b/packages/report-ui/src/app/components/CaseDrawer.tsx index 82e2ab5..3cd72fa 100644 --- a/packages/report-ui/src/app/components/CaseDrawer.tsx +++ b/packages/report-ui/src/app/components/CaseDrawer.tsx @@ -1,9 +1,13 @@ -import { useEffect, useRef } from "react"; import type { ReportCase, ReportRun } from "@vitest-evals/core"; -import { formatDuration } from "../model"; +import { useEffect, useRef } from "react"; +import { caseToMarkdown } from "../case-markdown"; +import { judgeTally } from "../judge-score"; +import { caseModel, formatDuration, formatJson } from "../model"; import type { DetailTab } from "../types"; import { TabButton } from "../ui"; +import { CopyButton } from "./CopyButton"; import { OverviewTab } from "./OverviewTab"; +import { PathLabel } from "./PathLabel"; import { RawTab } from "./RawTab"; import { Fact, FactsGrid, ScoreValue, StatusMark } from "./ReportPrimitives"; import { TranscriptTab } from "./TranscriptTab"; @@ -77,6 +81,7 @@ export function CaseDrawer({ const run = runs.find((candidate) => candidate.id === testCase.runId); const harnessRun = testCase.harness?.run; + const model = caseModel(testCase); return ( -

    - {testCase.displayFile} -

    -
    - - Score - - +
    + +
    +
    + + +
    + {model ? ( +
    + + Model + + + {model} + +
    + ) : null} +
    + + Score + + +
    +
    + {model ? ( +
    + + Model + + + {model} + +
    + ) : null}
    Score - +
    + ); +} From 324a66378cc989fdefb288451ec1ef3109adaccb Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 18:03:00 +0200 Subject: [PATCH 17/47] feat(report-ui): copy the absolute path from the editor menu --- .../src/app/components/FileOpenMenu.tsx | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 packages/report-ui/src/app/components/FileOpenMenu.tsx diff --git a/packages/report-ui/src/app/components/FileOpenMenu.tsx b/packages/report-ui/src/app/components/FileOpenMenu.tsx new file mode 100644 index 0000000..d1bfd8c --- /dev/null +++ b/packages/report-ui/src/app/components/FileOpenMenu.tsx @@ -0,0 +1,131 @@ +import { useEffect, useId, useRef, useState } from "react"; +import { type OpenFileTarget, editorTargets } from "../file-open"; +import { cx } from "../ui"; + +export function FileOpenMenu({ file, line, column }: OpenFileTarget) { + const [open, setOpen] = useState(false); + const menuId = useId(); + const rootRef = useRef(null); + + useEffect(() => { + if (!open) { + return; + } + const onPointerDown = (event: PointerEvent) => { + if ( + event.target instanceof Node && + !rootRef.current?.contains(event.target) + ) { + setOpen(false); + } + }; + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") { + setOpen(false); + } + }; + document.addEventListener("pointerdown", onPointerDown); + document.addEventListener("keydown", onKeyDown); + return () => { + document.removeEventListener("pointerdown", onPointerDown); + document.removeEventListener("keydown", onKeyDown); + }; + }, [open]); + + return ( +
    + + {open ? ( + + ) : null} +
    + ); +} + +function CopyAbsPathButton({ file }: { file: string }) { + const [copied, setCopied] = useState(false); + + useEffect(() => { + if (!copied) { + return; + } + const timeout = window.setTimeout(() => setCopied(false), 1500); + return () => window.clearTimeout(timeout); + }, [copied]); + + return ( + + ); +} + +function ExternalLinkIcon() { + return ( + + ); +} From 232f83d1a4b4d453b321b4635276eec338c01292 Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 18:03:00 +0200 Subject: [PATCH 18/47] feat(report-ui): copy a case failure without the whole JSON --- .../src/app/components/CaseDrawer.tsx | 93 ++++++++++++++++--- .../src/app/components/FailureList.tsx | 73 +++++++++++++++ 2 files changed, 154 insertions(+), 12 deletions(-) create mode 100644 packages/report-ui/src/app/components/FailureList.tsx diff --git a/packages/report-ui/src/app/components/CaseDrawer.tsx b/packages/report-ui/src/app/components/CaseDrawer.tsx index 82e2ab5..e8e7c4c 100644 --- a/packages/report-ui/src/app/components/CaseDrawer.tsx +++ b/packages/report-ui/src/app/components/CaseDrawer.tsx @@ -1,9 +1,16 @@ -import { useEffect, useRef } from "react"; import type { ReportCase, ReportRun } from "@vitest-evals/core"; -import { formatDuration } from "../model"; +import { useEffect, useRef } from "react"; +import { caseToMarkdown } from "../case-markdown"; +import { judgeTally } from "../judge-score"; +import { caseModel, formatDuration, formatJson } from "../model"; +import { estimateUsageCost, formatUsd } from "../pricing"; +import { useReportMeta } from "../report-meta"; import type { DetailTab } from "../types"; import { TabButton } from "../ui"; +import { CopyButton } from "./CopyButton"; +import { CostHelp } from "./CostHelp"; import { OverviewTab } from "./OverviewTab"; +import { PathLabel } from "./PathLabel"; import { RawTab } from "./RawTab"; import { Fact, FactsGrid, ScoreValue, StatusMark } from "./ReportPrimitives"; import { TranscriptTab } from "./TranscriptTab"; @@ -75,8 +82,12 @@ export function CaseDrawer({ return null; } + const { pricing } = useReportMeta(); const run = runs.find((candidate) => candidate.id === testCase.runId); const harnessRun = testCase.harness?.run; + const model = caseModel(testCase); + const usageCost = estimateUsageCost(harnessRun?.usage ?? {}, pricing); + const failureText = testCase.failureMessages.join("\n\n"); return (
    -

    - {testCase.displayFile} -

    -
    - - Score - - +
    + +
    +
    + + + {failureText ? ( + + ) : null} +
    + {model ? ( +
    + + Model + + + {model} + +
    + ) : null} +
    + + Score + + +
    +
    + {model ? ( +
    + + Model + + + {model} + +
    + ) : null} + {usageCost ? ( +
    + + Cost + + + + {formatUsd(usageCost.totalUsd)} + +
    + ) : null}
    Score - +
    + + ); } +export function nextSortSearch( + currentColumn: CaseSortColumn | undefined, + currentDirection: "asc" | "desc", + column: CaseSortColumn, +) { + if (currentColumn !== column) { + return { + sort: column, + dir: defaultSortDirection(column), + }; + } + + return { + sort: column, + dir: currentDirection === "asc" ? ("desc" as const) : ("asc" as const), + }; +} + +function defaultSortDirection(column: CaseSortColumn): "asc" | "desc" { + return column === "case" || column === "model" || column === "status" + ? "asc" + : "desc"; +} + export function resolveSelectedCase( selectedCaseId: string | undefined, - filteredCases: ReportWorkspace["cases"], + cases: ReportWorkspace["cases"], ) { - return filteredCases.find((testCase) => testCase.id === selectedCaseId); + return cases.find((testCase) => testCase.id === selectedCaseId); } export function resolveSelectedCaseId( @@ -175,7 +239,8 @@ function hasActiveCaseFilters(filters: CaseFilters) { return ( filters.query.trim().length > 0 || filters.status !== "all" || - filters.runId !== "all" + filters.runId !== "all" || + filters.model !== "all" ); } @@ -193,12 +258,20 @@ export async function loadWorkspace( signal: AbortSignal, ): Promise { try { - const response = await fetch("/data/workspace.json", { signal }); - if (!response.ok) { - throw new Error(`HTTP ${response.status}`); + const [workspaceResponse, metaResponse] = await Promise.all([ + fetch("/data/workspace.json", { signal }), + fetch("/data/meta.json", { signal }), + ]); + if (!workspaceResponse.ok) { + throw new Error(`HTTP ${workspaceResponse.status}`); } - const workspace = ReportWorkspaceSchema.parse(await response.json()); - return { status: "ready", workspace }; + const workspace = ReportWorkspaceSchema.parse( + await workspaceResponse.json(), + ); + const meta = metaResponse.ok + ? readReportMeta(await metaResponse.json()) + : DEFAULT_REPORT_META; + return { status: "ready", meta, workspace }; } catch (error) { if (signal.aborted) { return undefined; diff --git a/packages/report-ui/src/app/components/CaseWorkbench.tsx b/packages/report-ui/src/app/components/CaseWorkbench.tsx index 6dea4c9..f6ff5f1 100644 --- a/packages/report-ui/src/app/components/CaseWorkbench.tsx +++ b/packages/report-ui/src/app/components/CaseWorkbench.tsx @@ -1,18 +1,28 @@ -import type { ReactNode } from "react"; import type { ReportCase, ReportRun } from "@vitest-evals/core"; +import type { ReactNode } from "react"; +import { relativeDisplayPath, resolveOpenPath } from "../display-path"; +import { judgeTally } from "../judge-score"; import { + type CaseFilters, + type CaseSortColumn, + type CaseSortDirection, + type CaseStatusFilter, + caseModel, caseToolCallCount, caseTotalTokens, formatDuration, formatNumber, - type CaseFilters, - type CaseStatusFilter, } from "../model"; +import { estimateUsageCost, formatUsd } from "../pricing"; +import { useReportMeta } from "../report-meta"; import { EmptyState, Field, Input, Select, cx } from "../ui"; +import { CostHelp } from "./CostHelp"; +import { FileOpenMenu } from "./FileOpenMenu"; +import { InstantTooltip } from "./InstantTooltip"; import { ScoreValue, StatusMark } from "./ReportPrimitives"; type CaseColumn = { - id: string; + id: CaseSortColumn; header: string; className: string; }; @@ -38,6 +48,11 @@ const CASE_COLUMNS: CaseColumn[] = [ header: "Case", className: "min-w-[220px]", }, + { + id: "model", + header: "Model", + className: "w-[148px]", + }, { id: "score", header: "Score", @@ -53,6 +68,11 @@ const CASE_COLUMNS: CaseColumn[] = [ header: "Tokens", className: "w-[92px] text-right", }, + { + id: "cost", + header: "Cost", + className: "w-[88px] text-right", + }, { id: "tools", header: "Tools", @@ -63,19 +83,27 @@ const CASE_COLUMNS: CaseColumn[] = [ export function CaseWorkbench({ cases, filters, + modelOptions, runs, selectedCaseId, + sortColumn, + sortDirection, totalCases, onFiltersChange, onSelectCase, + onSortChange, }: { cases: ReportCase[]; filters: CaseFilters; + modelOptions: string[]; runs: ReportRun[]; selectedCaseId: string | undefined; + sortColumn: CaseSortColumn | undefined; + sortDirection: CaseSortDirection; totalCases: number; onFiltersChange: (filters: CaseFilters) => void; onSelectCase: (testCase: ReportCase) => void; + onSortChange: (column: CaseSortColumn) => void; }) { return (
    @@ -92,6 +120,7 @@ export function CaseWorkbench({ @@ -99,7 +128,10 @@ export function CaseWorkbench({
    ); @@ -107,15 +139,18 @@ export function CaseWorkbench({ function CaseFilterControls({ filters, + modelOptions, runs, onFiltersChange, }: { filters: CaseFilters; + modelOptions: string[]; runs: ReportRun[]; onFiltersChange: (filters: CaseFilters) => void; }) { + const { workspaceRoot } = useReportMeta(); return ( -
    +
    onFiltersChange({ ...filters, query: event.target.value }) } - placeholder="Case, file, judge" + placeholder="Case, file, judge, model" /> @@ -144,6 +179,25 @@ function CaseFilterControls({ ))} + + + @@ -167,11 +222,17 @@ function CaseFilterControls({ function CaseTable({ cases, selectedCaseId, + sortColumn, + sortDirection, onSelectCase, + onSortChange, }: { cases: ReportCase[]; selectedCaseId: string | undefined; + sortColumn: CaseSortColumn | undefined; + sortDirection: CaseSortDirection; onSelectCase: (testCase: ReportCase) => void; + onSortChange: (column: CaseSortColumn) => void; }) { if (cases.length === 0) { return No matching eval cases; @@ -179,17 +240,45 @@ function CaseTable({ return (
    - +
    - {CASE_COLUMNS.map((column) => ( - - ))} + {CASE_COLUMNS.map((column) => { + const active = sortColumn === column.id; + return ( + + ); + })} @@ -216,6 +305,15 @@ function CaseRow({ testCase: ReportCase; onSelectCase: (testCase: ReportCase) => void; }) { + const { pricing, workspaceRoot } = useReportMeta(); + const model = caseModel(testCase); + const displayFile = + relativeDisplayPath(testCase.displayFile, workspaceRoot) || + testCase.displayFile; + const usageCost = estimateUsageCost( + testCase.harness?.run?.usage ?? {}, + pricing, + ); const selectCase = () => onSelectCase(testCase); return ( @@ -232,19 +330,39 @@ function CaseRow({ + @@ -254,7 +372,10 @@ function CaseRow({ label={`Open ${testCase.displayName}`} onClick={selectCase} > - + +
    - {column.header} - + +
    +
    + + + {testCase.displayName} + + + {displayFile} + + +
    + +
    +
    +     		- - {testCase.displayName} - - - {testCase.displayFile} + + {model ?? "n/a"} @@ -272,9 +393,31 @@ function CaseRow({ label={`Open ${testCase.displayName}`} onClick={selectCase} > - {formatNumber(caseTotalTokens(testCase))} + {usageCost ? ( + + {formatNumber(caseTotalTokens(testCase))} + + ) : ( + formatNumber(caseTotalTokens(testCase)) + )} +
    + + {usageCost ? formatUsd(usageCost.totalUsd) : "n/a"} + + {usageCost ? ( +
    + +
    + ) : null} +
    +     		{ }); }); +describe("sortReportCases", () => { + test("orders failed cases first when sorting by status", () => { + expect( + sortReportCases(workspace.cases, "status", "asc").map( + (testCase) => testCase.status, + ), + ).toEqual(["failed", "passed"]); + }); + + test("orders higher scores first when sorting score descending", () => { + expect( + sortReportCases(workspace.cases, "score", "desc").map( + (testCase) => testCase.eval?.avgScore, + ), + ).toEqual([1, 0.2]); + }); + + test("orders models alphabetically", () => { + expect( + sortReportCases( + [ + { + ...workspace.cases[0]!, + harness: { + name: "pi-ai", + run: { + errors: [], + output: {}, + session: { events: [] }, + usage: { model: "gpt-4o" }, + }, + }, + }, + { + ...workspace.cases[1]!, + harness: { + name: "pi-ai", + run: { + errors: [], + output: {}, + session: { events: [] }, + usage: { model: "gemini-2.5-flash" }, + }, + }, + }, + ], + "model", + "asc", + ).map((testCase) => testCase.harness?.run?.usage?.model), + ).toEqual(["gemini-2.5-flash", "gpt-4o"]); + }); +}); + describe("filterReportCases", () => { test("filters by status, run, and search query", () => { expect( filterReportCases(workspace.cases, { status: "failed", runId: "shard-a.json", + model: "all", query: "fraud", }), ).toEqual([workspace.cases[0]]); @@ -249,6 +304,7 @@ describe("filterReportCases", () => { filterReportCases(workspace.cases, { status: "all", runId: "all", + model: "all", query: "StructuredOutputJudge", }), ).toEqual(workspace.cases); @@ -257,10 +313,66 @@ describe("filterReportCases", () => { filterReportCases(workspace.cases, { status: "all", runId: "all", + model: "all", query: "pi-ai", }), ).toEqual([]); }); + + test("searches recorded model ids", () => { + expect( + filterReportCases( + [ + { + ...workspace.cases[0]!, + harness: { + name: "pi-ai", + run: { + errors: [], + output: {}, + session: { events: [] }, + usage: { model: "gemini-2.5-flash" }, + }, + }, + }, + ], + { + status: "all", + runId: "all", + model: "all", + query: "gemini-2.5", + }, + ), + ).toHaveLength(1); + }); + + test("filters by recorded model id", () => { + expect( + filterReportCases( + [ + { + ...workspace.cases[0]!, + harness: { + name: "pi-ai", + run: { + errors: [], + output: {}, + session: { events: [] }, + usage: { model: "gemini-2.5-flash" }, + }, + }, + }, + workspace.cases[1]!, + ], + { + status: "all", + runId: "all", + model: "gemini-2.5-flash", + query: "", + }, + ), + ).toHaveLength(1); + }); }); describe("case helpers", () => { diff --git a/packages/report-ui/src/app/model.ts b/packages/report-ui/src/app/model.ts index eadf324..f336aa9 100644 --- a/packages/report-ui/src/app/model.ts +++ b/packages/report-ui/src/app/model.ts @@ -1,5 +1,4 @@ import { - toolCalls, type HarnessRun, type JsonValue, type NormalizedError, @@ -9,7 +8,9 @@ import { type TranscriptMessageEvent, type TranscriptToolCallEvent, type TranscriptToolResultEvent, + toolCalls, } from "@vitest-evals/core"; +import { type PricingTable, estimateUsageCost } from "./pricing"; export type CaseStatusFilter = "all" | ReportCase["status"]; @@ -17,6 +18,28 @@ export type CaseFilters = { query: string; status: CaseStatusFilter; runId: string; + model: string; +}; + +export type CaseSortColumn = + | "status" + | "case" + | "model" + | "score" + | "duration" + | "tokens" + | "cost" + | "tools"; + +export type CaseSortDirection = "asc" | "desc"; + +const STATUS_RANK: Record = { + failed: 0, + passed: 1, + pending: 2, + todo: 3, + skipped: 4, + disabled: 5, }; export type WorkspaceSummary = { @@ -95,6 +118,80 @@ export function summarizeWorkspace( }; } +/** Sorts filtered cases for the report ledger. */ +export function sortReportCases( + cases: ReportCase[], + column: CaseSortColumn | undefined, + direction: CaseSortDirection, + pricing?: PricingTable, +) { + if (!column) { + return cases; + } + + const ranked = [...cases].sort((left, right) => { + const comparison = compareCaseColumn(left, right, column, pricing); + return direction === "desc" ? -comparison : comparison; + }); + return ranked; +} + +function compareCaseColumn( + left: ReportCase, + right: ReportCase, + column: CaseSortColumn, + pricing?: PricingTable, +) { + switch (column) { + case "status": + return STATUS_RANK[left.status] - STATUS_RANK[right.status]; + case "case": + return left.displayName.localeCompare(right.displayName); + case "model": + return (caseModel(left) ?? "").localeCompare(caseModel(right) ?? ""); + case "score": + return compareNullableNumber(left.eval?.avgScore, right.eval?.avgScore); + case "duration": + return compareNullableNumber(left.durationMs, right.durationMs); + case "tokens": + return compareNullableNumber( + caseTotalTokens(left), + caseTotalTokens(right), + ); + case "cost": + return compareNullableNumber( + pricing + ? estimateUsageCost(left.harness?.run?.usage ?? {}, pricing)?.totalUsd + : undefined, + pricing + ? estimateUsageCost(right.harness?.run?.usage ?? {}, pricing) + ?.totalUsd + : undefined, + ); + case "tools": + return compareNullableNumber( + caseToolCallCount(left), + caseToolCallCount(right), + ); + } +} + +function compareNullableNumber( + left: number | null | undefined, + right: number | null | undefined, +) { + if (left == null && right == null) { + return 0; + } + if (left == null) { + return 1; + } + if (right == null) { + return -1; + } + return left - right; +} + /** Filters cases for the report explorer. */ export function filterReportCases(cases: ReportCase[], filters: CaseFilters) { const query = filters.query.trim().toLowerCase(); @@ -105,6 +202,9 @@ export function filterReportCases(cases: ReportCase[], filters: CaseFilters) { if (filters.runId !== "all" && testCase.runId !== filters.runId) { return false; } + if (filters.model !== "all" && caseModel(testCase) !== filters.model) { + return false; + } if (!query) { return true; } @@ -118,6 +218,22 @@ export function caseToolCalls(testCase: ReportCase) { return toolCallsForCase(testCase); } +/** Returns the recorded application model for a report case. */ +export function caseModel(testCase: ReportCase) { + return testCase.harness?.run?.usage?.model; +} + +/** Unique recorded models, sorted for the ledger filter. */ +export function uniqueCaseModels(cases: ReportCase[]) { + return [ + ...new Set( + cases + .map((testCase) => caseModel(testCase)) + .filter((model): model is string => Boolean(model)), + ), + ].sort(); +} + /** Returns the best available token total for a report case. */ export function caseTotalTokens(testCase: ReportCase) { const run = testCase.harness?.run; @@ -390,6 +506,7 @@ function searchableCaseText(testCase: ReportCase) { testCase.fullName, testCase.displayFile, testCase.source, + caseModel(testCase), ...(testCase.eval?.scores ?? []).map((score) => score.name ?? ""), ] .filter(Boolean) diff --git a/packages/report-ui/src/app/router.tsx b/packages/report-ui/src/app/router.tsx new file mode 100644 index 0000000..d1d6658 --- /dev/null +++ b/packages/report-ui/src/app/router.tsx @@ -0,0 +1,46 @@ +import { + Outlet, + createRootRoute, + createRoute, + createRouter, + stripSearchParams, +} from "@tanstack/react-router"; +import { App } from "./App"; +import { validateReportSearch } from "./search"; + +const rootRoute = createRootRoute({ + component: Outlet, +}); + +export const indexRoute = createRoute({ + getParentRoute: () => rootRoute, + path: "/", + validateSearch: (search: Record) => + validateReportSearch(search), + search: { + middlewares: [ + stripSearchParams({ + q: "", + status: "all", + run: "all", + model: "all", + dir: "asc", + tab: "overview", + }), + ], + }, + component: App, +}); + +const routeTree = rootRoute.addChildren([indexRoute]); + +export const router = createRouter({ + routeTree, + trailingSlash: "never", +}); + +declare module "@tanstack/react-router" { + interface Register { + router: typeof router; + } +} diff --git a/packages/report-ui/src/app/search.test.ts b/packages/report-ui/src/app/search.test.ts new file mode 100644 index 0000000..102f2e4 --- /dev/null +++ b/packages/report-ui/src/app/search.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, test } from "vitest"; +import { + compactReportSearch, + toReportSearch, + validateReportSearch, +} from "./search"; + +describe("validateReportSearch", () => { + test("fills defaults for an empty query string", () => { + expect(validateReportSearch({})).toEqual({ + q: "", + status: "all", + run: "all", + model: "all", + sort: undefined, + dir: "asc", + case: undefined, + tab: "overview", + }); + }); + + test("keeps recognized values and ignores unknown ones", () => { + expect( + validateReportSearch({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "model", + dir: "desc", + case: "case-1", + tab: "raw", + extra: "nope", + }), + ).toEqual({ + q: "fraud", + status: "failed", + run: "run-1", + model: "all", + sort: "model", + dir: "desc", + case: "case-1", + tab: "raw", + }); + + expect( + validateReportSearch({ + status: "bogus", + sort: "nope", + dir: "sideways", + tab: "secret", + }), + ).toMatchObject({ + status: "all", + sort: undefined, + dir: "asc", + tab: "overview", + }); + }); +}); + +describe("toReportSearch", () => { + test("fills defaults for a partial search patch", () => { + expect(toReportSearch({ status: "failed" })).toEqual({ + q: "", + status: "failed", + run: "all", + model: "all", + sort: undefined, + dir: "asc", + case: undefined, + tab: "overview", + }); + }); +}); + +describe("compactReportSearch", () => { + test("omits default values from the serialized search", () => { + expect( + compactReportSearch({ + q: "", + status: "all", + run: "all", + dir: "asc", + tab: "overview", + }), + ).toEqual({}); + }); + + test("keeps only the active inspection state", () => { + expect( + compactReportSearch({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "model", + dir: "desc", + case: "case-1", + tab: "raw", + }), + ).toEqual({ + q: "fraud", + status: "failed", + run: "run-1", + sort: "model", + dir: "desc", + case: "case-1", + tab: "raw", + }); + }); + + test("drops the drawer tab when no case is open", () => { + expect( + compactReportSearch({ + q: "", + status: "all", + run: "all", + dir: "asc", + tab: "raw", + }), + ).toEqual({}); + }); +}); diff --git a/packages/report-ui/src/app/search.ts b/packages/report-ui/src/app/search.ts new file mode 100644 index 0000000..534f95b --- /dev/null +++ b/packages/report-ui/src/app/search.ts @@ -0,0 +1,125 @@ +import type { CaseSortColumn, CaseStatusFilter } from "./model"; +import type { DetailTab } from "./types"; + +export type ReportSearch = { + q: string; + status: CaseStatusFilter; + run: string; + model: string; + sort?: CaseSortColumn; + dir: "asc" | "desc"; + case?: string; + tab: DetailTab; +}; + +const STATUS_VALUES = new Set([ + "all", + "passed", + "failed", + "skipped", + "pending", + "todo", + "disabled", +]); + +const SORT_VALUES = new Set([ + "status", + "case", + "model", + "score", + "duration", + "tokens", + "cost", + "tools", +]); + +const TAB_VALUES = new Set(["overview", "transcript", "raw"]); + +/** Parses report UI search params and fills defaults for missing keys. */ +export function validateReportSearch( + search: Record, +): ReportSearch { + return { + q: readString(search.q) ?? "", + status: readStatus(search.status), + run: readString(search.run) ?? "all", + model: readString(search.model) ?? "all", + sort: readSort(search.sort), + dir: search.dir === "desc" ? "desc" : "asc", + case: readString(search.case), + tab: readTab(search.tab), + }; +} + +/** Fills missing search keys so router navigations stay fully typed. */ +export function toReportSearch(search: Partial): ReportSearch { + return { + q: search.q ?? "", + status: search.status ?? "all", + run: search.run ?? "all", + model: search.model ?? "all", + sort: search.sort, + dir: search.dir ?? "asc", + case: search.case, + tab: search.tab ?? "overview", + }; +} + +/** Drops default search values so shared URLs stay short. */ +export function compactReportSearch( + search: Partial, +): Record { + return compactFilledSearch(toReportSearch(search)); +} + +function compactFilledSearch(search: ReportSearch): Record { + const next: Record = {}; + if (search.q.trim().length > 0) { + next.q = search.q; + } + if (search.status !== "all") { + next.status = search.status; + } + if (search.run !== "all") { + next.run = search.run; + } + if (search.model !== "all") { + next.model = search.model; + } + if (search.sort) { + next.sort = search.sort; + if (search.dir !== "asc") { + next.dir = search.dir; + } + } + if (search.case) { + next.case = search.case; + if (search.tab !== "overview") { + next.tab = search.tab; + } + } + return next; +} + +function readString(value: unknown) { + return typeof value === "string" && value.length > 0 ? value : undefined; +} + +function readStatus(value: unknown): CaseStatusFilter { + return typeof value === "string" && + STATUS_VALUES.has(value as CaseStatusFilter) + ? (value as CaseStatusFilter) + : "all"; +} + +function readSort(value: unknown): CaseSortColumn | undefined { + return typeof value === "string" && SORT_VALUES.has(value as CaseSortColumn) + ? (value as CaseSortColumn) + : undefined; +} + +function readTab(value: unknown): DetailTab { + return typeof value === "string" && TAB_VALUES.has(value as DetailTab) + ? (value as DetailTab) + : "overview"; +} From 32c8040efd0da11e31e09cd8a4fb17f77dd1bfcf Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 18:03:08 +0200 Subject: [PATCH 20/47] feat(report-ui): price cached tokens when usage splits them --- packages/report-ui/src/app/pricing.test.ts | 90 ++++++ packages/report-ui/src/app/pricing.ts | 267 ++++++++++++++++++ .../report-ui/src/app/report-meta.test.ts | 42 +++ packages/report-ui/src/app/report-meta.ts | 57 ++++ .../report-ui/src/pricing-catalog.test.ts | 86 ++++++ packages/report-ui/src/pricing-catalog.ts | 184 ++++++++++++ 6 files changed, 726 insertions(+) create mode 100644 packages/report-ui/src/app/pricing.test.ts create mode 100644 packages/report-ui/src/app/pricing.ts create mode 100644 packages/report-ui/src/app/report-meta.test.ts create mode 100644 packages/report-ui/src/app/report-meta.ts create mode 100644 packages/report-ui/src/pricing-catalog.test.ts create mode 100644 packages/report-ui/src/pricing-catalog.ts diff --git a/packages/report-ui/src/app/pricing.test.ts b/packages/report-ui/src/app/pricing.test.ts new file mode 100644 index 0000000..29ae490 --- /dev/null +++ b/packages/report-ui/src/app/pricing.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, test } from "vitest"; +import { + FALLBACK_PRICING, + estimateUsageCost, + estimateWorkspaceCost, + formatUsd, + matchModelRate, +} from "./pricing"; + +describe("matchModelRate", () => { + test("matches a bare model id and a provider-prefixed id", () => { + expect(matchModelRate("gemini-2.5-flash", FALLBACK_PRICING)?.id).toBe( + "gemini-2.5-flash", + ); + expect( + matchModelRate("google/gemini-2.5-flash", FALLBACK_PRICING)?.id, + ).toBe("gemini-2.5-flash"); + }); +}); + +describe("estimateUsageCost", () => { + test("prices one million input and output tokens at the catalog rate", () => { + expect( + estimateUsageCost( + { model: "gpt-4o", inputTokens: 1_000_000, outputTokens: 1_000_000 }, + FALLBACK_PRICING, + ), + ).toMatchObject({ + matchedId: "gpt-4o", + inputUsd: 2.5, + outputUsd: 10, + totalUsd: 12.5, + }); + }); +}); + +describe("estimateWorkspaceCost", () => { + test("sums matched cases and treats totalTokens as input when split is missing", () => { + expect( + estimateUsageCost( + { model: "gpt-4o-mini", totalTokens: 1_000_000 }, + FALLBACK_PRICING, + ), + ).toMatchObject({ + inputUsd: 0.15, + outputUsd: 0, + totalUsd: 0.15, + }); + + expect( + estimateWorkspaceCost( + [ + { model: "gpt-4o-mini", inputTokens: 1_000_000, outputTokens: 0 }, + { model: "unknown-model", inputTokens: 1_000_000 }, + ], + FALLBACK_PRICING, + ), + ).toBe(0.15); + }); +}); + +describe("cached token pricing", () => { + test("bills cache hits at the cache-read rate and the rest at input", () => { + expect( + estimateUsageCost( + { + model: "gpt-4o", + inputTokens: 1_000_000, + outputTokens: 0, + metadata: { cachedInputTokens: 400_000 }, + }, + FALLBACK_PRICING, + ), + ).toMatchObject({ + inputTokens: 600_000, + cachedReadTokens: 400_000, + pricedCachedReads: true, + inputUsd: 1.5, + cachedReadUsd: 0.5, + totalUsd: 2, + }); + }); +}); + +describe("formatUsd", () => { + test("uses four digits for sub-cent eval spends", () => { + expect(formatUsd(0.009)).toBe("$0.0090"); + expect(formatUsd(0.12)).toBe("$0.120"); + }); +}); diff --git a/packages/report-ui/src/app/pricing.ts b/packages/report-ui/src/app/pricing.ts new file mode 100644 index 0000000..a4c9621 --- /dev/null +++ b/packages/report-ui/src/app/pricing.ts @@ -0,0 +1,267 @@ +export type ModelRate = { + id: string; + inputPerMillionUsd: number; + outputPerMillionUsd: number; + cacheReadPerMillionUsd?: number; + cacheWritePerMillionUsd?: number; +}; + +export type PricingSource = { + label: string; + url: string; +}; + +export type PricingTable = { + fetchedAt?: string; + source: string; + sources: PricingSource[]; + models: ModelRate[]; +}; + +export type PricedUsage = { + model?: string; + inputTokens?: number; + outputTokens?: number; + totalTokens?: number; + metadata?: Record; +}; + +export type UsageCost = { + model?: string; + matchedId?: string; + inputTokens: number; + cachedReadTokens: number; + cacheWriteTokens: number; + outputTokens: number; + inputUsd: number; + cachedReadUsd: number; + cacheWriteUsd: number; + outputUsd: number; + totalUsd: number; + inputPerMillionUsd: number; + outputPerMillionUsd: number; + cacheReadPerMillionUsd?: number; + cacheWritePerMillionUsd?: number; + pricedCachedReads: boolean; + usedTotalTokensFallback: boolean; +}; + +export type ReportUiMeta = { + workspaceRoot?: string; + pricing: PricingTable; +}; + +export const PRICING_SOURCE_LINKS: PricingSource[] = [ + { label: "models.dev", url: "https://models.dev" }, + { label: "models.dev API", url: "https://models.dev/api.json" }, + { + label: "LiteLLM prices", + url: "https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json", + }, +]; + +/** Bundled first-party rates used when the live catalog is unavailable. */ +export const FALLBACK_PRICING: PricingTable = { + source: "fallback", + sources: PRICING_SOURCE_LINKS, + models: [ + { + id: "gemini-1.5-flash", + inputPerMillionUsd: 0.075, + outputPerMillionUsd: 0.3, + cacheReadPerMillionUsd: 0.01875, + }, + { id: "gemini-1.5-pro", inputPerMillionUsd: 1.25, outputPerMillionUsd: 5 }, + { + id: "gemini-2.0-flash", + inputPerMillionUsd: 0.1, + outputPerMillionUsd: 0.4, + cacheReadPerMillionUsd: 0.025, + }, + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + cacheReadPerMillionUsd: 0.075, + }, + { + id: "gemini-2.5-flash-lite", + inputPerMillionUsd: 0.1, + outputPerMillionUsd: 0.4, + cacheReadPerMillionUsd: 0.025, + }, + { + id: "gpt-4o", + inputPerMillionUsd: 2.5, + outputPerMillionUsd: 10, + cacheReadPerMillionUsd: 1.25, + }, + { + id: "gpt-4o-mini", + inputPerMillionUsd: 0.15, + outputPerMillionUsd: 0.6, + cacheReadPerMillionUsd: 0.075, + }, + { id: "gpt-5.6-terra", inputPerMillionUsd: 1.25, outputPerMillionUsd: 10 }, + { + id: "claude-sonnet-4.5", + inputPerMillionUsd: 3, + outputPerMillionUsd: 15, + cacheReadPerMillionUsd: 0.3, + cacheWritePerMillionUsd: 3.75, + }, + { + id: "claude-sonnet-4", + inputPerMillionUsd: 3, + outputPerMillionUsd: 15, + cacheReadPerMillionUsd: 0.3, + cacheWritePerMillionUsd: 3.75, + }, + { + id: "claude-3-5-sonnet", + inputPerMillionUsd: 3, + outputPerMillionUsd: 15, + cacheReadPerMillionUsd: 0.3, + cacheWritePerMillionUsd: 3.75, + }, + ], +}; + +/** Finds the best catalog row for a recorded model id. */ +export function matchModelRate( + model: string | undefined, + table: PricingTable, +): ModelRate | undefined { + if (!model) { + return undefined; + } + const exact = table.models.find((row) => row.id === model); + if (exact) { + return exact; + } + const bare = model.split("/").pop() ?? model; + return ( + table.models.find((row) => row.id === bare) ?? + table.models.find( + (row) => row.id.endsWith(`/${bare}`) || row.id.endsWith(`/${model}`), + ) + ); +} + +/** Reads cache-hit tokens from harness usage or provider metadata. */ +export function cachedReadTokens(usage: PricedUsage): number { + return ( + firstNumber( + usage.metadata?.cachedInputTokens, + usage.metadata?.cacheReadTokens, + usage.metadata?.cacheReadInputTokens, + usage.metadata?.cached_tokens, + usage.metadata?.cache_read_input_tokens, + usage.metadata?.promptCacheHitTokens, + ) ?? 0 + ); +} + +/** Reads cache-write / cache-creation tokens from provider metadata. */ +export function cacheWriteTokens(usage: PricedUsage): number { + return ( + firstNumber( + usage.metadata?.cacheWriteTokens, + usage.metadata?.cacheCreationTokens, + usage.metadata?.cache_creation_input_tokens, + usage.metadata?.cacheWriteInputTokens, + ) ?? 0 + ); +} + +/** Estimates USD from token counts, cache hits, and a matched model rate. */ +export function estimateUsageCost( + usage: PricedUsage, + table: PricingTable, +): UsageCost | undefined { + const rate = matchModelRate(usage.model, table); + if (!rate) { + return undefined; + } + const outputTokens = usage.outputTokens ?? 0; + const reportedInput = usage.inputTokens ?? 0; + const usedTotalTokensFallback = + reportedInput === 0 && outputTokens === 0 && (usage.totalTokens ?? 0) > 0; + const rawInputTokens = usedTotalTokensFallback + ? (usage.totalTokens ?? 0) + : reportedInput; + const cachedReads = Math.min(cachedReadTokens(usage), rawInputTokens); + const uncachedInput = Math.max(0, rawInputTokens - cachedReads); + const writes = cacheWriteTokens(usage); + const pricedCachedReads = + cachedReads > 0 && rate.cacheReadPerMillionUsd !== undefined; + const cachedReadUsd = pricedCachedReads + ? (cachedReads / 1_000_000) * (rate.cacheReadPerMillionUsd ?? 0) + : (cachedReads / 1_000_000) * rate.inputPerMillionUsd; + const inputUsd = (uncachedInput / 1_000_000) * rate.inputPerMillionUsd; + const cacheWriteUsd = + writes > 0 && rate.cacheWritePerMillionUsd !== undefined + ? (writes / 1_000_000) * rate.cacheWritePerMillionUsd + : 0; + const outputUsd = (outputTokens / 1_000_000) * rate.outputPerMillionUsd; + return { + model: usage.model, + matchedId: rate.id, + inputTokens: uncachedInput, + cachedReadTokens: cachedReads, + cacheWriteTokens: writes, + outputTokens, + inputUsd, + cachedReadUsd, + cacheWriteUsd, + outputUsd, + totalUsd: inputUsd + cachedReadUsd + cacheWriteUsd + outputUsd, + inputPerMillionUsd: rate.inputPerMillionUsd, + outputPerMillionUsd: rate.outputPerMillionUsd, + cacheReadPerMillionUsd: rate.cacheReadPerMillionUsd, + cacheWritePerMillionUsd: rate.cacheWritePerMillionUsd, + pricedCachedReads, + usedTotalTokensFallback, + }; +} + +/** Sums priced cases. Unmatched models are skipped. */ +export function estimateWorkspaceCost( + usages: PricedUsage[], + table: PricingTable, +): number | undefined { + let total = 0; + let matched = false; + for (const usage of usages) { + const cost = estimateUsageCost(usage, table); + if (!cost) { + continue; + } + matched = true; + total += cost.totalUsd; + } + return matched ? total : undefined; +} + +/** Formats a USD estimate with enough digits for eval-sized spends. */ +export function formatUsd(amount: number | undefined): string { + if (amount === undefined) { + return "n/a"; + } + if (amount === 0) { + return "$0.00"; + } + if (amount < 0.01) { + return `$${amount.toFixed(4)}`; + } + return `$${amount.toFixed(3)}`; +} + +function firstNumber(...values: unknown[]): number | undefined { + for (const value of values) { + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + } + return undefined; +} diff --git a/packages/report-ui/src/app/report-meta.test.ts b/packages/report-ui/src/app/report-meta.test.ts new file mode 100644 index 0000000..16c377e --- /dev/null +++ b/packages/report-ui/src/app/report-meta.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, test } from "vitest"; +import { FALLBACK_PRICING } from "./pricing"; +import { readReportMeta } from "./report-meta"; + +describe("readReportMeta", () => { + test("keeps a valid workspace root and pricing table", () => { + expect( + readReportMeta({ + workspaceRoot: "/repo", + pricing: { + source: "https://models.dev/api.json", + models: [ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + ], + }, + }), + ).toEqual({ + workspaceRoot: "/repo", + pricing: { + source: "https://models.dev/api.json", + sources: FALLBACK_PRICING.sources, + models: [ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + ], + }, + }); + }); + + test("falls back when the payload is empty", () => { + expect(readReportMeta({})).toEqual({ + pricing: FALLBACK_PRICING, + }); + }); +}); diff --git a/packages/report-ui/src/app/report-meta.ts b/packages/report-ui/src/app/report-meta.ts new file mode 100644 index 0000000..7d4f7a6 --- /dev/null +++ b/packages/report-ui/src/app/report-meta.ts @@ -0,0 +1,57 @@ +import { createContext, useContext } from "react"; +import { FALLBACK_PRICING, type ReportUiMeta } from "./pricing"; + +export type ReportMeta = ReportUiMeta; + +export const DEFAULT_REPORT_META: ReportMeta = { + pricing: FALLBACK_PRICING, +}; + +export const ReportMetaContext = createContext(DEFAULT_REPORT_META); + +/** Reads the CLI workspace root and live/fallback pricing for the report UI. */ +export function useReportMeta(): ReportMeta { + return useContext(ReportMetaContext); +} + +/** Accepts /data/meta.json or falls back to bundled pricing. */ +export function readReportMeta(value: unknown): ReportMeta { + if (!isRecord(value)) { + return DEFAULT_REPORT_META; + } + + const workspaceRoot = + typeof value.workspaceRoot === "string" && value.workspaceRoot.length > 0 + ? value.workspaceRoot + : undefined; + const pricing = isPricingTable(value.pricing) + ? { + ...value.pricing, + sources: + value.pricing.sources?.length > 0 + ? value.pricing.sources + : FALLBACK_PRICING.sources, + } + : FALLBACK_PRICING; + return { workspaceRoot, pricing }; +} + +function isPricingTable(value: unknown): value is ReportMeta["pricing"] { + if (!isRecord(value) || typeof value.source !== "string") { + return false; + } + if (!Array.isArray(value.models)) { + return false; + } + return value.models.every( + (row) => + isRecord(row) && + typeof row.id === "string" && + typeof row.inputPerMillionUsd === "number" && + typeof row.outputPerMillionUsd === "number", + ); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/report-ui/src/pricing-catalog.test.ts b/packages/report-ui/src/pricing-catalog.test.ts new file mode 100644 index 0000000..755800a --- /dev/null +++ b/packages/report-ui/src/pricing-catalog.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, test } from "vitest"; +import { + flattenLiteLlm, + flattenModelsDev, + mergeModelRates, +} from "./pricing-catalog"; + +describe("flattenModelsDev", () => { + test("prefers first-party google rates over reseller copies", () => { + expect( + flattenModelsDev({ + google: { + models: { + "gemini-2.5-flash": { cost: { input: 0.3, output: 2.5 } }, + }, + }, + openrouter: { + models: { + "gemini-2.5-flash": { cost: { input: 9, output: 9 } }, + }, + }, + }), + ).toEqual([ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + ]); + }); +}); + +describe("flattenLiteLlm", () => { + test("converts per-token prices into per-million USD", () => { + const [rate] = flattenLiteLlm({ + "gemini/gemini-2.0-flash": { + input_cost_per_token: 1e-7, + output_cost_per_token: 4e-7, + cache_read_input_token_cost: 2.5e-8, + }, + }); + expect(rate?.id).toBe("gemini-2.0-flash"); + expect(rate?.inputPerMillionUsd).toBeCloseTo(0.1); + expect(rate?.outputPerMillionUsd).toBeCloseTo(0.4); + expect(rate?.cacheReadPerMillionUsd).toBeCloseTo(0.025); + }); +}); + +describe("mergeModelRates", () => { + test("keeps primary rows and fills missing LiteLLM models", () => { + expect( + mergeModelRates( + [ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + ], + [ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 9, + outputPerMillionUsd: 9, + }, + { + id: "gpt-4o-mini", + inputPerMillionUsd: 0.15, + outputPerMillionUsd: 0.6, + }, + ], + ), + ).toEqual([ + { + id: "gemini-2.5-flash", + inputPerMillionUsd: 0.3, + outputPerMillionUsd: 2.5, + }, + { + id: "gpt-4o-mini", + inputPerMillionUsd: 0.15, + outputPerMillionUsd: 0.6, + }, + ]); + }); +}); diff --git a/packages/report-ui/src/pricing-catalog.ts b/packages/report-ui/src/pricing-catalog.ts new file mode 100644 index 0000000..8ebe1a9 --- /dev/null +++ b/packages/report-ui/src/pricing-catalog.ts @@ -0,0 +1,184 @@ +import { + FALLBACK_PRICING, + type ModelRate, + PRICING_SOURCE_LINKS, + type PricingTable, +} from "./app/pricing"; + +const MODELS_DEV_URL = "https://models.dev/api.json"; +const LITELLM_URL = + "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; +const FIRST_PARTY = new Set(["google", "openai", "anthropic"]); + +/** Loads models.dev + LiteLLM pricing, falling back to the bundled table. */ +export async function loadPricingTable(): Promise { + const [modelsDev, liteLlm] = await Promise.all([ + fetchJson(MODELS_DEV_URL), + fetchJson(LITELLM_URL), + ]); + const models = mergeModelRates( + modelsDev ? flattenModelsDev(modelsDev) : [], + liteLlm ? flattenLiteLlm(liteLlm) : [], + ); + if (models.length === 0) { + return FALLBACK_PRICING; + } + + const sources = [ + modelsDev ? MODELS_DEV_URL : undefined, + liteLlm ? LITELLM_URL : undefined, + ].filter((source): source is string => Boolean(source)); + + return { + fetchedAt: new Date().toISOString(), + source: sources.join(" + "), + sources: PRICING_SOURCE_LINKS, + models, + }; +} + +/** Flattens models.dev provider catalogs into bare-id first-party rates. */ +export function flattenModelsDev(catalog: unknown): ModelRate[] { + if (!isRecord(catalog)) { + return []; + } + + const ranked: Array = []; + for (const [providerId, provider] of Object.entries(catalog)) { + if (!isRecord(provider) || !isRecord(provider.models)) { + continue; + } + const priority = FIRST_PARTY.has(providerId) ? 0 : 1; + for (const [modelId, model] of Object.entries(provider.models)) { + if (!isRecord(model) || !isRecord(model.cost)) { + continue; + } + const input = asFiniteNumber(model.cost.input); + const output = asFiniteNumber(model.cost.output); + if (input === undefined || output === undefined) { + continue; + } + ranked.push({ + id: modelId.includes("/") ? modelId : `${providerId}/${modelId}`, + inputPerMillionUsd: input, + outputPerMillionUsd: output, + ...optionalRate("cacheReadPerMillionUsd", model.cost.cache_read), + ...optionalRate("cacheWritePerMillionUsd", model.cost.cache_write), + priority, + }); + } + } + + ranked.sort((left, right) => left.priority - right.priority); + const seen = new Set(); + const models: ModelRate[] = []; + for (const row of ranked) { + const bare = row.id.split("/").pop() ?? row.id; + if (seen.has(row.id) || seen.has(bare)) { + continue; + } + seen.add(row.id); + seen.add(bare); + models.push({ + id: bare, + inputPerMillionUsd: row.inputPerMillionUsd, + outputPerMillionUsd: row.outputPerMillionUsd, + ...optionalRate("cacheReadPerMillionUsd", row.cacheReadPerMillionUsd), + ...optionalRate("cacheWritePerMillionUsd", row.cacheWritePerMillionUsd), + }); + } + return models; +} + +/** Flattens LiteLLM per-token prices into per-million USD rates. */ +export function flattenLiteLlm(catalog: unknown): ModelRate[] { + if (!isRecord(catalog)) { + return []; + } + + const models: ModelRate[] = []; + const seen = new Set(); + for (const [id, row] of Object.entries(catalog)) { + if (!isRecord(row)) { + continue; + } + const input = asFiniteNumber(row.input_cost_per_token); + const output = asFiniteNumber(row.output_cost_per_token); + if (input === undefined || output === undefined) { + continue; + } + const bare = id.split("/").pop() ?? id; + if (seen.has(bare)) { + continue; + } + seen.add(bare); + const cacheRead = asFiniteNumber(row.cache_read_input_token_cost); + const cacheWrite = asFiniteNumber(row.cache_creation_input_token_cost); + models.push({ + id: bare, + inputPerMillionUsd: input * 1_000_000, + outputPerMillionUsd: output * 1_000_000, + ...optionalRate( + "cacheReadPerMillionUsd", + cacheRead === undefined ? undefined : cacheRead * 1_000_000, + ), + ...optionalRate( + "cacheWritePerMillionUsd", + cacheWrite === undefined ? undefined : cacheWrite * 1_000_000, + ), + }); + } + return models; +} + +/** Prefers models.dev rows, then fills gaps from LiteLLM. */ +export function mergeModelRates( + primary: ModelRate[], + secondary: ModelRate[], +): ModelRate[] { + const seen = new Set(); + const models: ModelRate[] = []; + for (const row of [...primary, ...secondary]) { + const bare = row.id.split("/").pop() ?? row.id; + if (seen.has(row.id) || seen.has(bare)) { + continue; + } + seen.add(row.id); + seen.add(bare); + models.push(row); + } + return models; +} + +async function fetchJson(url: string): Promise { + try { + const response = await fetch(url, { + headers: { Accept: "application/json" }, + signal: AbortSignal.timeout(8000), + }); + if (!response.ok) { + return undefined; + } + return await response.json(); + } catch { + return undefined; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function asFiniteNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) + ? value + : undefined; +} + +function optionalRate( + key: "cacheReadPerMillionUsd" | "cacheWritePerMillionUsd", + value: unknown, +): Partial { + const amount = asFiniteNumber(value); + return amount === undefined ? {} : { [key]: amount }; +} From 05320383621bc072acde8272d53305168d1ea59c Mon Sep 17 00:00:00 2001 From: Alexandre Stahmer Date: Mon, 31 Aug 2026 18:03:08 +0200 Subject: [PATCH 21/47] feat(report-ui): explain how estimated cost is computed --- .../report-ui/src/app/components/CostHelp.tsx | 160 +++++++++++++++ .../src/app/components/OverviewTab.tsx | 36 ++-- .../src/app/components/ReportChrome.tsx | 183 ++++++++++++++++-- 3 files changed, 348 insertions(+), 31 deletions(-) create mode 100644 packages/report-ui/src/app/components/CostHelp.tsx diff --git a/packages/report-ui/src/app/components/CostHelp.tsx b/packages/report-ui/src/app/components/CostHelp.tsx new file mode 100644 index 0000000..40915a4 --- /dev/null +++ b/packages/report-ui/src/app/components/CostHelp.tsx @@ -0,0 +1,160 @@ +import { useEffect, useId, useRef, useState } from "react"; +import { formatNumber } from "../model"; +import { type PricingTable, type UsageCost, formatUsd } from "../pricing"; +import { cx } from "../ui"; + +export function CostHelp({ + cost, + pricing, + align = "left", +}: { + cost?: UsageCost; + pricing: PricingTable; + align?: "left" | "right"; +}) { + const [open, setOpen] = useState(false); + const menuId = useId(); + const rootRef = useRef(null); + + useEffect(() => { + if (!open) { + return; + } + const onPointerDown = (event: PointerEvent) => { + if ( + event.target instanceof Node && + !rootRef.current?.contains(event.target) + ) { + setOpen(false); + } + }; + const onKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") { + setOpen(false); + } + }; + document.addEventListener("pointerdown", onPointerDown); + document.addEventListener("keydown", onKeyDown); + return () => { + document.removeEventListener("pointerdown", onPointerDown); + document.removeEventListener("keydown", onKeyDown); + }; + }, [open]); + + return ( +
    + + {open ? ( +
    +

    + How cost is estimated +

    +

    + Uncached input × input rate + cached input × cache-read rate + cache + write × write rate + output × output rate. Rates are USD per 1M + tokens. models.dev first-party rows win, then LiteLLM. Unmatched + models are skipped. +

    + {cost ? : null} + {pricing.source === "fallback" ? ( +

    + Live catalog unavailable — using the bundled fallback table. +

    + ) : null} + {pricing.fetchedAt ? ( +

    + Fetched {pricing.fetchedAt} +

    + ) : null} + +
    + ) : null} +
    + ); +} + +function CostBreakdown({ cost }: { cost: UsageCost }) { + return ( +
    + + {cost.cachedReadTokens > 0 ? ( + + ) : ( +

    + No cache-hit split recorded — input billed at the full input rate. +

    + )} + {cost.cacheWriteTokens > 0 ? ( + + ) : null} + + +

    + Matched {cost.matchedId ?? "n/a"} + {cost.usedTotalTokensFallback ? " · used totalTokens as input" : ""} + {cost.cachedReadTokens > 0 && !cost.pricedCachedReads + ? " · cache hits billed at input rate" + : ""} +

    +
    + ); +} + +function BreakdownRow({ label, value }: { label: string; value: string }) { + return ( +
    +
    {label}
    +
    {value}
    +
    + ); +} diff --git a/packages/report-ui/src/app/components/OverviewTab.tsx b/packages/report-ui/src/app/components/OverviewTab.tsx index c8585f6..31f7036 100644 --- a/packages/report-ui/src/app/components/OverviewTab.tsx +++ b/packages/report-ui/src/app/components/OverviewTab.tsx @@ -1,8 +1,13 @@ import type { HarnessRun, ReportCase } from "@vitest-evals/core"; import { formatDuration, formatNumber } from "../model"; +import { estimateUsageCost, formatUsd } from "../pricing"; +import { useReportMeta } from "../report-meta"; import { EmptyState } from "../ui"; +import { CostHelp } from "./CostHelp"; import { DetailContent, DetailSection } from "./DetailLayout"; -import { Fact, FactsGrid, JsonBlock, ScoreValue } from "./ReportPrimitives"; +import { FailureList } from "./FailureList"; +import { JsonInspector } from "./JsonInspector"; +import { Fact, FactsGrid, ScoreValue } from "./ReportPrimitives"; export function OverviewTab({ testCase, @@ -14,7 +19,7 @@ export function OverviewTab({ return ( - + @@ -23,17 +28,7 @@ export function OverviewTab({ - {testCase.failureMessages.length > 0 ? ( -
      - {testCase.failureMessages.map((message) => ( -
    • - {message} -
      • - ))} -
    - ) : ( - No failure messages - )} +     ); @@ -140,7 +135,9 @@ function EvidenceLine({ label, value }: { label: string; value: string }) { } function UsageGrid({ run }: { run: HarnessRun | undefined }) { + const { pricing } = useReportMeta(); const usage = run?.usage; + const cost = estimateUsageCost(usage ?? {}, pricing); return ( @@ -149,6 +146,19 @@ function UsageGrid({ run }: { run: HarnessRun | undefined }) { + + {formatUsd(cost.totalUsd)} + + + ) : ( + "n/a" + ) + } + /> diff --git a/packages/report-ui/src/app/components/ReportChrome.tsx b/packages/report-ui/src/app/components/ReportChrome.tsx index 8749caa..a1ac284 100644 --- a/packages/report-ui/src/app/components/ReportChrome.tsx +++ b/packages/report-ui/src/app/components/ReportChrome.tsx @@ -1,11 +1,19 @@ +import { Link } from "@tanstack/react-router"; import type { ReportRun } from "@vitest-evals/core"; +import type { ReactNode } from "react"; import { + type CaseStatusFilter, formatDuration, formatNumber, formatScore, type summarizeWorkspace, } from "../model"; -import { cx, toneTextClass, type Tone } from "../ui"; +import { formatUsd } from "../pricing"; +import { useReportMeta } from "../report-meta"; +import { toReportSearch } from "../search"; +import { type Tone, cx, toneTextClass } from "../ui"; +import { CostHelp } from "./CostHelp"; +import { PathLabel } from "./PathLabel"; import { executedCaseCount, passRate, @@ -15,15 +23,23 @@ import { export function ReportHeader({ caseCount, runCount, + visibleCaseCount, }: { caseCount: number; runCount: number; + visibleCaseCount: number; }) { return (
    - vitest-evals + toReportSearch({})} + to="/" + > + vitest-evals +
    @@ -43,6 +59,15 @@ export function ReportHeader({ {" "} cases + {visibleCaseCount !== caseCount ? ( + + showing{" "} + + {formatNumber(visibleCaseCount)} + {" "} + of {formatNumber(caseCount)} + + ) : null}
    @@ -50,10 +75,15 @@ export function ReportHeader({ } export function SummaryBar({ + currentStatus, + estimatedCostUsd, summary, }: { + currentStatus: CaseStatusFilter; + estimatedCostUsd?: number; summary: ReturnType; }) { + const { pricing } = useReportMeta(); const verdictTone = passRateTone(summary); return ( @@ -80,12 +110,15 @@ export function SummaryBar({
    - + {summary.failed} {" "} failed - + {summary.caseCount} @@ -112,13 +145,31 @@ export function SummaryBar({
    - - - + + +
    -
    +
    + } + label="Est. cost" + value={formatUsd(estimatedCostUsd)} + />