From 7d3669546d6f950f5c7dca9e7371380eaf690d0a Mon Sep 17 00:00:00 2001 From: Hardik Bhatia Date: Sat, 26 Sep 2026 21:15:47 +0530 Subject: [PATCH 1/2] feat(evaluations): compare saved revisions with reproducible regression reports --- .github/workflows/ci.yml | 2 + apps/control-plane/package.json | 1 + apps/control-plane/src/access.ts | 2 +- apps/control-plane/src/app.ts | 8 +- apps/control-plane/src/evaluations.ts | 131 +++ apps/control-plane/src/reviews.ts | 2 +- apps/control-plane/test/evaluations.test.ts | 38 + apps/dashboard/src/App.tsx | 5 +- apps/dashboard/src/pages/EvaluationsPage.tsx | 39 + apps/gateway/src/app.ts | 144 +-- docs/control-plane.openapi.yaml | 152 +++ docs/deployment.md | 10 + docs/evaluations/README.md | 59 + docs/evaluations/local-smoke-report.json | 1017 ++++++++++++++++++ evaluations/local-smoke.jsonl | 20 + evaluations/semantic-starter.jsonl | 10 + package.json | 3 +- packages/classifiers/src/engine.ts | 130 +++ packages/classifiers/src/evaluation.test.ts | 8 + packages/classifiers/src/evaluation.ts | 25 + packages/classifiers/src/index.ts | 3 + packages/cli/test/contract.test.ts | 2 +- pnpm-lock.yaml | 3 + scripts/evaluate.mjs | 33 + 24 files changed, 1701 insertions(+), 146 deletions(-) create mode 100644 apps/control-plane/src/evaluations.ts create mode 100644 apps/control-plane/test/evaluations.test.ts create mode 100644 apps/dashboard/src/pages/EvaluationsPage.tsx create mode 100644 docs/evaluations/README.md create mode 100644 docs/evaluations/local-smoke-report.json create mode 100644 evaluations/local-smoke.jsonl create mode 100644 evaluations/semantic-starter.jsonl create mode 100644 packages/classifiers/src/engine.ts create mode 100644 packages/classifiers/src/evaluation.test.ts create mode 100644 packages/classifiers/src/evaluation.ts create mode 100644 scripts/evaluate.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9fb659a..5338e4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,6 +38,8 @@ jobs: - run: pnpm install --frozen-lockfile - run: pnpm audit --audit-level=high - run: pnpm run check + - name: Local evaluation regression budget + run: pnpm evaluate --min-accuracy 1 --max-false-positive-rate 0 - name: Install CLI from its packed release outside the repository run: pnpm --filter @delvisor/pyro run test:install - uses: actions/setup-python@v7 diff --git a/apps/control-plane/package.json b/apps/control-plane/package.json index 6fbe4c0..d86e601 100644 --- a/apps/control-plane/package.json +++ b/apps/control-plane/package.json @@ -14,6 +14,7 @@ "dependencies": { "@fastify/cookie": "^11.0.2", "@fastify/websocket": "^11.2.0", + "@pyro/classifiers": "workspace:*", "@pyro/contracts": "workspace:*", "@pyro/integrations": "workspace:*", "@pyro/storage": "workspace:*", diff --git a/apps/control-plane/src/access.ts b/apps/control-plane/src/access.ts index b94075a..f412b01 100644 --- a/apps/control-plane/src/access.ts +++ b/apps/control-plane/src/access.ts @@ -32,7 +32,7 @@ export function accessGuard(database: Database) { if (method === "GET" && ["/api/overview", "/api/usage", "/api/activity", "/api/activity/:id", "/api/apps", "/api/profiles", "/api/profile-presets", "/api/reviews", "/api/reviews/:id", "/api/evaluations", "/api/evaluations/:id", "/api/datasets"].includes(path)) return grant(); if (path.startsWith("/api/reviews/") && user.role === "reviewer") return grant(); if (user.role === "operator") { - if (["/api/evaluations", "/api/evaluations/:id", "/api/datasets", "/api/reviews/:id"].includes(path)) return grant(); + if (["/api/evaluations", "/api/evaluations/:id", "/api/datasets", "/api/datasets/:id", "/api/reviews/:id"].includes(path)) return grant(); if (path === "/api/keys" && method === "GET") return; const body = request.body as { appId?: string } | undefined; const params = request.params as { id?: string }; diff --git a/apps/control-plane/src/app.ts b/apps/control-plane/src/app.ts index 28f1c1b..89d3f27 100644 --- a/apps/control-plane/src/app.ts +++ b/apps/control-plane/src/app.ts @@ -23,6 +23,7 @@ import { createSession, ensureAdmin, sessionUserId, sha256, verifyAdminPassword import type { ControlPlaneConfig } from "./config.js"; import { accessGuard, appScope, canAccessApp, visibleEvent, visibleUser, allowedProfiles } from "./access.js"; +import { registerEvaluations } from "./evaluations.js"; import { registerReviews } from "./reviews.js"; import { registerTeam, verifyPassword } from "./team.js"; import { registerOidc } from "./oidc.js"; @@ -104,7 +105,8 @@ export async function buildControlPlane(config: ControlPlaneConfig): Promise profile.id === request.params.id)) return reply.code(404).send({ error: "Profile not found." }); const [applications, apiKeys] = await Promise.all([appsStore.read(), keysStore.read()]); - const referencedByApp = applications.some((record) => record.defaultProfileId === request.params.id || record.allowedProfileIds.includes(request.params.id)); + const referencedByApp = applications.some((record) => record.defaultProfileId === request.params.id || record.allowedProfileIds.includes(request.params.id) || Boolean(record.profileRevisions?.[request.params.id]) || record.canary?.profileId === request.params.id); const referencedByKey = apiKeys.some((key) => !key.revokedAt && (key.defaultProfileId === request.params.id || key.allowedProfileIds?.includes(request.params.id))); const referencedByShadow = profiles.some((profile) => profile.id !== request.params.id && profile.shadowProfileIds.includes(request.params.id)); if (referencedByApp || referencedByKey || referencedByShadow) { @@ -553,6 +555,8 @@ export async function buildControlPlane(config: ControlPlaneConfig): Promise { clearInterval(poller); + await stopEvaluations(); + await stopReviews(); await database.close(); }); diff --git a/apps/control-plane/src/evaluations.ts b/apps/control-plane/src/evaluations.ts new file mode 100644 index 0000000..b8bc02d --- /dev/null +++ b/apps/control-plane/src/evaluations.ts @@ -0,0 +1,131 @@ +import { createHash, randomUUID } from "node:crypto"; +import { z } from "zod"; +import type { FastifyInstance, FastifyReply, FastifyRequest } from "fastify"; +import type { AppRecord, Profile, ProviderSettings, StoredSecret } from "@pyro/contracts"; +import { createDefaultProviderSettings } from "@pyro/contracts"; +import { DurableJobs, DurableWorker, decryptText, encryptText, policyHash, revisionOf, type Database, type PolicyRecord } from "@pyro/storage"; +import { evaluatePolicy, evaluationReport, type EvaluationRow } from "@pyro/classifiers"; +import type { ControlPlaneConfig } from "./config.js"; +import { canAccessApp } from "./access.js"; +const digest = (value: unknown) => createHash("sha256").update(JSON.stringify(value)).digest("hex"); +const CaseSchema = z.object({ id: z.string().min(1).max(100), input: z.unknown().refine((value) => value !== undefined, "input is required"), expected: z.enum(["allow", "review", "block"]), category: z.string().min(1).max(100).default("general") }); +type Case = z.infer; +interface Dataset { id: string; appId: string; name: string; version: number; contentHash: string; count: number; createdAt: string; expiresAt: string; actorId: string; cases: StoredSecret } +interface Run { id: string; appId: string; datasetId: string; datasetHash: string; status: "queued" | "running" | "cancelled" | "complete" | "failed"; createdAt: string; actorId: string; expiresAt: string; profiles: Profile[]; app: AppRecord; provider: ProviderSettings; configurationHash: string; credential?: StoredSecret; rows: EvaluationRow[]; total: number; error?: string; report?: ReturnType; estimate: { maximumProviderCalls: number; cost: string }; generation: number; jobId?: string } +const publicDataset = ({ cases, ...dataset }: Dataset) => dataset; +const publicRun = ({ credential, ...run }: Run) => run; +export function registerEvaluations(app: FastifyInstance, database: Database, config: ControlPlaneConfig, guard: (r: FastifyRequest, p: FastifyReply) => Promise) { + const datasets = database.document("evaluation_datasets", () => []); + const runs = database.document("evaluation_runs", () => []); + const jobs = new DurableJobs(database, config.controlPlaneSecret, "evaluation_jobs", 20); + const enqueue = async (run: Run) => { + const job = await jobs.enqueue({ id: randomUUID(), appId: run.appId, fingerprint: `${run.id}:${run.generation}`, idempotencyKey: `${run.id}:${run.generation}`, input: { runId: run.id, generation: run.generation } }); + await runs.update((rows) => rows.map((r) => r.id === run.id && r.generation === run.generation ? { ...r, jobId: job.id } : r)); + }; + app.get("/api/datasets", { preHandler: guard }, async (request) => ({ datasets: (await datasets.read()).filter((d) => canAccessApp(request.user!, d.appId) && Date.parse(d.expiresAt) > Date.now()).map(publicDataset) })); + app.post("/api/datasets", { preHandler: guard }, async (request, reply) => { + const parsed = z.object({ appId: z.string(), name: z.string().trim().min(1).max(100), jsonl: z.string().min(1).max(500_000), retentionDays: z.union([z.literal(1), z.literal(7), z.literal(30)]), retainInputs: z.literal(true) }).safeParse(request.body); + if (!parsed.success) return reply.code(400).send({ error: "Provide appId, name, JSONL, retentionDays (1, 7 or 30), and explicit retainInputs: true." }); + const body = parsed.data; + if (!canAccessApp(request.user!, body.appId)) return reply.code(403).send({ error: "Application access required." }); + if (!(await database.document("apps", () => []).read()).some((a) => a.id === body.appId)) return reply.code(404).send({ error: "Application not found." }); + let cases: Case[]; + try { cases = body.jsonl.split(/\r?\n/).filter((line) => line.trim()).map((line) => CaseSchema.parse(JSON.parse(line))); if (!cases.length || cases.length > 500 || new Set(cases.map((c) => c.id)).size !== cases.length) throw new Error("Use 1–500 cases with unique IDs."); } + catch (e) { return reply.code(400).send({ error: e instanceof Error ? e.message : "Invalid dataset." }); } + let dataset!: Dataset; + await datasets.update((rows) => { + dataset = { id: randomUUID(), appId: body.appId, name: body.name, version: 1 + Math.max(0, ...rows.filter((d) => d.appId === body.appId && d.name === body.name).map((d) => d.version)), contentHash: digest(cases), count: cases.length, createdAt: new Date().toISOString(), expiresAt: new Date(Date.now() + body.retentionDays * 86400_000).toISOString(), actorId: request.user!.id, cases: encryptText(JSON.stringify(cases), config.controlPlaneSecret) }; + return [...rows, dataset]; + }); + return reply.code(201).send({ dataset: publicDataset(dataset) }); + }); + app.delete<{ Params: { id: string } }>("/api/datasets/:id", { preHandler: guard }, async (request, reply) => { + const dataset = (await datasets.read()).find((d) => d.id === request.params.id && canAccessApp(request.user!, d.appId)); + if (!dataset) return reply.code(404).send({ error: "Dataset not found." }); + await runs.update((rows) => rows.map((r) => r.datasetId === dataset.id && ["queued", "running"].includes(r.status) ? { ...r, status: "cancelled", credential: undefined, error: "Dataset deleted." } : r)); + await datasets.update((rows) => rows.filter((d) => d.id !== dataset.id)); + return reply.code(204).send(); + }); + app.get("/api/evaluations", { preHandler: guard }, async (request) => ({ runs: (await runs.read()).filter((r) => canAccessApp(request.user!, r.appId)).map((r) => ({ ...publicRun(r), rows: undefined })) })); + app.get<{ Params: { id: string } }>("/api/evaluations/:id", { preHandler: guard }, async (request, reply) => { + const run = (await runs.read()).find((r) => r.id === request.params.id && canAccessApp(request.user!, r.appId)); + if (!run) return reply.code(404).send({ error: "Evaluation not found." }); + return { run: publicRun(run) }; + }); + app.post("/api/evaluations", { preHandler: guard }, async (request, reply) => { + const parsed = z.object({ datasetId: z.string(), policies: z.array(z.object({ id: z.string(), revision: z.number().int().positive() })).min(1).max(2), allowPaid: z.boolean().default(false) }).safeParse(request.body); + if (!parsed.success) return reply.code(400).send({ error: "Choose a dataset and one or two published policy revisions." }); + const body = parsed.data; + if (new Set(body.policies.map((p) => `${p.id}@${p.revision}`)).size !== body.policies.length) return reply.code(400).send({ error: "Choose distinct policy revisions." }); + const dataset = (await datasets.read()).find((d) => d.id === body.datasetId && Date.parse(d.expiresAt) > Date.now() && canAccessApp(request.user!, d.appId)); + if (!dataset) return reply.code(404).send({ error: "Dataset not found or expired." }); + const application = (await database.document("apps", () => []).read()).find((a) => a.id === dataset.appId && a.enabled); + if (!application) return reply.code(400).send({ error: "Application is unavailable." }); + const records = await database.document("profiles", () => []).read(); + const profiles = body.policies.map((p) => { const record = records.find((r) => r.id === p.id); return record && revisionOf(record, p.revision); }); + if (profiles.some((p) => !p || application.allowedProfileIds.length && !application.allowedProfileIds.includes(p.id))) return reply.code(400).send({ error: "All revisions must be published and allowed by the application." }); + const provider = await database.document("provider_settings", createDefaultProviderSettings).read(); + const semanticCount = profiles.filter((p) => p!.detectors.some((d) => d.enabled)).length; + const maximumProviderCalls = semanticCount * dataset.count * (1 + provider.maxRetries); + const paid = semanticCount > 0 && provider.mode !== "mock"; + if (paid && !body.allowPaid) return reply.code(400).send({ error: `This run may send dataset inputs to TypeSafe in up to ${maximumProviderCalls} provider attempts. Cost is unknown. Explicit allowPaid: true is required.` }); + const credential = paid ? config.typesafeApiKey ? encryptText(config.typesafeApiKey, config.controlPlaneSecret) : (await database.document<{ typesafeApiKey?: StoredSecret }>("provider_secrets", () => ({})).read()).typesafeApiKey : undefined; + if (paid && !credential) return reply.code(400).send({ error: "Configure the TypeSafe API key before running semantic evaluations." }); + const run: Run = { id: randomUUID(), appId: dataset.appId, datasetId: dataset.id, datasetHash: dataset.contentHash, createdAt: new Date().toISOString(), actorId: request.user!.id, expiresAt: dataset.expiresAt, profiles: profiles as Profile[], app: application, provider, configurationHash: digest({ profiles, application, provider }), credential, status: "queued", rows: [], total: dataset.count * profiles.length, estimate: { maximumProviderCalls: provider.mode === "mock" ? 0 : maximumProviderCalls, cost: paid ? "Unknown; provider billing applies." : "No external provider charges." }, generation: 1 }; + await runs.update((rows) => [...rows, run]); + // A maintenance pass repairs an interrupted enqueue from the persisted run. + try { await enqueue(run); } catch (e) { app.log.error(e, "Evaluation enqueue will retry"); } + return reply.code(202).send({ run: publicRun(run) }); + }); + app.put<{ Params: { id: string } }>("/api/evaluations/:id", { preHandler: guard }, async (request, reply) => { + const parsed = z.object({ action: z.enum(["cancel", "resume"]) }).safeParse(request.body); + if (!parsed.success) return reply.code(400).send({ error: "Choose cancel or resume." }); + let updated: Run | undefined; + await runs.update((rows) => rows.map((r) => { + if (r.id !== request.params.id || !canAccessApp(request.user!, r.appId) || r.status === "complete") return r; + if (parsed.data.action === "resume" && !["cancelled", "failed"].includes(r.status)) return r; + updated = parsed.data.action === "cancel" ? { ...r, status: "cancelled" } : { ...r, status: "queued", generation: r.generation + 1, jobId: undefined, error: undefined }; + return updated; + })); + if (!updated) return reply.code(409).send({ error: "Evaluation is unavailable or already in the requested state." }); + return { run: publicRun(updated) }; + }); + const worker = new DurableWorker(jobs, 1, async (job) => { + const { runId, generation } = jobs.input<{ runId: string; generation: number }>(job); + let run = (await runs.read()).find((r) => r.id === runId); + if (!run || run.generation !== generation || run.status === "cancelled") return; + const dataset = (await datasets.read()).find((d) => d.id === run!.datasetId && Date.parse(d.expiresAt) > Date.now()); + if (!dataset) throw new Error("Dataset expired or was deleted."); + const cases = JSON.parse(decryptText(dataset.cases, config.controlPlaneSecret)) as Case[]; + await runs.update((rows) => rows.map((r) => r.id === runId && r.generation === generation && r.status === "queued" ? { ...r, status: "running" } : r)); + for (const sample of cases) for (const profile of run.profiles) { + run = (await runs.read()).find((r) => r.id === runId)!; + if (!run || run.generation !== generation || run.status === "cancelled") return; + if (Date.parse(run.expiresAt) <= Date.now() || Date.parse(job.expiresAt) <= Date.now()) throw new Error("Evaluation deadline or dataset retention expired. Resume with a retained dataset."); + if (run.rows.some((r) => r.caseId === sample.id && r.profileId === profile.id && r.revision === profile.revision)) continue; + if (JSON.stringify(sample.input).length > profile.maxInputChars) throw new Error(`Case ${sample.id} exceeds policy input limit.`); + const result = await evaluatePolicy({ id: randomUUID(), envelope: { input: sample.input }, profile, traceId: runId, firewallApp: run.app, provider: run.provider, apiKey: async () => run!.credential ? decryptText(run!.credential, config.controlPlaneSecret) : undefined }); + delete result.decision.metadata; + result.decision.policyRevision = profile.revision; result.decision.policyHash = profile.contentHash ?? policyHash(profile); + const row: EvaluationRow = { caseId: sample.id, category: sample.category, expected: sample.expected, profileId: profile.id, revision: profile.revision!, inputHash: digest(sample.input), decision: result.decision }; + await runs.update((rows) => rows.map((r) => r.id === runId && r.generation === generation && !r.rows.some((old) => old.caseId === row.caseId && old.profileId === row.profileId && old.revision === row.revision) ? { ...r, rows: [...r.rows, row] } : r)); + } + await runs.update((rows) => rows.map((r) => r.id === runId && r.generation === generation && r.status !== "cancelled" ? { ...r, status: "complete", report: evaluationReport(r.rows), credential: undefined } : r)); + return { runId }; + }, (e) => app.log.error(e, "Evaluation worker failed")); + worker.start(); + let maintenance: Promise | undefined; + const maintain = async () => { + await datasets.update((rows) => rows.filter((d) => Date.parse(d.expiresAt) > Date.now())); + await runs.update((rows) => rows.filter((r) => Date.parse(r.expiresAt) > Date.now())); + for (const run of await runs.read()) { + if (run.status === "queued" && !run.jobId) await enqueue(run); + if (run.jobId && ["queued", "running"].includes(run.status)) { + const job = await jobs.get(run.jobId, run.appId); + if (!job || job.status === "failed") await runs.update((rows) => rows.map((r) => r.id === run.id && r.generation === run.generation ? { ...r, status: "failed", error: job?.error ?? "Worker result expired; resume to continue saved rows." } : r)); + } + } + }; + const timer = setInterval(() => { if (!maintenance) maintenance = maintain().catch((e) => app.log.error(e, "Evaluation maintenance failed")).finally(() => { maintenance = undefined; }); }, 500); timer.unref(); + return async () => { clearInterval(timer); await maintenance; await worker.stop(); }; +} diff --git a/apps/control-plane/src/reviews.ts b/apps/control-plane/src/reviews.ts index 76af3d5..839a707 100644 --- a/apps/control-plane/src/reviews.ts +++ b/apps/control-plane/src/reviews.ts @@ -73,5 +73,5 @@ export function registerReviews(app: FastifyInstance, database: Database, guard: await store.update((rows) => rows.filter((r) => r.updatedAt >= cutoff || r.pendingCallbacks?.length)); }; const timer = setInterval(() => { if (!active) { active = flush().catch((e) => app.log.error(e, "Review callback handoff failed")).finally(() => { active = undefined; }); } }, 1000); timer.unref(); - app.addHook("onClose", async () => { clearInterval(timer); await active; }); + return async () => { clearInterval(timer); await active; }; } diff --git a/apps/control-plane/test/evaluations.test.ts b/apps/control-plane/test/evaluations.test.ts new file mode 100644 index 0000000..de98d26 --- /dev/null +++ b/apps/control-plane/test/evaluations.test.ts @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { randomUUID } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import { openDatabase } from "@pyro/storage"; +import { buildControlPlane } from "../src/app.js"; +test("evaluation compares immutable revisions, resumes checkpoints, encrypts inputs and requires paid consent", async (t) => { + const config = { host: "127.0.0.1", port: 0, databaseUrl: `memory://evaluation-${randomUUID()}`, adminPassword: "correct-horse-battery-staple", controlPlaneSecret: "control-plane-test-secret", gatewayInternalUrl: "http://127.0.0.1:1", gatewayApiKey: "test-key", typesafeEndpoint: "https://api.typesafe.ai/v1/systemone", typesafeModel: "jev-latest" }; + const app = await buildControlPlane(config); t.after(() => app.close()); const db = await openDatabase(config.databaseUrl); + const login = await app.inject({ method: "POST", url: "/api/auth/login", payload: { password: config.adminPassword } }); const headers = { cookie: login.headers["set-cookie"]!.split(";")[0]! }; + const post = (url: string, payload: unknown) => app.inject({ method: "POST", url, headers, payload: payload as object }); + const yaml = await readFile(new URL("../../../profiles/local-secrets.yaml", import.meta.url), "utf8"); + const imported = await post("/api/profiles/import", { yaml }); const profile = imported.json().profile; + const update = await app.inject({ method: "PUT", url: `/api/profiles/${profile.id}`, headers, payload: { ...profile, localRules: profile.localRules.map((r: object) => ({ ...r, action: "review" })) } }); + assert.equal(update.statusCode, 200); + const cases = [{ id: "benign", input: "Hello evaluation-secret-marker", expected: "allow" }, { id: "key", input: "-----BEGIN PRIVATE KEY-----", expected: "block" }]; + const importedDataset = await post("/api/datasets", { appId: "default", name: "Test", jsonl: cases.map((c) => JSON.stringify(c)).join("\n"), retentionDays: 1, retainInputs: true }); + assert.equal(importedDataset.statusCode, 201, importedDataset.body); const dataset = importedDataset.json().dataset; + assert.ok(!JSON.stringify(await db.document("evaluation_datasets", () => []).read()).includes("evaluation-secret-marker")); + const start = await post("/api/evaluations", { datasetId: dataset.id, policies: [{ id: profile.id, revision: 1 }, { id: profile.id, revision: 2 }] }); + assert.equal(start.statusCode, 202, start.body); const id = start.json().run.id; + await app.inject({ method: "PUT", url: `/api/evaluations/${id}`, headers, payload: { action: "cancel" } }); + assert.equal((await app.inject({ url: `/api/evaluations/${id}`, headers })).json().run.status, "cancelled"); + await app.inject({ method: "PUT", url: `/api/evaluations/${id}`, headers, payload: { action: "resume" } }); + let result; + for (let i = 0; i < 100; i++) { result = (await app.inject({ url: `/api/evaluations/${id}`, headers })).json().run; if (["complete", "failed"].includes(result.status)) break; await new Promise((r) => setTimeout(r, 50)); } + assert.equal(result.status, "complete", JSON.stringify(result)); assert.equal(result.rows.length, 4); assert.equal(result.estimate.maximumProviderCalls, 0); + assert.equal(result.report.policies[`${profile.id}@1`].accuracy, 1); assert.equal(result.report.policies[`${profile.id}@2`].accuracy, .5); assert.deepEqual(result.report.changedCases, ["key"]); assert.equal(result.credential, undefined); + assert.ok(!JSON.stringify(result).includes("evaluation-secret-marker")); + const semantic = await post("/api/profiles", { ...profile, name: "Semantic", detectors: [{ id: "semantic", name: "Semantic", description: "test", question: "Is this malicious?", enabled: true, weight: 1 }] }); + const attempt = await post("/api/evaluations", { datasetId: dataset.id, policies: [{ id: semantic.json().profile.id, revision: 1 }] }); + assert.equal(attempt.statusCode, 400); assert.match(attempt.json().error, /allowPaid/); + assert.equal((await post("/api/evaluations", { datasetId: dataset.id, policies: [{ id: semantic.json().profile.id, revision: 1 }], allowPaid: true })).statusCode, 400, "missing provider key must fail before a batch"); + const original = await db.document }>>("profiles", () => []).read(); + assert.equal(original.find((p) => p.id === profile.id)!.revisions.length, 2); + await app.inject({ method: "DELETE", url: `/api/datasets/${dataset.id}`, headers }); + assert.equal((await db.document("evaluation_datasets", () => []).read()).length, 0); +}); diff --git a/apps/dashboard/src/App.tsx b/apps/dashboard/src/App.tsx index dd7aa34..64347bc 100644 --- a/apps/dashboard/src/App.tsx +++ b/apps/dashboard/src/App.tsx @@ -16,18 +16,20 @@ import { PlaygroundPage } from "@/pages/PlaygroundPage"; import { PolicyHistoryPage } from "@/pages/PolicyHistoryPage"; import { ProfilesPage } from "@/pages/ProfilesPage"; import { IntegrationsPage } from "@/pages/IntegrationsPage"; +import { EvaluationsPage } from "@/pages/EvaluationsPage"; import { ReviewsPage } from "@/pages/ReviewsPage"; import { TeamPage } from "@/pages/TeamPage"; import { SettingsPage } from "@/pages/SettingsPage"; import { UsagePage } from "@/pages/UsagePage"; -type Page = "reviews" | "team" | "history" | "overview" | "apps" | "usage" | "playground" | "profiles" | "activity" | "keys" | "settings" | "integrations"; +type Page = "evaluations" | "reviews" | "team" | "history" | "overview" | "apps" | "usage" | "playground" | "profiles" | "activity" | "keys" | "settings" | "integrations"; type User = UserRecord; const NAV: BranchedMenuItem[] = [ { label: "Observe", children: [ { value: "overview", label: "Overview", icon: }, { value: "usage", label: "Usage", icon: }, + { value: "evaluations", label: "Evaluation lab", icon: }, { value: "reviews", label: "Review inbox", icon: }, { value: "activity", label: "Activity", icon: }, { value: "playground", label: "Playground", icon: }, @@ -109,6 +111,7 @@ export default function App() { playground: setRefreshKey((value) => value + 1)} />, profiles: , history: , + evaluations: , reviews: , activity: , keys: , diff --git a/apps/dashboard/src/pages/EvaluationsPage.tsx b/apps/dashboard/src/pages/EvaluationsPage.tsx new file mode 100644 index 0000000..ddd7968 --- /dev/null +++ b/apps/dashboard/src/pages/EvaluationsPage.tsx @@ -0,0 +1,39 @@ +import { useEffect, useState } from "react"; +import type { AppRecord, Profile } from "@pyro/contracts"; +import { api } from "@/lib/api"; +import { PageHeader } from "@/components/shared"; +import { Button } from "@/components/ui/button"; +import { Card, CardContent } from "@/components/ui/card"; +import { Input } from "@/components/ui/input"; +import { Textarea } from "@/components/ui/textarea"; +interface Dataset { id: string; name: string; version: number; count: number; appId: string; contentHash: string; expiresAt: string } +interface Stats { count: number; accuracy: number | null; falsePositiveRate: number | null; indeterminate: number; p50LatencyMs: number | null; p95LatencyMs: number | null; reportedCosts: Record; costUnreported: number; confusion: Record> } +interface Run { id: string; status: string; total: number; datasetHash: string; configurationHash: string; createdAt: string; error?: string; rows?: unknown[]; report?: { policies: Record; disagreements: unknown[]; changedCases: string[] }; estimate: { maximumProviderCalls: number; cost: string } } +const selectStyle = "border border-line bg-surface p-2 text-sm"; +export function EvaluationsPage({ canRun }: { canRun: boolean }) { + const [datasets, setDatasets] = useState([]), [runs, setRuns] = useState([]), [apps, setApps] = useState([]), [profiles, setProfiles] = useState([]); + const [appId, setAppId] = useState(""), [name, setName] = useState(""), [jsonl, setJsonl] = useState(""), [retention, setRetention] = useState(7), [retain, setRetain] = useState(false); + const [datasetId, setDatasetId] = useState(""), [first, setFirst] = useState(""), [second, setSecond] = useState(""), [revision, setRevision] = useState(1), [otherRevision, setOtherRevision] = useState(1), [paid, setPaid] = useState(false); + const [selected, setSelected] = useState(), [message, setMessage] = useState(""), [busy, setBusy] = useState(false); + const load = async () => { const [d, r, a, p] = await Promise.all([api.get<{ datasets: Dataset[] }>("/api/datasets"), api.get<{ runs: Run[] }>("/api/evaluations"), api.get<{ apps: AppRecord[] }>("/api/apps"), api.get<{ profiles: Profile[] }>("/api/profiles")]); setDatasets(d.datasets); setRuns(r.runs); setApps(a.apps); setProfiles(p.profiles); setAppId((id) => id || a.apps[0]?.id || ""); setDatasetId((id) => id || d.datasets[0]?.id || ""); }; + const inspect = async (id: string) => { const { run } = await api.get<{ run: Run }>(`/api/evaluations/${id}`); setSelected(run); }; + useEffect(() => { void load().catch((e) => setMessage(e.message)); }, []); + useEffect(() => { if (!selected || !["queued", "running"].includes(selected.status)) return; const timer = setInterval(() => void inspect(selected.id).catch((e) => setMessage(e.message)), 1500); return () => clearInterval(timer); }, [selected?.id, selected?.status]); + const work = async (fn: () => Promise) => { setBusy(true); setMessage(""); try { await fn(); await load(); } catch (e) { setMessage(e instanceof Error ? e.message : "Request failed."); } finally { setBusy(false); } }; + const changeProfile = (id: string, other = false) => { const rev = profiles.find((p) => p.id === id)?.revision ?? 1; if (other) { setSecond(id); setOtherRevision(rev); } else { setFirst(id); setRevision(rev); } }; + const dataset = datasets.find((d) => d.id === datasetId); + const semantic = [first, second].some((id) => profiles.find((p) => p.id === id)?.detectors.some((d) => d.enabled)); + const download = () => { const url = URL.createObjectURL(new Blob([JSON.stringify(selected, null, 2)], { type: "application/json" })); const link = document.createElement("a"); link.href = url; link.download = `pyro-evaluation-${selected!.id}.json`; link.click(); URL.revokeObjectURL(url); }; + const percent = (v: number | null) => v === null ? "—" : `${(v * 100).toFixed(1)}%`; + return
+ {message &&

{message}

} + {canRun &&

Import a dataset version

JSONL: one object per line with id, input, expected (allow / review / block), and optional category. Maximum 500 cases. Reimport the same name to create a new version.

{ const file = e.target.files?.[0]; if (file) void file.text().then(setJsonl).catch((e) => setMessage(e.message)); }} />