diff --git a/docs/.vitepress/config.mts b/docs/.vitepress/config.mts index ad520f64..cd1b3855 100644 --- a/docs/.vitepress/config.mts +++ b/docs/.vitepress/config.mts @@ -73,6 +73,7 @@ export default defineConfig({ { text: 'Agent Tracing', link: '/guide/tracing' }, { text: 'Guardrails', link: '/guide/guardrails' }, { text: 'PII Service', link: '/guide/pii' }, + { text: 'Evaluation & Analysis', link: '/guide/evaluation-and-analysis' }, { text: 'RAG', link: '/guide/rag' }, { text: 'Prompts', link: '/guide/prompts' }, { text: 'Memory', link: '/guide/memory' }, @@ -131,6 +132,8 @@ export default defineConfig({ { text: 'Files', link: '/api/files' }, { text: 'Guardrails', link: '/api/guardrails' }, { text: 'PII', link: '/api/pii' }, + { text: 'Evaluation', link: '/api/evaluation' }, + { text: 'Analysis', link: '/api/analysis' }, { text: 'Prompts', link: '/api/prompts' }, { text: 'RAG', link: '/api/rag' }, { text: 'Memory', link: '/api/memory' }, diff --git a/docs/api/analysis.md b/docs/api/analysis.md new file mode 100644 index 00000000..a97e727a --- /dev/null +++ b/docs/api/analysis.md @@ -0,0 +1,166 @@ +# Analysis API + +Conversation analysis: extract structured fields from transcripts, judge +quality against a rubric, and score extraction accuracy against ground truth — +on demand or on a nightly cron. Built for use cases like IVR/call-center review. + +All endpoints are under `/api/analysis/*` and are session-authenticated. Requests +are tenant- and project-scoped from the session. + +## Concepts + +``` +definition → fieldSet + extraction prompt + modes (+ models, + schedule) +conversation → an ingested transcript { role, content }[] (+ referenceFields) +run → one execution of a definition over a set of conversations +``` + +### Modes + +| Mode | Effect | +|---|---| +| `extract` | Always on — pulls the `fieldSet` out of each transcript as typed JSON. | +| `store` | Writes the extracted fields back onto each conversation (`extractedFields`, `lastAnalyzedAt`). | +| `judge` | An LLM grades each conversation against `judge.rubric` (0–1). | +| `accuracy` | Compares extracted fields to the conversation's `referenceFields`, per field. | + +## Definitions + +```http +GET /api/analysis/definitions?search=intent +POST /api/analysis/definitions +GET /api/analysis/definitions/:id +PATCH /api/analysis/definitions/:id +DELETE /api/analysis/definitions/:id +``` + +### Create + +```json +{ + "name": "Call intent & resolution", + "fieldSet": [ + { "key": "intent", "type": "enum", "enumValues": ["billing", "support"], "required": true }, + { "key": "resolved", "type": "boolean" } + ], + "extractionInstructions": "Focus on the caller's primary reason.", + "modes": { "store": true, "accuracy": true, "judge": { "rubric": "Was the caller helped politely?" } }, + "extractionModelKey": "gpt-4o-mini", + "judgeModelKey": "gpt-4o", + "schedule": { "cron": "0 2 * * *", "enabled": true } +} +``` + +| Field | Type | Required | Notes | +|---|---|---|---| +| `name` | string | yes | `key` is slugified from it. | +| `fieldSet` | array | yes | Each `{ key, type, description?, enumValues?, required? }`; `type` ∈ `string \| number \| boolean \| enum`. `enum` needs `enumValues`. | +| `modes` | object | yes | `{ store?, accuracy?, judge?: { rubric, threshold? } }`. | +| `extractionModelKey` | string | recommended | Model used for extraction (required at run time). | +| `judgeModelKey` | string | when `modes.judge` | Model used for grading. | +| `schedule` | object | no | `{ cron, enabled }`. Validated with a standard 5-field cron expression (UTC). | + +## Conversations + +```http +GET /api/analysis/conversations?search=refund&limit=100 +POST /api/analysis/conversations +GET /api/analysis/conversations/:id +DELETE /api/analysis/conversations/:id +``` + +### Ingest + +Accepts a single conversation or `{ "conversations": [...] }` for bulk import +(e.g. an external call export). + +```json +{ + "conversations": [ + { + "name": "Call 1042", + "transcript": [ + { "role": "caller", "content": "I was charged twice." }, + { "role": "agent", "content": "I've issued a refund." } + ], + "referenceFields": { "intent": "billing", "resolved": true } + } + ] +} +``` + +| Field | Type | Required | Notes | +|---|---|---|---| +| `transcript` | array | yes | Non-empty `{ role, content }[]`. | +| `referenceFields` | object | no | Ground truth for the accuracy mode. | +| `name` | string | no | Display name; `key` is slugified/auto-generated. | +| `source` | `imported \| platform \| manual` | no | Defaults to `imported`. | + +## Runs + +### Run a definition + +```http +POST /api/analysis/definitions/:key/run +``` + +```json +{ "conversationKeys": ["call-1042", "call-1043"] } +``` + +Omit `conversationKeys` to analyze the most recent corpus (up to 500 +conversations). Extraction runs per conversation with bounded concurrency, then +optional judge and accuracy; the run is persisted with an aggregate. + +#### Response + +```json +{ + "run": { + "id": "…", + "definitionKey": "call-intent-resolution", + "status": "completed", + "aggregate": { "total": 120, "completed": 118, "failed": 2, "passed": 110, "passRate": 0.932, "avgJudgeScore": 0.88, "avgExtractionAccuracy": 0.91 }, + "items": [ + { "conversationKey": "call-1042", "passed": true, "extractedFields": { "intent": "billing", "resolved": true }, "missing": [], "judge": { "score": 0.9, "passed": true }, "accuracy": { "score": 1, "comparedCount": 2, "perField": { "intent": { "expected": "billing", "actual": "billing", "match": true } } } } + ] + } +} +``` + +`avgJudgeScore` / `avgExtractionAccuracy` are `null` when no item used that mode. + +### List / get runs + +```http +GET /api/analysis/runs?definitionKey=call-intent-resolution&limit=50 +GET /api/analysis/runs/:id +``` + +## Scheduling + +When a definition has `schedule.enabled` with a valid cron, the background +**analysis scheduler** fires it automatically (e.g. `0 2 * * *` = 02:00 UTC +nightly). Each cron slot fires at most once, decided against the most recent +run. See the [guide](/guide/evaluation-and-analysis#automation). + +## Alerting + +Run aggregates feed the alert system through the `analysis` module: + +| Metric | Source | +|---|---| +| `analysis_pass_rate` | `aggregate.passRate` × 100, averaged over completed runs in the window. | +| `analysis_avg_judge_score` | `aggregate.avgJudgeScore` × 100. | +| `analysis_avg_accuracy` | `aggregate.avgExtractionAccuracy` × 100. | + +A rule like `analysis_avg_accuracy lt 85` over 24h notifies you when extraction +quality drops on the nightly run. + +## Errors + +| Status | Cause | +|---|---| +| 400 | Missing `name`, empty/invalid `fieldSet`, enum without `enumValues`, bad cron, transcript missing `role`/`content`. | +| 404 | Definition / conversation / run not found (or unknown definition `key` on run). | +| 500 | Internal error. | diff --git a/docs/api/evaluation.md b/docs/api/evaluation.md new file mode 100644 index 00000000..11460c21 --- /dev/null +++ b/docs/api/evaluation.md @@ -0,0 +1,173 @@ +# Evaluation API + +Offline testing for models and agents. Define a **target** (what to test), a +**dataset** (the test cases), and a **suite** (binds a target + dataset + one or +more scorers), then **run** the suite to produce a scored result. + +All endpoints are under `/api/evaluation/*` and are session-authenticated +(dashboard surface). Requests are tenant- and project-scoped from the session. + +## Concepts + +``` +target → a model | agent | external endpoint under test +dataset → an ordered list of items { input messages, expected? } +suite → target + dataset + scorers[] (+ judge model) +run → one execution of a suite over its dataset, with aggregate + per-item scores +``` + +### Scorers + +| Type | What it checks | +|---|---| +| `assertion` | Deterministic checks against `expected`: `mustContain`, `equals`, `regex`, `jsonSchema`, `jsonPath`. | +| `llm-judge` | An LLM grades the output against a `rubric` (0–1), backed by `judgeModelKey`. | + +A run's per-item `score` is the weighted mean of its scorer scores; `passed` is +true when every scorer passes. The aggregate reports `passRate`, `avgScore`, and +`avgLatencyMs`. + +## Targets + +```http +GET /api/evaluation/targets?kind=model&search=gpt +POST /api/evaluation/targets +GET /api/evaluation/targets/:id +PATCH /api/evaluation/targets/:id +DELETE /api/evaluation/targets/:id +``` + +### Create + +```json +{ + "name": "GPT-4o production", + "kind": "model", + "modelKey": "gpt-4o" +} +``` + +| Field | Type | Required | Notes | +|---|---|---|---| +| `name` | string | yes | Display name; `key` is slugified from it. | +| `kind` | `model \| agent \| external` | yes | `model` is live; `agent`/`external` are recorded as per-item errors until their adapters ship. | +| `modelKey` | string | for `model` | Registered model key. | +| `agentKey` | string | for `agent` | Registered agent key. | + +## Datasets + +```http +GET /api/evaluation/datasets?search=faq +POST /api/evaluation/datasets +GET /api/evaluation/datasets/:id +PATCH /api/evaluation/datasets/:id +DELETE /api/evaluation/datasets/:id +``` + +### Create + +```json +{ + "name": "FAQ regression", + "items": [ + { + "id": "q1", + "input": [{ "role": "user", "content": "What is 2+2?" }], + "expected": { "mustContain": ["4"] } + } + ] +} +``` + +`items[].input` is an array of `{ role, content }` chat messages. `expected` is +optional and consumed by the assertion scorer. + +## Suites + +```http +GET /api/evaluation/suites?search=faq +POST /api/evaluation/suites +GET /api/evaluation/suites/:id +PATCH /api/evaluation/suites/:id +DELETE /api/evaluation/suites/:id +``` + +### Create + +```json +{ + "name": "FAQ accuracy", + "targetKey": "gpt-4o-production", + "datasetKey": "faq-regression", + "scorers": [ + { "type": "assertion" }, + { "type": "llm-judge", "rubric": "Answer is correct and concise." } + ], + "judgeModelKey": "gpt-4o" +} +``` + +| Field | Type | Required | Notes | +|---|---|---|---| +| `targetKey` / `datasetKey` | string | yes | Keys of an existing target and dataset. | +| `scorers` | array | yes | Non-empty; each `{ type, weight?, rubric?, threshold? }`. | +| `judgeModelKey` | string | when an `llm-judge` scorer is present | Model used for grading. | +| `runConfig.concurrency` | number | no | Parallel items per run. | + +## Runs + +### Run a suite + +```http +POST /api/evaluation/suites/:key/run +``` + +Loads the suite, target, and dataset, executes the target over every item with +bounded concurrency, scores each output, persists the run, and returns it. + +#### Response + +```json +{ + "run": { + "id": "…", + "suiteKey": "faq-accuracy", + "status": "completed", + "aggregate": { "total": 12, "completed": 12, "failed": 0, "passed": 11, "passRate": 0.916, "avgScore": 0.94, "avgLatencyMs": 410 }, + "items": [ + { "itemId": "q1", "passed": true, "score": 1, "scores": [ { "scorerType": "assertion", "score": 1, "passed": true, "weight": 1 } ], "output": { "text": "4", "latencyMs": 380 } } + ] + } +} +``` + +A target/judge error on an item is recorded on that item (`error`) and counted +in `aggregate.failed`; it does not abort the run. + +### List / get runs + +```http +GET /api/evaluation/runs?suiteKey=faq-accuracy&limit=50 +GET /api/evaluation/runs/:id +``` + +## Alerting + +Run aggregates feed the alert system through the `evaluation` module: + +| Metric | Source | +|---|---| +| `evaluation_pass_rate` | `aggregate.passRate` × 100, averaged over completed runs in the window. | +| `evaluation_avg_score` | `aggregate.avgScore` × 100, same. | + +Create an alert rule (module `evaluation`) such as `evaluation_pass_rate lt 80` +to be notified when quality regresses. See the +[Evaluation & Analysis guide](/guide/evaluation-and-analysis#automation). + +## Errors + +| Status | Cause | +|---|---| +| 400 | Missing `name`, bad `kind`, empty `scorers`, non-array `items`. | +| 404 | Target / dataset / suite / run not found (or unknown `suiteKey` on run). | +| 500 | Internal error. | diff --git a/docs/guide/evaluation-and-analysis.md b/docs/guide/evaluation-and-analysis.md new file mode 100644 index 00000000..a962bae6 --- /dev/null +++ b/docs/guide/evaluation-and-analysis.md @@ -0,0 +1,171 @@ +# Evaluation & Analysis + +Two related "operate" services for measuring AI quality offline: + +- **Evaluation** — regression-test a model or agent against a fixed dataset with + deterministic and LLM-judge scorers. Answers *"did this change make the model + better or worse?"* +- **Analysis** — extract structured fields from real conversations, judge their + quality, and score extraction accuracy against ground truth — on demand or on + a nightly schedule. Answers *"what is happening across yesterday's calls, and + is quality holding?"* + +They are independent services that share the same architectural shape and plug +into the same alerting pipeline. + +## Architecture + +Both services are built in the same four layers, each independently testable: + +``` +┌─ Dashboard UI ────────────── /dashboard/evaluations, /dashboard/analysis +│ tabbed pages, create modals, run viewers +├─ REST API (Fastify plugin) ── /api/evaluation/*, /api/analysis/* +│ validation, session + project scope +├─ Service ──────────────────── src/lib/services/{evaluation,analysis}/service.ts +│ tenant-scoped CRUD + run orchestration + live model adapters +└─ Engine core (pure, DI) ───── runner / scorers / extraction / judge / accuracy + no DB, queue, or model-runtime imports — the model is injected +``` + +The **engine core** is deliberately free of platform coupling: the model call is +passed in as an `invoker` function. This keeps scoring logic unit-testable +without a database or live model, and lets the service layer inject either the +real `handleChatCompletion` adapter or a fake in tests. + +## Evaluation + +### Data model + +| Entity | Purpose | +|---|---| +| Target | What is under test: a `model`, `agent`, or `external` endpoint. | +| Dataset | Ordered test items: `input` messages + optional `expected`. | +| Suite | Binds a target + dataset + `scorers[]` (+ `judgeModelKey`). | +| Run | One execution of a suite: per-item scores + an aggregate. | + +### Scorers + +- **assertion** — deterministic checks against `expected`: `mustContain`, + `equals`, `regex`, `jsonSchema`, `jsonPath`. +- **llm-judge** — an LLM grades the output against a `rubric`, normalised to + 0–1, backed by the suite's `judgeModelKey`. + +Per-item `score` is the weighted mean of the scorers; the item `passed` when all +scorers pass. The run aggregate exposes `passRate`, `avgScore`, `avgLatencyMs`. + +### Walkthrough + +1. **Targets → New target** — pick `model` and a registered model key. +2. **Datasets → New dataset** — paste a JSON array of items. +3. **Suites → New suite** — choose the target and dataset, enable assertion + and/or LLM-judge (with a rubric and judge model). +4. **Run** from the suite row → the run viewer shows pass/fail, score, the + per-scorer breakdown, and the model output for each item. + +> Model targets are live today. Agent and external targets can be created now; +> their execution adapters are recorded as per-item errors until they ship, so a +> run never aborts midway. + +## Analysis + +### Data model + +| Entity | Purpose | +|---|---| +| Definition | The recipe: `fieldSet`, extraction prompt, `modes`, models, optional `schedule`. | +| Conversation | An ingested transcript (`{ role, content }[]`) with optional `referenceFields`. | +| Run | One execution of a definition over a set of conversations + an aggregate. | + +### The four modes + +| Mode | Effect | +|---|---| +| **extract** | Always on. Pulls the `fieldSet` from each transcript as typed JSON; each field is coerced to its declared type (`string`/`number`/`boolean`/`enum`) and required fields are validated. | +| **store** | Writes the extracted fields back onto the conversation (`extractedFields`, `lastAnalyzedAt`) so they can be browsed and queried later. | +| **judge** | An LLM grades each conversation against a rubric (0–1). | +| **accuracy** | Compares extracted fields to the conversation's `referenceFields`, per field, returning a 0–1 score and a per-field match map. | + +The aggregate exposes `passRate` (extraction success + judge pass), plus +`avgJudgeScore` and `avgExtractionAccuracy` (averaged only over items that used +those modes). + +### Walkthrough + +1. **Definitions → New definition** — build the field-set (key/type/required, + enum values), choose modes, set the extraction model (and judge model + + rubric if judging). Optionally set a cron `schedule`. +2. **Conversations → Ingest** — paste a JSON array of transcripts. Add + `referenceFields` to any conversation you want to score for accuracy. +3. **Run analysis** from a definition row → the run viewer shows the extracted + fields, judge score, and accuracy per conversation. + +## Automation + +The IVR use case — *"every night, analyze the day's calls and alert me if quality +drops"* — is covered by two independent, composable mechanisms. + +### Scheduled runs + +A definition with `schedule: { cron, enabled }` is fired automatically by the +background **analysis scheduler**: + +- The scheduler runs on a 60s interval, guarded by a distributed lock (use + `CACHE_PROVIDER=redis` for multi-instance deployments) so a single instance + fires each tick. It is started from the server bootstrap. +- For each tenant it loads scheduled definitions and fires any that are **due**. + "Due" is decided by the pure `schedulePlanner`: a cron slot fires at most once, + compared against the definition's most recent run. So `0 2 * * *` runs once + per night even though the scheduler ticks every minute. +- Scheduled runs analyze the recent conversation corpus with `createdBy: + "system"`. + +Cron expressions are standard 5-field, evaluated in **UTC**. Set the schedule via +the definition create/update API (`schedule`) or the dashboard. + +### Threshold alerts + +Both services expose their run aggregates to the existing alert pipeline as +metric collectors — no new alert logic is involved. Create an alert rule (in the +Alerts service) on the `analysis` or `evaluation` module: + +| Module | Metric | Meaning (0–100) | +|---|---|---| +| `analysis` | `analysis_pass_rate` | Mean pass rate over completed runs in the window. | +| `analysis` | `analysis_avg_judge_score` | Mean judge score. | +| `analysis` | `analysis_avg_accuracy` | Mean extraction accuracy. | +| `evaluation` | `evaluation_pass_rate` | Mean pass rate. | +| `evaluation` | `evaluation_avg_score` | Mean weighted score. | + +The collectors average the persisted run aggregate over completed runs in the +rule's window (excluding runs where the metric is null), honouring the project +scope. The existing alert scheduler/evaluator then applies the rule's condition +and fires through its channels. + +**Putting it together:** a definition scheduled at `0 2 * * *` plus an alert rule +`analysis_avg_accuracy lt 85 over 1440 minutes` gives you a nightly analysis that +pages you when extraction quality slips below 85%. + +## Multi-tenancy & persistence + +Every entity is tenant-scoped and persisted through the dual-provider database +layer (MongoDB documents or SQLite JSON columns) with full parity. Runs embed +their per-item results and aggregate. Reads and writes always go through +`switchToTenant`, so one tenant never sees another's targets, datasets, +conversations, or runs. + +## Where things live + +| Area | Path | +|---|---| +| Evaluation engine | `src/lib/services/evaluation/` | +| Analysis engine | `src/lib/services/analysis/` | +| Schedule planner | `src/lib/services/analysis/schedulePlanner.ts` | +| Analysis scheduler | `src/lib/services/analysis/analysisScheduler.ts` | +| Alert collectors | `src/lib/services/alerts/metrics/{analysis,evaluation}Collector.ts` | +| DB mixins | `src/lib/database/{mongodb,sqlite}/{evaluation,analysis}.mixin.ts` | +| API plugins | `src/server/api/plugins/{evaluations,analysis}.ts` | +| Dashboard UI | `src/app/dashboard/{evaluations,analysis}/` | + +See the API references for [Evaluation](/api/evaluation) and +[Analysis](/api/analysis). diff --git a/src/__tests__/integration/alert-run-collectors.test.ts b/src/__tests__/integration/alert-run-collectors.test.ts new file mode 100644 index 00000000..40a76f26 --- /dev/null +++ b/src/__tests__/integration/alert-run-collectors.test.ts @@ -0,0 +1,107 @@ +/** + * Integration test for the analysis & evaluation alert metric collectors. + * + * Seeds completed/failed runs with known aggregates in a real SQLite tenant DB + * and asserts the collectors average the right aggregate field (as a 0–100 + * percentage), respect the status filter, exclude null metrics, and honour the + * projectId scope. + */ + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +const tmpRoot = mkdtempSync(path.join(tmpdir(), 'cognipeer-alert-collectors-')); +process.env.DB_PROVIDER = 'sqlite'; +process.env.SQLITE_DATA_DIR = tmpRoot; +process.env.MAIN_DB_NAME = 'alert_collectors_main'; + +import { reloadConfig } from '@/lib/core/config'; +import { disconnectDatabase, getDatabase } from '@/lib/database'; +import { AnalysisCollector } from '@/lib/services/alerts/metrics/analysisCollector'; +import { EvaluationCollector } from '@/lib/services/alerts/metrics/evaluationCollector'; + +const TENANT_DB_NAME = 'alert_collectors_tenant'; +const TENANT_ID = 'tenant-alert-collectors'; + +beforeAll(async () => { + reloadConfig(); + const db = await getDatabase(); + await db.switchToTenant(TENANT_DB_NAME); + + // Analysis runs (project p1) + await db.createAnalysisRun({ + tenantId: TENANT_ID, projectId: 'p1', definitionKey: 'd1', status: 'completed', mode: 'sync', + progress: { total: 2, completed: 2, failed: 0 }, items: [], createdBy: 'sys', + aggregate: { total: 2, completed: 2, failed: 0, passed: 2, passRate: 1, avgJudgeScore: 0.8, avgExtractionAccuracy: 1 }, + }); + await db.createAnalysisRun({ + tenantId: TENANT_ID, projectId: 'p1', definitionKey: 'd1', status: 'completed', mode: 'sync', + progress: { total: 2, completed: 2, failed: 0 }, items: [], createdBy: 'sys', + aggregate: { total: 2, completed: 2, failed: 0, passed: 1, passRate: 0.5, avgJudgeScore: 0.6, avgExtractionAccuracy: null }, + }); + await db.createAnalysisRun({ + tenantId: TENANT_ID, projectId: 'p1', definitionKey: 'd1', status: 'failed', mode: 'sync', + progress: { total: 1, completed: 0, failed: 1 }, items: [], createdBy: 'sys', error: 'boom', + }); + await db.createAnalysisRun({ + tenantId: TENANT_ID, projectId: 'p2', definitionKey: 'd2', status: 'completed', mode: 'sync', + progress: { total: 1, completed: 1, failed: 0 }, items: [], createdBy: 'sys', + aggregate: { total: 1, completed: 1, failed: 0, passed: 0, passRate: 0, avgJudgeScore: 0, avgExtractionAccuracy: 0 }, + }); + + // Evaluation run (project p1) + await db.createEvaluationRun({ + tenantId: TENANT_ID, projectId: 'p1', suiteKey: 's1', targetKey: 't1', datasetKey: 'ds1', + status: 'completed', mode: 'sync', progress: { total: 2, completed: 2, failed: 0 }, items: [], createdBy: 'sys', + aggregate: { total: 2, completed: 2, failed: 0, passed: 1, passRate: 0.6, avgScore: 0.9, avgLatencyMs: 120 }, + }); +}); + +afterAll(async () => { + await disconnectDatabase(); + rmSync(tmpRoot, { recursive: true, force: true }); +}); + +describe('AnalysisCollector', () => { + const collector = new AnalysisCollector(); + const base = { tenantDbName: TENANT_DB_NAME, tenantId: TENANT_ID, windowMinutes: 1440 }; + + it('averages passRate over completed runs (excludes failed)', async () => { + const r = await collector.collect({ ...base, metric: 'analysis_pass_rate', scope: { projectId: 'p1' } }); + expect(r.value).toBeCloseTo(75, 5); // avg(1, 0.5) * 100 + expect(r.sampleCount).toBe(2); + }); + + it('averages judge score', async () => { + const r = await collector.collect({ ...base, metric: 'analysis_avg_judge_score', scope: { projectId: 'p1' } }); + expect(r.value).toBeCloseTo(70, 5); // avg(0.8, 0.6) * 100 + expect(r.sampleCount).toBe(2); + }); + + it('excludes null metrics from accuracy average', async () => { + const r = await collector.collect({ ...base, metric: 'analysis_avg_accuracy', scope: { projectId: 'p1' } }); + expect(r.value).toBeCloseTo(100, 5); // only the non-null run (1) counts + expect(r.sampleCount).toBe(1); + }); + + it('honours the projectId scope', async () => { + const r = await collector.collect({ ...base, metric: 'analysis_pass_rate', scope: { projectId: 'p2' } }); + expect(r.value).toBeCloseTo(0, 5); + expect(r.sampleCount).toBe(1); + }); +}); + +describe('EvaluationCollector', () => { + const collector = new EvaluationCollector(); + const base = { tenantDbName: TENANT_DB_NAME, tenantId: TENANT_ID, windowMinutes: 1440 }; + + it('averages passRate and score', async () => { + const pass = await collector.collect({ ...base, metric: 'evaluation_pass_rate', scope: { projectId: 'p1' } }); + expect(pass.value).toBeCloseTo(60, 5); + const score = await collector.collect({ ...base, metric: 'evaluation_avg_score', scope: { projectId: 'p1' } }); + expect(score.value).toBeCloseTo(90, 5); + expect(score.sampleCount).toBe(1); + }); +}); diff --git a/src/__tests__/integration/analysis-e2e.test.ts b/src/__tests__/integration/analysis-e2e.test.ts new file mode 100644 index 00000000..794e28da --- /dev/null +++ b/src/__tests__/integration/analysis-e2e.test.ts @@ -0,0 +1,187 @@ +/** + * End-to-end test for the Analysis service vertical. + * + * Backed by a real SQLiteProvider in a temp directory. Exercises CRUD for + * definitions / conversations and a full `runDefinition` flow whose model + * invokers are injected (fakes) so no live model calls are made — verifying + * persistence, extraction/judge/accuracy aggregation, store-mode write-back, + * and run retrieval against the real DB layer. + */ + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +// SQLite + temp dir must be configured BEFORE getDatabase() is ever called. +const tmpRoot = mkdtempSync(path.join(tmpdir(), 'cognipeer-analysis-e2e-')); +process.env.DB_PROVIDER = 'sqlite'; +process.env.SQLITE_DATA_DIR = tmpRoot; +process.env.MAIN_DB_NAME = 'analysis_e2e_main'; + +import { reloadConfig } from '@/lib/core/config'; +import { disconnectDatabase, getDatabase } from '@/lib/database'; +import { + createDefinition, + deleteConversation, + getConversation, + getRun, + ingestConversations, + listConversations, + listDefinitions, + listRuns, + runDefinition, + runScheduledAnalyses, + updateDefinition, +} from '@/lib/services/analysis/service'; +import type { AnalysisMessage, ModelInvoker } from '@/lib/services/analysis/types'; + +const TENANT_DB_NAME = 'analysis_e2e_tenant'; +const TENANT_ID = 'tenant-analysis-e2e'; +const ACTOR = 'tester@example.com'; + +/** Fake invoker factory: extraction branches on a transcript marker; judge approves. */ +const fakeBuildModelInvoker = ( + _modelKey: string | undefined, + _ctx: unknown, + role: 'extraction' | 'judge', +): ModelInvoker => { + if (role === 'judge') { + return async () => '{"score":0.9,"passed":true,"reasoning":"ok"}'; + } + return async (messages: AnalysisMessage[]) => { + const text = messages.map((m) => m.content).join('\n'); + if (text.includes('BILLING')) return '{"intent":"billing","resolved":true}'; + return '{"intent":"support","resolved":false}'; + }; +}; + +beforeAll(async () => { + reloadConfig(); + const db = await getDatabase(); + await db.switchToTenant(TENANT_DB_NAME); +}); + +afterAll(async () => { + await disconnectDatabase(); + rmSync(tmpRoot, { recursive: true, force: true }); +}); + +describe('Analysis service — full vertical (SQLite)', () => { + it('persists a definition + conversations then runs extraction, judge & accuracy', async () => { + const definition = await createDefinition(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Call Intent Analysis', + fieldSet: [ + { key: 'intent', type: 'enum', enumValues: ['billing', 'support'], required: true }, + { key: 'resolved', type: 'boolean' }, + ], + modes: { store: true, judge: { rubric: 'Was the caller helped?' }, accuracy: true }, + extractionModelKey: 'extract-model', + judgeModelKey: 'judge-model', + }); + expect(definition.id).toBeTruthy(); + expect(definition.key).toBe('call-intent-analysis'); + expect(definition.modes.judge?.rubric).toContain('caller'); + + const [c1, c2] = await ingestConversations(TENANT_DB_NAME, TENANT_ID, ACTOR, [ + { + name: 'Call 1', + transcript: [{ role: 'caller', content: 'I have a BILLING issue' }, { role: 'agent', content: 'Refunded.' }], + referenceFields: { intent: 'billing', resolved: true }, + }, + { + name: 'Call 2', + transcript: [{ role: 'caller', content: 'A SUPPORT request' }], + referenceFields: { intent: 'billing' }, // extracted 'support' → mismatch + }, + ]); + expect(c1.id).toBeTruthy(); + expect(c2.key).toBe('call-2'); + + const run = await runDefinition( + { tenantDbName: TENANT_DB_NAME, tenantId: TENANT_ID, createdBy: ACTOR, definitionKey: definition.key }, + { buildModelInvoker: fakeBuildModelInvoker }, + ); + + expect(run.status).toBe('completed'); + expect(run.definitionKey).toBe(definition.key); + expect(run.aggregate?.total).toBe(2); + expect(run.aggregate?.completed).toBe(2); + expect(run.aggregate?.failed).toBe(0); + expect(run.aggregate?.passed).toBe(2); // both extract required intent + judge approves + expect(run.aggregate?.avgJudgeScore).toBeCloseTo(0.9, 5); + expect(run.aggregate?.avgExtractionAccuracy).toBeCloseTo(0.5, 5); // c1=1.0, c2=0.0 + expect(run.items).toHaveLength(2); + + // Run is retrievable by id with persisted per-item detail. + const fetched = await getRun(TENANT_DB_NAME, run.id); + const item1 = fetched?.items.find((i) => i.conversationKey === c1.key); + expect(item1?.extractedFields.intent).toBe('billing'); + expect(item1?.accuracy?.score).toBe(1); + expect(item1?.judge?.passed).toBe(true); + + // Store mode wrote the extracted fields back onto the conversation. + const storedC1 = await getConversation(TENANT_DB_NAME, c1.id); + expect(storedC1?.extractedFields?.intent).toBe('billing'); + expect(storedC1?.lastAnalyzedAt).toBeTruthy(); + }); + + it('lists entities and round-trips definition update / conversation delete', async () => { + expect((await listDefinitions(TENANT_DB_NAME)).length).toBeGreaterThanOrEqual(1); + expect((await listConversations(TENANT_DB_NAME)).length).toBeGreaterThanOrEqual(2); + expect((await listRuns(TENANT_DB_NAME)).length).toBeGreaterThanOrEqual(1); + + const def = (await listDefinitions(TENANT_DB_NAME))[0]; + const updated = await updateDefinition(TENANT_DB_NAME, def.id, ACTOR, { description: 'nightly IVR analysis' }); + expect(updated?.description).toBe('nightly IVR analysis'); + + const [conv] = await ingestConversations(TENANT_DB_NAME, TENANT_ID, ACTOR, [ + { name: 'Disposable', transcript: [{ role: 'caller', content: 'temp' }] }, + ]); + expect(await deleteConversation(TENANT_DB_NAME, conv.id)).toBe(true); + expect(await getConversation(TENANT_DB_NAME, conv.id)).toBeNull(); + }); + + it('records per-item errors when the extraction model throws', async () => { + const definition = await createDefinition(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Erroring Analysis', + fieldSet: [{ key: 'intent', type: 'string', required: true }], + modes: {}, + extractionModelKey: 'x', + }); + await ingestConversations(TENANT_DB_NAME, TENANT_ID, ACTOR, [ + { key: 'err-only-conv', transcript: [{ role: 'caller', content: 'hi' }] }, + ]); + + const run = await runDefinition( + { tenantDbName: TENANT_DB_NAME, tenantId: TENANT_ID, createdBy: ACTOR, definitionKey: definition.key, conversationKeys: ['err-only-conv'] }, + { buildModelInvoker: () => async () => { throw new Error('model exploded'); } }, + ); + + expect(run.status).toBe('completed'); + expect(run.aggregate?.total).toBe(1); + expect(run.aggregate?.failed).toBe(1); + expect(run.aggregate?.completed).toBe(0); + expect(run.items[0].error).toMatch(/exploded/); + }); + + it('persists a cron schedule and fires it via runScheduledAnalyses', async () => { + const definition = await createDefinition(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Nightly Scheduled', + fieldSet: [{ key: 'intent', type: 'string' }], + modes: {}, + extractionModelKey: 'extract-model', + schedule: { cron: '* * * * *', enabled: true }, + }); + expect(definition.schedule?.enabled).toBe(true); // round-trips through SQLite + + const result = await runScheduledAnalyses(TENANT_DB_NAME, TENANT_ID, new Date(), { + buildModelInvoker: fakeBuildModelInvoker, + }); + expect(result.fired).toContain(definition.key); + + const runs = await listRuns(TENANT_DB_NAME, { definitionKey: definition.key }); + expect(runs.length).toBeGreaterThanOrEqual(1); + expect(runs[0].status).toBe('completed'); + }); +}); diff --git a/src/__tests__/integration/evaluation-e2e.test.ts b/src/__tests__/integration/evaluation-e2e.test.ts new file mode 100644 index 00000000..b0d844bd --- /dev/null +++ b/src/__tests__/integration/evaluation-e2e.test.ts @@ -0,0 +1,158 @@ +/** + * End-to-end test for the Evaluation service vertical. + * + * Backed by a real SQLiteProvider in a temp directory. Exercises CRUD for + * targets / datasets / suites and a full `runSuite` flow whose target & judge + * invokers are injected (fakes) so no live model calls are made — verifying + * persistence, aggregation, and run retrieval against the real DB layer. + */ + +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +// SQLite + temp dir must be configured BEFORE getDatabase() is ever called. +const tmpRoot = mkdtempSync(path.join(tmpdir(), 'cognipeer-eval-e2e-')); +process.env.DB_PROVIDER = 'sqlite'; +process.env.SQLITE_DATA_DIR = tmpRoot; +process.env.MAIN_DB_NAME = 'eval_e2e_main'; + +import { reloadConfig } from '@/lib/core/config'; +import { disconnectDatabase, getDatabase } from '@/lib/database'; +import { + createDataset, + createSuite, + createTarget, + deleteTarget, + getRun, + listDatasets, + listRuns, + listSuites, + listTargets, + runSuite, + updateTarget, +} from '@/lib/services/evaluation/service'; + +const TENANT_DB_NAME = 'eval_e2e_tenant'; +const TENANT_ID = 'tenant-eval-e2e'; +const ACTOR = 'tester@example.com'; + +beforeAll(async () => { + reloadConfig(); + const db = await getDatabase(); + await db.switchToTenant(TENANT_DB_NAME); +}); + +afterAll(async () => { + await disconnectDatabase(); + rmSync(tmpRoot, { recursive: true, force: true }); +}); + +describe('Evaluation service — full vertical (SQLite)', () => { + it('persists targets, datasets and suites then runs an evaluation', async () => { + const target = await createTarget(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'GPT Eval Target', + kind: 'model', + modelKey: 'gpt-test', + }); + expect(target.id).toBeTruthy(); + expect(target.key).toBe('gpt-eval-target'); + expect(target.kind).toBe('model'); + + const dataset = await createDataset(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Smoke Dataset', + items: [ + { id: 'q1', input: [{ role: 'user', content: 'say ok' }], expected: { mustContain: ['ok'] } }, + { id: 'q2', input: [{ role: 'user', content: 'say ok too' }], expected: { mustContain: ['ok'] } }, + ], + }); + expect(dataset.id).toBeTruthy(); + expect(dataset.items).toHaveLength(2); + + const suite = await createSuite(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Smoke Suite', + targetKey: target.key, + datasetKey: dataset.key, + scorers: [{ type: 'assertion' }, { type: 'llm-judge', rubric: 'Answer must contain ok.' }], + judgeModelKey: 'judge-test', + }); + expect(suite.id).toBeTruthy(); + expect(suite.scorers).toHaveLength(2); + + // Injected fakes: target echoes "ok" only for q1; judge always approves. + const targetFn = vi.fn(async (item: { id: string }) => ({ text: item.id === 'q1' ? 'ok' : 'nope' })); + const judgeFn = vi.fn(async () => '{"score":1,"passed":true}'); + + const run = await runSuite( + { tenantDbName: TENANT_DB_NAME, tenantId: TENANT_ID, createdBy: ACTOR, suiteKey: suite.key }, + { buildTargetInvoker: () => targetFn, buildJudgeInvoker: () => judgeFn }, + ); + + expect(run.status).toBe('completed'); + expect(run.suiteKey).toBe(suite.key); + expect(run.aggregate?.total).toBe(2); + expect(run.aggregate?.completed).toBe(2); + expect(run.aggregate?.passed).toBe(1); // only q1 passes the assertion + expect(run.aggregate?.passRate).toBeCloseTo(0.5, 5); + expect(run.aggregate?.avgScore).toBeCloseTo(0.75, 5); // q1=1.0, q2=0.5 + expect(run.items).toHaveLength(2); + expect(targetFn).toHaveBeenCalledTimes(2); + expect(judgeFn).toHaveBeenCalledTimes(2); + + // Run is retrievable by id with its persisted items. + const fetched = await getRun(TENANT_DB_NAME, run.id); + expect(fetched?.id).toBe(run.id); + expect(fetched?.items).toHaveLength(2); + const q1 = fetched?.items.find((i) => i.itemId === 'q1'); + expect(q1?.passed).toBe(true); + expect(q1?.scores).toHaveLength(2); + }); + + it('lists entities and round-trips target update/delete', async () => { + const targets = await listTargets(TENANT_DB_NAME); + const datasets = await listDatasets(TENANT_DB_NAME); + const suites = await listSuites(TENANT_DB_NAME); + const runs = await listRuns(TENANT_DB_NAME); + expect(targets.length).toBeGreaterThanOrEqual(1); + expect(datasets.length).toBeGreaterThanOrEqual(1); + expect(suites.length).toBeGreaterThanOrEqual(1); + expect(runs.length).toBeGreaterThanOrEqual(1); + + const extra = await createTarget(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Disposable Target', + kind: 'model', + modelKey: 'tmp', + }); + const updated = await updateTarget(TENANT_DB_NAME, extra.id, ACTOR, { description: 'updated desc' }); + expect(updated?.description).toBe('updated desc'); + + const deleted = await deleteTarget(TENANT_DB_NAME, extra.id); + expect(deleted).toBe(true); + expect(await listTargets(TENANT_DB_NAME, { search: 'Disposable' })).toHaveLength(0); + }); + + it('records a per-item error when the target invoker throws', async () => { + const target = await createTarget(TENANT_DB_NAME, TENANT_ID, ACTOR, { name: 'Erroring Target', kind: 'model', modelKey: 'x' }); + const dataset = await createDataset(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Error Dataset', + items: [{ id: 'e1', input: [{ role: 'user', content: 'hi' }] }], + }); + const suite = await createSuite(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Error Suite', + targetKey: target.key, + datasetKey: dataset.key, + scorers: [{ type: 'assertion' }], + }); + + const run = await runSuite( + { tenantDbName: TENANT_DB_NAME, tenantId: TENANT_ID, createdBy: ACTOR, suiteKey: suite.key }, + { buildTargetInvoker: () => async () => { throw new Error('model exploded'); } }, + ); + + expect(run.status).toBe('completed'); + expect(run.aggregate?.failed).toBe(1); + expect(run.aggregate?.completed).toBe(0); + expect(run.items[0].error).toMatch(/model exploded/); + }); +}); diff --git a/src/__tests__/unit/analysis-accuracy.test.ts b/src/__tests__/unit/analysis-accuracy.test.ts new file mode 100644 index 00000000..98c3adec --- /dev/null +++ b/src/__tests__/unit/analysis-accuracy.test.ts @@ -0,0 +1,57 @@ +/** + * Unit tests — analysis reference-based accuracy scoring. + */ + +import { describe, it, expect } from 'vitest'; +import { scoreAccuracy, valuesMatch } from '@/lib/services/analysis/accuracy'; +import type { FieldSet } from '@/lib/services/analysis/types'; + +const FIELDS: FieldSet = [ + { key: 'intent', type: 'enum', enumValues: ['billing', 'support'] }, + { key: 'resolved', type: 'boolean' }, + { key: 'amount', type: 'number' }, +]; + +describe('valuesMatch', () => { + it('compares numbers regardless of string/number form', () => { + expect(valuesMatch('100', 100, 'number')).toBe(true); + expect(valuesMatch(100, 101, 'number')).toBe(false); + }); + it('compares strings case/space-insensitively', () => { + expect(valuesMatch('Billing ', 'billing', 'string')).toBe(true); + }); + it('compares booleans across representations', () => { + expect(valuesMatch('yes', true, 'boolean')).toBe(true); + expect(valuesMatch('no', true, 'boolean')).toBe(false); + }); +}); + +describe('scoreAccuracy', () => { + it('only compares fields present in the reference', () => { + const r = scoreAccuracy( + { intent: 'billing', resolved: true, amount: 50 }, + { intent: 'Billing', resolved: 'yes' }, + FIELDS, + ); + expect(r.comparedCount).toBe(2); + expect(r.score).toBe(1); + expect(r.perField.intent.match).toBe(true); + expect(r.perField.amount).toBeUndefined(); + }); + it('computes a partial score and flags mismatches', () => { + const r = scoreAccuracy( + { intent: 'support', resolved: true }, + { intent: 'billing', resolved: true }, + FIELDS, + ); + expect(r.comparedCount).toBe(2); + expect(r.score).toBeCloseTo(0.5, 5); + expect(r.perField.intent.match).toBe(false); + expect(r.perField.resolved.match).toBe(true); + }); + it('returns score 1 when there is nothing to compare', () => { + const r = scoreAccuracy({ intent: 'billing' }, {}, FIELDS); + expect(r.comparedCount).toBe(0); + expect(r.score).toBe(1); + }); +}); diff --git a/src/__tests__/unit/analysis-extraction.test.ts b/src/__tests__/unit/analysis-extraction.test.ts new file mode 100644 index 00000000..fb3b5842 --- /dev/null +++ b/src/__tests__/unit/analysis-extraction.test.ts @@ -0,0 +1,99 @@ +/** + * Unit tests — analysis field extraction. + * Covers type coercion, required-field detection, JSON parsing (incl. fenced), + * prompt construction, and graceful failure. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { + buildExtractionPrompt, + coerceField, + parseExtraction, + extractFields, +} from '@/lib/services/analysis/extraction'; +import type { AnalysisConversation, FieldSet } from '@/lib/services/analysis/types'; + +const FIELDS: FieldSet = [ + { key: 'intent', type: 'enum', enumValues: ['billing', 'support'], required: true }, + { key: 'resolved', type: 'boolean' }, + { key: 'amount', type: 'number' }, + { key: 'summary', type: 'string' }, +]; + +const CONV: AnalysisConversation = { + id: 'c1', + transcript: [ + { role: 'caller', content: 'I was double charged.' }, + { role: 'agent', content: 'I refunded 50 dollars.' }, + ], +}; + +describe('coerceField', () => { + it('coerces numbers from strings', () => { + expect(coerceField('42', { key: 'n', type: 'number' })).toEqual({ value: 42, valid: true }); + expect(coerceField('nan', { key: 'n', type: 'number' })).toEqual({ value: null, valid: false }); + }); + it('coerces booleans from yes/no/true', () => { + expect(coerceField('yes', { key: 'b', type: 'boolean' }).value).toBe(true); + expect(coerceField(false, { key: 'b', type: 'boolean' })).toEqual({ value: false, valid: true }); + expect(coerceField('maybe', { key: 'b', type: 'boolean' }).valid).toBe(false); + }); + it('matches enums case-insensitively and rejects unknowns', () => { + expect(coerceField('Billing', { key: 'e', type: 'enum', enumValues: ['billing', 'support'] })).toEqual({ value: 'billing', valid: true }); + expect(coerceField('other', { key: 'e', type: 'enum', enumValues: ['billing'] }).valid).toBe(false); + }); + it('treats null/empty as invalid', () => { + expect(coerceField(null, { key: 's', type: 'string' }).valid).toBe(false); + expect(coerceField('', { key: 's', type: 'string' }).valid).toBe(false); + }); +}); + +describe('parseExtraction', () => { + it('parses and coerces a complete object', () => { + const r = parseExtraction('{"intent":"billing","resolved":"yes","amount":"50","summary":"refund"}', FIELDS); + expect(r.error).toBeUndefined(); + expect(r.missing).toEqual([]); + expect(r.fields).toEqual({ intent: 'billing', resolved: true, amount: 50, summary: 'refund' }); + }); + it('flags missing required fields', () => { + const r = parseExtraction('{"resolved":true}', FIELDS); + expect(r.missing).toEqual(['intent']); + expect(r.fields.intent).toBeNull(); + }); + it('reports an error for non-JSON / non-object output', () => { + expect(parseExtraction('totally not json', FIELDS).error).toBeTruthy(); + expect(parseExtraction('[1,2,3]', FIELDS).error).toMatch(/not a JSON object/); + expect(parseExtraction('not json', FIELDS).missing).toEqual(['intent']); + }); + it('extracts JSON from a fenced block', () => { + const r = parseExtraction('```json\n{"intent":"support"}\n```', FIELDS); + expect(r.fields.intent).toBe('support'); + }); +}); + +describe('buildExtractionPrompt', () => { + it('includes the field schema, transcript and instructions', () => { + const messages = buildExtractionPrompt(CONV, FIELDS, 'Focus on refunds.'); + expect(messages[0].role).toBe('system'); + expect(messages[0].content).toContain('"intent"'); + expect(messages[0].content).toContain('billing | support'); + expect(messages[1].content).toContain('Focus on refunds.'); + expect(messages[1].content).toContain('caller: I was double charged.'); + }); +}); + +describe('extractFields', () => { + it('returns coerced fields from the invoker output', async () => { + const invoke = vi.fn().mockResolvedValue('{"intent":"billing","resolved":true,"amount":50,"summary":"x"}'); + const r = await extractFields(CONV, FIELDS, undefined, invoke); + expect(invoke).toHaveBeenCalledOnce(); + expect(r.fields.intent).toBe('billing'); + expect(r.missing).toEqual([]); + }); + it('fails gracefully when the model throws', async () => { + const invoke = vi.fn().mockRejectedValue(new Error('timeout')); + const r = await extractFields(CONV, FIELDS, undefined, invoke); + expect(r.error).toMatch(/timeout/); + expect(r.missing).toEqual(['intent']); + }); +}); diff --git a/src/__tests__/unit/analysis-runner.test.ts b/src/__tests__/unit/analysis-runner.test.ts new file mode 100644 index 00000000..b4998afe --- /dev/null +++ b/src/__tests__/unit/analysis-runner.test.ts @@ -0,0 +1,93 @@ +/** + * Unit tests — analysis runner. + * Covers extraction aggregation, judge + accuracy wiring, extraction failures, + * concurrency, and the progress hook. Model invokers are mocked. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { runAnalysis } from '@/lib/services/analysis/runner'; +import type { AnalysisConversation, AnalysisSpec, FieldSet, ModelInvoker } from '@/lib/services/analysis/types'; + +const FIELD_SET: FieldSet = [ + { key: 'intent', type: 'enum', enumValues: ['billing', 'support'], required: true }, + { key: 'resolved', type: 'boolean' }, +]; + +/** Extraction invoker that branches on a marker embedded in the transcript. */ +const extractionInvoker: ModelInvoker = async (messages) => { + const text = messages.map((m) => m.content).join('\n'); + if (text.includes('BOOM')) throw new Error('model exploded'); + if (text.includes('MISSING')) return '{"resolved":true}'; // omits required intent + if (text.includes('SUPPORT')) return '{"intent":"support","resolved":false}'; + return '{"intent":"billing","resolved":true}'; +}; + +function conv(id: string, marker: string, referenceFields?: Record): AnalysisConversation { + return { id, transcript: [{ role: 'caller', content: marker }], referenceFields }; +} + +const BASE_SPEC: AnalysisSpec = { fieldSet: FIELD_SET, modes: {} }; + +describe('runAnalysis', () => { + it('extracts and aggregates over a batch', async () => { + const conversations = [conv('a', 'BILLING'), conv('b', 'SUPPORT'), conv('c', 'MISSING')]; + const result = await runAnalysis({ conversations, spec: BASE_SPEC, invokeExtraction: extractionInvoker }); + expect(result.aggregate.total).toBe(3); + expect(result.aggregate.completed).toBe(3); // no extraction errors + expect(result.aggregate.failed).toBe(0); + expect(result.aggregate.passed).toBe(2); // 'c' is missing required intent + const cItem = result.items.find((i) => i.conversationId === 'c'); + expect(cItem?.missing).toEqual(['intent']); + expect(cItem?.passed).toBe(false); + }); + + it('records extraction errors without aborting', async () => { + const conversations = [conv('a', 'BILLING'), conv('b', 'BOOM')]; + const result = await runAnalysis({ conversations, spec: BASE_SPEC, invokeExtraction: extractionInvoker }); + expect(result.aggregate.failed).toBe(1); + expect(result.aggregate.completed).toBe(1); + expect(result.items.find((i) => i.conversationId === 'b')?.error).toMatch(/exploded/); + }); + + it('wires the judge and averages its score', async () => { + const conversations = [conv('a', 'BILLING'), conv('b', 'BILLING')]; + const invokeJudge = vi.fn().mockResolvedValue('{"score":0.8,"passed":true}'); + const spec: AnalysisSpec = { fieldSet: FIELD_SET, modes: { judge: { rubric: 'Be polite' } } }; + const result = await runAnalysis({ conversations, spec, invokeExtraction: extractionInvoker, invokeJudge }); + expect(invokeJudge).toHaveBeenCalledTimes(2); + expect(result.aggregate.avgJudgeScore).toBeCloseTo(0.8, 5); + expect(result.items[0].judge?.passed).toBe(true); + }); + + it('scores accuracy against reference fields', async () => { + const conversations = [ + conv('a', 'BILLING', { intent: 'billing', resolved: true }), + conv('b', 'SUPPORT', { intent: 'billing' }), // extracted support → mismatch + ]; + const spec: AnalysisSpec = { fieldSet: FIELD_SET, modes: { accuracy: true } }; + const result = await runAnalysis({ conversations, spec, invokeExtraction: extractionInvoker }); + const a = result.items.find((i) => i.conversationId === 'a'); + const b = result.items.find((i) => i.conversationId === 'b'); + expect(a?.accuracy?.score).toBe(1); + expect(b?.accuracy?.score).toBe(0); + expect(result.aggregate.avgExtractionAccuracy).toBeCloseTo(0.5, 5); + }); + + it('fails an item when the judge rejects it', async () => { + const conversations = [conv('a', 'BILLING')]; + const invokeJudge = vi.fn().mockResolvedValue('{"score":0.1,"passed":false}'); + const spec: AnalysisSpec = { fieldSet: FIELD_SET, modes: { judge: { rubric: 'strict' } } }; + const result = await runAnalysis({ conversations, spec, invokeExtraction: extractionInvoker, invokeJudge }); + expect(result.items[0].passed).toBe(false); + expect(result.aggregate.passed).toBe(0); + }); + + it('processes every conversation once under bounded concurrency and calls onItem', async () => { + const conversations = Array.from({ length: 9 }, (_, i) => conv(`c${i}`, 'BILLING')); + const onItem = vi.fn(); + const result = await runAnalysis({ conversations, spec: BASE_SPEC, invokeExtraction: extractionInvoker, config: { concurrency: 3 }, onItem }); + expect(result.items.every((i) => i)).toBe(true); + expect(result.items).toHaveLength(9); + expect(onItem).toHaveBeenCalledTimes(9); + }); +}); diff --git a/src/__tests__/unit/analysis-schedule-planner.test.ts b/src/__tests__/unit/analysis-schedule-planner.test.ts new file mode 100644 index 00000000..70fae360 --- /dev/null +++ b/src/__tests__/unit/analysis-schedule-planner.test.ts @@ -0,0 +1,45 @@ +/** + * Unit tests — analysis cron schedule planner. + */ + +import { describe, it, expect } from 'vitest'; +import { computeNextRun, isDue, validateCron, type AnalysisSchedule } from '@/lib/services/analysis/schedulePlanner'; + +const nightly: AnalysisSchedule = { cron: '0 2 * * *', enabled: true }; // 02:00 UTC daily + +describe('validateCron', () => { + it('accepts valid expressions and rejects bad/empty ones', () => { + expect(validateCron('0 2 * * *')).toBeNull(); + expect(validateCron('')).toBeTruthy(); + expect(validateCron('not a cron')).toBeTruthy(); + }); +}); + +describe('computeNextRun', () => { + it('returns the next slot at/after the reference', () => { + const from = new Date('2026-06-03T03:00:00Z'); // just after 02:00 + const next = computeNextRun(nightly, null, from); + expect(next?.toISOString()).toBe('2026-06-04T02:00:00.000Z'); + }); + it('returns null when disabled or invalid', () => { + expect(computeNextRun({ cron: '0 2 * * *', enabled: false }, null)).toBeNull(); + expect(computeNextRun({ cron: 'bad', enabled: true }, null)).toBeNull(); + }); +}); + +describe('isDue', () => { + const now = new Date('2026-06-03T03:00:00Z'); // 02:00 slot has passed today + + it('is due when it has never run and a slot has passed', () => { + expect(isDue(nightly, null, now)).toBe(true); + }); + it('is due when the last run predates the most recent slot', () => { + expect(isDue(nightly, new Date('2026-06-02T02:30:00Z'), now)).toBe(true); + }); + it('is not due when already run after the most recent slot', () => { + expect(isDue(nightly, new Date('2026-06-03T02:30:00Z'), now)).toBe(false); + }); + it('is not due when disabled', () => { + expect(isDue({ cron: '0 2 * * *', enabled: false }, null, now)).toBe(false); + }); +}); diff --git a/src/__tests__/unit/evaluation-assertion-scorer.test.ts b/src/__tests__/unit/evaluation-assertion-scorer.test.ts new file mode 100644 index 00000000..b0458377 --- /dev/null +++ b/src/__tests__/unit/evaluation-assertion-scorer.test.ts @@ -0,0 +1,80 @@ +/** + * Unit tests — evaluation assertion scorer. + * Covers equals, contains/notContains, regex, json-schema, json-path, and the + * no-assertion no-op case. + */ + +import { describe, it, expect } from 'vitest'; +import { scoreAssertion } from '@/lib/services/evaluation/scorers/assertionScorer'; +import type { AssertionScorerConfig, DatasetItem, TargetOutput } from '@/lib/services/evaluation/types'; + +const CONFIG: AssertionScorerConfig = { type: 'assertion' }; + +function item(expected: DatasetItem['expected']): DatasetItem { + return { id: 'i1', input: [{ role: 'user', content: 'hi' }], expected }; +} +function out(text: string): TargetOutput { + return { text }; +} + +describe('assertionScorer', () => { + it('treats absence of expectations as a passing no-op', () => { + const r = scoreAssertion(item(undefined), out('anything'), CONFIG); + expect(r.passed).toBe(true); + expect(r.score).toBe(1); + expect(r.detail?.total).toBe(0); + }); + + it('passes exact equals (trimmed) and fails otherwise', () => { + expect(scoreAssertion(item({ equals: 'yes' }), out(' yes \n'), CONFIG).passed).toBe(true); + expect(scoreAssertion(item({ equals: 'yes' }), out('no'), CONFIG).passed).toBe(false); + }); + + it('handles mustContain / mustNotContain', () => { + const r = scoreAssertion(item({ mustContain: ['foo', 'bar'], mustNotContain: ['baz'] }), out('foo and bar'), CONFIG); + expect(r.passed).toBe(true); + const r2 = scoreAssertion(item({ mustContain: ['foo'], mustNotContain: ['bar'] }), out('foo bar'), CONFIG); + expect(r2.passed).toBe(false); + }); + + it('computes a partial score from the fraction of checks passed', () => { + const r = scoreAssertion(item({ mustContain: ['a', 'b', 'c', 'd'] }), out('a b'), CONFIG); + expect(r.score).toBeCloseTo(0.5, 5); + expect(r.passed).toBe(false); + }); + + it('evaluates regex and reports invalid patterns as failed', () => { + expect(scoreAssertion(item({ regex: '^\\d{3}$' }), out('123'), CONFIG).passed).toBe(true); + expect(scoreAssertion(item({ regex: '(' }), out('123'), CONFIG).passed).toBe(false); + }); + + it('validates a minimal JSON schema against parsed output', () => { + const schema = { type: 'object' as const, required: ['name', 'age'], properties: { name: { type: 'string' as const }, age: { type: 'integer' as const } } }; + expect(scoreAssertion(item({ jsonSchema: schema }), out('{"name":"x","age":3}'), CONFIG).passed).toBe(true); + expect(scoreAssertion(item({ jsonSchema: schema }), out('{"name":"x","age":"old"}'), CONFIG).passed).toBe(false); + expect(scoreAssertion(item({ jsonSchema: schema }), out('not json'), CONFIG).passed).toBe(false); + }); + + it('extracts JSON from fenced / chatty output for schema checks', () => { + const schema = { type: 'object' as const, required: ['ok'] }; + const text = 'Sure! Here you go:\n```json\n{"ok": true}\n```'; + expect(scoreAssertion(item({ jsonSchema: schema }), out(text), CONFIG).passed).toBe(true); + }); + + it('evaluates json-path existence and equality', () => { + const text = '{"data":{"items":[{"name":"alpha"}]}}'; + const r = scoreAssertion( + item({ jsonPath: [{ path: 'data.items[0].name', equals: 'alpha' }, { path: 'data.missing', exists: false }] }), + out(text), + CONFIG, + ); + expect(r.passed).toBe(true); + const r2 = scoreAssertion(item({ jsonPath: [{ path: 'data.items[0].name', equals: 'beta' }] }), out(text), CONFIG); + expect(r2.passed).toBe(false); + }); + + it('respects the configured weight', () => { + const r = scoreAssertion(item({ equals: 'x' }), out('x'), { type: 'assertion', weight: 3 }); + expect(r.weight).toBe(3); + }); +}); diff --git a/src/__tests__/unit/evaluation-llm-judge-scorer.test.ts b/src/__tests__/unit/evaluation-llm-judge-scorer.test.ts new file mode 100644 index 00000000..59c504b5 --- /dev/null +++ b/src/__tests__/unit/evaluation-llm-judge-scorer.test.ts @@ -0,0 +1,100 @@ +/** + * Unit tests — evaluation LLM-judge scorer (with a mocked judge invoker). + * Covers score normalisation, threshold/pass handling, prompt construction, + * and graceful failure on unparseable verdicts. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { + scoreLlmJudge, + parseJudgeResponse, + normaliseScore, + buildJudgePrompt, +} from '@/lib/services/evaluation/scorers/llmJudgeScorer'; +import type { DatasetItem, LlmJudgeScorerConfig, TargetOutput } from '@/lib/services/evaluation/types'; + +const ITEM: DatasetItem = { + id: 'i1', + input: [ + { role: 'system', content: 'be helpful' }, + { role: 'user', content: 'What is 2+2?' }, + ], + expected: { reference: '4' }, +}; +const OUTPUT: TargetOutput = { text: 'The answer is 4.' }; +const CONFIG: LlmJudgeScorerConfig = { type: 'llm-judge', rubric: 'Correct and concise.' }; + +describe('normaliseScore', () => { + it('passes through a 0..1 score', () => { + expect(normaliseScore(0.7)).toBeCloseTo(0.7, 5); + }); + it('auto-detects and rescales a 0..10 score', () => { + expect(normaliseScore(8)).toBeCloseTo(0.8, 5); + }); + it('clamps out-of-range values', () => { + expect(normaliseScore(-2)).toBe(0); + expect(normaliseScore(50)).toBe(1); + }); +}); + +describe('parseJudgeResponse', () => { + it('parses a plain JSON verdict', () => { + expect(parseJudgeResponse('{"score":0.9,"passed":true,"reasoning":"good"}')).toEqual({ + score: 0.9, + passed: true, + reasoning: 'good', + }); + }); + it('parses a fenced verdict', () => { + const v = parseJudgeResponse('```json\n{"score": 1}\n```'); + expect(v.score).toBe(1); + }); + it('throws when no numeric score is present', () => { + expect(() => parseJudgeResponse('{"reasoning":"n/a"}')).toThrow(/score/); + expect(() => parseJudgeResponse('totally not json')).toThrow(); + }); +}); + +describe('buildJudgePrompt', () => { + it('includes rubric, the last user message, the reference and the output', () => { + const messages = buildJudgePrompt(ITEM, OUTPUT, CONFIG); + expect(messages[0].role).toBe('system'); + const body = messages[1].content; + expect(body).toContain('Correct and concise.'); + expect(body).toContain('What is 2+2?'); + expect(body).toContain('4'); + expect(body).toContain('The answer is 4.'); + }); +}); + +describe('scoreLlmJudge', () => { + it('uses the judge verdict and explicit passed flag', async () => { + const invokeJudge = vi.fn().mockResolvedValue('{"score":0.95,"passed":true,"reasoning":"correct"}'); + const r = await scoreLlmJudge(ITEM, OUTPUT, CONFIG, invokeJudge); + expect(invokeJudge).toHaveBeenCalledOnce(); + expect(r.score).toBeCloseTo(0.95, 5); + expect(r.passed).toBe(true); + expect(r.detail?.reasoning).toBe('correct'); + }); + + it('derives passed from the threshold when not given', async () => { + const invokeJudge = vi.fn().mockResolvedValue('{"score":0.4}'); + const lenient = await scoreLlmJudge(ITEM, OUTPUT, { ...CONFIG, threshold: 0.3 }, invokeJudge); + const strict = await scoreLlmJudge(ITEM, OUTPUT, { ...CONFIG, threshold: 0.5 }, invokeJudge); + expect(lenient.passed).toBe(true); + expect(strict.passed).toBe(false); + }); + + it('fails gracefully when the judge errors or is unparseable', async () => { + const boom = vi.fn().mockRejectedValue(new Error('rate limited')); + const r = await scoreLlmJudge(ITEM, OUTPUT, CONFIG, boom); + expect(r.passed).toBe(false); + expect(r.score).toBe(0); + expect(r.error).toMatch(/rate limited/); + + const garbage = vi.fn().mockResolvedValue('no json here'); + const r2 = await scoreLlmJudge(ITEM, OUTPUT, CONFIG, garbage); + expect(r2.passed).toBe(false); + expect(r2.error).toBeTruthy(); + }); +}); diff --git a/src/__tests__/unit/evaluation-runner.test.ts b/src/__tests__/unit/evaluation-runner.test.ts new file mode 100644 index 00000000..841074a6 --- /dev/null +++ b/src/__tests__/unit/evaluation-runner.test.ts @@ -0,0 +1,84 @@ +/** + * Unit tests — evaluation runner. + * Covers aggregation (pass-rate / avg score / latency), per-item target + * failures, judge wiring, concurrency correctness, and the progress hook. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { runEvaluation } from '@/lib/services/evaluation/runner'; +import type { DatasetItem, ScorerConfig, TargetInvoker } from '@/lib/services/evaluation/types'; + +function items(n: number): DatasetItem[] { + return Array.from({ length: n }, (_, i) => ({ + id: `i${i}`, + input: [{ role: 'user', content: `q${i}` }], + expected: { mustContain: ['ok'] }, + })); +} + +const ASSERTION: ScorerConfig[] = [{ type: 'assertion' }]; + +describe('runEvaluation', () => { + it('aggregates pass-rate, score and latency across items', async () => { + const invokeTarget: TargetInvoker = async (item) => ({ + text: item.id === 'i1' ? 'nope' : 'ok', + latencyMs: 10, + }); + const result = await runEvaluation({ items: items(4), scorers: ASSERTION, invokeTarget }); + expect(result.aggregate.total).toBe(4); + expect(result.aggregate.completed).toBe(4); + expect(result.aggregate.failed).toBe(0); + expect(result.aggregate.passed).toBe(3); + expect(result.aggregate.passRate).toBeCloseTo(0.75, 5); + expect(result.aggregate.avgLatencyMs).toBe(10); + expect(result.items).toHaveLength(4); + }); + + it('records target failures without aborting the run', async () => { + const invokeTarget: TargetInvoker = async (item) => { + if (item.id === 'i2') throw new Error('boom'); + return { text: 'ok' }; + }; + const result = await runEvaluation({ items: items(4), scorers: ASSERTION, invokeTarget }); + expect(result.aggregate.failed).toBe(1); + expect(result.aggregate.completed).toBe(3); + const failedItem = result.items.find((i) => i.itemId === 'i2'); + expect(failedItem?.error).toMatch(/boom/); + expect(failedItem?.passed).toBe(false); + }); + + it('wires the judge invoker into llm-judge scorers', async () => { + const invokeTarget: TargetInvoker = async () => ({ text: 'ok' }); + const invokeJudge = vi.fn().mockResolvedValue('{"score":1,"passed":true}'); + const scorers: ScorerConfig[] = [{ type: 'assertion' }, { type: 'llm-judge', rubric: 'r' }]; + const result = await runEvaluation({ items: items(2), scorers, invokeTarget, invokeJudge }); + expect(invokeJudge).toHaveBeenCalledTimes(2); + expect(result.aggregate.passed).toBe(2); + expect(result.items[0].scores).toHaveLength(2); + }); + + it('processes every item exactly once under bounded concurrency', async () => { + const seen = new Set(); + let active = 0; + let maxActive = 0; + const invokeTarget: TargetInvoker = async (item) => { + active += 1; + maxActive = Math.max(maxActive, active); + await new Promise((r) => setTimeout(r, 1)); + seen.add(item.id); + active -= 1; + return { text: 'ok' }; + }; + const result = await runEvaluation({ items: items(10), scorers: ASSERTION, invokeTarget, config: { concurrency: 3 } }); + expect(seen.size).toBe(10); + expect(result.items.every((i) => i)).toBe(true); + expect(maxActive).toBeLessThanOrEqual(3); + }); + + it('invokes the progress hook once per item', async () => { + const invokeTarget: TargetInvoker = async () => ({ text: 'ok' }); + const onItem = vi.fn(); + await runEvaluation({ items: items(5), scorers: ASSERTION, invokeTarget, onItem }); + expect(onItem).toHaveBeenCalledTimes(5); + }); +}); diff --git a/src/app/dashboard/analysis/page.tsx b/src/app/dashboard/analysis/page.tsx new file mode 100644 index 00000000..bd4a727e --- /dev/null +++ b/src/app/dashboard/analysis/page.tsx @@ -0,0 +1,286 @@ +'use client'; + +import { useEffect, useMemo, useState } from 'react'; +import { useRouter } from 'next/navigation'; +import { Button, Group, Modal, Tabs, Text } from '@mantine/core'; +import { notifications } from '@mantine/notifications'; +import { + IconClipboardText, + IconMessages, + IconPlayerPlay, + IconPlus, + IconReportAnalytics, + IconTrash, + IconUpload, +} from '@tabler/icons-react'; +import PageContainer, { PageHeader } from '@/components/common/ui/PageContainer'; +import StatTile from '@/components/common/ui/StatTile'; +import DataGrid, { type DataGridColumn } from '@/components/common/ui/DataGrid'; +import CreateDefinitionModal from '@/components/analysis/CreateDefinitionModal'; +import IngestConversationsModal from '@/components/analysis/IngestConversationsModal'; +import type { + AnalysisConversationView, + AnalysisDefinitionView, + AnalysisRunView, + ModelOption, +} from '@/components/analysis/types'; + +type TabKey = 'definitions' | 'conversations' | 'runs'; + +const RUN_STATUS_BADGE: Record = { + completed: 'ds-badge-teal', + running: 'ds-badge-info', + failed: 'ds-badge-err', + pending: 'ds-badge', + cancelled: 'ds-badge-warn', +}; + +function fmtDate(value?: string): string { + if (!value) return '—'; + const d = new Date(value); + return Number.isFinite(d.getTime()) ? d.toLocaleString() : '—'; +} + +function pct(value?: number | null): string { + return value === undefined || value === null ? '—' : `${Math.round(value * 100)}%`; +} + +export default function AnalysisPage() { + const router = useRouter(); + const [tab, setTab] = useState('definitions'); + const [definitions, setDefinitions] = useState([]); + const [conversations, setConversations] = useState([]); + const [runs, setRuns] = useState([]); + const [models, setModels] = useState([]); + const [loading, setLoading] = useState(true); + const [refreshing, setRefreshing] = useState(false); + const [runningKey, setRunningKey] = useState(null); + + const [definitionModal, setDefinitionModal] = useState(false); + const [ingestModal, setIngestModal] = useState(false); + const [deleteItem, setDeleteItem] = useState<{ kind: TabKey; id: string; name: string } | null>(null); + const [deleting, setDeleting] = useState(false); + + const loadAll = async () => { + setRefreshing(true); + try { + const [dRes, cRes, rRes, mRes] = await Promise.all([ + fetch('/api/analysis/definitions', { cache: 'no-store' }), + fetch('/api/analysis/conversations', { cache: 'no-store' }), + fetch('/api/analysis/runs', { cache: 'no-store' }), + fetch('/api/models?category=llm', { cache: 'no-store' }), + ]); + if (dRes.ok) setDefinitions((await dRes.json()).definitions ?? []); + if (cRes.ok) setConversations((await cRes.json()).conversations ?? []); + if (rRes.ok) setRuns((await rRes.json()).runs ?? []); + if (mRes.ok) setModels(((await mRes.json()).models ?? []).map((m: { key: string; name: string }) => ({ value: m.key, label: m.name }))); + } catch (err) { + console.error('Failed to load analysis', err); + } finally { + setLoading(false); + setRefreshing(false); + } + }; + + const loadRuns = async () => { + const res = await fetch('/api/analysis/runs', { cache: 'no-store' }); + if (res.ok) setRuns((await res.json()).runs ?? []); + }; + + useEffect(() => { + void loadAll(); + }, []); + + const runDefinitionNow = async (def: AnalysisDefinitionView) => { + setRunningKey(def.key); + try { + const res = await fetch(`/api/analysis/definitions/${encodeURIComponent(def.key)}/run`, { method: 'POST' }); + const data = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(data.error || 'Run failed'); + const agg = data.run?.aggregate; + notifications.show({ + title: 'Analysis complete', + message: agg ? `${agg.completed}/${agg.total} analyzed · pass ${pct(agg.passRate)}` : 'Run finished', + color: 'teal', + }); + await loadRuns(); + setTab('runs'); + if (data.run?.id) router.push(`/dashboard/analysis/runs/${data.run.id}`); + } catch (err) { + notifications.show({ title: 'Run failed', message: err instanceof Error ? err.message : 'Run failed', color: 'red' }); + } finally { + setRunningKey(null); + } + }; + + const confirmDelete = async () => { + if (!deleteItem) return; + const path = deleteItem.kind === 'definitions' ? 'definitions' : 'conversations'; + setDeleting(true); + try { + const res = await fetch(`/api/analysis/${path}/${deleteItem.id}`, { method: 'DELETE' }); + if (!res.ok) throw new Error('Failed to delete'); + notifications.show({ title: 'Deleted', message: `"${deleteItem.name}" was deleted`, color: 'red' }); + setDeleteItem(null); + await loadAll(); + } catch (err) { + notifications.show({ title: 'Error', message: err instanceof Error ? err.message : 'Failed to delete', color: 'red' }); + } finally { + setDeleting(false); + } + }; + + const definitionColumns: DataGridColumn[] = [ + { key: 'name', label: 'Name', render: (d) => ( +
+ {d.name} + {d.key} +
+ ) }, + { key: 'fields', label: 'Fields', render: (d) => {d.fieldSet.length} }, + { key: 'modes', label: 'Modes', render: (d) => ( + + extract + {d.modes.store ? store : null} + {d.modes.judge ? judge : null} + {d.modes.accuracy ? accuracy : null} + + ) }, + { key: 'model', label: 'Extraction model', render: (d) => {d.extractionModelKey ?? '—'} }, + { key: 'schedule', label: 'Schedule', render: (d) => (d.schedule?.enabled ? {d.schedule.cron} : —) }, + ]; + + const conversationColumns: DataGridColumn[] = [ + { key: 'name', label: 'Name', render: (c) => ( +
+ {c.name || c.key} + {c.key} +
+ ) }, + { key: 'turns', label: 'Turns', render: (c) => {c.transcript.length} }, + { key: 'source', label: 'Source', render: (c) => {c.source} }, + { key: 'analyzed', label: 'Last analyzed', render: (c) => {fmtDate(c.lastAnalyzedAt)} }, + { key: 'created', label: 'Ingested', render: (c) => {fmtDate(c.createdAt)} }, + ]; + + const runColumns: DataGridColumn[] = [ + { key: 'def', label: 'Definition', render: (r) => {r.definitionKey} }, + { key: 'status', label: 'Status', render: (r) => {r.status} }, + { key: 'pass', label: 'Analyzed', render: (r) => {r.aggregate ? `${r.aggregate.completed}/${r.aggregate.total} (${pct(r.aggregate.passRate)})` : '—'} }, + { key: 'judge', label: 'Avg judge', render: (r) => {pct(r.aggregate?.avgJudgeScore)} }, + { key: 'acc', label: 'Avg accuracy', render: (r) => {pct(r.aggregate?.avgExtractionAccuracy)} }, + { key: 'created', label: 'Started', render: (r) => {fmtDate(r.startedAt ?? r.createdAt)} }, + ]; + + const actionButton = useMemo(() => { + if (tab === 'runs') return null; + if (tab === 'definitions') { + return ( + + ); + } + return ( + + ); + }, [tab]); + + return ( + + + +
+ } value={definitions.length} /> + } value={conversations.length} /> + } value={runs.length} /> + c.lastAnalyzedAt).length} /> +
+ + setTab((v as TabKey) ?? 'definitions')}> + + }>Definitions + }>Conversations + }>Runs + + + + + records={definitions} + loading={loading} + rowKey={(d) => d.id} + columns={definitionColumns} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No definitions yet', + description: 'A definition declares the fields to extract and which modes (judge / accuracy / store) to apply.', + primaryAction: { label: 'New definition', icon: , onClick: () => setDefinitionModal(true) }, + }} + rowActions={(d) => [ + { id: 'run', label: runningKey === d.key ? 'Running…' : 'Run analysis', icon: , onClick: () => void runDefinitionNow(d) }, + { divider: true }, + { id: 'delete', label: 'Delete', icon: , color: 'red', onClick: () => setDeleteItem({ kind: 'definitions', id: d.id, name: d.name }) }, + ]} + /> + + + + + records={conversations} + loading={loading} + rowKey={(c) => c.id} + columns={conversationColumns} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No conversations yet', + description: 'Ingest transcripts (from an external export or platform traffic) to analyze them.', + primaryAction: { label: 'Ingest conversations', icon: , onClick: () => setIngestModal(true) }, + }} + rowActions={(c) => [ + { id: 'delete', label: 'Delete', icon: , color: 'red', onClick: () => setDeleteItem({ kind: 'conversations', id: c.id, name: c.name || c.key }) }, + ]} + /> + + + + + records={runs} + loading={loading} + rowKey={(r) => r.id} + columns={runColumns} + onRowClick={(r) => router.push(`/dashboard/analysis/runs/${r.id}`)} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No runs yet', + description: 'Run a definition from the Definitions tab to see results here.', + }} + /> + + + + setDeleteItem(null)} title="Delete" centered size="sm"> + Delete {deleteItem?.name}? This action cannot be undone. + + + + + + + setDefinitionModal(false)} models={models} onCreated={() => void loadAll()} /> + setIngestModal(false)} onIngested={() => void loadAll()} /> + + ); +} diff --git a/src/app/dashboard/analysis/runs/[id]/page.tsx b/src/app/dashboard/analysis/runs/[id]/page.tsx new file mode 100644 index 00000000..d398bf7b --- /dev/null +++ b/src/app/dashboard/analysis/runs/[id]/page.tsx @@ -0,0 +1,121 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { useParams, useRouter } from 'next/navigation'; +import { Button, Group, Loader, Text } from '@mantine/core'; +import { IconArrowLeft } from '@tabler/icons-react'; +import PageContainer, { PageHeader } from '@/components/common/ui/PageContainer'; +import StatTile from '@/components/common/ui/StatTile'; +import DataGrid, { type DataGridColumn } from '@/components/common/ui/DataGrid'; +import type { AnalysisRunItemView, AnalysisRunView } from '@/components/analysis/types'; + +const RUN_STATUS_BADGE: Record = { + completed: 'ds-badge-teal', + running: 'ds-badge-info', + failed: 'ds-badge-err', + pending: 'ds-badge', + cancelled: 'ds-badge-warn', +}; + +function pct(value?: number | null): string { + return value === undefined || value === null ? '—' : `${Math.round(value * 100)}%`; +} + +function fieldsSummary(fields: Record): string { + const entries = Object.entries(fields); + if (entries.length === 0) return '—'; + return entries.map(([k, v]) => `${k}=${v === null || v === undefined ? '∅' : String(v)}`).join(', '); +} + +export default function AnalysisRunDetailPage() { + const router = useRouter(); + const params = useParams<{ id: string }>(); + const runId = params?.id; + const [run, setRun] = useState(null); + const [loading, setLoading] = useState(true); + const [notFound, setNotFound] = useState(false); + + useEffect(() => { + if (!runId) return; + let cancelled = false; + (async () => { + try { + const res = await fetch(`/api/analysis/runs/${runId}`, { cache: 'no-store' }); + if (res.status === 404) { if (!cancelled) setNotFound(true); return; } + const data = await res.json(); + if (!cancelled) setRun(data.run ?? null); + } catch { + if (!cancelled) setNotFound(true); + } finally { + if (!cancelled) setLoading(false); + } + })(); + return () => { cancelled = true; }; + }, [runId]); + + const itemColumns: DataGridColumn[] = [ + { key: 'conv', label: 'Conversation', render: (i) => {i.conversationKey} }, + { + key: 'result', + label: 'Result', + render: (i) => + i.error + ? error + : {i.passed ? 'pass' : i.missing.length ? `missing ${i.missing.length}` : 'fail'}, + }, + { key: 'fields', label: 'Extracted', render: (i) => {i.error ? i.error : fieldsSummary(i.extractedFields)} }, + { key: 'judge', label: 'Judge', render: (i) => (i.judge ? {i.judge.error ? 'err' : pct(i.judge.score)} : —) }, + { key: 'acc', label: 'Accuracy', render: (i) => (i.accuracy && i.accuracy.comparedCount > 0 ? {pct(i.accuracy.score)} : —) }, + ]; + + const backButton = ( + + ); + + if (loading) { + return ; + } + + if (notFound || !run) { + return ( + + + This analysis run could not be found. + + ); + } + + const agg = run.aggregate; + + return ( + + {run.status}} + actions={backButton} + /> + + {run.error ? ( +
{run.error}
+ ) : null} + +
+ + + + +
+ + + records={run.items} + rowKey={(i) => i.conversationKey} + columns={itemColumns} + footerLeft={`${run.items.length} conversations`} + empty={{ title: 'No items', description: 'This run produced no conversation results.' }} + /> + + ); +} diff --git a/src/app/dashboard/evaluations/page.tsx b/src/app/dashboard/evaluations/page.tsx new file mode 100644 index 00000000..4191a9e8 --- /dev/null +++ b/src/app/dashboard/evaluations/page.tsx @@ -0,0 +1,338 @@ +'use client'; + +import { useEffect, useMemo, useState } from 'react'; +import { useRouter } from 'next/navigation'; +import { Button, Group, Modal, Tabs, Text } from '@mantine/core'; +import { notifications } from '@mantine/notifications'; +import { + IconChecklist, + IconDatabase, + IconPlayerPlay, + IconPlus, + IconRobot, + IconTrash, +} from '@tabler/icons-react'; +import PageContainer, { PageHeader } from '@/components/common/ui/PageContainer'; +import StatTile from '@/components/common/ui/StatTile'; +import DataGrid, { type DataGridColumn } from '@/components/common/ui/DataGrid'; +import CreateTargetModal from '@/components/evaluations/CreateTargetModal'; +import CreateDatasetModal from '@/components/evaluations/CreateDatasetModal'; +import CreateSuiteModal from '@/components/evaluations/CreateSuiteModal'; +import type { + EvalDatasetView, + EvalRunView, + EvalSuiteView, + EvalTargetView, + ModelOption, +} from '@/components/evaluations/types'; + +type TabKey = 'targets' | 'datasets' | 'suites' | 'runs'; + +const RUN_STATUS_BADGE: Record = { + completed: 'ds-badge-teal', + running: 'ds-badge-info', + failed: 'ds-badge-err', + pending: 'ds-badge', + cancelled: 'ds-badge-warn', +}; + +function fmtDate(value?: string): string { + if (!value) return '—'; + const d = new Date(value); + return Number.isFinite(d.getTime()) ? d.toLocaleString() : '—'; +} + +function pct(value?: number): string { + return value === undefined ? '—' : `${Math.round(value * 100)}%`; +} + +export default function EvaluationsPage() { + const router = useRouter(); + const [tab, setTab] = useState('targets'); + const [targets, setTargets] = useState([]); + const [datasets, setDatasets] = useState([]); + const [suites, setSuites] = useState([]); + const [runs, setRuns] = useState([]); + const [models, setModels] = useState([]); + const [loading, setLoading] = useState(true); + const [refreshing, setRefreshing] = useState(false); + const [runningKey, setRunningKey] = useState(null); + + const [targetModal, setTargetModal] = useState(false); + const [datasetModal, setDatasetModal] = useState(false); + const [suiteModal, setSuiteModal] = useState(false); + const [deleteItem, setDeleteItem] = useState<{ kind: TabKey; id: string; name: string } | null>(null); + const [deleting, setDeleting] = useState(false); + + const loadAll = async () => { + setRefreshing(true); + try { + const [tRes, dRes, sRes, rRes, mRes] = await Promise.all([ + fetch('/api/evaluation/targets', { cache: 'no-store' }), + fetch('/api/evaluation/datasets', { cache: 'no-store' }), + fetch('/api/evaluation/suites', { cache: 'no-store' }), + fetch('/api/evaluation/runs', { cache: 'no-store' }), + fetch('/api/models?category=llm', { cache: 'no-store' }), + ]); + if (tRes.ok) setTargets((await tRes.json()).targets ?? []); + if (dRes.ok) setDatasets((await dRes.json()).datasets ?? []); + if (sRes.ok) setSuites((await sRes.json()).suites ?? []); + if (rRes.ok) setRuns((await rRes.json()).runs ?? []); + if (mRes.ok) { + setModels(((await mRes.json()).models ?? []).map((m: { key: string; name: string }) => ({ value: m.key, label: m.name }))); + } + } catch (err) { + console.error('Failed to load evaluations', err); + } finally { + setLoading(false); + setRefreshing(false); + } + }; + + const loadRuns = async () => { + const res = await fetch('/api/evaluation/runs', { cache: 'no-store' }); + if (res.ok) setRuns((await res.json()).runs ?? []); + }; + + useEffect(() => { + void loadAll(); + }, []); + + const runSuiteNow = async (suite: EvalSuiteView) => { + setRunningKey(suite.key); + try { + const res = await fetch(`/api/evaluation/suites/${encodeURIComponent(suite.key)}/run`, { method: 'POST' }); + const data = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(data.error || 'Run failed'); + const agg = data.run?.aggregate; + notifications.show({ + title: 'Evaluation complete', + message: agg ? `${agg.passed}/${agg.total} passed · avg score ${pct(agg.avgScore)}` : 'Run finished', + color: 'teal', + }); + await loadRuns(); + setTab('runs'); + if (data.run?.id) router.push(`/dashboard/evaluations/runs/${data.run.id}`); + } catch (err) { + notifications.show({ title: 'Run failed', message: err instanceof Error ? err.message : 'Run failed', color: 'red' }); + } finally { + setRunningKey(null); + } + }; + + const confirmDelete = async () => { + if (!deleteItem) return; + const path = + deleteItem.kind === 'targets' ? 'targets' : deleteItem.kind === 'datasets' ? 'datasets' : 'suites'; + setDeleting(true); + try { + const res = await fetch(`/api/evaluation/${path}/${deleteItem.id}`, { method: 'DELETE' }); + if (!res.ok) throw new Error('Failed to delete'); + notifications.show({ title: 'Deleted', message: `"${deleteItem.name}" was deleted`, color: 'red' }); + setDeleteItem(null); + await loadAll(); + } catch (err) { + notifications.show({ title: 'Error', message: err instanceof Error ? err.message : 'Failed to delete', color: 'red' }); + } finally { + setDeleting(false); + } + }; + + const targetName = (key: string) => targets.find((t) => t.key === key)?.name ?? key; + const datasetName = (key: string) => datasets.find((d) => d.key === key)?.name ?? key; + + const targetColumns: DataGridColumn[] = [ + { key: 'name', label: 'Name', render: (t) => ( +
+ {t.name} + {t.key} +
+ ) }, + { key: 'kind', label: 'Kind', render: (t) => {t.kind} }, + { key: 'ref', label: 'Model / Agent', render: (t) => ( + {t.modelKey ?? t.agentKey ?? '—'} + ) }, + { key: 'created', label: 'Created', render: (t) => {fmtDate(t.createdAt)} }, + ]; + + const datasetColumns: DataGridColumn[] = [ + { key: 'name', label: 'Name', render: (d) => ( +
+ {d.name} + {d.key} +
+ ) }, + { key: 'items', label: 'Items', render: (d) => {d.items.length} }, + { key: 'source', label: 'Source', render: (d) => {d.source} }, + { key: 'created', label: 'Created', render: (d) => {fmtDate(d.createdAt)} }, + ]; + + const suiteColumns: DataGridColumn[] = [ + { key: 'name', label: 'Name', render: (s) => ( +
+ {s.name} + {s.key} +
+ ) }, + { key: 'target', label: 'Target', render: (s) => {targetName(s.targetKey)} }, + { key: 'dataset', label: 'Dataset', render: (s) => {datasetName(s.datasetKey)} }, + { key: 'scorers', label: 'Scorers', render: (s) => ( + {s.scorers.map((sc) => {sc.type})} + ) }, + ]; + + const runColumns: DataGridColumn[] = [ + { key: 'suite', label: 'Suite', render: (r) => {r.suiteKey} }, + { key: 'status', label: 'Status', render: (r) => {r.status} }, + { key: 'pass', label: 'Pass rate', render: (r) => {r.aggregate ? `${r.aggregate.passed}/${r.aggregate.total} (${pct(r.aggregate.passRate)})` : '—'} }, + { key: 'score', label: 'Avg score', render: (r) => {r.aggregate ? pct(r.aggregate.avgScore) : '—'} }, + { key: 'created', label: 'Started', render: (r) => {fmtDate(r.startedAt ?? r.createdAt)} }, + ]; + + const actionButton = useMemo(() => { + if (tab === 'runs') return null; + const label = tab === 'targets' ? 'New target' : tab === 'datasets' ? 'New dataset' : 'New suite'; + const onClick = () => { + if (tab === 'targets') setTargetModal(true); + else if (tab === 'datasets') setDatasetModal(true); + else setSuiteModal(true); + }; + return ( + + ); + }, [tab]); + + return ( + + + +
+ } value={targets.length} /> + } value={datasets.length} /> + } value={suites.length} /> + +
+ + setTab((v as TabKey) ?? 'targets')}> + + }>Targets + }>Datasets + }>Suites + }>Runs + + + + + records={targets} + loading={loading} + rowKey={(t) => t.id} + columns={targetColumns} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No targets yet', + description: 'A target is the agent, model, or endpoint under test.', + primaryAction: { label: 'New target', icon: , onClick: () => setTargetModal(true) }, + }} + rowActions={(t) => [ + { id: 'delete', label: 'Delete', icon: , color: 'red', onClick: () => setDeleteItem({ kind: 'targets', id: t.id, name: t.name }) }, + ]} + /> + + + + + records={datasets} + loading={loading} + rowKey={(d) => d.id} + columns={datasetColumns} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No datasets yet', + description: 'A dataset is a set of test cases (inputs and optional expectations).', + primaryAction: { label: 'New dataset', icon: , onClick: () => setDatasetModal(true) }, + }} + rowActions={(d) => [ + { id: 'delete', label: 'Delete', icon: , color: 'red', onClick: () => setDeleteItem({ kind: 'datasets', id: d.id, name: d.name }) }, + ]} + /> + + + + + records={suites} + loading={loading} + rowKey={(s) => s.id} + columns={suiteColumns} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No suites yet', + description: 'A suite binds a target to a dataset with one or more scorers.', + primaryAction: { label: 'New suite', icon: , onClick: () => setSuiteModal(true) }, + }} + rowActions={(s) => [ + { + id: 'run', + label: runningKey === s.key ? 'Running…' : 'Run', + icon: , + onClick: () => void runSuiteNow(s), + }, + { divider: true }, + { id: 'delete', label: 'Delete', icon: , color: 'red', onClick: () => setDeleteItem({ kind: 'suites', id: s.id, name: s.name }) }, + ]} + /> + + + + + records={runs} + loading={loading} + rowKey={(r) => r.id} + columns={runColumns} + onRowClick={(r) => router.push(`/dashboard/evaluations/runs/${r.id}`)} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No runs yet', + description: 'Run a suite from the Suites tab to see results here.', + }} + /> + + + + setDeleteItem(null)} title="Delete" centered size="sm"> + + Delete {deleteItem?.name}? This action cannot be undone. + + + + + + + + setTargetModal(false)} models={models} onCreated={() => void loadAll()} /> + setDatasetModal(false)} onCreated={() => void loadAll()} /> + setSuiteModal(false)} + targets={targets} + datasets={datasets} + models={models} + onCreated={() => void loadAll()} + /> + + ); +} diff --git a/src/app/dashboard/evaluations/runs/[id]/page.tsx b/src/app/dashboard/evaluations/runs/[id]/page.tsx new file mode 100644 index 00000000..6b23df22 --- /dev/null +++ b/src/app/dashboard/evaluations/runs/[id]/page.tsx @@ -0,0 +1,151 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { useParams, useRouter } from 'next/navigation'; +import { Button, Group, Loader, Text } from '@mantine/core'; +import { IconArrowLeft } from '@tabler/icons-react'; +import PageContainer, { PageHeader } from '@/components/common/ui/PageContainer'; +import StatTile from '@/components/common/ui/StatTile'; +import DataGrid, { type DataGridColumn } from '@/components/common/ui/DataGrid'; +import type { EvalRunItemView, EvalRunView } from '@/components/evaluations/types'; + +const RUN_STATUS_BADGE: Record = { + completed: 'ds-badge-teal', + running: 'ds-badge-info', + failed: 'ds-badge-err', + pending: 'ds-badge', + cancelled: 'ds-badge-warn', +}; + +function pct(value?: number): string { + return value === undefined || value === null ? '—' : `${Math.round(value * 100)}%`; +} + +export default function EvaluationRunDetailPage() { + const router = useRouter(); + const params = useParams<{ id: string }>(); + const runId = params?.id; + const [run, setRun] = useState(null); + const [loading, setLoading] = useState(true); + const [notFound, setNotFound] = useState(false); + + useEffect(() => { + if (!runId) return; + let cancelled = false; + (async () => { + try { + const res = await fetch(`/api/evaluation/runs/${runId}`, { cache: 'no-store' }); + if (res.status === 404) { + if (!cancelled) setNotFound(true); + return; + } + const data = await res.json(); + if (!cancelled) setRun(data.run ?? null); + } catch { + if (!cancelled) setNotFound(true); + } finally { + if (!cancelled) setLoading(false); + } + })(); + return () => { + cancelled = true; + }; + }, [runId]); + + const itemColumns: DataGridColumn[] = [ + { key: 'item', label: 'Item', render: (i) => {i.itemId} }, + { + key: 'result', + label: 'Result', + render: (i) => + i.error + ? error + : {i.passed ? 'pass' : 'fail'}, + }, + { key: 'score', label: 'Score', render: (i) => {pct(i.score)} }, + { + key: 'scorers', + label: 'Scorers', + render: (i) => ( + + {i.scores.map((s, idx) => ( + + {s.scorerType}: {s.error ? 'err' : pct(s.score)} + + ))} + + ), + }, + { + key: 'output', + label: 'Output', + render: (i) => { + const text = i.error ? i.error : (i.output?.text ?? ''); + const short = text.length > 120 ? `${text.slice(0, 120)}…` : text; + return {short || '—'}; + }, + }, + ]; + + const backButton = ( + + ); + + if (loading) { + return ( + + + + ); + } + + if (notFound || !run) { + return ( + + + This evaluation run could not be found. + + ); + } + + const agg = run.aggregate; + + return ( + + + Target {run.targetKey} · Dataset {run.datasetKey}{' '} + · {run.status} + + } + actions={backButton} + /> + + {run.error ? ( +
+ {run.error} +
+ ) : null} + +
+ + + + +
+ + + records={run.items} + rowKey={(i) => i.itemId} + columns={itemColumns} + footerLeft={`${run.items.length} items`} + empty={{ title: 'No items', description: 'This run produced no item results.' }} + /> + + ); +} diff --git a/src/components/analysis/CreateDefinitionModal.tsx b/src/components/analysis/CreateDefinitionModal.tsx new file mode 100644 index 00000000..0b2c935e --- /dev/null +++ b/src/components/analysis/CreateDefinitionModal.tsx @@ -0,0 +1,189 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { + ActionIcon, + Button, + Checkbox, + Divider, + Group, + Modal, + Select, + Stack, + Text, + Textarea, + TextInput, +} from '@mantine/core'; +import { notifications } from '@mantine/notifications'; +import { IconPlus, IconTrash } from '@tabler/icons-react'; +import type { AnalysisDefinitionView, AnalysisFieldType, ModelOption } from './types'; + +interface CreateDefinitionModalProps { + opened: boolean; + onClose: () => void; + onCreated: (definition: AnalysisDefinitionView) => void; + models?: ModelOption[]; +} + +interface FieldRow { + key: string; + type: AnalysisFieldType; + required: boolean; + enumValues: string; +} + +const TYPE_OPTIONS = [ + { value: 'string', label: 'String' }, + { value: 'number', label: 'Number' }, + { value: 'boolean', label: 'Boolean' }, + { value: 'enum', label: 'Enum' }, +]; + +const emptyField = (): FieldRow => ({ key: '', type: 'string', required: false, enumValues: '' }); + +export default function CreateDefinitionModal({ opened, onClose, onCreated, models = [] }: CreateDefinitionModalProps) { + const [loading, setLoading] = useState(false); + const [name, setName] = useState(''); + const [description, setDescription] = useState(''); + const [instructions, setInstructions] = useState(''); + const [fields, setFields] = useState([emptyField()]); + const [extractionModelKey, setExtractionModelKey] = useState(''); + const [modeStore, setModeStore] = useState(true); + const [modeAccuracy, setModeAccuracy] = useState(false); + const [modeJudge, setModeJudge] = useState(false); + const [judgeRubric, setJudgeRubric] = useState(''); + const [judgeModelKey, setJudgeModelKey] = useState(''); + const [scheduleEnabled, setScheduleEnabled] = useState(false); + const [scheduleCron, setScheduleCron] = useState('0 2 * * *'); + const [error, setError] = useState(null); + + useEffect(() => { + if (!opened) { + setName(''); setDescription(''); setInstructions(''); + setFields([emptyField()]); setExtractionModelKey(''); + setModeStore(true); setModeAccuracy(false); setModeJudge(false); + setJudgeRubric(''); setJudgeModelKey(''); + setScheduleEnabled(false); setScheduleCron('0 2 * * *'); setError(null); + } + }, [opened]); + + const updateField = (idx: number, patch: Partial) => { + setFields((prev) => prev.map((f, i) => (i === idx ? { ...f, ...patch } : f))); + }; + + const handleSubmit = async () => { + const cleanFields = fields.filter((f) => f.key.trim()); + if (!name.trim()) return setError('Name is required'); + if (cleanFields.length === 0) return setError('Add at least one field with a key'); + if (!extractionModelKey) return setError('An extraction model is required'); + if (modeJudge && !judgeRubric.trim()) return setError('A judge rubric is required when the judge mode is on'); + if (modeJudge && !judgeModelKey) return setError('A judge model is required when the judge mode is on'); + for (const f of cleanFields) { + if (f.type === 'enum' && !f.enumValues.split(',').map((v) => v.trim()).filter(Boolean).length) { + return setError(`Enum field "${f.key}" needs comma-separated values`); + } + } + setError(null); + setLoading(true); + try { + const fieldSet = cleanFields.map((f) => ({ + key: f.key.trim(), + type: f.type, + required: f.required || undefined, + enumValues: f.type === 'enum' ? f.enumValues.split(',').map((v) => v.trim()).filter(Boolean) : undefined, + })); + const modes = { + store: modeStore || undefined, + accuracy: modeAccuracy || undefined, + judge: modeJudge ? { rubric: judgeRubric.trim() } : undefined, + }; + const res = await fetch('/api/analysis/definitions', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + name: name.trim(), + description: description || undefined, + fieldSet, + extractionInstructions: instructions || undefined, + modes, + extractionModelKey, + judgeModelKey: modeJudge ? judgeModelKey : undefined, + schedule: scheduleEnabled ? { cron: scheduleCron.trim(), enabled: true } : undefined, + }), + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.error || 'Failed to create definition'); + } + const data = await res.json(); + notifications.show({ title: 'Definition created', message: `"${data.definition.name}" was created`, color: 'teal' }); + onCreated(data.definition); + onClose(); + } catch (err) { + notifications.show({ title: 'Error', message: err instanceof Error ? err.message : 'Failed to create', color: 'red' }); + } finally { + setLoading(false); + } + }; + + return ( + + + setName(e.currentTarget.value)} /> +