From 5024b52f3bf88a491706bb9089d388c301db5ac2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 15:16:26 +0000 Subject: [PATCH 01/10] feat(evaluation): add dependency-injected scoring engine core Introduce the Evaluation service engine as a self-contained core with no database, queue, or model-runtime coupling, so it compiles and is fully unit-testable on its own. Persistence, live target/judge adapters, the HTTP API, and the dashboard layer on top of this engine separately. - types: dataset items, scorer configs (assertion | llm-judge), run and aggregate results, and injected target/judge invokers - assertion scorer: equals / contains / notContains / regex / minimal JSON-schema / JSON-path checks, dependency-free - llm-judge scorer: rubric-driven grading through an injected judge invoker, with 0..1 score normalisation and graceful failure handling - runner: bounded-concurrency orchestration with pass-rate / score / latency aggregation and a per-item progress hook Covered by 24 unit tests; tsc --noEmit and eslint are clean. https://claude.ai/code/session_01UDGtTEyau4AoGC5eQuQKK3 --- .../unit/evaluation-assertion-scorer.test.ts | 80 ++++++++ .../unit/evaluation-llm-judge-scorer.test.ts | 100 ++++++++++ src/__tests__/unit/evaluation-runner.test.ts | 84 ++++++++ src/lib/services/evaluation/index.ts | 13 ++ src/lib/services/evaluation/runner.ts | 112 +++++++++++ .../evaluation/scorers/assertionScorer.ts | 82 ++++++++ src/lib/services/evaluation/scorers/index.ts | 52 +++++ src/lib/services/evaluation/scorers/json.ts | 182 ++++++++++++++++++ .../evaluation/scorers/llmJudgeScorer.ts | 104 ++++++++++ src/lib/services/evaluation/types.ts | 145 ++++++++++++++ 10 files changed, 954 insertions(+) create mode 100644 src/__tests__/unit/evaluation-assertion-scorer.test.ts create mode 100644 src/__tests__/unit/evaluation-llm-judge-scorer.test.ts create mode 100644 src/__tests__/unit/evaluation-runner.test.ts create mode 100644 src/lib/services/evaluation/index.ts create mode 100644 src/lib/services/evaluation/runner.ts create mode 100644 src/lib/services/evaluation/scorers/assertionScorer.ts create mode 100644 src/lib/services/evaluation/scorers/index.ts create mode 100644 src/lib/services/evaluation/scorers/json.ts create mode 100644 src/lib/services/evaluation/scorers/llmJudgeScorer.ts create mode 100644 src/lib/services/evaluation/types.ts diff --git a/src/__tests__/unit/evaluation-assertion-scorer.test.ts b/src/__tests__/unit/evaluation-assertion-scorer.test.ts new file mode 100644 index 00000000..b0458377 --- /dev/null +++ b/src/__tests__/unit/evaluation-assertion-scorer.test.ts @@ -0,0 +1,80 @@ +/** + * Unit tests — evaluation assertion scorer. + * Covers equals, contains/notContains, regex, json-schema, json-path, and the + * no-assertion no-op case. + */ + +import { describe, it, expect } from 'vitest'; +import { scoreAssertion } from '@/lib/services/evaluation/scorers/assertionScorer'; +import type { AssertionScorerConfig, DatasetItem, TargetOutput } from '@/lib/services/evaluation/types'; + +const CONFIG: AssertionScorerConfig = { type: 'assertion' }; + +function item(expected: DatasetItem['expected']): DatasetItem { + return { id: 'i1', input: [{ role: 'user', content: 'hi' }], expected }; +} +function out(text: string): TargetOutput { + return { text }; +} + +describe('assertionScorer', () => { + it('treats absence of expectations as a passing no-op', () => { + const r = scoreAssertion(item(undefined), out('anything'), CONFIG); + expect(r.passed).toBe(true); + expect(r.score).toBe(1); + expect(r.detail?.total).toBe(0); + }); + + it('passes exact equals (trimmed) and fails otherwise', () => { + expect(scoreAssertion(item({ equals: 'yes' }), out(' yes \n'), CONFIG).passed).toBe(true); + expect(scoreAssertion(item({ equals: 'yes' }), out('no'), CONFIG).passed).toBe(false); + }); + + it('handles mustContain / mustNotContain', () => { + const r = scoreAssertion(item({ mustContain: ['foo', 'bar'], mustNotContain: ['baz'] }), out('foo and bar'), CONFIG); + expect(r.passed).toBe(true); + const r2 = scoreAssertion(item({ mustContain: ['foo'], mustNotContain: ['bar'] }), out('foo bar'), CONFIG); + expect(r2.passed).toBe(false); + }); + + it('computes a partial score from the fraction of checks passed', () => { + const r = scoreAssertion(item({ mustContain: ['a', 'b', 'c', 'd'] }), out('a b'), CONFIG); + expect(r.score).toBeCloseTo(0.5, 5); + expect(r.passed).toBe(false); + }); + + it('evaluates regex and reports invalid patterns as failed', () => { + expect(scoreAssertion(item({ regex: '^\\d{3}$' }), out('123'), CONFIG).passed).toBe(true); + expect(scoreAssertion(item({ regex: '(' }), out('123'), CONFIG).passed).toBe(false); + }); + + it('validates a minimal JSON schema against parsed output', () => { + const schema = { type: 'object' as const, required: ['name', 'age'], properties: { name: { type: 'string' as const }, age: { type: 'integer' as const } } }; + expect(scoreAssertion(item({ jsonSchema: schema }), out('{"name":"x","age":3}'), CONFIG).passed).toBe(true); + expect(scoreAssertion(item({ jsonSchema: schema }), out('{"name":"x","age":"old"}'), CONFIG).passed).toBe(false); + expect(scoreAssertion(item({ jsonSchema: schema }), out('not json'), CONFIG).passed).toBe(false); + }); + + it('extracts JSON from fenced / chatty output for schema checks', () => { + const schema = { type: 'object' as const, required: ['ok'] }; + const text = 'Sure! Here you go:\n```json\n{"ok": true}\n```'; + expect(scoreAssertion(item({ jsonSchema: schema }), out(text), CONFIG).passed).toBe(true); + }); + + it('evaluates json-path existence and equality', () => { + const text = '{"data":{"items":[{"name":"alpha"}]}}'; + const r = scoreAssertion( + item({ jsonPath: [{ path: 'data.items[0].name', equals: 'alpha' }, { path: 'data.missing', exists: false }] }), + out(text), + CONFIG, + ); + expect(r.passed).toBe(true); + const r2 = scoreAssertion(item({ jsonPath: [{ path: 'data.items[0].name', equals: 'beta' }] }), out(text), CONFIG); + expect(r2.passed).toBe(false); + }); + + it('respects the configured weight', () => { + const r = scoreAssertion(item({ equals: 'x' }), out('x'), { type: 'assertion', weight: 3 }); + expect(r.weight).toBe(3); + }); +}); diff --git a/src/__tests__/unit/evaluation-llm-judge-scorer.test.ts b/src/__tests__/unit/evaluation-llm-judge-scorer.test.ts new file mode 100644 index 00000000..59c504b5 --- /dev/null +++ b/src/__tests__/unit/evaluation-llm-judge-scorer.test.ts @@ -0,0 +1,100 @@ +/** + * Unit tests — evaluation LLM-judge scorer (with a mocked judge invoker). + * Covers score normalisation, threshold/pass handling, prompt construction, + * and graceful failure on unparseable verdicts. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { + scoreLlmJudge, + parseJudgeResponse, + normaliseScore, + buildJudgePrompt, +} from '@/lib/services/evaluation/scorers/llmJudgeScorer'; +import type { DatasetItem, LlmJudgeScorerConfig, TargetOutput } from '@/lib/services/evaluation/types'; + +const ITEM: DatasetItem = { + id: 'i1', + input: [ + { role: 'system', content: 'be helpful' }, + { role: 'user', content: 'What is 2+2?' }, + ], + expected: { reference: '4' }, +}; +const OUTPUT: TargetOutput = { text: 'The answer is 4.' }; +const CONFIG: LlmJudgeScorerConfig = { type: 'llm-judge', rubric: 'Correct and concise.' }; + +describe('normaliseScore', () => { + it('passes through a 0..1 score', () => { + expect(normaliseScore(0.7)).toBeCloseTo(0.7, 5); + }); + it('auto-detects and rescales a 0..10 score', () => { + expect(normaliseScore(8)).toBeCloseTo(0.8, 5); + }); + it('clamps out-of-range values', () => { + expect(normaliseScore(-2)).toBe(0); + expect(normaliseScore(50)).toBe(1); + }); +}); + +describe('parseJudgeResponse', () => { + it('parses a plain JSON verdict', () => { + expect(parseJudgeResponse('{"score":0.9,"passed":true,"reasoning":"good"}')).toEqual({ + score: 0.9, + passed: true, + reasoning: 'good', + }); + }); + it('parses a fenced verdict', () => { + const v = parseJudgeResponse('```json\n{"score": 1}\n```'); + expect(v.score).toBe(1); + }); + it('throws when no numeric score is present', () => { + expect(() => parseJudgeResponse('{"reasoning":"n/a"}')).toThrow(/score/); + expect(() => parseJudgeResponse('totally not json')).toThrow(); + }); +}); + +describe('buildJudgePrompt', () => { + it('includes rubric, the last user message, the reference and the output', () => { + const messages = buildJudgePrompt(ITEM, OUTPUT, CONFIG); + expect(messages[0].role).toBe('system'); + const body = messages[1].content; + expect(body).toContain('Correct and concise.'); + expect(body).toContain('What is 2+2?'); + expect(body).toContain('4'); + expect(body).toContain('The answer is 4.'); + }); +}); + +describe('scoreLlmJudge', () => { + it('uses the judge verdict and explicit passed flag', async () => { + const invokeJudge = vi.fn().mockResolvedValue('{"score":0.95,"passed":true,"reasoning":"correct"}'); + const r = await scoreLlmJudge(ITEM, OUTPUT, CONFIG, invokeJudge); + expect(invokeJudge).toHaveBeenCalledOnce(); + expect(r.score).toBeCloseTo(0.95, 5); + expect(r.passed).toBe(true); + expect(r.detail?.reasoning).toBe('correct'); + }); + + it('derives passed from the threshold when not given', async () => { + const invokeJudge = vi.fn().mockResolvedValue('{"score":0.4}'); + const lenient = await scoreLlmJudge(ITEM, OUTPUT, { ...CONFIG, threshold: 0.3 }, invokeJudge); + const strict = await scoreLlmJudge(ITEM, OUTPUT, { ...CONFIG, threshold: 0.5 }, invokeJudge); + expect(lenient.passed).toBe(true); + expect(strict.passed).toBe(false); + }); + + it('fails gracefully when the judge errors or is unparseable', async () => { + const boom = vi.fn().mockRejectedValue(new Error('rate limited')); + const r = await scoreLlmJudge(ITEM, OUTPUT, CONFIG, boom); + expect(r.passed).toBe(false); + expect(r.score).toBe(0); + expect(r.error).toMatch(/rate limited/); + + const garbage = vi.fn().mockResolvedValue('no json here'); + const r2 = await scoreLlmJudge(ITEM, OUTPUT, CONFIG, garbage); + expect(r2.passed).toBe(false); + expect(r2.error).toBeTruthy(); + }); +}); diff --git a/src/__tests__/unit/evaluation-runner.test.ts b/src/__tests__/unit/evaluation-runner.test.ts new file mode 100644 index 00000000..841074a6 --- /dev/null +++ b/src/__tests__/unit/evaluation-runner.test.ts @@ -0,0 +1,84 @@ +/** + * Unit tests — evaluation runner. + * Covers aggregation (pass-rate / avg score / latency), per-item target + * failures, judge wiring, concurrency correctness, and the progress hook. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { runEvaluation } from '@/lib/services/evaluation/runner'; +import type { DatasetItem, ScorerConfig, TargetInvoker } from '@/lib/services/evaluation/types'; + +function items(n: number): DatasetItem[] { + return Array.from({ length: n }, (_, i) => ({ + id: `i${i}`, + input: [{ role: 'user', content: `q${i}` }], + expected: { mustContain: ['ok'] }, + })); +} + +const ASSERTION: ScorerConfig[] = [{ type: 'assertion' }]; + +describe('runEvaluation', () => { + it('aggregates pass-rate, score and latency across items', async () => { + const invokeTarget: TargetInvoker = async (item) => ({ + text: item.id === 'i1' ? 'nope' : 'ok', + latencyMs: 10, + }); + const result = await runEvaluation({ items: items(4), scorers: ASSERTION, invokeTarget }); + expect(result.aggregate.total).toBe(4); + expect(result.aggregate.completed).toBe(4); + expect(result.aggregate.failed).toBe(0); + expect(result.aggregate.passed).toBe(3); + expect(result.aggregate.passRate).toBeCloseTo(0.75, 5); + expect(result.aggregate.avgLatencyMs).toBe(10); + expect(result.items).toHaveLength(4); + }); + + it('records target failures without aborting the run', async () => { + const invokeTarget: TargetInvoker = async (item) => { + if (item.id === 'i2') throw new Error('boom'); + return { text: 'ok' }; + }; + const result = await runEvaluation({ items: items(4), scorers: ASSERTION, invokeTarget }); + expect(result.aggregate.failed).toBe(1); + expect(result.aggregate.completed).toBe(3); + const failedItem = result.items.find((i) => i.itemId === 'i2'); + expect(failedItem?.error).toMatch(/boom/); + expect(failedItem?.passed).toBe(false); + }); + + it('wires the judge invoker into llm-judge scorers', async () => { + const invokeTarget: TargetInvoker = async () => ({ text: 'ok' }); + const invokeJudge = vi.fn().mockResolvedValue('{"score":1,"passed":true}'); + const scorers: ScorerConfig[] = [{ type: 'assertion' }, { type: 'llm-judge', rubric: 'r' }]; + const result = await runEvaluation({ items: items(2), scorers, invokeTarget, invokeJudge }); + expect(invokeJudge).toHaveBeenCalledTimes(2); + expect(result.aggregate.passed).toBe(2); + expect(result.items[0].scores).toHaveLength(2); + }); + + it('processes every item exactly once under bounded concurrency', async () => { + const seen = new Set(); + let active = 0; + let maxActive = 0; + const invokeTarget: TargetInvoker = async (item) => { + active += 1; + maxActive = Math.max(maxActive, active); + await new Promise((r) => setTimeout(r, 1)); + seen.add(item.id); + active -= 1; + return { text: 'ok' }; + }; + const result = await runEvaluation({ items: items(10), scorers: ASSERTION, invokeTarget, config: { concurrency: 3 } }); + expect(seen.size).toBe(10); + expect(result.items.every((i) => i)).toBe(true); + expect(maxActive).toBeLessThanOrEqual(3); + }); + + it('invokes the progress hook once per item', async () => { + const invokeTarget: TargetInvoker = async () => ({ text: 'ok' }); + const onItem = vi.fn(); + await runEvaluation({ items: items(5), scorers: ASSERTION, invokeTarget, onItem }); + expect(onItem).toHaveBeenCalledTimes(5); + }); +}); diff --git a/src/lib/services/evaluation/index.ts b/src/lib/services/evaluation/index.ts new file mode 100644 index 00000000..36dcce74 --- /dev/null +++ b/src/lib/services/evaluation/index.ts @@ -0,0 +1,13 @@ +/** + * Evaluation service — public surface of the engine core. + * + * This module is a self-contained, dependency-injected scoring engine: it has + * no database, queue, or model-runtime imports. Higher layers (persistence, + * live target/judge adapters, HTTP API, dashboard) build on top of it. + */ + +export * from './types'; +export { runEvaluation } from './runner'; +export type { RunEvaluationParams } from './runner'; +export { runScorers, SUPPORTED_SCORERS, scoreAssertion, scoreLlmJudge } from './scorers'; +export type { ScorerDeps } from './scorers'; diff --git a/src/lib/services/evaluation/runner.ts b/src/lib/services/evaluation/runner.ts new file mode 100644 index 00000000..e8f71059 --- /dev/null +++ b/src/lib/services/evaluation/runner.ts @@ -0,0 +1,112 @@ +/** + * Evaluation runner — orchestrates target invocation + scoring across a + * dataset with bounded concurrency, then aggregates pass-rate / score / + * latency. Pure with respect to the platform: targets and judges are injected. + */ + +import type { + DatasetItem, + JudgeInvoker, + RunAggregate, + RunConfig, + RunItemResult, + RunResult, + ScorerConfig, + ScoreResult, + TargetInvoker, +} from './types'; +import { runScorers } from './scorers'; + +export interface RunEvaluationParams { + items: DatasetItem[]; + scorers: ScorerConfig[]; + invokeTarget: TargetInvoker; + invokeJudge?: JudgeInvoker; + config?: RunConfig; + /** Progress hook, invoked once per completed item. */ + onItem?: (result: RunItemResult, index: number) => void; +} + +export async function runEvaluation(params: RunEvaluationParams): Promise { + const { items, scorers, invokeTarget, invokeJudge, config, onItem } = params; + const concurrency = Math.max(1, config?.concurrency ?? 4); + const results = new Array(items.length); + + let cursor = 0; + const worker = async (): Promise => { + for (;;) { + const index = cursor; + cursor += 1; + if (index >= items.length) return; + const result = await runItem(items[index], scorers, invokeTarget, invokeJudge); + results[index] = result; + onItem?.(result, index); + } + }; + + const poolSize = Math.min(concurrency, items.length || 1); + await Promise.all(Array.from({ length: poolSize }, () => worker())); + + return { aggregate: aggregate(results), items: results }; +} + +async function runItem( + item: DatasetItem, + scorers: ScorerConfig[], + invokeTarget: TargetInvoker, + invokeJudge: JudgeInvoker | undefined, +): Promise { + const started = Date.now(); + try { + const output = await invokeTarget(item); + const scores = await runScorers(item, output, scorers, { invokeJudge }); + const { score, passed } = combine(scores); + return { + itemId: item.id, + output, + scores, + score, + passed, + latencyMs: output.latencyMs ?? Date.now() - started, + }; + } catch (err) { + return { + itemId: item.id, + scores: [], + score: 0, + passed: false, + error: (err as Error).message, + latencyMs: Date.now() - started, + }; + } +} + +/** Weighted mean of scorer scores; an item passes only if every scorer did. */ +function combine(scores: ScoreResult[]): { score: number; passed: boolean } { + if (scores.length === 0) return { score: 0, passed: false }; + const totalWeight = scores.reduce((sum, s) => sum + (s.weight ?? 1), 0) || scores.length; + const weighted = scores.reduce((sum, s) => sum + s.score * (s.weight ?? 1), 0); + return { score: weighted / totalWeight, passed: scores.every((s) => s.passed) }; +} + +function aggregate(items: RunItemResult[]): RunAggregate { + const total = items.length; + const completedItems = items.filter((i) => !i.error); + const completed = completedItems.length; + const failed = total - completed; + const passed = items.filter((i) => i.passed).length; + + const latencies = items.map((i) => i.latencyMs).filter((v): v is number => typeof v === 'number'); + const avgLatencyMs = latencies.length ? latencies.reduce((a, b) => a + b, 0) / latencies.length : null; + const avgScore = completed ? completedItems.reduce((a, b) => a + b.score, 0) / completed : 0; + + return { + total, + completed, + failed, + passed, + passRate: completed ? passed / completed : 0, + avgScore, + avgLatencyMs, + }; +} diff --git a/src/lib/services/evaluation/scorers/assertionScorer.ts b/src/lib/services/evaluation/scorers/assertionScorer.ts new file mode 100644 index 00000000..3a7b4443 --- /dev/null +++ b/src/lib/services/evaluation/scorers/assertionScorer.ts @@ -0,0 +1,82 @@ +/** + * Assertion scorer — pure, deterministic checks against the target output. + * + * Supports: exact equals, substring inclusion/exclusion, regex match, + * minimal JSON-schema validation, and JSON-path value/existence assertions. + * The score is the fraction of checks that passed; with no checks present the + * scorer is a no-op (score 1, passed). + */ + +import type { AssertionScorerConfig, DatasetItem, ScoreResult, TargetOutput } from '../types'; +import { deepEqual, extractJson, getByPath, validateSchema } from './json'; + +interface Check { + name: string; + passed: boolean; + detail?: string; +} + +export function scoreAssertion( + item: DatasetItem, + output: TargetOutput, + config: AssertionScorerConfig, +): ScoreResult { + const weight = config.weight ?? 1; + const expected = item.expected ?? {}; + const text = output.text ?? ''; + const checks: Check[] = []; + + if (expected.equals !== undefined) { + checks.push({ name: 'equals', passed: text.trim() === expected.equals.trim() }); + } + + for (const sub of expected.mustContain ?? []) { + checks.push({ name: `contains:${sub}`, passed: text.includes(sub) }); + } + + for (const sub of expected.mustNotContain ?? []) { + checks.push({ name: `notContains:${sub}`, passed: !text.includes(sub) }); + } + + if (expected.regex !== undefined) { + try { + checks.push({ name: 'regex', passed: new RegExp(expected.regex).test(text) }); + } catch (err) { + checks.push({ name: 'regex', passed: false, detail: `invalid regex: ${(err as Error).message}` }); + } + } + + if (expected.jsonSchema) { + const parsed = extractJson(text); + if (!parsed.ok) { + checks.push({ name: 'jsonSchema', passed: false, detail: parsed.error }); + } else { + const schemaErrors = validateSchema(parsed.value, expected.jsonSchema); + checks.push({ name: 'jsonSchema', passed: schemaErrors.length === 0, detail: schemaErrors[0] }); + } + } + + for (const assertion of expected.jsonPath ?? []) { + const parsed = extractJson(text); + if (!parsed.ok) { + checks.push({ name: `jsonPath:${assertion.path}`, passed: false, detail: parsed.error }); + continue; + } + const lookup = getByPath(parsed.value, assertion.path); + let passed = true; + if (assertion.exists !== undefined) passed = passed && lookup.exists === assertion.exists; + if (assertion.equals !== undefined) passed = passed && lookup.exists && deepEqual(lookup.value, assertion.equals); + checks.push({ name: `jsonPath:${assertion.path}`, passed }); + } + + const total = checks.length; + const passedCount = checks.filter((c) => c.passed).length; + + return { + scorerType: 'assertion', + score: total === 0 ? 1 : passedCount / total, + passed: total === 0 ? true : passedCount === total, + weight, + detail: { total, passedCount, checks }, + }; +} diff --git a/src/lib/services/evaluation/scorers/index.ts b/src/lib/services/evaluation/scorers/index.ts new file mode 100644 index 00000000..165c3c8e --- /dev/null +++ b/src/lib/services/evaluation/scorers/index.ts @@ -0,0 +1,52 @@ +/** + * Scorer dispatch. Runs the configured scorers for one item against the + * target output, returning one ScoreResult per scorer. + */ + +import type { DatasetItem, JudgeInvoker, ScorerConfig, ScorerType, ScoreResult, TargetOutput } from '../types'; +import { scoreAssertion } from './assertionScorer'; +import { scoreLlmJudge } from './llmJudgeScorer'; + +export const SUPPORTED_SCORERS: ScorerType[] = ['assertion', 'llm-judge']; + +export interface ScorerDeps { + invokeJudge?: JudgeInvoker; +} + +export async function runScorers( + item: DatasetItem, + output: TargetOutput, + scorers: ScorerConfig[], + deps: ScorerDeps = {}, +): Promise { + const results: ScoreResult[] = []; + for (const config of scorers) { + switch (config.type) { + case 'assertion': + results.push(scoreAssertion(item, output, config)); + break; + case 'llm-judge': + if (!deps.invokeJudge) { + results.push({ + scorerType: 'llm-judge', + score: 0, + passed: false, + weight: config.weight ?? 1, + error: 'no judge invoker configured', + }); + } else { + results.push(await scoreLlmJudge(item, output, config, deps.invokeJudge)); + } + break; + default: { + // Exhaustiveness guard — a new ScorerConfig variant must be handled. + const _never: never = config; + throw new Error(`unsupported scorer: ${JSON.stringify(_never)}`); + } + } + } + return results; +} + +export { scoreAssertion } from './assertionScorer'; +export { scoreLlmJudge, buildJudgePrompt, parseJudgeResponse, normaliseScore } from './llmJudgeScorer'; diff --git a/src/lib/services/evaluation/scorers/json.ts b/src/lib/services/evaluation/scorers/json.ts new file mode 100644 index 00000000..66d8fccf --- /dev/null +++ b/src/lib/services/evaluation/scorers/json.ts @@ -0,0 +1,182 @@ +/** + * Dependency-free JSON helpers shared by the assertion and LLM-judge scorers: + * lenient extraction (handles ```json fences / surrounding prose), dot/bracket + * path resolution, deep equality, and a minimal JSON-schema subset validator. + */ + +import type { JsonSchema } from '../types'; + +export type ParseResult = + | { ok: true; value: unknown } + | { ok: false; error: string }; + +/** + * Parse JSON from arbitrary model output. Tries a direct parse first, then + * falls back to the first balanced `{...}` / `[...]` block found in the text + * (covering fenced code blocks and chatty preambles). + */ +export function extractJson(text: string): ParseResult { + const trimmed = (text ?? '').trim(); + if (!trimmed) return { ok: false, error: 'empty output' }; + + const direct = tryParse(trimmed); + if (direct.ok) return direct; + + const block = findFirstBalancedBlock(trimmed); + if (block) { + const parsed = tryParse(block); + if (parsed.ok) return parsed; + } + return { ok: false, error: 'no valid JSON found in output' }; +} + +function tryParse(s: string): ParseResult { + try { + return { ok: true, value: JSON.parse(s) }; + } catch (err) { + return { ok: false, error: (err as Error).message }; + } +} + +/** Find the first balanced object or array literal, respecting strings. */ +function findFirstBalancedBlock(text: string): string | null { + const start = text.search(/[{[]/); + if (start === -1) return null; + const open = text[start]; + const close = open === '{' ? '}' : ']'; + let depth = 0; + let inString = false; + let escaped = false; + for (let i = start; i < text.length; i += 1) { + const ch = text[i]; + if (inString) { + if (escaped) escaped = false; + else if (ch === '\\') escaped = true; + else if (ch === '"') inString = false; + continue; + } + if (ch === '"') inString = true; + else if (ch === open) depth += 1; + else if (ch === close) { + depth -= 1; + if (depth === 0) return text.slice(start, i + 1); + } + } + return null; +} + +export interface PathLookup { + exists: boolean; + value: unknown; +} + +/** Resolve a dot / bracket path (e.g. `data.items[0].name`) against a value. */ +export function getByPath(root: unknown, path: string): PathLookup { + const tokens = tokenizePath(path); + let current: unknown = root; + for (const token of tokens) { + if (current === null || current === undefined) return { exists: false, value: undefined }; + if (typeof token === 'number') { + if (!Array.isArray(current) || token < 0 || token >= current.length) { + return { exists: false, value: undefined }; + } + current = current[token]; + } else { + if (typeof current !== 'object' || Array.isArray(current)) { + return { exists: false, value: undefined }; + } + const obj = current as Record; + if (!(token in obj)) return { exists: false, value: undefined }; + current = obj[token]; + } + } + return { exists: true, value: current }; +} + +function tokenizePath(path: string): Array { + const tokens: Array = []; + // Split on dots that are not inside brackets, then expand [n] indices. + const segments = path.split('.').filter((s) => s.length > 0); + for (const segment of segments) { + const bracketMatch = segment.match(/^([^[]*)((\[\d+\])*)$/); + if (!bracketMatch) { + tokens.push(segment); + continue; + } + const [, name, indices] = bracketMatch; + if (name) tokens.push(name); + for (const idx of indices.match(/\[(\d+)\]/g) ?? []) { + tokens.push(Number(idx.slice(1, -1))); + } + } + return tokens; +} + +/** Structural deep-equality good enough for JSON values. */ +export function deepEqual(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (typeof a !== typeof b) return false; + if (a === null || b === null) return a === b; + if (Array.isArray(a) || Array.isArray(b)) { + if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false; + return a.every((v, i) => deepEqual(v, b[i])); + } + if (typeof a === 'object') { + const ao = a as Record; + const bo = b as Record; + const ak = Object.keys(ao); + const bk = Object.keys(bo); + if (ak.length !== bk.length) return false; + return ak.every((k) => k in bo && deepEqual(ao[k], bo[k])); + } + return false; +} + +/** Validate a value against the minimal JSON-schema subset; returns errors. */ +export function validateSchema(value: unknown, schema: JsonSchema, path = '$'): string[] { + const errors: string[] = []; + if (schema.type) { + if (!matchesType(value, schema.type)) { + errors.push(`${path}: expected ${schema.type}, got ${describeType(value)}`); + return errors; // type mismatch — deeper checks are meaningless + } + } + if (schema.type === 'object' || (schema.properties && isPlainObject(value))) { + const obj = value as Record; + for (const key of schema.required ?? []) { + if (!(key in obj)) errors.push(`${path}.${key}: required`); + } + if (schema.properties) { + for (const [key, sub] of Object.entries(schema.properties)) { + if (key in obj) errors.push(...validateSchema(obj[key], sub, `${path}.${key}`)); + } + } + } + if (schema.type === 'array' && schema.items && Array.isArray(value)) { + value.forEach((item, i) => errors.push(...validateSchema(item, schema.items as JsonSchema, `${path}[${i}]`))); + } + return errors; +} + +function matchesType(value: unknown, type: NonNullable): boolean { + switch (type) { + case 'object': return isPlainObject(value); + case 'array': return Array.isArray(value); + case 'string': return typeof value === 'string'; + case 'number': return typeof value === 'number' && Number.isFinite(value); + case 'integer': return typeof value === 'number' && Number.isInteger(value); + case 'boolean': return typeof value === 'boolean'; + case 'null': return value === null; + default: return false; + } +} + +function isPlainObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function describeType(value: unknown): string { + if (value === null) return 'null'; + if (Array.isArray(value)) return 'array'; + return typeof value; +} diff --git a/src/lib/services/evaluation/scorers/llmJudgeScorer.ts b/src/lib/services/evaluation/scorers/llmJudgeScorer.ts new file mode 100644 index 00000000..1abd518e --- /dev/null +++ b/src/lib/services/evaluation/scorers/llmJudgeScorer.ts @@ -0,0 +1,104 @@ +/** + * LLM-as-judge scorer. + * + * Builds a rubric-driven grading prompt, calls an injected judge invoker, and + * parses a `{ score, passed?, reasoning? }` verdict. Scores are normalised to + * [0, 1] (a 0..10 scale is auto-detected). The judge invoker is injected so + * this scorer stays free of any model-runtime coupling and is unit-testable. + */ + +import type { + DatasetItem, + EvalMessage, + JudgeInvoker, + LlmJudgeScorerConfig, + ScoreResult, + TargetOutput, +} from '../types'; +import { extractJson } from './json'; + +const JUDGE_SYSTEM = [ + 'You are a strict, fair evaluation judge.', + 'Grade the ASSISTANT OUTPUT against the rubric and (when provided) the reference answer.', + 'Respond with ONLY a JSON object of the form:', + '{"score": , "passed": , "reasoning": ""}', + 'Do not include any text outside the JSON object.', +].join('\n'); + +export function buildJudgePrompt( + item: DatasetItem, + output: TargetOutput, + config: LlmJudgeScorerConfig, +): EvalMessage[] { + const lastUser = [...item.input].reverse().find((m) => m.role === 'user'); + const sections = [ + `# Rubric\n${config.rubric}`, + lastUser ? `# User input\n${lastUser.content}` : '', + item.expected?.reference ? `# Reference answer\n${item.expected.reference}` : '', + `# Assistant output\n${output.text ?? ''}`, + ].filter(Boolean); + + return [ + { role: 'system', content: JUDGE_SYSTEM }, + { role: 'user', content: sections.join('\n\n') }, + ]; +} + +export interface JudgeVerdict { + score: number; + passed?: boolean; + reasoning?: string; +} + +/** Parse + normalise a judge completion into a verdict. */ +export function parseJudgeResponse(raw: string): JudgeVerdict { + const parsed = extractJson(raw); + if (!parsed.ok || typeof parsed.value !== 'object' || parsed.value === null) { + throw new Error(`could not parse judge response: ${parsed.ok ? 'not an object' : parsed.error}`); + } + const obj = parsed.value as Record; + if (typeof obj.score !== 'number' || !Number.isFinite(obj.score)) { + throw new Error('judge response missing numeric "score"'); + } + return { + score: normaliseScore(obj.score), + passed: typeof obj.passed === 'boolean' ? obj.passed : undefined, + reasoning: typeof obj.reasoning === 'string' ? obj.reasoning : undefined, + }; +} + +/** Map a raw judge score onto [0, 1] (auto-detecting a 0..10 scale). */ +export function normaliseScore(raw: number): number { + const scaled = raw > 1 ? raw / 10 : raw; + return Math.min(1, Math.max(0, scaled)); +} + +export async function scoreLlmJudge( + item: DatasetItem, + output: TargetOutput, + config: LlmJudgeScorerConfig, + invokeJudge: JudgeInvoker, +): Promise { + const weight = config.weight ?? 1; + const threshold = config.threshold ?? 0.5; + try { + const raw = await invokeJudge(buildJudgePrompt(item, output, config)); + const verdict = parseJudgeResponse(raw); + const passed = verdict.passed ?? verdict.score >= threshold; + return { + scorerType: 'llm-judge', + score: verdict.score, + passed, + weight, + detail: { reasoning: verdict.reasoning, threshold }, + }; + } catch (err) { + return { + scorerType: 'llm-judge', + score: 0, + passed: false, + weight, + error: (err as Error).message, + }; + } +} diff --git a/src/lib/services/evaluation/types.ts b/src/lib/services/evaluation/types.ts new file mode 100644 index 00000000..88f0e4df --- /dev/null +++ b/src/lib/services/evaluation/types.ts @@ -0,0 +1,145 @@ +/** + * Core types for the Evaluation service engine. + * + * The engine is intentionally free of any database, queue, or model-runtime + * coupling: the target under test and the LLM judge are supplied as injected + * invokers, so the whole scoring pipeline is pure and unit-testable. + * Persistence, live model / agent / external adapters, and the HTTP API are + * layered on top of this core separately (see service + plugin layers). + */ + +export type EvalRole = 'system' | 'user' | 'assistant'; + +export interface EvalMessage { + role: EvalRole; + content: string; +} + +/** Minimal JSON-schema subset supported without an external dependency. */ +export interface JsonSchema { + type?: 'object' | 'array' | 'string' | 'number' | 'integer' | 'boolean' | 'null'; + required?: string[]; + properties?: Record; + items?: JsonSchema; +} + +/** A single JSON-path assertion against the (parsed) target output. */ +export interface JsonPathAssertion { + /** Dot / bracket path, e.g. `data.items[0].name`. */ + path: string; + /** Require the path to resolve (or explicitly to be absent). */ + exists?: boolean; + /** Require the resolved value to deep-equal this. */ + equals?: unknown; +} + +/** Reference data / expectations attached to a dataset item. */ +export interface ExpectedOutput { + /** Gold answer (used by judge / similarity scorers). */ + reference?: string; + /** Substrings that MUST appear in the output. */ + mustContain?: string[]; + /** Substrings that must NOT appear in the output. */ + mustNotContain?: string[]; + /** Exact (trimmed) match. */ + equals?: string; + /** Output must match this regular expression. */ + regex?: string; + /** Parsed output must satisfy this schema. */ + jsonSchema?: JsonSchema; + /** Parsed-output path assertions. */ + jsonPath?: JsonPathAssertion[]; +} + +/** A single test case. */ +export interface DatasetItem { + id: string; + /** Conversation / prompt fed to the target under test. */ + input: EvalMessage[]; + /** Optional reference / expectations consumed by scorers. */ + expected?: ExpectedOutput; + tags?: string[]; +} + +/** Output produced by a target for one item. */ +export interface TargetOutput { + text: string; + latencyMs?: number; + /** Raw provider payload, kept for debugging / trajectory scoring. */ + raw?: unknown; +} + +/** Injected: how to run the target under test for one item. */ +export type TargetInvoker = (item: DatasetItem) => Promise; + +/** Injected: how to call a judge model. Returns the raw completion text. */ +export type JudgeInvoker = (messages: EvalMessage[]) => Promise; + +// ── Scorer configs (discriminated union) ────────────────────────────── + +export interface AssertionScorerConfig { + type: 'assertion'; + /** Weight in the aggregate item score (default 1). */ + weight?: number; +} + +export interface LlmJudgeScorerConfig { + type: 'llm-judge'; + weight?: number; + /** Rubric describing what a good answer looks like. */ + rubric: string; + /** Pass threshold on the normalised 0..1 judge score (default 0.5). */ + threshold?: number; +} + +export type ScorerConfig = AssertionScorerConfig | LlmJudgeScorerConfig; +export type ScorerType = ScorerConfig['type']; + +// ── Results ──────────────────────────────────────────────────────────── + +export interface ScoreResult { + scorerType: ScorerType; + /** Normalised score in [0, 1]. */ + score: number; + passed: boolean; + weight: number; + detail?: Record; + error?: string; +} + +export interface RunItemResult { + itemId: string; + output?: TargetOutput; + scores: ScoreResult[]; + /** Weighted mean of scorer scores, in [0, 1]. */ + score: number; + /** True when every scorer passed (and the target did not error). */ + passed: boolean; + latencyMs?: number; + error?: string; +} + +export interface RunAggregate { + total: number; + /** Items that produced a result (i.e. the target did not throw). */ + completed: number; + /** Items whose target invocation threw. */ + failed: number; + /** Items where `passed === true`. */ + passed: number; + /** passed / completed, in [0, 1]. */ + passRate: number; + /** Mean item score over completed items, in [0, 1]. */ + avgScore: number; + avgLatencyMs: number | null; +} + +export interface RunResult { + aggregate: RunAggregate; + items: RunItemResult[]; +} + +export interface RunConfig { + /** Parallel item executions (default 4). */ + concurrency?: number; +} From 5e66cada174113c2d11a2de3d8de8e9013cba9d8 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 15:53:14 +0000 Subject: [PATCH 02/10] feat(evaluation): persistence, model/judge adapters, and REST API Build the Evaluation service end-to-end on top of the engine core, wiring it through the platform's dual-provider DB layer and HTTP API. Data model (tenant-scoped, both SQLite + MongoDB providers): - evaluation_targets, evaluation_datasets (items embedded as JSON), evaluation_suites, evaluation_runs (result items + aggregate embedded) - domain interfaces + DatabaseProvider contract methods + SQLite/Mongo mixins composed into both providers; types re-exported from @/lib/database Service + adapters: - tenant-scoped CRUD for targets/datasets/suites and run listing/retrieval - runSuite(): loads a suite, builds live invokers, drives the engine runner, and persists the run + aggregate. Target/judge invokers are injectable so orchestration is testable without live model calls - live model target + llm-judge invokers via handleChatCompletion; agent and external targets recognised but stubbed (recorded as per-item errors) REST API: /evaluation/{targets,datasets,suites,runs} CRUD plus POST /evaluation/suites/:key/run, registered in the API plugin. Integration test runs the full vertical against a real SQLite provider with injected fakes. Full suite green (2082 passed); tsc and eslint clean. https://claude.ai/code/session_01UDGtTEyau4AoGC5eQuQKK3 --- .../integration/evaluation-e2e.test.ts | 158 ++++++ src/lib/database/index.ts | 15 + src/lib/database/mongodb.provider.ts | 3 +- src/lib/database/mongodb/base.ts | 4 + src/lib/database/mongodb/evaluation.mixin.ts | 259 ++++++++++ src/lib/database/provider/contract.ts | 73 +++ src/lib/database/provider/types.domain.ts | 136 ++++++ src/lib/database/sqlite.provider.ts | 3 +- src/lib/database/sqlite/base.ts | 4 + src/lib/database/sqlite/evaluation.mixin.ts | 455 ++++++++++++++++++ src/lib/database/sqlite/schema.ts | 80 +++ src/lib/services/evaluation/adapters.ts | 73 +++ src/lib/services/evaluation/service.ts | 386 +++++++++++++++ src/server/api/plugin.ts | 2 + src/server/api/plugins/evaluations.ts | 383 +++++++++++++++ 15 files changed, 2032 insertions(+), 2 deletions(-) create mode 100644 src/__tests__/integration/evaluation-e2e.test.ts create mode 100644 src/lib/database/mongodb/evaluation.mixin.ts create mode 100644 src/lib/database/sqlite/evaluation.mixin.ts create mode 100644 src/lib/services/evaluation/adapters.ts create mode 100644 src/lib/services/evaluation/service.ts create mode 100644 src/server/api/plugins/evaluations.ts diff --git a/src/__tests__/integration/evaluation-e2e.test.ts b/src/__tests__/integration/evaluation-e2e.test.ts new file mode 100644 index 00000000..b0d844bd --- /dev/null +++ b/src/__tests__/integration/evaluation-e2e.test.ts @@ -0,0 +1,158 @@ +/** + * End-to-end test for the Evaluation service vertical. + * + * Backed by a real SQLiteProvider in a temp directory. Exercises CRUD for + * targets / datasets / suites and a full `runSuite` flow whose target & judge + * invokers are injected (fakes) so no live model calls are made — verifying + * persistence, aggregation, and run retrieval against the real DB layer. + */ + +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; + +// SQLite + temp dir must be configured BEFORE getDatabase() is ever called. +const tmpRoot = mkdtempSync(path.join(tmpdir(), 'cognipeer-eval-e2e-')); +process.env.DB_PROVIDER = 'sqlite'; +process.env.SQLITE_DATA_DIR = tmpRoot; +process.env.MAIN_DB_NAME = 'eval_e2e_main'; + +import { reloadConfig } from '@/lib/core/config'; +import { disconnectDatabase, getDatabase } from '@/lib/database'; +import { + createDataset, + createSuite, + createTarget, + deleteTarget, + getRun, + listDatasets, + listRuns, + listSuites, + listTargets, + runSuite, + updateTarget, +} from '@/lib/services/evaluation/service'; + +const TENANT_DB_NAME = 'eval_e2e_tenant'; +const TENANT_ID = 'tenant-eval-e2e'; +const ACTOR = 'tester@example.com'; + +beforeAll(async () => { + reloadConfig(); + const db = await getDatabase(); + await db.switchToTenant(TENANT_DB_NAME); +}); + +afterAll(async () => { + await disconnectDatabase(); + rmSync(tmpRoot, { recursive: true, force: true }); +}); + +describe('Evaluation service — full vertical (SQLite)', () => { + it('persists targets, datasets and suites then runs an evaluation', async () => { + const target = await createTarget(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'GPT Eval Target', + kind: 'model', + modelKey: 'gpt-test', + }); + expect(target.id).toBeTruthy(); + expect(target.key).toBe('gpt-eval-target'); + expect(target.kind).toBe('model'); + + const dataset = await createDataset(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Smoke Dataset', + items: [ + { id: 'q1', input: [{ role: 'user', content: 'say ok' }], expected: { mustContain: ['ok'] } }, + { id: 'q2', input: [{ role: 'user', content: 'say ok too' }], expected: { mustContain: ['ok'] } }, + ], + }); + expect(dataset.id).toBeTruthy(); + expect(dataset.items).toHaveLength(2); + + const suite = await createSuite(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Smoke Suite', + targetKey: target.key, + datasetKey: dataset.key, + scorers: [{ type: 'assertion' }, { type: 'llm-judge', rubric: 'Answer must contain ok.' }], + judgeModelKey: 'judge-test', + }); + expect(suite.id).toBeTruthy(); + expect(suite.scorers).toHaveLength(2); + + // Injected fakes: target echoes "ok" only for q1; judge always approves. + const targetFn = vi.fn(async (item: { id: string }) => ({ text: item.id === 'q1' ? 'ok' : 'nope' })); + const judgeFn = vi.fn(async () => '{"score":1,"passed":true}'); + + const run = await runSuite( + { tenantDbName: TENANT_DB_NAME, tenantId: TENANT_ID, createdBy: ACTOR, suiteKey: suite.key }, + { buildTargetInvoker: () => targetFn, buildJudgeInvoker: () => judgeFn }, + ); + + expect(run.status).toBe('completed'); + expect(run.suiteKey).toBe(suite.key); + expect(run.aggregate?.total).toBe(2); + expect(run.aggregate?.completed).toBe(2); + expect(run.aggregate?.passed).toBe(1); // only q1 passes the assertion + expect(run.aggregate?.passRate).toBeCloseTo(0.5, 5); + expect(run.aggregate?.avgScore).toBeCloseTo(0.75, 5); // q1=1.0, q2=0.5 + expect(run.items).toHaveLength(2); + expect(targetFn).toHaveBeenCalledTimes(2); + expect(judgeFn).toHaveBeenCalledTimes(2); + + // Run is retrievable by id with its persisted items. + const fetched = await getRun(TENANT_DB_NAME, run.id); + expect(fetched?.id).toBe(run.id); + expect(fetched?.items).toHaveLength(2); + const q1 = fetched?.items.find((i) => i.itemId === 'q1'); + expect(q1?.passed).toBe(true); + expect(q1?.scores).toHaveLength(2); + }); + + it('lists entities and round-trips target update/delete', async () => { + const targets = await listTargets(TENANT_DB_NAME); + const datasets = await listDatasets(TENANT_DB_NAME); + const suites = await listSuites(TENANT_DB_NAME); + const runs = await listRuns(TENANT_DB_NAME); + expect(targets.length).toBeGreaterThanOrEqual(1); + expect(datasets.length).toBeGreaterThanOrEqual(1); + expect(suites.length).toBeGreaterThanOrEqual(1); + expect(runs.length).toBeGreaterThanOrEqual(1); + + const extra = await createTarget(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Disposable Target', + kind: 'model', + modelKey: 'tmp', + }); + const updated = await updateTarget(TENANT_DB_NAME, extra.id, ACTOR, { description: 'updated desc' }); + expect(updated?.description).toBe('updated desc'); + + const deleted = await deleteTarget(TENANT_DB_NAME, extra.id); + expect(deleted).toBe(true); + expect(await listTargets(TENANT_DB_NAME, { search: 'Disposable' })).toHaveLength(0); + }); + + it('records a per-item error when the target invoker throws', async () => { + const target = await createTarget(TENANT_DB_NAME, TENANT_ID, ACTOR, { name: 'Erroring Target', kind: 'model', modelKey: 'x' }); + const dataset = await createDataset(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Error Dataset', + items: [{ id: 'e1', input: [{ role: 'user', content: 'hi' }] }], + }); + const suite = await createSuite(TENANT_DB_NAME, TENANT_ID, ACTOR, { + name: 'Error Suite', + targetKey: target.key, + datasetKey: dataset.key, + scorers: [{ type: 'assertion' }], + }); + + const run = await runSuite( + { tenantDbName: TENANT_DB_NAME, tenantId: TENANT_ID, createdBy: ACTOR, suiteKey: suite.key }, + { buildTargetInvoker: () => async () => { throw new Error('model exploded'); } }, + ); + + expect(run.status).toBe('completed'); + expect(run.aggregate?.failed).toBe(1); + expect(run.aggregate?.completed).toBe(0); + expect(run.items[0].error).toMatch(/model exploded/); + }); +}); diff --git a/src/lib/database/index.ts b/src/lib/database/index.ts index 8455e64b..5738ce5e 100644 --- a/src/lib/database/index.ts +++ b/src/lib/database/index.ts @@ -169,6 +169,21 @@ export type { IGuardrailPromptShieldPolicy, IGuardrailEvaluationLog, IGuardrailEvalAggregate, + IEvaluationTarget, + IEvaluationExternalTarget, + IEvaluationDataset, + IEvaluationDatasetItem, + IEvaluationSuite, + IEvaluationScorerConfig, + IEvaluationScore, + IEvaluationRun, + IEvaluationRunItem, + IEvaluationRunAggregate, + EvaluationTargetKind, + EvaluationRunStatus, + EvaluationRunMode, + EvaluationDatasetSource, + EvaluationScorerType, IPiiPolicy, IPiiCustomPattern, PiiAction, diff --git a/src/lib/database/mongodb.provider.ts b/src/lib/database/mongodb.provider.ts index cc2169a4..24147d20 100644 --- a/src/lib/database/mongodb.provider.ts +++ b/src/lib/database/mongodb.provider.ts @@ -25,6 +25,7 @@ import { FileMixin } from './mongodb/file.mixin'; import { ProviderRecordMixin } from './mongodb/provider-record.mixin'; import { InferenceMixin } from './mongodb/inference.mixin'; import { GuardrailMixin } from './mongodb/guardrail.mixin'; +import { EvaluationMixin } from './mongodb/evaluation.mixin'; import { PiiPolicyMixin } from './mongodb/pii-policy.mixin'; import { AlertMixin } from './mongodb/alert.mixin'; import { IncidentMixin } from './mongodb/incident.mixin'; @@ -61,7 +62,7 @@ const AIBase = VectorMixin(ModelMixin(TracingMixin(ContentBase))); const StorageBase = ProviderRecordMixin(FileMixin(AIBase)); // Group 5 – Advanced features -const AdvancedBase = CrawlerMixin(AuditMixin(BrowserMixin(VectorMigrationMixin(AgentMixin(ToolMixin(JsSandboxMixin(McpServerMixin(ConfigMixin(MemoryMixin(RerankerMixin(RagMixin(IncidentMixin(AlertMixin(PiiPolicyMixin(GuardrailMixin(InferenceMixin(StorageBase))))))))))))))))); +const AdvancedBase = CrawlerMixin(AuditMixin(BrowserMixin(VectorMigrationMixin(AgentMixin(ToolMixin(JsSandboxMixin(McpServerMixin(ConfigMixin(MemoryMixin(RerankerMixin(RagMixin(IncidentMixin(AlertMixin(PiiPolicyMixin(EvaluationMixin(GuardrailMixin(InferenceMixin(StorageBase)))))))))))))))))); // Group 6 – Cluster (system-wide; uses main DB) const ClusterBase = ClusterMixin(AdvancedBase); diff --git a/src/lib/database/mongodb/base.ts b/src/lib/database/mongodb/base.ts index dbca58c6..93bbd4b4 100644 --- a/src/lib/database/mongodb/base.ts +++ b/src/lib/database/mongodb/base.ts @@ -34,6 +34,10 @@ export const COLLECTIONS = { inferenceServerMetrics: 'inference_server_metrics', guardrails: 'guardrails', guardrailEvalLogs: 'guardrail_evaluation_logs', + evaluationTargets: 'evaluation_targets', + evaluationDatasets: 'evaluation_datasets', + evaluationSuites: 'evaluation_suites', + evaluationRuns: 'evaluation_runs', piiPolicies: 'pii_policies', alertRules: 'alert_rules', alertEvents: 'alert_events', diff --git a/src/lib/database/mongodb/evaluation.mixin.ts b/src/lib/database/mongodb/evaluation.mixin.ts new file mode 100644 index 00000000..43a97568 --- /dev/null +++ b/src/lib/database/mongodb/evaluation.mixin.ts @@ -0,0 +1,259 @@ +/** + * MongoDB Provider – Evaluation operations mixin + * + * CRUD for evaluation targets, datasets, suites, and runs. Documents store + * nested structures natively (no JSON stringification). Mirrors the guardrail + * mixin conventions. + */ + +import { ObjectId } from 'mongodb'; +import type { + IEvaluationTarget, + IEvaluationDataset, + IEvaluationSuite, + IEvaluationRun, + EvaluationTargetKind, + EvaluationDatasetSource, + EvaluationRunStatus, +} from '../provider.interface'; +import type { Constructor } from './types'; +import { MongoDBProviderBase, COLLECTIONS } from './base'; + +export function EvaluationMixin>(Base: TBase) { + return class EvaluationOps extends Base { + // ── Targets ────────────────────────────────────────────────────── + + async createEvaluationTarget( + target: Omit, + ): Promise { + const db = this.getTenantDb(); + const now = new Date(); + const doc = { ...target, createdAt: now, updatedAt: now }; + const result = await db.collection(COLLECTIONS.evaluationTargets).insertOne(doc); + return { ...doc, _id: result.insertedId.toString() }; + } + + async updateEvaluationTarget( + id: string, + data: Partial>, + ): Promise { + const db = this.getTenantDb(); + const updateData: Record = { ...data, updatedAt: new Date() }; + delete updateData._id; + const result = await db + .collection(COLLECTIONS.evaluationTargets) + .findOneAndUpdate({ _id: new ObjectId(id) }, { $set: updateData }, { returnDocument: 'after' }); + if (!result) return null; + return { ...result, _id: result._id?.toString() } as IEvaluationTarget; + } + + async deleteEvaluationTarget(id: string): Promise { + const db = this.getTenantDb(); + const result = await db.collection(COLLECTIONS.evaluationTargets).deleteOne({ _id: new ObjectId(id) }); + return result.deletedCount === 1; + } + + async findEvaluationTargetById(id: string): Promise { + const db = this.getTenantDb(); + const doc = await db.collection(COLLECTIONS.evaluationTargets).findOne({ _id: new ObjectId(id) }); + return doc as unknown as IEvaluationTarget | null; + } + + async findEvaluationTargetByKey(key: string, projectId?: string): Promise { + const db = this.getTenantDb(); + const filter: Record = { key }; + if (projectId !== undefined) filter.projectId = projectId; + const doc = await db.collection(COLLECTIONS.evaluationTargets).findOne(filter); + return doc as unknown as IEvaluationTarget | null; + } + + async listEvaluationTargets(filters?: { projectId?: string; kind?: EvaluationTargetKind; search?: string }): Promise { + const db = this.getTenantDb(); + const filter: Record = {}; + if (filters?.projectId !== undefined) filter.projectId = filters.projectId; + if (filters?.kind !== undefined) filter.kind = filters.kind; + if (filters?.search) { + filter.$or = [ + { name: { $regex: filters.search, $options: 'i' } }, + { description: { $regex: filters.search, $options: 'i' } }, + { key: { $regex: filters.search, $options: 'i' } }, + ]; + } + const docs = await db.collection(COLLECTIONS.evaluationTargets).find(filter).sort({ createdAt: -1 }).toArray(); + return docs as unknown as IEvaluationTarget[]; + } + + // ── Datasets ───────────────────────────────────────────────────── + + async createEvaluationDataset( + dataset: Omit, + ): Promise { + const db = this.getTenantDb(); + const now = new Date(); + const doc = { ...dataset, createdAt: now, updatedAt: now }; + const result = await db.collection(COLLECTIONS.evaluationDatasets).insertOne(doc); + return { ...doc, _id: result.insertedId.toString() }; + } + + async updateEvaluationDataset( + id: string, + data: Partial>, + ): Promise { + const db = this.getTenantDb(); + const updateData: Record = { ...data, updatedAt: new Date() }; + delete updateData._id; + const result = await db + .collection(COLLECTIONS.evaluationDatasets) + .findOneAndUpdate({ _id: new ObjectId(id) }, { $set: updateData }, { returnDocument: 'after' }); + if (!result) return null; + return { ...result, _id: result._id?.toString() } as IEvaluationDataset; + } + + async deleteEvaluationDataset(id: string): Promise { + const db = this.getTenantDb(); + const result = await db.collection(COLLECTIONS.evaluationDatasets).deleteOne({ _id: new ObjectId(id) }); + return result.deletedCount === 1; + } + + async findEvaluationDatasetById(id: string): Promise { + const db = this.getTenantDb(); + const doc = await db.collection(COLLECTIONS.evaluationDatasets).findOne({ _id: new ObjectId(id) }); + return doc as unknown as IEvaluationDataset | null; + } + + async findEvaluationDatasetByKey(key: string, projectId?: string): Promise { + const db = this.getTenantDb(); + const filter: Record = { key }; + if (projectId !== undefined) filter.projectId = projectId; + const doc = await db.collection(COLLECTIONS.evaluationDatasets).findOne(filter); + return doc as unknown as IEvaluationDataset | null; + } + + async listEvaluationDatasets(filters?: { projectId?: string; source?: EvaluationDatasetSource; search?: string }): Promise { + const db = this.getTenantDb(); + const filter: Record = {}; + if (filters?.projectId !== undefined) filter.projectId = filters.projectId; + if (filters?.source !== undefined) filter.source = filters.source; + if (filters?.search) { + filter.$or = [ + { name: { $regex: filters.search, $options: 'i' } }, + { description: { $regex: filters.search, $options: 'i' } }, + { key: { $regex: filters.search, $options: 'i' } }, + ]; + } + const docs = await db.collection(COLLECTIONS.evaluationDatasets).find(filter).sort({ createdAt: -1 }).toArray(); + return docs as unknown as IEvaluationDataset[]; + } + + // ── Suites ─────────────────────────────────────────────────────── + + async createEvaluationSuite( + suite: Omit, + ): Promise { + const db = this.getTenantDb(); + const now = new Date(); + const doc = { ...suite, createdAt: now, updatedAt: now }; + const result = await db.collection(COLLECTIONS.evaluationSuites).insertOne(doc); + return { ...doc, _id: result.insertedId.toString() }; + } + + async updateEvaluationSuite( + id: string, + data: Partial>, + ): Promise { + const db = this.getTenantDb(); + const updateData: Record = { ...data, updatedAt: new Date() }; + delete updateData._id; + const result = await db + .collection(COLLECTIONS.evaluationSuites) + .findOneAndUpdate({ _id: new ObjectId(id) }, { $set: updateData }, { returnDocument: 'after' }); + if (!result) return null; + return { ...result, _id: result._id?.toString() } as IEvaluationSuite; + } + + async deleteEvaluationSuite(id: string): Promise { + const db = this.getTenantDb(); + const result = await db.collection(COLLECTIONS.evaluationSuites).deleteOne({ _id: new ObjectId(id) }); + return result.deletedCount === 1; + } + + async findEvaluationSuiteById(id: string): Promise { + const db = this.getTenantDb(); + const doc = await db.collection(COLLECTIONS.evaluationSuites).findOne({ _id: new ObjectId(id) }); + return doc as unknown as IEvaluationSuite | null; + } + + async findEvaluationSuiteByKey(key: string, projectId?: string): Promise { + const db = this.getTenantDb(); + const filter: Record = { key }; + if (projectId !== undefined) filter.projectId = projectId; + const doc = await db.collection(COLLECTIONS.evaluationSuites).findOne(filter); + return doc as unknown as IEvaluationSuite | null; + } + + async listEvaluationSuites(filters?: { projectId?: string; targetKey?: string; datasetKey?: string; search?: string }): Promise { + const db = this.getTenantDb(); + const filter: Record = {}; + if (filters?.projectId !== undefined) filter.projectId = filters.projectId; + if (filters?.targetKey !== undefined) filter.targetKey = filters.targetKey; + if (filters?.datasetKey !== undefined) filter.datasetKey = filters.datasetKey; + if (filters?.search) { + filter.$or = [ + { name: { $regex: filters.search, $options: 'i' } }, + { description: { $regex: filters.search, $options: 'i' } }, + { key: { $regex: filters.search, $options: 'i' } }, + ]; + } + const docs = await db.collection(COLLECTIONS.evaluationSuites).find(filter).sort({ createdAt: -1 }).toArray(); + return docs as unknown as IEvaluationSuite[]; + } + + // ── Runs ───────────────────────────────────────────────────────── + + async createEvaluationRun( + run: Omit, + ): Promise { + const db = this.getTenantDb(); + const now = new Date(); + const doc = { ...run, createdAt: now, updatedAt: now }; + const result = await db.collection(COLLECTIONS.evaluationRuns).insertOne(doc); + return { ...doc, _id: result.insertedId.toString() }; + } + + async updateEvaluationRun( + id: string, + data: Partial>, + ): Promise { + const db = this.getTenantDb(); + const updateData: Record = { ...data, updatedAt: new Date() }; + delete updateData._id; + const result = await db + .collection(COLLECTIONS.evaluationRuns) + .findOneAndUpdate({ _id: new ObjectId(id) }, { $set: updateData }, { returnDocument: 'after' }); + if (!result) return null; + return { ...result, _id: result._id?.toString() } as IEvaluationRun; + } + + async findEvaluationRunById(id: string): Promise { + const db = this.getTenantDb(); + const doc = await db.collection(COLLECTIONS.evaluationRuns).findOne({ _id: new ObjectId(id) }); + return doc as unknown as IEvaluationRun | null; + } + + async listEvaluationRuns(filters?: { projectId?: string; suiteKey?: string; status?: EvaluationRunStatus; limit?: number; skip?: number }): Promise { + const db = this.getTenantDb(); + const filter: Record = {}; + if (filters?.projectId !== undefined) filter.projectId = filters.projectId; + if (filters?.suiteKey !== undefined) filter.suiteKey = filters.suiteKey; + if (filters?.status !== undefined) filter.status = filters.status; + const docs = await db + .collection(COLLECTIONS.evaluationRuns) + .find(filter) + .sort({ createdAt: -1 }) + .skip(filters?.skip ?? 0) + .limit(filters?.limit ?? 50) + .toArray(); + return docs as unknown as IEvaluationRun[]; + } + }; +} diff --git a/src/lib/database/provider/contract.ts b/src/lib/database/provider/contract.ts index 5042f946..7a53bb45 100644 --- a/src/lib/database/provider/contract.ts +++ b/src/lib/database/provider/contract.ts @@ -109,6 +109,13 @@ import type { ISandboxSettings, SandboxInstanceState, SandboxCommandStatus, + IEvaluationTarget, + IEvaluationDataset, + IEvaluationSuite, + IEvaluationRun, + EvaluationTargetKind, + EvaluationDatasetSource, + EvaluationRunStatus, } from './types'; export interface DatabaseProvider { @@ -538,6 +545,72 @@ export interface DatabaseProvider { options?: { from?: Date; to?: Date; groupBy?: 'hour' | 'day' | 'month' }, ): Promise; + // ── Evaluation operations (tenant-specific) ── + createEvaluationTarget( + target: Omit, + ): Promise; + updateEvaluationTarget( + id: string, + data: Partial>, + ): Promise; + deleteEvaluationTarget(id: string): Promise; + findEvaluationTargetById(id: string): Promise; + findEvaluationTargetByKey(key: string, projectId?: string): Promise; + listEvaluationTargets(filters?: { + projectId?: string; + kind?: EvaluationTargetKind; + search?: string; + }): Promise; + + createEvaluationDataset( + dataset: Omit, + ): Promise; + updateEvaluationDataset( + id: string, + data: Partial>, + ): Promise; + deleteEvaluationDataset(id: string): Promise; + findEvaluationDatasetById(id: string): Promise; + findEvaluationDatasetByKey(key: string, projectId?: string): Promise; + listEvaluationDatasets(filters?: { + projectId?: string; + source?: EvaluationDatasetSource; + search?: string; + }): Promise; + + createEvaluationSuite( + suite: Omit, + ): Promise; + updateEvaluationSuite( + id: string, + data: Partial>, + ): Promise; + deleteEvaluationSuite(id: string): Promise; + findEvaluationSuiteById(id: string): Promise; + findEvaluationSuiteByKey(key: string, projectId?: string): Promise; + listEvaluationSuites(filters?: { + projectId?: string; + targetKey?: string; + datasetKey?: string; + search?: string; + }): Promise; + + createEvaluationRun( + run: Omit, + ): Promise; + updateEvaluationRun( + id: string, + data: Partial>, + ): Promise; + findEvaluationRunById(id: string): Promise; + listEvaluationRuns(filters?: { + projectId?: string; + suiteKey?: string; + status?: EvaluationRunStatus; + limit?: number; + skip?: number; + }): Promise; + // ── PII policy operations (tenant-specific) ── createPiiPolicy( policy: Omit, diff --git a/src/lib/database/provider/types.domain.ts b/src/lib/database/provider/types.domain.ts index 32f4bcda..b7adc4c0 100644 --- a/src/lib/database/provider/types.domain.ts +++ b/src/lib/database/provider/types.domain.ts @@ -53,6 +53,142 @@ export interface IGuardrail { updatedAt?: Date; } +// ── Evaluation types ───────────────────────────────────────────────────────── + +export type EvaluationTargetKind = 'agent' | 'model' | 'external'; +export type EvaluationRunStatus = 'pending' | 'running' | 'completed' | 'failed' | 'cancelled'; +export type EvaluationRunMode = 'sync' | 'async'; +export type EvaluationDatasetSource = 'manual' | 'file' | 'generated'; +export type EvaluationScorerType = 'assertion' | 'llm-judge'; + +export interface IEvaluationExternalTarget { + protocol: 'openai-chat' | 'webhook'; + url: string; + headers?: Record; + /** Provider key holding encrypted credentials for the external endpoint. */ + credentialProviderKey?: string; + /** Dot-path used to pull the assistant text out of a webhook response. */ + responsePath?: string; +} + +export interface IEvaluationTarget { + _id?: ObjectId | string; + tenantId: string; + projectId?: string; + key: string; + name: string; + description?: string; + kind: EvaluationTargetKind; + agentKey?: string; + modelKey?: string; + external?: IEvaluationExternalTarget; + defaultParams?: Record; + metadata?: Record; + createdBy: string; + updatedBy?: string; + createdAt?: Date; + updatedAt?: Date; +} + +export interface IEvaluationDatasetItem { + id: string; + input: Array<{ role: 'system' | 'user' | 'assistant'; content: string }>; + expected?: Record; + tags?: string[]; +} + +export interface IEvaluationDataset { + _id?: ObjectId | string; + tenantId: string; + projectId?: string; + key: string; + name: string; + description?: string; + source: EvaluationDatasetSource; + items: IEvaluationDatasetItem[]; + metadata?: Record; + createdBy: string; + updatedBy?: string; + createdAt?: Date; + updatedAt?: Date; +} + +export interface IEvaluationScorerConfig { + type: EvaluationScorerType; + weight?: number; + rubric?: string; + threshold?: number; +} + +export interface IEvaluationSuite { + _id?: ObjectId | string; + tenantId: string; + projectId?: string; + key: string; + name: string; + description?: string; + targetKey: string; + datasetKey: string; + scorers: IEvaluationScorerConfig[]; + /** Model used to back any llm-judge scorers. */ + judgeModelKey?: string; + runConfig?: { concurrency?: number }; + metadata?: Record; + createdBy: string; + updatedBy?: string; + createdAt?: Date; + updatedAt?: Date; +} + +export interface IEvaluationScore { + scorerType: EvaluationScorerType; + score: number; + passed: boolean; + weight: number; + detail?: Record; + error?: string; +} + +export interface IEvaluationRunItem { + itemId: string; + output?: { text: string; latencyMs?: number }; + scores: IEvaluationScore[]; + score: number; + passed: boolean; + latencyMs?: number; + error?: string; +} + +export interface IEvaluationRunAggregate { + total: number; + completed: number; + failed: number; + passed: number; + passRate: number; + avgScore: number; + avgLatencyMs: number | null; +} + +export interface IEvaluationRun { + _id?: ObjectId | string; + tenantId: string; + projectId?: string; + suiteKey: string; + targetKey: string; + datasetKey: string; + status: EvaluationRunStatus; + mode: EvaluationRunMode; + progress: { total: number; completed: number; failed: number }; + aggregate?: IEvaluationRunAggregate; + items: IEvaluationRunItem[]; + error?: string; + startedAt?: Date; + finishedAt?: Date; + createdBy: string; + createdAt?: Date; + updatedAt?: Date; +} + export interface IInferenceServerMetrics { _id?: ObjectId | string; tenantId: string; diff --git a/src/lib/database/sqlite.provider.ts b/src/lib/database/sqlite.provider.ts index 59859a07..acd2a548 100644 --- a/src/lib/database/sqlite.provider.ts +++ b/src/lib/database/sqlite.provider.ts @@ -24,6 +24,7 @@ import { FileMixin } from './sqlite/file.mixin'; import { ProviderRecordMixin } from './sqlite/provider-record.mixin'; import { InferenceMixin } from './sqlite/inference.mixin'; import { GuardrailMixin } from './sqlite/guardrail.mixin'; +import { EvaluationMixin } from './sqlite/evaluation.mixin'; import { PiiPolicyMixin } from './sqlite/pii-policy.mixin'; import { AlertMixin } from './sqlite/alert.mixin'; import { IncidentMixin } from './sqlite/incident.mixin'; @@ -60,7 +61,7 @@ const AIBase = VectorMixin(ModelMixin(TracingMixin(ContentBase))); const StorageBase = ProviderRecordMixin(FileMixin(AIBase)); // Group 5 – Advanced features -const AdvancedBase = CrawlerMixin(AuditMixin(BrowserMixin(VectorMigrationMixin(AgentMixin(ToolMixin(JsSandboxMixin(McpServerMixin(ConfigMixin(MemoryMixin(RerankerMixin(RagMixin(IncidentMixin(AlertMixin(PiiPolicyMixin(GuardrailMixin(InferenceMixin(StorageBase))))))))))))))))); +const AdvancedBase = CrawlerMixin(AuditMixin(BrowserMixin(VectorMigrationMixin(AgentMixin(ToolMixin(JsSandboxMixin(McpServerMixin(ConfigMixin(MemoryMixin(RerankerMixin(RagMixin(IncidentMixin(AlertMixin(PiiPolicyMixin(EvaluationMixin(GuardrailMixin(InferenceMixin(StorageBase)))))))))))))))))); // Group 6 – Cluster (system-wide; uses main DB) const ClusterBase = ClusterMixin(AdvancedBase); diff --git a/src/lib/database/sqlite/base.ts b/src/lib/database/sqlite/base.ts index 02ef8c36..d83e0a78 100644 --- a/src/lib/database/sqlite/base.ts +++ b/src/lib/database/sqlite/base.ts @@ -41,6 +41,10 @@ export const TABLES = { inferenceServerMetrics: 'inference_server_metrics', guardrails: 'guardrails', guardrailEvalLogs: 'guardrail_evaluation_logs', + evaluationTargets: 'evaluation_targets', + evaluationDatasets: 'evaluation_datasets', + evaluationSuites: 'evaluation_suites', + evaluationRuns: 'evaluation_runs', piiPolicies: 'pii_policies', alertRules: 'alert_rules', alertEvents: 'alert_events', diff --git a/src/lib/database/sqlite/evaluation.mixin.ts b/src/lib/database/sqlite/evaluation.mixin.ts new file mode 100644 index 00000000..63b4287c --- /dev/null +++ b/src/lib/database/sqlite/evaluation.mixin.ts @@ -0,0 +1,455 @@ +/** + * SQLite Provider – Evaluation operations mixin + * + * CRUD for evaluation targets, datasets (items embedded as JSON), suites, and + * runs (result items + aggregate embedded as JSON). Mirrors the guardrail + * mixin conventions (prepared statements, JSON columns, row mappers). + */ + +import type { + IEvaluationTarget, + IEvaluationDataset, + IEvaluationDatasetItem, + IEvaluationSuite, + IEvaluationScorerConfig, + IEvaluationRun, + IEvaluationRunItem, + IEvaluationRunAggregate, + EvaluationTargetKind, + EvaluationDatasetSource, + EvaluationRunStatus, +} from '../provider.interface'; +import type { Constructor, SqliteRow } from './types'; +import { SQLiteProviderBase, TABLES } from './base'; + +function toIso(value: Date | string | undefined | null): string | null { + if (!value) return null; + const d = value instanceof Date ? value : new Date(value); + return Number.isFinite(d.getTime()) ? d.toISOString() : null; +} + +export function EvaluationMixin>(Base: TBase) { + return class EvaluationOps extends Base { + // ── Targets ────────────────────────────────────────────────────── + + async createEvaluationTarget( + target: Omit, + ): Promise { + const db = this.getTenantDb(); + const id = this.newId(); + const now = this.now(); + db.prepare(` + INSERT INTO ${TABLES.evaluationTargets} + (id, tenantId, projectId, key, name, description, kind, agentKey, modelKey, + external, defaultParams, metadata, createdBy, updatedBy, createdAt, updatedAt) + VALUES (@id, @tenantId, @projectId, @key, @name, @description, @kind, @agentKey, @modelKey, + @external, @defaultParams, @metadata, @createdBy, @updatedBy, @createdAt, @updatedAt) + `).run({ + id, + tenantId: target.tenantId, + projectId: target.projectId ?? null, + key: target.key, + name: target.name, + description: target.description ?? null, + kind: target.kind, + agentKey: target.agentKey ?? null, + modelKey: target.modelKey ?? null, + external: target.external ? this.toJson(target.external) : null, + defaultParams: this.toJson(target.defaultParams ?? {}), + metadata: this.toJson(target.metadata ?? {}), + createdBy: target.createdBy, + updatedBy: target.updatedBy ?? null, + createdAt: now, + updatedAt: now, + }); + return { ...target, _id: id, createdAt: new Date(now), updatedAt: new Date(now) }; + } + + async updateEvaluationTarget( + id: string, + data: Partial>, + ): Promise { + const db = this.getTenantDb(); + const sets: string[] = ['updatedAt = @updatedAt']; + const params: Record = { id, updatedAt: this.now() }; + if (data.name !== undefined) { sets.push('name = @name'); params.name = data.name; } + if (data.description !== undefined) { sets.push('description = @description'); params.description = data.description; } + if (data.kind !== undefined) { sets.push('kind = @kind'); params.kind = data.kind; } + if (data.agentKey !== undefined) { sets.push('agentKey = @agentKey'); params.agentKey = data.agentKey; } + if (data.modelKey !== undefined) { sets.push('modelKey = @modelKey'); params.modelKey = data.modelKey; } + if (data.external !== undefined) { sets.push('external = @external'); params.external = data.external ? this.toJson(data.external) : null; } + if (data.defaultParams !== undefined) { sets.push('defaultParams = @defaultParams'); params.defaultParams = this.toJson(data.defaultParams); } + if (data.metadata !== undefined) { sets.push('metadata = @metadata'); params.metadata = this.toJson(data.metadata); } + if (data.updatedBy !== undefined) { sets.push('updatedBy = @updatedBy'); params.updatedBy = data.updatedBy; } + if (data.projectId !== undefined) { sets.push('projectId = @projectId'); params.projectId = data.projectId; } + db.prepare(`UPDATE ${TABLES.evaluationTargets} SET ${sets.join(', ')} WHERE id = @id`).run(params); + return this.findEvaluationTargetById(id); + } + + async deleteEvaluationTarget(id: string): Promise { + const db = this.getTenantDb(); + return db.prepare(`DELETE FROM ${TABLES.evaluationTargets} WHERE id = @id`).run({ id }).changes === 1; + } + + async findEvaluationTargetById(id: string): Promise { + const db = this.getTenantDb(); + const row = db.prepare(`SELECT * FROM ${TABLES.evaluationTargets} WHERE id = @id`).get({ id }) as SqliteRow | undefined; + return row ? this.mapTargetRow(row) : null; + } + + async findEvaluationTargetByKey(key: string, projectId?: string): Promise { + const db = this.getTenantDb(); + const clauses = ['key = @key']; + const params: Record = { key }; + if (projectId !== undefined) { clauses.push('projectId = @projectId'); params.projectId = projectId; } + const row = db.prepare(`SELECT * FROM ${TABLES.evaluationTargets} WHERE ${clauses.join(' AND ')}`).get(params) as SqliteRow | undefined; + return row ? this.mapTargetRow(row) : null; + } + + async listEvaluationTargets(filters?: { projectId?: string; kind?: EvaluationTargetKind; search?: string }): Promise { + const db = this.getTenantDb(); + const clauses: string[] = []; + const params: Record = {}; + if (filters?.projectId !== undefined) { clauses.push('projectId = @projectId'); params.projectId = filters.projectId; } + if (filters?.kind !== undefined) { clauses.push('kind = @kind'); params.kind = filters.kind; } + if (filters?.search) { clauses.push('(name LIKE @search OR description LIKE @search OR key LIKE @search)'); params.search = this.likePattern(filters.search); } + const where = clauses.length ? `WHERE ${clauses.join(' AND ')}` : ''; + const rows = db.prepare(`SELECT * FROM ${TABLES.evaluationTargets} ${where} ORDER BY createdAt DESC`).all(params) as SqliteRow[]; + return rows.map((r) => this.mapTargetRow(r)); + } + + // ── Datasets ───────────────────────────────────────────────────── + + async createEvaluationDataset( + dataset: Omit, + ): Promise { + const db = this.getTenantDb(); + const id = this.newId(); + const now = this.now(); + db.prepare(` + INSERT INTO ${TABLES.evaluationDatasets} + (id, tenantId, projectId, key, name, description, source, items, metadata, + createdBy, updatedBy, createdAt, updatedAt) + VALUES (@id, @tenantId, @projectId, @key, @name, @description, @source, @items, @metadata, + @createdBy, @updatedBy, @createdAt, @updatedAt) + `).run({ + id, + tenantId: dataset.tenantId, + projectId: dataset.projectId ?? null, + key: dataset.key, + name: dataset.name, + description: dataset.description ?? null, + source: dataset.source, + items: this.toJson(dataset.items ?? []), + metadata: this.toJson(dataset.metadata ?? {}), + createdBy: dataset.createdBy, + updatedBy: dataset.updatedBy ?? null, + createdAt: now, + updatedAt: now, + }); + return { ...dataset, _id: id, createdAt: new Date(now), updatedAt: new Date(now) }; + } + + async updateEvaluationDataset( + id: string, + data: Partial>, + ): Promise { + const db = this.getTenantDb(); + const sets: string[] = ['updatedAt = @updatedAt']; + const params: Record = { id, updatedAt: this.now() }; + if (data.name !== undefined) { sets.push('name = @name'); params.name = data.name; } + if (data.description !== undefined) { sets.push('description = @description'); params.description = data.description; } + if (data.source !== undefined) { sets.push('source = @source'); params.source = data.source; } + if (data.items !== undefined) { sets.push('items = @items'); params.items = this.toJson(data.items); } + if (data.metadata !== undefined) { sets.push('metadata = @metadata'); params.metadata = this.toJson(data.metadata); } + if (data.updatedBy !== undefined) { sets.push('updatedBy = @updatedBy'); params.updatedBy = data.updatedBy; } + if (data.projectId !== undefined) { sets.push('projectId = @projectId'); params.projectId = data.projectId; } + db.prepare(`UPDATE ${TABLES.evaluationDatasets} SET ${sets.join(', ')} WHERE id = @id`).run(params); + return this.findEvaluationDatasetById(id); + } + + async deleteEvaluationDataset(id: string): Promise { + const db = this.getTenantDb(); + return db.prepare(`DELETE FROM ${TABLES.evaluationDatasets} WHERE id = @id`).run({ id }).changes === 1; + } + + async findEvaluationDatasetById(id: string): Promise { + const db = this.getTenantDb(); + const row = db.prepare(`SELECT * FROM ${TABLES.evaluationDatasets} WHERE id = @id`).get({ id }) as SqliteRow | undefined; + return row ? this.mapDatasetRow(row) : null; + } + + async findEvaluationDatasetByKey(key: string, projectId?: string): Promise { + const db = this.getTenantDb(); + const clauses = ['key = @key']; + const params: Record = { key }; + if (projectId !== undefined) { clauses.push('projectId = @projectId'); params.projectId = projectId; } + const row = db.prepare(`SELECT * FROM ${TABLES.evaluationDatasets} WHERE ${clauses.join(' AND ')}`).get(params) as SqliteRow | undefined; + return row ? this.mapDatasetRow(row) : null; + } + + async listEvaluationDatasets(filters?: { projectId?: string; source?: EvaluationDatasetSource; search?: string }): Promise { + const db = this.getTenantDb(); + const clauses: string[] = []; + const params: Record = {}; + if (filters?.projectId !== undefined) { clauses.push('projectId = @projectId'); params.projectId = filters.projectId; } + if (filters?.source !== undefined) { clauses.push('source = @source'); params.source = filters.source; } + if (filters?.search) { clauses.push('(name LIKE @search OR description LIKE @search OR key LIKE @search)'); params.search = this.likePattern(filters.search); } + const where = clauses.length ? `WHERE ${clauses.join(' AND ')}` : ''; + const rows = db.prepare(`SELECT * FROM ${TABLES.evaluationDatasets} ${where} ORDER BY createdAt DESC`).all(params) as SqliteRow[]; + return rows.map((r) => this.mapDatasetRow(r)); + } + + // ── Suites ─────────────────────────────────────────────────────── + + async createEvaluationSuite( + suite: Omit, + ): Promise { + const db = this.getTenantDb(); + const id = this.newId(); + const now = this.now(); + db.prepare(` + INSERT INTO ${TABLES.evaluationSuites} + (id, tenantId, projectId, key, name, description, targetKey, datasetKey, scorers, + judgeModelKey, runConfig, metadata, createdBy, updatedBy, createdAt, updatedAt) + VALUES (@id, @tenantId, @projectId, @key, @name, @description, @targetKey, @datasetKey, @scorers, + @judgeModelKey, @runConfig, @metadata, @createdBy, @updatedBy, @createdAt, @updatedAt) + `).run({ + id, + tenantId: suite.tenantId, + projectId: suite.projectId ?? null, + key: suite.key, + name: suite.name, + description: suite.description ?? null, + targetKey: suite.targetKey, + datasetKey: suite.datasetKey, + scorers: this.toJson(suite.scorers ?? []), + judgeModelKey: suite.judgeModelKey ?? null, + runConfig: this.toJson(suite.runConfig ?? {}), + metadata: this.toJson(suite.metadata ?? {}), + createdBy: suite.createdBy, + updatedBy: suite.updatedBy ?? null, + createdAt: now, + updatedAt: now, + }); + return { ...suite, _id: id, createdAt: new Date(now), updatedAt: new Date(now) }; + } + + async updateEvaluationSuite( + id: string, + data: Partial>, + ): Promise { + const db = this.getTenantDb(); + const sets: string[] = ['updatedAt = @updatedAt']; + const params: Record = { id, updatedAt: this.now() }; + if (data.name !== undefined) { sets.push('name = @name'); params.name = data.name; } + if (data.description !== undefined) { sets.push('description = @description'); params.description = data.description; } + if (data.targetKey !== undefined) { sets.push('targetKey = @targetKey'); params.targetKey = data.targetKey; } + if (data.datasetKey !== undefined) { sets.push('datasetKey = @datasetKey'); params.datasetKey = data.datasetKey; } + if (data.scorers !== undefined) { sets.push('scorers = @scorers'); params.scorers = this.toJson(data.scorers); } + if (data.judgeModelKey !== undefined) { sets.push('judgeModelKey = @judgeModelKey'); params.judgeModelKey = data.judgeModelKey; } + if (data.runConfig !== undefined) { sets.push('runConfig = @runConfig'); params.runConfig = this.toJson(data.runConfig); } + if (data.metadata !== undefined) { sets.push('metadata = @metadata'); params.metadata = this.toJson(data.metadata); } + if (data.updatedBy !== undefined) { sets.push('updatedBy = @updatedBy'); params.updatedBy = data.updatedBy; } + if (data.projectId !== undefined) { sets.push('projectId = @projectId'); params.projectId = data.projectId; } + db.prepare(`UPDATE ${TABLES.evaluationSuites} SET ${sets.join(', ')} WHERE id = @id`).run(params); + return this.findEvaluationSuiteById(id); + } + + async deleteEvaluationSuite(id: string): Promise { + const db = this.getTenantDb(); + return db.prepare(`DELETE FROM ${TABLES.evaluationSuites} WHERE id = @id`).run({ id }).changes === 1; + } + + async findEvaluationSuiteById(id: string): Promise { + const db = this.getTenantDb(); + const row = db.prepare(`SELECT * FROM ${TABLES.evaluationSuites} WHERE id = @id`).get({ id }) as SqliteRow | undefined; + return row ? this.mapSuiteRow(row) : null; + } + + async findEvaluationSuiteByKey(key: string, projectId?: string): Promise { + const db = this.getTenantDb(); + const clauses = ['key = @key']; + const params: Record = { key }; + if (projectId !== undefined) { clauses.push('projectId = @projectId'); params.projectId = projectId; } + const row = db.prepare(`SELECT * FROM ${TABLES.evaluationSuites} WHERE ${clauses.join(' AND ')}`).get(params) as SqliteRow | undefined; + return row ? this.mapSuiteRow(row) : null; + } + + async listEvaluationSuites(filters?: { projectId?: string; targetKey?: string; datasetKey?: string; search?: string }): Promise { + const db = this.getTenantDb(); + const clauses: string[] = []; + const params: Record = {}; + if (filters?.projectId !== undefined) { clauses.push('projectId = @projectId'); params.projectId = filters.projectId; } + if (filters?.targetKey !== undefined) { clauses.push('targetKey = @targetKey'); params.targetKey = filters.targetKey; } + if (filters?.datasetKey !== undefined) { clauses.push('datasetKey = @datasetKey'); params.datasetKey = filters.datasetKey; } + if (filters?.search) { clauses.push('(name LIKE @search OR description LIKE @search OR key LIKE @search)'); params.search = this.likePattern(filters.search); } + const where = clauses.length ? `WHERE ${clauses.join(' AND ')}` : ''; + const rows = db.prepare(`SELECT * FROM ${TABLES.evaluationSuites} ${where} ORDER BY createdAt DESC`).all(params) as SqliteRow[]; + return rows.map((r) => this.mapSuiteRow(r)); + } + + // ── Runs ───────────────────────────────────────────────────────── + + async createEvaluationRun( + run: Omit, + ): Promise { + const db = this.getTenantDb(); + const id = this.newId(); + const now = this.now(); + db.prepare(` + INSERT INTO ${TABLES.evaluationRuns} + (id, tenantId, projectId, suiteKey, targetKey, datasetKey, status, mode, progress, + aggregate, items, error, startedAt, finishedAt, createdBy, createdAt, updatedAt) + VALUES (@id, @tenantId, @projectId, @suiteKey, @targetKey, @datasetKey, @status, @mode, @progress, + @aggregate, @items, @error, @startedAt, @finishedAt, @createdBy, @createdAt, @updatedAt) + `).run({ + id, + tenantId: run.tenantId, + projectId: run.projectId ?? null, + suiteKey: run.suiteKey, + targetKey: run.targetKey, + datasetKey: run.datasetKey, + status: run.status, + mode: run.mode, + progress: this.toJson(run.progress ?? { total: 0, completed: 0, failed: 0 }), + aggregate: run.aggregate !== undefined ? this.toJson(run.aggregate) : null, + items: this.toJson(run.items ?? []), + error: run.error ?? null, + startedAt: toIso(run.startedAt), + finishedAt: toIso(run.finishedAt), + createdBy: run.createdBy, + createdAt: now, + updatedAt: now, + }); + return { ...run, _id: id, createdAt: new Date(now), updatedAt: new Date(now) }; + } + + async updateEvaluationRun( + id: string, + data: Partial>, + ): Promise { + const db = this.getTenantDb(); + const sets: string[] = ['updatedAt = @updatedAt']; + const params: Record = { id, updatedAt: this.now() }; + if (data.status !== undefined) { sets.push('status = @status'); params.status = data.status; } + if (data.mode !== undefined) { sets.push('mode = @mode'); params.mode = data.mode; } + if (data.progress !== undefined) { sets.push('progress = @progress'); params.progress = this.toJson(data.progress); } + if (data.aggregate !== undefined) { sets.push('aggregate = @aggregate'); params.aggregate = this.toJson(data.aggregate); } + if (data.items !== undefined) { sets.push('items = @items'); params.items = this.toJson(data.items); } + if (data.error !== undefined) { sets.push('error = @error'); params.error = data.error; } + if (data.startedAt !== undefined) { sets.push('startedAt = @startedAt'); params.startedAt = toIso(data.startedAt); } + if (data.finishedAt !== undefined) { sets.push('finishedAt = @finishedAt'); params.finishedAt = toIso(data.finishedAt); } + if (data.projectId !== undefined) { sets.push('projectId = @projectId'); params.projectId = data.projectId; } + db.prepare(`UPDATE ${TABLES.evaluationRuns} SET ${sets.join(', ')} WHERE id = @id`).run(params); + return this.findEvaluationRunById(id); + } + + async findEvaluationRunById(id: string): Promise { + const db = this.getTenantDb(); + const row = db.prepare(`SELECT * FROM ${TABLES.evaluationRuns} WHERE id = @id`).get({ id }) as SqliteRow | undefined; + return row ? this.mapRunRow(row) : null; + } + + async listEvaluationRuns(filters?: { projectId?: string; suiteKey?: string; status?: EvaluationRunStatus; limit?: number; skip?: number }): Promise { + const db = this.getTenantDb(); + const clauses: string[] = []; + const params: Record = {}; + if (filters?.projectId !== undefined) { clauses.push('projectId = @projectId'); params.projectId = filters.projectId; } + if (filters?.suiteKey !== undefined) { clauses.push('suiteKey = @suiteKey'); params.suiteKey = filters.suiteKey; } + if (filters?.status !== undefined) { clauses.push('status = @status'); params.status = filters.status; } + const where = clauses.length ? `WHERE ${clauses.join(' AND ')}` : ''; + const limit = filters?.limit ?? 50; + const skip = filters?.skip ?? 0; + const rows = db.prepare( + `SELECT * FROM ${TABLES.evaluationRuns} ${where} ORDER BY createdAt DESC LIMIT ${limit} OFFSET ${skip}`, + ).all(params) as SqliteRow[]; + return rows.map((r) => this.mapRunRow(r)); + } + + // ── Row mappers ────────────────────────────────────────────────── + + protected mapTargetRow(r: SqliteRow): IEvaluationTarget { + return { + _id: r.id as string, + tenantId: r.tenantId as string, + projectId: (r.projectId as string | null) ?? undefined, + key: r.key as string, + name: r.name as string, + description: (r.description as string | null) ?? undefined, + kind: r.kind as EvaluationTargetKind, + agentKey: (r.agentKey as string | null) ?? undefined, + modelKey: (r.modelKey as string | null) ?? undefined, + external: this.parseJson(r.external, undefined as IEvaluationTarget['external']), + defaultParams: this.parseJson(r.defaultParams, {}), + metadata: this.parseJson(r.metadata, {}), + createdBy: r.createdBy as string, + updatedBy: (r.updatedBy as string | null) ?? undefined, + createdAt: this.toDate(r.createdAt), + updatedAt: this.toDate(r.updatedAt), + }; + } + + protected mapDatasetRow(r: SqliteRow): IEvaluationDataset { + return { + _id: r.id as string, + tenantId: r.tenantId as string, + projectId: (r.projectId as string | null) ?? undefined, + key: r.key as string, + name: r.name as string, + description: (r.description as string | null) ?? undefined, + source: r.source as EvaluationDatasetSource, + items: this.parseJson(r.items, []), + metadata: this.parseJson(r.metadata, {}), + createdBy: r.createdBy as string, + updatedBy: (r.updatedBy as string | null) ?? undefined, + createdAt: this.toDate(r.createdAt), + updatedAt: this.toDate(r.updatedAt), + }; + } + + protected mapSuiteRow(r: SqliteRow): IEvaluationSuite { + return { + _id: r.id as string, + tenantId: r.tenantId as string, + projectId: (r.projectId as string | null) ?? undefined, + key: r.key as string, + name: r.name as string, + description: (r.description as string | null) ?? undefined, + targetKey: r.targetKey as string, + datasetKey: r.datasetKey as string, + scorers: this.parseJson(r.scorers, []), + judgeModelKey: (r.judgeModelKey as string | null) ?? undefined, + runConfig: this.parseJson(r.runConfig, {}), + metadata: this.parseJson(r.metadata, {}), + createdBy: r.createdBy as string, + updatedBy: (r.updatedBy as string | null) ?? undefined, + createdAt: this.toDate(r.createdAt), + updatedAt: this.toDate(r.updatedAt), + }; + } + + protected mapRunRow(r: SqliteRow): IEvaluationRun { + const aggregate = this.parseJson(r.aggregate, null); + return { + _id: r.id as string, + tenantId: r.tenantId as string, + projectId: (r.projectId as string | null) ?? undefined, + suiteKey: r.suiteKey as string, + targetKey: r.targetKey as string, + datasetKey: r.datasetKey as string, + status: r.status as EvaluationRunStatus, + mode: r.mode as IEvaluationRun['mode'], + progress: this.parseJson(r.progress, { total: 0, completed: 0, failed: 0 }), + aggregate: aggregate ?? undefined, + items: this.parseJson(r.items, []), + error: (r.error as string | null) ?? undefined, + startedAt: this.toDate(r.startedAt), + finishedAt: this.toDate(r.finishedAt), + createdBy: r.createdBy as string, + createdAt: this.toDate(r.createdAt), + updatedAt: this.toDate(r.updatedAt), + }; + } + }; +} diff --git a/src/lib/database/sqlite/schema.ts b/src/lib/database/sqlite/schema.ts index a1e0a805..ff38decd 100644 --- a/src/lib/database/sqlite/schema.ts +++ b/src/lib/database/sqlite/schema.ts @@ -533,6 +533,86 @@ export const TENANT_SCHEMA_SQL = ` CREATE INDEX IF NOT EXISTS idx_guardrail_eval_guardrailId ON guardrail_evaluation_logs(guardrailId); CREATE INDEX IF NOT EXISTS idx_guardrail_eval_createdAt ON guardrail_evaluation_logs(createdAt); + -- Evaluation service (offline agent/model testing) + CREATE TABLE IF NOT EXISTS evaluation_targets ( + id TEXT PRIMARY KEY, + tenantId TEXT NOT NULL, + projectId TEXT, + key TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT, + kind TEXT NOT NULL DEFAULT 'model', + agentKey TEXT, + modelKey TEXT, + external TEXT, + defaultParams TEXT DEFAULT '{}', + metadata TEXT DEFAULT '{}', + createdBy TEXT NOT NULL, + updatedBy TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_eval_targets_key ON evaluation_targets(key); + + CREATE TABLE IF NOT EXISTS evaluation_datasets ( + id TEXT PRIMARY KEY, + tenantId TEXT NOT NULL, + projectId TEXT, + key TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT, + source TEXT NOT NULL DEFAULT 'manual', + items TEXT DEFAULT '[]', + metadata TEXT DEFAULT '{}', + createdBy TEXT NOT NULL, + updatedBy TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_eval_datasets_key ON evaluation_datasets(key); + + CREATE TABLE IF NOT EXISTS evaluation_suites ( + id TEXT PRIMARY KEY, + tenantId TEXT NOT NULL, + projectId TEXT, + key TEXT NOT NULL, + name TEXT NOT NULL, + description TEXT, + targetKey TEXT NOT NULL, + datasetKey TEXT NOT NULL, + scorers TEXT DEFAULT '[]', + judgeModelKey TEXT, + runConfig TEXT DEFAULT '{}', + metadata TEXT DEFAULT '{}', + createdBy TEXT NOT NULL, + updatedBy TEXT, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_eval_suites_key ON evaluation_suites(key); + + CREATE TABLE IF NOT EXISTS evaluation_runs ( + id TEXT PRIMARY KEY, + tenantId TEXT NOT NULL, + projectId TEXT, + suiteKey TEXT NOT NULL, + targetKey TEXT NOT NULL, + datasetKey TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + mode TEXT NOT NULL DEFAULT 'sync', + progress TEXT DEFAULT '{}', + aggregate TEXT, + items TEXT DEFAULT '[]', + error TEXT, + startedAt TEXT, + finishedAt TEXT, + createdBy TEXT NOT NULL, + createdAt TEXT NOT NULL, + updatedAt TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_eval_runs_suiteKey ON evaluation_runs(suiteKey); + CREATE INDEX IF NOT EXISTS idx_eval_runs_createdAt ON evaluation_runs(createdAt); + -- PII policies (standalone service) CREATE TABLE IF NOT EXISTS pii_policies ( id TEXT PRIMARY KEY, diff --git a/src/lib/services/evaluation/adapters.ts b/src/lib/services/evaluation/adapters.ts new file mode 100644 index 00000000..71c0b46a --- /dev/null +++ b/src/lib/services/evaluation/adapters.ts @@ -0,0 +1,73 @@ +/** + * Live invokers that bridge the pure evaluation engine to the platform model + * runtime. `model` targets (and the llm-judge) call `handleChatCompletion`. + * `agent` and `external` targets are recognised but not yet wired — they throw + * a descriptive error, which the runner records as a per-item failure rather + * than aborting the whole run. + */ + +import { handleChatCompletion } from '@/lib/services/models/inferenceService'; +import type { IEvaluationTarget } from '@/lib/database'; +import type { DatasetItem, JudgeInvoker, TargetInvoker, TargetOutput } from './types'; + +export interface EvaluationModelContext { + tenantDbName: string; + tenantId: string; + projectId: string; +} + +interface ChatLikeResponse { + choices?: Array<{ message?: { content?: unknown } }>; +} + +/** Pull the assistant text out of an OpenAI-shaped chat completion response. */ +export function extractAssistantText(response: unknown): string { + const content = (response as ChatLikeResponse | null | undefined)?.choices?.[0]?.message?.content; + if (typeof content === 'string') return content; + if (content === null || content === undefined) return ''; + return JSON.stringify(content); +} + +async function invokeModel( + ctx: EvaluationModelContext, + modelKey: string, + messages: Array<{ role: string; content: string }>, +): Promise<{ text: string; latencyMs?: number; raw: unknown }> { + const result = (await handleChatCompletion({ + tenantDbName: ctx.tenantDbName, + tenantId: ctx.tenantId, + modelKey, + projectId: ctx.projectId, + body: { messages }, + })) as { response?: unknown; latencyMs?: number }; + return { text: extractAssistantText(result.response), latencyMs: result.latencyMs, raw: result.response }; +} + +export function buildTargetInvoker(target: IEvaluationTarget, ctx: EvaluationModelContext): TargetInvoker { + return async (item: DatasetItem): Promise => { + if (target.kind === 'model') { + if (!target.modelKey) throw new Error(`Evaluation target "${target.key}" has no modelKey configured`); + const messages = item.input.map((m) => ({ role: m.role, content: m.content })); + const { text, latencyMs, raw } = await invokeModel(ctx, target.modelKey, messages); + return { text, latencyMs, raw }; + } + if (target.kind === 'agent') { + throw new Error('agent evaluation targets are not yet supported (model targets only in this release)'); + } + throw new Error('external evaluation targets are not yet supported (model targets only in this release)'); + }; +} + +export function buildJudgeInvoker(judgeModelKey: string | undefined, ctx: EvaluationModelContext): JudgeInvoker { + return async (messages) => { + if (!judgeModelKey) { + throw new Error('judgeModelKey is required when a suite uses llm-judge scorers'); + } + const { text } = await invokeModel( + ctx, + judgeModelKey, + messages.map((m) => ({ role: m.role, content: m.content })), + ); + return text; + }; +} diff --git a/src/lib/services/evaluation/service.ts b/src/lib/services/evaluation/service.ts new file mode 100644 index 00000000..dca8b023 --- /dev/null +++ b/src/lib/services/evaluation/service.ts @@ -0,0 +1,386 @@ +/** + * Evaluation service — tenant-scoped CRUD over targets / datasets / suites / + * runs, plus `runSuite` which loads a suite, builds live invokers, drives the + * pure engine runner, and persists the run + aggregate. + * + * Target/judge invokers are injectable (RunSuiteDeps) so the orchestration is + * testable without live model calls. + */ + +import slugify from 'slugify'; +import { getDatabase } from '@/lib/database'; +import type { + IEvaluationTarget, + IEvaluationDataset, + IEvaluationDatasetItem, + IEvaluationSuite, + IEvaluationScorerConfig, + IEvaluationRun, + IEvaluationRunItem, + EvaluationTargetKind, + EvaluationDatasetSource, +} from '@/lib/database'; +import { runEvaluation } from './runner'; +import type { DatasetItem, RunItemResult, ScorerConfig } from './types'; +import { buildJudgeInvoker, buildTargetInvoker, type EvaluationModelContext } from './adapters'; + +const SLUG_OPTIONS = { lower: true, strict: true, trim: true }; +const MAX_KEY_ATTEMPTS = 50; + +export type WithId = Omit & { id: string }; + +function toView(record: T): WithId { + const { _id, ...rest } = record as T & { _id?: unknown }; + const id = + typeof _id === 'string' + ? _id + : _id && typeof (_id as { toString?: () => string }).toString === 'function' + ? (_id as { toString: () => string }).toString() + : ''; + return { ...(rest as Omit), id }; +} + +async function generateUniqueKey(desired: string, exists: (key: string) => Promise): Promise { + const base = slugify(desired?.trim() || 'item', SLUG_OPTIONS) || 'item'; + let candidate = base; + let attempt = 0; + while (attempt < MAX_KEY_ATTEMPTS) { + if (!(await exists(candidate))) return candidate; + attempt += 1; + candidate = `${base}-${attempt}`; + } + throw new Error(`Could not generate a unique key for "${desired}"`); +} + +function mapScorer(config: IEvaluationScorerConfig): ScorerConfig { + if (config.type === 'llm-judge') { + return { type: 'llm-judge', weight: config.weight, rubric: config.rubric ?? '', threshold: config.threshold }; + } + return { type: 'assertion', weight: config.weight }; +} + +function toRunItem(result: RunItemResult): IEvaluationRunItem { + return { + itemId: result.itemId, + output: result.output ? { text: result.output.text, latencyMs: result.output.latencyMs } : undefined, + scores: result.scores.map((s) => ({ + scorerType: s.scorerType, + score: s.score, + passed: s.passed, + weight: s.weight, + detail: s.detail, + error: s.error, + })), + score: result.score, + passed: result.passed, + latencyMs: result.latencyMs, + error: result.error, + }; +} + +// ── Targets ──────────────────────────────────────────────────────────────── + +export interface CreateTargetInput { + name: string; + description?: string; + kind: EvaluationTargetKind; + agentKey?: string; + modelKey?: string; + external?: IEvaluationTarget['external']; + defaultParams?: Record; + projectId?: string; +} + +export async function createTarget( + tenantDbName: string, + tenantId: string, + createdBy: string, + input: CreateTargetInput, +): Promise> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + const key = await generateUniqueKey(input.name, async (k) => !!(await db.findEvaluationTargetByKey(k, input.projectId))); + const target = await db.createEvaluationTarget({ + tenantId, + projectId: input.projectId, + key, + name: input.name, + description: input.description, + kind: input.kind, + agentKey: input.agentKey, + modelKey: input.modelKey, + external: input.external, + defaultParams: input.defaultParams, + createdBy, + }); + return toView(target); +} + +export async function listTargets( + tenantDbName: string, + filters?: { projectId?: string; kind?: EvaluationTargetKind; search?: string }, +): Promise[]> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + return (await db.listEvaluationTargets(filters)).map(toView); +} + +export async function getTarget(tenantDbName: string, id: string): Promise | null> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + const record = await db.findEvaluationTargetById(id); + return record ? toView(record) : null; +} + +export async function updateTarget( + tenantDbName: string, + id: string, + updatedBy: string, + data: Partial>, +): Promise | null> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + const updated = await db.updateEvaluationTarget(id, { ...data, updatedBy }); + return updated ? toView(updated) : null; +} + +export async function deleteTarget(tenantDbName: string, id: string): Promise { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + return db.deleteEvaluationTarget(id); +} + +// ── Datasets ─────────────────────────────────────────────────────────────── + +export interface CreateDatasetInput { + name: string; + description?: string; + source?: EvaluationDatasetSource; + items?: IEvaluationDatasetItem[]; + projectId?: string; +} + +export async function createDataset( + tenantDbName: string, + tenantId: string, + createdBy: string, + input: CreateDatasetInput, +): Promise> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + const key = await generateUniqueKey(input.name, async (k) => !!(await db.findEvaluationDatasetByKey(k, input.projectId))); + const dataset = await db.createEvaluationDataset({ + tenantId, + projectId: input.projectId, + key, + name: input.name, + description: input.description, + source: input.source ?? 'manual', + items: input.items ?? [], + createdBy, + }); + return toView(dataset); +} + +export async function listDatasets( + tenantDbName: string, + filters?: { projectId?: string; source?: EvaluationDatasetSource; search?: string }, +): Promise[]> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + return (await db.listEvaluationDatasets(filters)).map(toView); +} + +export async function getDataset(tenantDbName: string, id: string): Promise | null> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + const record = await db.findEvaluationDatasetById(id); + return record ? toView(record) : null; +} + +export async function updateDataset( + tenantDbName: string, + id: string, + updatedBy: string, + data: Partial>, +): Promise | null> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + const updated = await db.updateEvaluationDataset(id, { ...data, updatedBy }); + return updated ? toView(updated) : null; +} + +export async function deleteDataset(tenantDbName: string, id: string): Promise { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + return db.deleteEvaluationDataset(id); +} + +// ── Suites ───────────────────────────────────────────────────────────────── + +export interface CreateSuiteInput { + name: string; + description?: string; + targetKey: string; + datasetKey: string; + scorers: IEvaluationScorerConfig[]; + judgeModelKey?: string; + runConfig?: { concurrency?: number }; + projectId?: string; +} + +export async function createSuite( + tenantDbName: string, + tenantId: string, + createdBy: string, + input: CreateSuiteInput, +): Promise> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + const key = await generateUniqueKey(input.name, async (k) => !!(await db.findEvaluationSuiteByKey(k, input.projectId))); + const suite = await db.createEvaluationSuite({ + tenantId, + projectId: input.projectId, + key, + name: input.name, + description: input.description, + targetKey: input.targetKey, + datasetKey: input.datasetKey, + scorers: input.scorers, + judgeModelKey: input.judgeModelKey, + runConfig: input.runConfig, + createdBy, + }); + return toView(suite); +} + +export async function listSuites( + tenantDbName: string, + filters?: { projectId?: string; targetKey?: string; datasetKey?: string; search?: string }, +): Promise[]> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + return (await db.listEvaluationSuites(filters)).map(toView); +} + +export async function getSuite(tenantDbName: string, id: string): Promise | null> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + const record = await db.findEvaluationSuiteById(id); + return record ? toView(record) : null; +} + +export async function updateSuite( + tenantDbName: string, + id: string, + updatedBy: string, + data: Partial>, +): Promise | null> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + const updated = await db.updateEvaluationSuite(id, { ...data, updatedBy }); + return updated ? toView(updated) : null; +} + +export async function deleteSuite(tenantDbName: string, id: string): Promise { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + return db.deleteEvaluationSuite(id); +} + +// ── Runs ─────────────────────────────────────────────────────────────────── + +export async function listRuns( + tenantDbName: string, + filters?: { projectId?: string; suiteKey?: string; limit?: number; skip?: number }, +): Promise[]> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + return (await db.listEvaluationRuns(filters)).map(toView); +} + +export async function getRun(tenantDbName: string, id: string): Promise | null> { + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + const record = await db.findEvaluationRunById(id); + return record ? toView(record) : null; +} + +export interface RunSuiteDeps { + buildTargetInvoker?: typeof buildTargetInvoker; + buildJudgeInvoker?: typeof buildJudgeInvoker; +} + +export async function runSuite( + params: { tenantDbName: string; tenantId: string; projectId?: string; createdBy: string; suiteKey: string }, + deps: RunSuiteDeps = {}, +): Promise> { + const { tenantDbName, tenantId, projectId, createdBy, suiteKey } = params; + const db = await getDatabase(); + await db.switchToTenant(tenantDbName); + + const suite = await db.findEvaluationSuiteByKey(suiteKey, projectId); + if (!suite) throw new Error(`Evaluation suite "${suiteKey}" not found`); + const target = await db.findEvaluationTargetByKey(suite.targetKey, projectId); + if (!target) throw new Error(`Evaluation target "${suite.targetKey}" not found`); + const dataset = await db.findEvaluationDatasetByKey(suite.datasetKey, projectId); + if (!dataset) throw new Error(`Evaluation dataset "${suite.datasetKey}" not found`); + + const items: DatasetItem[] = dataset.items.map((it) => ({ + id: it.id, + input: it.input, + expected: it.expected as DatasetItem['expected'], + tags: it.tags, + })); + const scorers: ScorerConfig[] = suite.scorers.map(mapScorer); + + const run = await db.createEvaluationRun({ + tenantId, + projectId, + suiteKey: suite.key, + targetKey: target.key, + datasetKey: dataset.key, + status: 'running', + mode: 'sync', + progress: { total: items.length, completed: 0, failed: 0 }, + items: [], + createdBy, + startedAt: new Date(), + }); + const runId = toView(run).id; + + try { + const ctx: EvaluationModelContext = { tenantDbName, tenantId, projectId: projectId ?? '' }; + const makeTarget = deps.buildTargetInvoker ?? buildTargetInvoker; + const makeJudge = deps.buildJudgeInvoker ?? buildJudgeInvoker; + const needJudge = scorers.some((s) => s.type === 'llm-judge'); + + const result = await runEvaluation({ + items, + scorers, + invokeTarget: makeTarget(target, ctx), + invokeJudge: needJudge ? makeJudge(suite.judgeModelKey, ctx) : undefined, + config: { concurrency: suite.runConfig?.concurrency }, + }); + + const updated = await db.updateEvaluationRun(runId, { + status: 'completed', + progress: { + total: result.aggregate.total, + completed: result.aggregate.completed, + failed: result.aggregate.failed, + }, + aggregate: result.aggregate, + items: result.items.map(toRunItem), + finishedAt: new Date(), + }); + return toView(updated ?? run); + } catch (err) { + await db.updateEvaluationRun(runId, { + status: 'failed', + error: (err as Error).message, + finishedAt: new Date(), + }); + throw err; + } +} diff --git a/src/server/api/plugin.ts b/src/server/api/plugin.ts index 9b91cfc4..bd84ac29 100644 --- a/src/server/api/plugin.ts +++ b/src/server/api/plugin.ts @@ -42,6 +42,7 @@ import { configApiPlugin } from './plugins/config'; import { dashboardApiPlugin } from './plugins/dashboard'; import { filesApiPlugin } from './plugins/files'; import { guardrailsApiPlugin } from './plugins/guardrails'; +import { evaluationsApiPlugin } from './plugins/evaluations'; import { piiApiPlugin } from './plugins/pii'; import { healthApiPlugin } from './plugins/health'; import { inferenceMonitoringApiPlugin } from './plugins/inference-monitoring'; @@ -351,6 +352,7 @@ export const fastifyApiPlugin: FastifyPluginAsync = async (app) => { await app.register(dashboardApiPlugin); await app.register(filesApiPlugin); await app.register(guardrailsApiPlugin); + await app.register(evaluationsApiPlugin); await app.register(piiApiPlugin); await app.register(healthApiPlugin); await app.register(inferenceMonitoringApiPlugin); diff --git a/src/server/api/plugins/evaluations.ts b/src/server/api/plugins/evaluations.ts new file mode 100644 index 00000000..e788c6d5 --- /dev/null +++ b/src/server/api/plugins/evaluations.ts @@ -0,0 +1,383 @@ +import type { FastifyPluginAsync } from 'fastify'; +import type { + EvaluationTargetKind, + IEvaluationScorerConfig, + IEvaluationDatasetItem, +} from '@/lib/database'; +import { createLogger } from '@/lib/core/logger'; +import { + createDataset, + createSuite, + createTarget, + deleteDataset, + deleteSuite, + deleteTarget, + getDataset, + getRun, + getSuite, + getTarget, + listDatasets, + listRuns, + listSuites, + listTargets, + runSuite, + updateDataset, + updateSuite, + updateTarget, +} from '@/lib/services/evaluation/service'; +import { + readJsonBody, + requireProjectContextForRequest, + requireSessionContext, + sendProjectContextError, + withApiRequestContext, +} from '../fastify-utils'; + +const logger = createLogger('api:evaluations'); + +const VALID_KINDS: EvaluationTargetKind[] = ['agent', 'model', 'external']; +const VALID_SCORERS = ['assertion', 'llm-judge']; + +function internalError(reply: import('fastify').FastifyReply, error: unknown) { + return ( + sendProjectContextError(reply, error) + ?? reply.code(500).send({ error: error instanceof Error ? error.message : 'Internal error' }) + ); +} + +function sanitizeScorers(raw: unknown): IEvaluationScorerConfig[] | null { + if (!Array.isArray(raw)) return null; + const scorers: IEvaluationScorerConfig[] = []; + for (const entry of raw) { + if (!entry || typeof entry !== 'object') return null; + const e = entry as Record; + if (typeof e.type !== 'string' || !VALID_SCORERS.includes(e.type)) return null; + scorers.push({ + type: e.type as IEvaluationScorerConfig['type'], + weight: typeof e.weight === 'number' ? e.weight : undefined, + rubric: typeof e.rubric === 'string' ? e.rubric : undefined, + threshold: typeof e.threshold === 'number' ? e.threshold : undefined, + }); + } + return scorers; +} + +export const evaluationsApiPlugin: FastifyPluginAsync = async (app) => { + // ── Targets ──────────────────────────────────────────────────────── + + app.get('/evaluation/targets', withApiRequestContext(async (request, reply) => { + try { + const { projectId, session } = await requireProjectContextForRequest(request); + const query = (request.query ?? {}) as { kind?: EvaluationTargetKind; search?: string }; + const targets = await listTargets(session.tenantDbName, { projectId, kind: query.kind, search: query.search }); + return reply.code(200).send({ targets }); + } catch (error) { + return internalError(reply, error); + } + })); + + app.post('/evaluation/targets', withApiRequestContext(async (request, reply) => { + try { + const { projectId, session } = await requireProjectContextForRequest(request); + const body = readJsonBody>(request); + if (typeof body.name !== 'string' || body.name.trim() === '') { + return reply.code(400).send({ error: 'name is required' }); + } + if (!VALID_KINDS.includes(body.kind as EvaluationTargetKind)) { + return reply.code(400).send({ error: 'kind must be "agent", "model", or "external"' }); + } + if (body.kind === 'model' && typeof body.modelKey !== 'string') { + return reply.code(400).send({ error: 'modelKey is required for model targets' }); + } + if (body.kind === 'agent' && typeof body.agentKey !== 'string') { + return reply.code(400).send({ error: 'agentKey is required for agent targets' }); + } + const target = await createTarget(session.tenantDbName, session.tenantId, session.userId, { + name: body.name.trim(), + description: typeof body.description === 'string' ? body.description : undefined, + kind: body.kind as EvaluationTargetKind, + agentKey: body.agentKey as string | undefined, + modelKey: body.modelKey as string | undefined, + external: body.external as never, + defaultParams: body.defaultParams as Record | undefined, + projectId, + }); + return reply.code(201).send({ target }); + } catch (error) { + logger.error('Create evaluation target error', { error }); + return internalError(reply, error); + } + })); + + app.get('/evaluation/targets/:id', withApiRequestContext(async (request, reply) => { + try { + const session = requireSessionContext(request); + const { id } = request.params as { id: string }; + const target = await getTarget(session.tenantDbName, id); + if (!target) return reply.code(404).send({ error: 'Target not found' }); + return reply.code(200).send({ target }); + } catch (error) { + return internalError(reply, error); + } + })); + + app.patch('/evaluation/targets/:id', withApiRequestContext(async (request, reply) => { + try { + const session = requireSessionContext(request); + const { id } = request.params as { id: string }; + const body = readJsonBody>(request); + const target = await updateTarget(session.tenantDbName, id, session.userId, { + name: body.name as string | undefined, + description: body.description as string | undefined, + agentKey: body.agentKey as string | undefined, + modelKey: body.modelKey as string | undefined, + defaultParams: body.defaultParams as Record | undefined, + }); + if (!target) return reply.code(404).send({ error: 'Target not found' }); + return reply.code(200).send({ target }); + } catch (error) { + logger.error('Update evaluation target error', { error }); + return internalError(reply, error); + } + })); + + app.delete('/evaluation/targets/:id', withApiRequestContext(async (request, reply) => { + try { + const session = requireSessionContext(request); + const { id } = request.params as { id: string }; + const deleted = await deleteTarget(session.tenantDbName, id); + if (!deleted) return reply.code(404).send({ error: 'Target not found' }); + return reply.code(200).send({ success: true }); + } catch (error) { + return internalError(reply, error); + } + })); + + // ── Datasets ─────────────────────────────────────────────────────── + + app.get('/evaluation/datasets', withApiRequestContext(async (request, reply) => { + try { + const { projectId, session } = await requireProjectContextForRequest(request); + const query = (request.query ?? {}) as { search?: string }; + const datasets = await listDatasets(session.tenantDbName, { projectId, search: query.search }); + return reply.code(200).send({ datasets }); + } catch (error) { + return internalError(reply, error); + } + })); + + app.post('/evaluation/datasets', withApiRequestContext(async (request, reply) => { + try { + const { projectId, session } = await requireProjectContextForRequest(request); + const body = readJsonBody>(request); + if (typeof body.name !== 'string' || body.name.trim() === '') { + return reply.code(400).send({ error: 'name is required' }); + } + if (body.items !== undefined && !Array.isArray(body.items)) { + return reply.code(400).send({ error: 'items must be an array' }); + } + const dataset = await createDataset(session.tenantDbName, session.tenantId, session.userId, { + name: body.name.trim(), + description: typeof body.description === 'string' ? body.description : undefined, + items: body.items as IEvaluationDatasetItem[] | undefined, + projectId, + }); + return reply.code(201).send({ dataset }); + } catch (error) { + logger.error('Create evaluation dataset error', { error }); + return internalError(reply, error); + } + })); + + app.get('/evaluation/datasets/:id', withApiRequestContext(async (request, reply) => { + try { + const session = requireSessionContext(request); + const { id } = request.params as { id: string }; + const dataset = await getDataset(session.tenantDbName, id); + if (!dataset) return reply.code(404).send({ error: 'Dataset not found' }); + return reply.code(200).send({ dataset }); + } catch (error) { + return internalError(reply, error); + } + })); + + app.patch('/evaluation/datasets/:id', withApiRequestContext(async (request, reply) => { + try { + const session = requireSessionContext(request); + const { id } = request.params as { id: string }; + const body = readJsonBody>(request); + if (body.items !== undefined && !Array.isArray(body.items)) { + return reply.code(400).send({ error: 'items must be an array' }); + } + const dataset = await updateDataset(session.tenantDbName, id, session.userId, { + name: body.name as string | undefined, + description: body.description as string | undefined, + items: body.items as IEvaluationDatasetItem[] | undefined, + }); + if (!dataset) return reply.code(404).send({ error: 'Dataset not found' }); + return reply.code(200).send({ dataset }); + } catch (error) { + logger.error('Update evaluation dataset error', { error }); + return internalError(reply, error); + } + })); + + app.delete('/evaluation/datasets/:id', withApiRequestContext(async (request, reply) => { + try { + const session = requireSessionContext(request); + const { id } = request.params as { id: string }; + const deleted = await deleteDataset(session.tenantDbName, id); + if (!deleted) return reply.code(404).send({ error: 'Dataset not found' }); + return reply.code(200).send({ success: true }); + } catch (error) { + return internalError(reply, error); + } + })); + + // ── Suites ───────────────────────────────────────────────────────── + + app.get('/evaluation/suites', withApiRequestContext(async (request, reply) => { + try { + const { projectId, session } = await requireProjectContextForRequest(request); + const query = (request.query ?? {}) as { search?: string }; + const suites = await listSuites(session.tenantDbName, { projectId, search: query.search }); + return reply.code(200).send({ suites }); + } catch (error) { + return internalError(reply, error); + } + })); + + app.post('/evaluation/suites', withApiRequestContext(async (request, reply) => { + try { + const { projectId, session } = await requireProjectContextForRequest(request); + const body = readJsonBody>(request); + if (typeof body.name !== 'string' || body.name.trim() === '') { + return reply.code(400).send({ error: 'name is required' }); + } + if (typeof body.targetKey !== 'string' || typeof body.datasetKey !== 'string') { + return reply.code(400).send({ error: 'targetKey and datasetKey are required' }); + } + const scorers = sanitizeScorers(body.scorers); + if (!scorers || scorers.length === 0) { + return reply.code(400).send({ error: 'scorers must be a non-empty array of { type: "assertion" | "llm-judge" }' }); + } + const runConfig = body.runConfig && typeof body.runConfig === 'object' + ? { concurrency: Number((body.runConfig as Record).concurrency) || undefined } + : undefined; + const suite = await createSuite(session.tenantDbName, session.tenantId, session.userId, { + name: body.name.trim(), + description: typeof body.description === 'string' ? body.description : undefined, + targetKey: body.targetKey, + datasetKey: body.datasetKey, + scorers, + judgeModelKey: typeof body.judgeModelKey === 'string' ? body.judgeModelKey : undefined, + runConfig, + projectId, + }); + return reply.code(201).send({ suite }); + } catch (error) { + logger.error('Create evaluation suite error', { error }); + return internalError(reply, error); + } + })); + + app.get('/evaluation/suites/:id', withApiRequestContext(async (request, reply) => { + try { + const session = requireSessionContext(request); + const { id } = request.params as { id: string }; + const suite = await getSuite(session.tenantDbName, id); + if (!suite) return reply.code(404).send({ error: 'Suite not found' }); + return reply.code(200).send({ suite }); + } catch (error) { + return internalError(reply, error); + } + })); + + app.patch('/evaluation/suites/:id', withApiRequestContext(async (request, reply) => { + try { + const session = requireSessionContext(request); + const { id } = request.params as { id: string }; + const body = readJsonBody>(request); + const scorers = body.scorers !== undefined ? sanitizeScorers(body.scorers) : undefined; + if (body.scorers !== undefined && !scorers) { + return reply.code(400).send({ error: 'scorers must be an array of { type: "assertion" | "llm-judge" }' }); + } + const suite = await updateSuite(session.tenantDbName, id, session.userId, { + name: body.name as string | undefined, + description: body.description as string | undefined, + targetKey: body.targetKey as string | undefined, + datasetKey: body.datasetKey as string | undefined, + scorers: scorers ?? undefined, + judgeModelKey: body.judgeModelKey as string | undefined, + }); + if (!suite) return reply.code(404).send({ error: 'Suite not found' }); + return reply.code(200).send({ suite }); + } catch (error) { + logger.error('Update evaluation suite error', { error }); + return internalError(reply, error); + } + })); + + app.delete('/evaluation/suites/:id', withApiRequestContext(async (request, reply) => { + try { + const session = requireSessionContext(request); + const { id } = request.params as { id: string }; + const deleted = await deleteSuite(session.tenantDbName, id); + if (!deleted) return reply.code(404).send({ error: 'Suite not found' }); + return reply.code(200).send({ success: true }); + } catch (error) { + return internalError(reply, error); + } + })); + + // ── Runs ─────────────────────────────────────────────────────────── + + app.post('/evaluation/suites/:key/run', withApiRequestContext(async (request, reply) => { + try { + const { projectId, session } = await requireProjectContextForRequest(request); + const { key } = request.params as { key: string }; + const run = await runSuite({ + tenantDbName: session.tenantDbName, + tenantId: session.tenantId, + projectId, + createdBy: session.userId, + suiteKey: key, + }); + return reply.code(201).send({ run }); + } catch (error) { + logger.error('Run evaluation suite error', { error }); + if (error instanceof Error && error.message.toLowerCase().includes('not found')) { + return reply.code(404).send({ error: error.message }); + } + return internalError(reply, error); + } + })); + + app.get('/evaluation/runs', withApiRequestContext(async (request, reply) => { + try { + const { projectId, session } = await requireProjectContextForRequest(request); + const query = (request.query ?? {}) as { suiteKey?: string; limit?: string; skip?: string }; + const runs = await listRuns(session.tenantDbName, { + projectId, + suiteKey: query.suiteKey, + limit: query.limit ? Math.min(Number.parseInt(query.limit, 10), 200) : undefined, + skip: query.skip ? Number.parseInt(query.skip, 10) : undefined, + }); + return reply.code(200).send({ runs }); + } catch (error) { + return internalError(reply, error); + } + })); + + app.get('/evaluation/runs/:id', withApiRequestContext(async (request, reply) => { + try { + const session = requireSessionContext(request); + const { id } = request.params as { id: string }; + const run = await getRun(session.tenantDbName, id); + if (!run) return reply.code(404).send({ error: 'Run not found' }); + return reply.code(200).send({ run }); + } catch (error) { + return internalError(reply, error); + } + })); +}; From eb5d3a3b6f01e2ffadc86443b00320022c6610bc Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 3 Jun 2026 16:11:32 +0000 Subject: [PATCH 03/10] feat(evaluation): dashboard UI for targets, datasets, suites and runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the Evaluations dashboard so the service is usable from the app, not just the API. UI: - /dashboard/evaluations — tabbed page (Targets / Datasets / Suites / Runs) with stat tiles, DataGrids, create modals, delete, and a one-click "Run" action on suites that navigates to the run detail - create modals: target (model/agent/external), dataset (JSON items with validation), suite (target + dataset + assertion / llm-judge scorers) - /dashboard/evaluations/runs/[id] — run detail with aggregate stats and a per-item table (pass/fail, score, per-scorer breakdown, output/error) Wiring: - platform-services.json: new "evaluations" service (operate category) - rbac.ts: evaluations PermissionService + definition + /api/evaluation route-prefix mapping - dashboardServices.ts: register IconChecklist - i18n: navigation labels (en + tr) tsc, eslint, and the full test suite (2082 passed) are clean. https://claude.ai/code/session_01UDGtTEyau4AoGC5eQuQKK3 --- src/app/dashboard/evaluations/page.tsx | 338 ++++++++++++++++++ .../dashboard/evaluations/runs/[id]/page.tsx | 151 ++++++++ .../evaluations/CreateDatasetModal.tsx | 121 +++++++ .../evaluations/CreateSuiteModal.tsx | 136 +++++++ .../evaluations/CreateTargetModal.tsx | 120 +++++++ src/components/evaluations/types.ts | 104 ++++++ src/config/platform-services.json | 11 + src/lib/i18n/messages/en.ts | 2 + src/lib/i18n/messages/tr.ts | 2 + src/lib/security/rbac.ts | 3 + src/lib/utils/dashboardServices.ts | 2 + 11 files changed, 990 insertions(+) create mode 100644 src/app/dashboard/evaluations/page.tsx create mode 100644 src/app/dashboard/evaluations/runs/[id]/page.tsx create mode 100644 src/components/evaluations/CreateDatasetModal.tsx create mode 100644 src/components/evaluations/CreateSuiteModal.tsx create mode 100644 src/components/evaluations/CreateTargetModal.tsx create mode 100644 src/components/evaluations/types.ts diff --git a/src/app/dashboard/evaluations/page.tsx b/src/app/dashboard/evaluations/page.tsx new file mode 100644 index 00000000..4191a9e8 --- /dev/null +++ b/src/app/dashboard/evaluations/page.tsx @@ -0,0 +1,338 @@ +'use client'; + +import { useEffect, useMemo, useState } from 'react'; +import { useRouter } from 'next/navigation'; +import { Button, Group, Modal, Tabs, Text } from '@mantine/core'; +import { notifications } from '@mantine/notifications'; +import { + IconChecklist, + IconDatabase, + IconPlayerPlay, + IconPlus, + IconRobot, + IconTrash, +} from '@tabler/icons-react'; +import PageContainer, { PageHeader } from '@/components/common/ui/PageContainer'; +import StatTile from '@/components/common/ui/StatTile'; +import DataGrid, { type DataGridColumn } from '@/components/common/ui/DataGrid'; +import CreateTargetModal from '@/components/evaluations/CreateTargetModal'; +import CreateDatasetModal from '@/components/evaluations/CreateDatasetModal'; +import CreateSuiteModal from '@/components/evaluations/CreateSuiteModal'; +import type { + EvalDatasetView, + EvalRunView, + EvalSuiteView, + EvalTargetView, + ModelOption, +} from '@/components/evaluations/types'; + +type TabKey = 'targets' | 'datasets' | 'suites' | 'runs'; + +const RUN_STATUS_BADGE: Record = { + completed: 'ds-badge-teal', + running: 'ds-badge-info', + failed: 'ds-badge-err', + pending: 'ds-badge', + cancelled: 'ds-badge-warn', +}; + +function fmtDate(value?: string): string { + if (!value) return '—'; + const d = new Date(value); + return Number.isFinite(d.getTime()) ? d.toLocaleString() : '—'; +} + +function pct(value?: number): string { + return value === undefined ? '—' : `${Math.round(value * 100)}%`; +} + +export default function EvaluationsPage() { + const router = useRouter(); + const [tab, setTab] = useState('targets'); + const [targets, setTargets] = useState([]); + const [datasets, setDatasets] = useState([]); + const [suites, setSuites] = useState([]); + const [runs, setRuns] = useState([]); + const [models, setModels] = useState([]); + const [loading, setLoading] = useState(true); + const [refreshing, setRefreshing] = useState(false); + const [runningKey, setRunningKey] = useState(null); + + const [targetModal, setTargetModal] = useState(false); + const [datasetModal, setDatasetModal] = useState(false); + const [suiteModal, setSuiteModal] = useState(false); + const [deleteItem, setDeleteItem] = useState<{ kind: TabKey; id: string; name: string } | null>(null); + const [deleting, setDeleting] = useState(false); + + const loadAll = async () => { + setRefreshing(true); + try { + const [tRes, dRes, sRes, rRes, mRes] = await Promise.all([ + fetch('/api/evaluation/targets', { cache: 'no-store' }), + fetch('/api/evaluation/datasets', { cache: 'no-store' }), + fetch('/api/evaluation/suites', { cache: 'no-store' }), + fetch('/api/evaluation/runs', { cache: 'no-store' }), + fetch('/api/models?category=llm', { cache: 'no-store' }), + ]); + if (tRes.ok) setTargets((await tRes.json()).targets ?? []); + if (dRes.ok) setDatasets((await dRes.json()).datasets ?? []); + if (sRes.ok) setSuites((await sRes.json()).suites ?? []); + if (rRes.ok) setRuns((await rRes.json()).runs ?? []); + if (mRes.ok) { + setModels(((await mRes.json()).models ?? []).map((m: { key: string; name: string }) => ({ value: m.key, label: m.name }))); + } + } catch (err) { + console.error('Failed to load evaluations', err); + } finally { + setLoading(false); + setRefreshing(false); + } + }; + + const loadRuns = async () => { + const res = await fetch('/api/evaluation/runs', { cache: 'no-store' }); + if (res.ok) setRuns((await res.json()).runs ?? []); + }; + + useEffect(() => { + void loadAll(); + }, []); + + const runSuiteNow = async (suite: EvalSuiteView) => { + setRunningKey(suite.key); + try { + const res = await fetch(`/api/evaluation/suites/${encodeURIComponent(suite.key)}/run`, { method: 'POST' }); + const data = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(data.error || 'Run failed'); + const agg = data.run?.aggregate; + notifications.show({ + title: 'Evaluation complete', + message: agg ? `${agg.passed}/${agg.total} passed · avg score ${pct(agg.avgScore)}` : 'Run finished', + color: 'teal', + }); + await loadRuns(); + setTab('runs'); + if (data.run?.id) router.push(`/dashboard/evaluations/runs/${data.run.id}`); + } catch (err) { + notifications.show({ title: 'Run failed', message: err instanceof Error ? err.message : 'Run failed', color: 'red' }); + } finally { + setRunningKey(null); + } + }; + + const confirmDelete = async () => { + if (!deleteItem) return; + const path = + deleteItem.kind === 'targets' ? 'targets' : deleteItem.kind === 'datasets' ? 'datasets' : 'suites'; + setDeleting(true); + try { + const res = await fetch(`/api/evaluation/${path}/${deleteItem.id}`, { method: 'DELETE' }); + if (!res.ok) throw new Error('Failed to delete'); + notifications.show({ title: 'Deleted', message: `"${deleteItem.name}" was deleted`, color: 'red' }); + setDeleteItem(null); + await loadAll(); + } catch (err) { + notifications.show({ title: 'Error', message: err instanceof Error ? err.message : 'Failed to delete', color: 'red' }); + } finally { + setDeleting(false); + } + }; + + const targetName = (key: string) => targets.find((t) => t.key === key)?.name ?? key; + const datasetName = (key: string) => datasets.find((d) => d.key === key)?.name ?? key; + + const targetColumns: DataGridColumn[] = [ + { key: 'name', label: 'Name', render: (t) => ( +
+ {t.name} + {t.key} +
+ ) }, + { key: 'kind', label: 'Kind', render: (t) => {t.kind} }, + { key: 'ref', label: 'Model / Agent', render: (t) => ( + {t.modelKey ?? t.agentKey ?? '—'} + ) }, + { key: 'created', label: 'Created', render: (t) => {fmtDate(t.createdAt)} }, + ]; + + const datasetColumns: DataGridColumn[] = [ + { key: 'name', label: 'Name', render: (d) => ( +
+ {d.name} + {d.key} +
+ ) }, + { key: 'items', label: 'Items', render: (d) => {d.items.length} }, + { key: 'source', label: 'Source', render: (d) => {d.source} }, + { key: 'created', label: 'Created', render: (d) => {fmtDate(d.createdAt)} }, + ]; + + const suiteColumns: DataGridColumn[] = [ + { key: 'name', label: 'Name', render: (s) => ( +
+ {s.name} + {s.key} +
+ ) }, + { key: 'target', label: 'Target', render: (s) => {targetName(s.targetKey)} }, + { key: 'dataset', label: 'Dataset', render: (s) => {datasetName(s.datasetKey)} }, + { key: 'scorers', label: 'Scorers', render: (s) => ( + {s.scorers.map((sc) => {sc.type})} + ) }, + ]; + + const runColumns: DataGridColumn[] = [ + { key: 'suite', label: 'Suite', render: (r) => {r.suiteKey} }, + { key: 'status', label: 'Status', render: (r) => {r.status} }, + { key: 'pass', label: 'Pass rate', render: (r) => {r.aggregate ? `${r.aggregate.passed}/${r.aggregate.total} (${pct(r.aggregate.passRate)})` : '—'} }, + { key: 'score', label: 'Avg score', render: (r) => {r.aggregate ? pct(r.aggregate.avgScore) : '—'} }, + { key: 'created', label: 'Started', render: (r) => {fmtDate(r.startedAt ?? r.createdAt)} }, + ]; + + const actionButton = useMemo(() => { + if (tab === 'runs') return null; + const label = tab === 'targets' ? 'New target' : tab === 'datasets' ? 'New dataset' : 'New suite'; + const onClick = () => { + if (tab === 'targets') setTargetModal(true); + else if (tab === 'datasets') setDatasetModal(true); + else setSuiteModal(true); + }; + return ( + + ); + }, [tab]); + + return ( + + + +
+ } value={targets.length} /> + } value={datasets.length} /> + } value={suites.length} /> + +
+ + setTab((v as TabKey) ?? 'targets')}> + + }>Targets + }>Datasets + }>Suites + }>Runs + + + + + records={targets} + loading={loading} + rowKey={(t) => t.id} + columns={targetColumns} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No targets yet', + description: 'A target is the agent, model, or endpoint under test.', + primaryAction: { label: 'New target', icon: , onClick: () => setTargetModal(true) }, + }} + rowActions={(t) => [ + { id: 'delete', label: 'Delete', icon: , color: 'red', onClick: () => setDeleteItem({ kind: 'targets', id: t.id, name: t.name }) }, + ]} + /> + + + + + records={datasets} + loading={loading} + rowKey={(d) => d.id} + columns={datasetColumns} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No datasets yet', + description: 'A dataset is a set of test cases (inputs and optional expectations).', + primaryAction: { label: 'New dataset', icon: , onClick: () => setDatasetModal(true) }, + }} + rowActions={(d) => [ + { id: 'delete', label: 'Delete', icon: , color: 'red', onClick: () => setDeleteItem({ kind: 'datasets', id: d.id, name: d.name }) }, + ]} + /> + + + + + records={suites} + loading={loading} + rowKey={(s) => s.id} + columns={suiteColumns} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No suites yet', + description: 'A suite binds a target to a dataset with one or more scorers.', + primaryAction: { label: 'New suite', icon: , onClick: () => setSuiteModal(true) }, + }} + rowActions={(s) => [ + { + id: 'run', + label: runningKey === s.key ? 'Running…' : 'Run', + icon: , + onClick: () => void runSuiteNow(s), + }, + { divider: true }, + { id: 'delete', label: 'Delete', icon: , color: 'red', onClick: () => setDeleteItem({ kind: 'suites', id: s.id, name: s.name }) }, + ]} + /> + + + + + records={runs} + loading={loading} + rowKey={(r) => r.id} + columns={runColumns} + onRowClick={(r) => router.push(`/dashboard/evaluations/runs/${r.id}`)} + onRefresh={loadAll} + refreshing={refreshing} + empty={{ + icon: , + title: 'No runs yet', + description: 'Run a suite from the Suites tab to see results here.', + }} + /> + + + + setDeleteItem(null)} title="Delete" centered size="sm"> + + Delete {deleteItem?.name}? This action cannot be undone. + + + + + + + + setTargetModal(false)} models={models} onCreated={() => void loadAll()} /> + setDatasetModal(false)} onCreated={() => void loadAll()} /> + setSuiteModal(false)} + targets={targets} + datasets={datasets} + models={models} + onCreated={() => void loadAll()} + /> + + ); +} diff --git a/src/app/dashboard/evaluations/runs/[id]/page.tsx b/src/app/dashboard/evaluations/runs/[id]/page.tsx new file mode 100644 index 00000000..6b23df22 --- /dev/null +++ b/src/app/dashboard/evaluations/runs/[id]/page.tsx @@ -0,0 +1,151 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { useParams, useRouter } from 'next/navigation'; +import { Button, Group, Loader, Text } from '@mantine/core'; +import { IconArrowLeft } from '@tabler/icons-react'; +import PageContainer, { PageHeader } from '@/components/common/ui/PageContainer'; +import StatTile from '@/components/common/ui/StatTile'; +import DataGrid, { type DataGridColumn } from '@/components/common/ui/DataGrid'; +import type { EvalRunItemView, EvalRunView } from '@/components/evaluations/types'; + +const RUN_STATUS_BADGE: Record = { + completed: 'ds-badge-teal', + running: 'ds-badge-info', + failed: 'ds-badge-err', + pending: 'ds-badge', + cancelled: 'ds-badge-warn', +}; + +function pct(value?: number): string { + return value === undefined || value === null ? '—' : `${Math.round(value * 100)}%`; +} + +export default function EvaluationRunDetailPage() { + const router = useRouter(); + const params = useParams<{ id: string }>(); + const runId = params?.id; + const [run, setRun] = useState(null); + const [loading, setLoading] = useState(true); + const [notFound, setNotFound] = useState(false); + + useEffect(() => { + if (!runId) return; + let cancelled = false; + (async () => { + try { + const res = await fetch(`/api/evaluation/runs/${runId}`, { cache: 'no-store' }); + if (res.status === 404) { + if (!cancelled) setNotFound(true); + return; + } + const data = await res.json(); + if (!cancelled) setRun(data.run ?? null); + } catch { + if (!cancelled) setNotFound(true); + } finally { + if (!cancelled) setLoading(false); + } + })(); + return () => { + cancelled = true; + }; + }, [runId]); + + const itemColumns: DataGridColumn[] = [ + { key: 'item', label: 'Item', render: (i) => {i.itemId} }, + { + key: 'result', + label: 'Result', + render: (i) => + i.error + ? error + : {i.passed ? 'pass' : 'fail'}, + }, + { key: 'score', label: 'Score', render: (i) => {pct(i.score)} }, + { + key: 'scorers', + label: 'Scorers', + render: (i) => ( + + {i.scores.map((s, idx) => ( + + {s.scorerType}: {s.error ? 'err' : pct(s.score)} + + ))} + + ), + }, + { + key: 'output', + label: 'Output', + render: (i) => { + const text = i.error ? i.error : (i.output?.text ?? ''); + const short = text.length > 120 ? `${text.slice(0, 120)}…` : text; + return {short || '—'}; + }, + }, + ]; + + const backButton = ( + + ); + + if (loading) { + return ( + + + + ); + } + + if (notFound || !run) { + return ( + + + This evaluation run could not be found. + + ); + } + + const agg = run.aggregate; + + return ( + + + Target {run.targetKey} · Dataset {run.datasetKey}{' '} + · {run.status} + + } + actions={backButton} + /> + + {run.error ? ( +
+ {run.error} +
+ ) : null} + +
+ + + + +
+ + + records={run.items} + rowKey={(i) => i.itemId} + columns={itemColumns} + footerLeft={`${run.items.length} items`} + empty={{ title: 'No items', description: 'This run produced no item results.' }} + /> + + ); +} diff --git a/src/components/evaluations/CreateDatasetModal.tsx b/src/components/evaluations/CreateDatasetModal.tsx new file mode 100644 index 00000000..91ad1647 --- /dev/null +++ b/src/components/evaluations/CreateDatasetModal.tsx @@ -0,0 +1,121 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { Button, Code, Group, Modal, Stack, Textarea, TextInput } from '@mantine/core'; +import { useForm } from '@mantine/form'; +import { notifications } from '@mantine/notifications'; +import type { EvalDatasetItemView, EvalDatasetView } from './types'; + +interface CreateDatasetModalProps { + opened: boolean; + onClose: () => void; + onCreated: (dataset: EvalDatasetView) => void; +} + +const EXAMPLE = `[ + { + "id": "q1", + "input": [{ "role": "user", "content": "What is 2+2?" }], + "expected": { "mustContain": ["4"] } + } +]`; + +interface FormValues { + name: string; + description: string; + itemsJson: string; +} + +/** Validate + normalise the items JSON into dataset items. */ +function parseItems(raw: string): { items: EvalDatasetItemView[] } | { error: string } { + const trimmed = raw.trim(); + if (!trimmed) return { items: [] }; + let parsed: unknown; + try { + parsed = JSON.parse(trimmed); + } catch (err) { + return { error: `Invalid JSON: ${(err as Error).message}` }; + } + if (!Array.isArray(parsed)) return { error: 'Items must be a JSON array' }; + const items: EvalDatasetItemView[] = []; + for (let i = 0; i < parsed.length; i += 1) { + const entry = parsed[i] as Record; + if (!entry || typeof entry !== 'object') return { error: `Item ${i} is not an object` }; + if (!Array.isArray(entry.input)) return { error: `Item ${i} must have an "input" array of messages` }; + items.push({ + id: typeof entry.id === 'string' && entry.id ? entry.id : `item-${i + 1}`, + input: entry.input as EvalDatasetItemView['input'], + expected: (entry.expected as Record | undefined) ?? undefined, + tags: Array.isArray(entry.tags) ? (entry.tags as string[]) : undefined, + }); + } + return { items }; +} + +export default function CreateDatasetModal({ opened, onClose, onCreated }: CreateDatasetModalProps) { + const [loading, setLoading] = useState(false); + const form = useForm({ + initialValues: { name: '', description: '', itemsJson: '' }, + validate: { + name: (v) => (!v.trim() ? 'Name is required' : null), + itemsJson: (v) => { + const result = parseItems(v); + return 'error' in result ? result.error : null; + }, + }, + }); + + useEffect(() => { + if (!opened) form.reset(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [opened]); + + const handleSubmit = async () => { + if (form.validate().hasErrors) return; + const v = form.getValues(); + const parsed = parseItems(v.itemsJson); + if ('error' in parsed) return; + setLoading(true); + try { + const res = await fetch('/api/evaluation/datasets', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name: v.name.trim(), description: v.description || undefined, items: parsed.items }), + }); + if (!res.ok) { + const data = await res.json().catch(() => ({})); + throw new Error(data.error || 'Failed to create dataset'); + } + const data = await res.json(); + notifications.show({ title: 'Dataset created', message: `"${data.dataset.name}" (${data.dataset.items.length} items)`, color: 'teal' }); + onCreated(data.dataset); + onClose(); + } catch (err) { + notifications.show({ title: 'Error', message: err instanceof Error ? err.message : 'Failed to create dataset', color: 'red' }); + } finally { + setLoading(false); + } + }; + + return ( + + + +