feat(evaluation): add dependency-injected scoring engine core

docs/.vitepress/config.mts

@@ -73,6 +73,7 @@ export default defineConfig({
             { text: 'Agent Tracing', link: '/guide/tracing' },
             { text: 'Guardrails', link: '/guide/guardrails' },
             { text: 'PII Service', link: '/guide/pii' },
+            { text: 'Evaluation & Analysis', link: '/guide/evaluation-and-analysis' },
             { text: 'RAG', link: '/guide/rag' },
             { text: 'Prompts', link: '/guide/prompts' },
             { text: 'Memory', link: '/guide/memory' },
@@ -131,6 +132,8 @@ export default defineConfig({
             { text: 'Files', link: '/api/files' },
             { text: 'Guardrails', link: '/api/guardrails' },
             { text: 'PII', link: '/api/pii' },
+            { text: 'Evaluation', link: '/api/evaluation' },
+            { text: 'Analysis', link: '/api/analysis' },
             { text: 'Prompts', link: '/api/prompts' },
             { text: 'RAG', link: '/api/rag' },
             { text: 'Memory', link: '/api/memory' },

docs/api/analysis.md

@@ -0,0 +1,166 @@
+# Analysis API
+
+Conversation analysis: extract structured fields from transcripts, judge
+quality against a rubric, and score extraction accuracy against ground truth —
+on demand or on a nightly cron. Built for use cases like IVR/call-center review.
+
+All endpoints are under `/api/analysis/*` and are session-authenticated. Requests
+are tenant- and project-scoped from the session.
+
+## Concepts
+
+```
+definition    →  fieldSet + extraction prompt + modes (+ models, + schedule)
+conversation  →  an ingested transcript { role, content }[] (+ referenceFields)
+run           →  one execution of a definition over a set of conversations
+```
+
+### Modes
+
+| Mode | Effect |
+|---|---|
+| `extract` | Always on — pulls the `fieldSet` out of each transcript as typed JSON. |
+| `store` | Writes the extracted fields back onto each conversation (`extractedFields`, `lastAnalyzedAt`). |
+| `judge` | An LLM grades each conversation against `judge.rubric` (0–1). |
+| `accuracy` | Compares extracted fields to the conversation's `referenceFields`, per field. |
+
+## Definitions
+
+```http
+GET    /api/analysis/definitions?search=intent
+POST   /api/analysis/definitions
+GET    /api/analysis/definitions/:id
+PATCH  /api/analysis/definitions/:id
+DELETE /api/analysis/definitions/:id
+```
+
+### Create
+
+```json
+{
+  "name": "Call intent & resolution",
+  "fieldSet": [
+    { "key": "intent", "type": "enum", "enumValues": ["billing", "support"], "required": true },
+    { "key": "resolved", "type": "boolean" }
+  ],
+  "extractionInstructions": "Focus on the caller's primary reason.",
+  "modes": { "store": true, "accuracy": true, "judge": { "rubric": "Was the caller helped politely?" } },
+  "extractionModelKey": "gpt-4o-mini",
+  "judgeModelKey": "gpt-4o",
+  "schedule": { "cron": "0 2 * * *", "enabled": true }
+}
+```
+
+| Field | Type | Required | Notes |
+|---|---|---|---|
+| `name` | string | yes | `key` is slugified from it. |
+| `fieldSet` | array | yes | Each `{ key, type, description?, enumValues?, required? }`; `type` ∈ `string \| number \| boolean \| enum`. `enum` needs `enumValues`. |
+| `modes` | object | yes | `{ store?, accuracy?, judge?: { rubric, threshold? } }`. |
+| `extractionModelKey` | string | recommended | Model used for extraction (required at run time). |
+| `judgeModelKey` | string | when `modes.judge` | Model used for grading. |
+| `schedule` | object | no | `{ cron, enabled }`. Validated with a standard 5-field cron expression (UTC). |
+
+## Conversations
+
+```http
+GET    /api/analysis/conversations?search=refund&limit=100
+POST   /api/analysis/conversations
+GET    /api/analysis/conversations/:id
+DELETE /api/analysis/conversations/:id
+```
+
+### Ingest
+
+Accepts a single conversation or `{ "conversations": [...] }` for bulk import
+(e.g. an external call export).
+
+```json
+{
+  "conversations": [
+    {
+      "name": "Call 1042",
+      "transcript": [
+        { "role": "caller", "content": "I was charged twice." },
+        { "role": "agent", "content": "I've issued a refund." }
+      ],
+      "referenceFields": { "intent": "billing", "resolved": true }
+    }
+  ]
+}
+```
+
+| Field | Type | Required | Notes |
+|---|---|---|---|
+| `transcript` | array | yes | Non-empty `{ role, content }[]`. |
+| `referenceFields` | object | no | Ground truth for the accuracy mode. |
+| `name` | string | no | Display name; `key` is slugified/auto-generated. |
+| `source` | `imported \| platform \| manual` | no | Defaults to `imported`. |
+
+## Runs
+
+### Run a definition
+
+```http
+POST /api/analysis/definitions/:key/run
+```
+
+```json
+{ "conversationKeys": ["call-1042", "call-1043"] }
+```
+
+Omit `conversationKeys` to analyze the most recent corpus (up to 500
+conversations). Extraction runs per conversation with bounded concurrency, then
+optional judge and accuracy; the run is persisted with an aggregate.
+
+#### Response
+
+```json
+{
+  "run": {
+    "id": "…",
+    "definitionKey": "call-intent-resolution",
+    "status": "completed",
+    "aggregate": { "total": 120, "completed": 118, "failed": 2, "passed": 110, "passRate": 0.932, "avgJudgeScore": 0.88, "avgExtractionAccuracy": 0.91 },
+    "items": [
+      { "conversationKey": "call-1042", "passed": true, "extractedFields": { "intent": "billing", "resolved": true }, "missing": [], "judge": { "score": 0.9, "passed": true }, "accuracy": { "score": 1, "comparedCount": 2, "perField": { "intent": { "expected": "billing", "actual": "billing", "match": true } } } }
+    ]
+  }
+}
+```
+
+`avgJudgeScore` / `avgExtractionAccuracy` are `null` when no item used that mode.
+
+### List / get runs
+
+```http
+GET /api/analysis/runs?definitionKey=call-intent-resolution&limit=50
+GET /api/analysis/runs/:id
+```
+
+## Scheduling
+
+When a definition has `schedule.enabled` with a valid cron, the background
+**analysis scheduler** fires it automatically (e.g. `0 2 * * *` = 02:00 UTC
+nightly). Each cron slot fires at most once, decided against the most recent
+run. See the [guide](/guide/evaluation-and-analysis#automation).
+
+## Alerting
+
+Run aggregates feed the alert system through the `analysis` module:
+
+| Metric | Source |
+|---|---|
+| `analysis_pass_rate` | `aggregate.passRate` × 100, averaged over completed runs in the window. |
+| `analysis_avg_judge_score` | `aggregate.avgJudgeScore` × 100. |
+| `analysis_avg_accuracy` | `aggregate.avgExtractionAccuracy` × 100. |
+
+A rule like `analysis_avg_accuracy lt 85` over 24h notifies you when extraction
+quality drops on the nightly run.
+
+## Errors
+
+| Status | Cause |
+|---|---|
+| 400 | Missing `name`, empty/invalid `fieldSet`, enum without `enumValues`, bad cron, transcript missing `role`/`content`. |
+| 404 | Definition / conversation / run not found (or unknown definition `key` on run). |
+| 500 | Internal error. |

docs/api/evaluation.md

@@ -0,0 +1,173 @@
+# Evaluation API
+
+Offline testing for models and agents. Define a **target** (what to test), a
+**dataset** (the test cases), and a **suite** (binds a target + dataset + one or
+more scorers), then **run** the suite to produce a scored result.
+
+All endpoints are under `/api/evaluation/*` and are session-authenticated
+(dashboard surface). Requests are tenant- and project-scoped from the session.
+
+## Concepts
+
+```
+target   →  a model | agent | external endpoint under test
+dataset  →  an ordered list of items { input messages, expected? }
+suite    →  target + dataset + scorers[] (+ judge model)
+run      →  one execution of a suite over its dataset, with aggregate + per-item scores
+```
+
+### Scorers
+
+| Type | What it checks |
+|---|---|
+| `assertion` | Deterministic checks against `expected`: `mustContain`, `equals`, `regex`, `jsonSchema`, `jsonPath`. |
+| `llm-judge` | An LLM grades the output against a `rubric` (0–1), backed by `judgeModelKey`. |
+
+A run's per-item `score` is the weighted mean of its scorer scores; `passed` is
+true when every scorer passes. The aggregate reports `passRate`, `avgScore`, and
+`avgLatencyMs`.
+
+## Targets
+
+```http
+GET    /api/evaluation/targets?kind=model&search=gpt
+POST   /api/evaluation/targets
+GET    /api/evaluation/targets/:id
+PATCH  /api/evaluation/targets/:id
+DELETE /api/evaluation/targets/:id
+```
+
+### Create
+
+```json
+{
+  "name": "GPT-4o production",
+  "kind": "model",
+  "modelKey": "gpt-4o"
+}
+```
+
+| Field | Type | Required | Notes |
+|---|---|---|---|
+| `name` | string | yes | Display name; `key` is slugified from it. |
+| `kind` | `model \| agent \| external` | yes | `model` is live; `agent`/`external` are recorded as per-item errors until their adapters ship. |
+| `modelKey` | string | for `model` | Registered model key. |
+| `agentKey` | string | for `agent` | Registered agent key. |
+
+## Datasets
+
+```http
+GET    /api/evaluation/datasets?search=faq
+POST   /api/evaluation/datasets
+GET    /api/evaluation/datasets/:id
+PATCH  /api/evaluation/datasets/:id
+DELETE /api/evaluation/datasets/:id
+```
+
+### Create
+
+```json
+{
+  "name": "FAQ regression",
+  "items": [
+    {
+      "id": "q1",
+      "input": [{ "role": "user", "content": "What is 2+2?" }],
+      "expected": { "mustContain": ["4"] }
+    }
+  ]
+}
+```
+
+`items[].input` is an array of `{ role, content }` chat messages. `expected` is
+optional and consumed by the assertion scorer.
+
+## Suites
+
+```http
+GET    /api/evaluation/suites?search=faq
+POST   /api/evaluation/suites
+GET    /api/evaluation/suites/:id
+PATCH  /api/evaluation/suites/:id
+DELETE /api/evaluation/suites/:id
+```
+
+### Create
+
+```json
+{
+  "name": "FAQ accuracy",
+  "targetKey": "gpt-4o-production",
+  "datasetKey": "faq-regression",
+  "scorers": [
+    { "type": "assertion" },
+    { "type": "llm-judge", "rubric": "Answer is correct and concise." }
+  ],
+  "judgeModelKey": "gpt-4o"
+}
+```
+
+| Field | Type | Required | Notes |
+|---|---|---|---|
+| `targetKey` / `datasetKey` | string | yes | Keys of an existing target and dataset. |
+| `scorers` | array | yes | Non-empty; each `{ type, weight?, rubric?, threshold? }`. |
+| `judgeModelKey` | string | when an `llm-judge` scorer is present | Model used for grading. |
+| `runConfig.concurrency` | number | no | Parallel items per run. |
+
+## Runs
+
+### Run a suite
+
+```http
+POST /api/evaluation/suites/:key/run
+```
+
+Loads the suite, target, and dataset, executes the target over every item with
+bounded concurrency, scores each output, persists the run, and returns it.
+
+#### Response
+
+```json
+{
+  "run": {
+    "id": "…",
+    "suiteKey": "faq-accuracy",
+    "status": "completed",
+    "aggregate": { "total": 12, "completed": 12, "failed": 0, "passed": 11, "passRate": 0.916, "avgScore": 0.94, "avgLatencyMs": 410 },
+    "items": [
+      { "itemId": "q1", "passed": true, "score": 1, "scores": [ { "scorerType": "assertion", "score": 1, "passed": true, "weight": 1 } ], "output": { "text": "4", "latencyMs": 380 } }
+    ]
+  }
+}
+```
+
+A target/judge error on an item is recorded on that item (`error`) and counted
+in `aggregate.failed`; it does not abort the run.
+
+### List / get runs
+
+```http
+GET /api/evaluation/runs?suiteKey=faq-accuracy&limit=50
+GET /api/evaluation/runs/:id
+```
+
+## Alerting
+
+Run aggregates feed the alert system through the `evaluation` module:
+
+| Metric | Source |
+|---|---|
+| `evaluation_pass_rate` | `aggregate.passRate` × 100, averaged over completed runs in the window. |
+| `evaluation_avg_score` | `aggregate.avgScore` × 100, same. |
+
+Create an alert rule (module `evaluation`) such as `evaluation_pass_rate lt 80`
+to be notified when quality regresses. See the
+[Evaluation & Analysis guide](/guide/evaluation-and-analysis#automation).
+
+## Errors
+
+| Status | Cause |
+|---|---|
+| 400 | Missing `name`, bad `kind`, empty `scorers`, non-array `items`. |
+| 404 | Target / dataset / suite / run not found (or unknown `suiteKey` on run). |
+| 500 | Internal error. |