From 0f802e55ed37cd9a2b827a9546a2ea7ffe9d0a0b Mon Sep 17 00:00:00 2001 From: Christian Schuetz Date: Wed, 15 Jul 2026 00:31:10 -0500 Subject: [PATCH 1/2] feat(evaluation-system): claude -p eval engine, envelope parsing, response cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundational layer of the prompt-eval suite (D1). Adds src/eval/ with a single-shot Claude Code CLI (`claude -p`) substrate — never the Anthropic SDK: - types.ts: EvalRequest/EvalResult/Envelope + injectable Spawn and Cache seams. - engine.ts: buildArgs (pure print-mode argv: --print --output-format json --json-schema --model --strict-mcp-config --settings + conditional system-prompt/tool-policy flags; prompt on stdin), parseEnvelope (graded, never throws — prefers structured_output, falls back to result, shallow schema check), runClaude (cache-first, spawn, cache-on-success), isAvailable and a skip signal distinct from graded failure for an absent CLI. - cache.ts: content-addressed ResponseCache keyed on a canonical SHA-256 over every output-affecting field. - tests/helpers/fake-claude.ts + tests/eval/*: hermetic engine/cache coverage via the injected Spawn stub, no real claude. Not a plugin deployable — no build.ts entry, no version bump. --- .gitignore | 1 + src/eval/cache.ts | 70 +++++++++++ src/eval/engine.ts | 228 +++++++++++++++++++++++++++++++++++ src/eval/types.ts | 113 +++++++++++++++++ tests/eval/cache.test.ts | 130 ++++++++++++++++++++ tests/eval/engine.test.ts | 181 +++++++++++++++++++++++++++ tests/helpers/fake-claude.ts | 82 +++++++++++++ 7 files changed, 805 insertions(+) create mode 100644 src/eval/cache.ts create mode 100644 src/eval/engine.ts create mode 100644 src/eval/types.ts create mode 100644 tests/eval/cache.test.ts create mode 100644 tests/eval/engine.test.ts create mode 100644 tests/helpers/fake-claude.ts diff --git a/.gitignore b/.gitignore index c2658d7..8c8f03b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules/ +.eval-cache/ diff --git a/src/eval/cache.ts b/src/eval/cache.ts new file mode 100644 index 0000000..811fd55 --- /dev/null +++ b/src/eval/cache.ts @@ -0,0 +1,70 @@ +// Content-addressed response cache for eval results. The key is a SHA-256 over a +// CANONICAL (sorted-key) JSON of every request field that changes the model's +// output — prompt, both system-prompt knobs, model, schema, tool policy, and +// settings. Include every such field or an A/B would collide; a changed prompt +// naturally misses. Results are stored one JSON file per key; no eviction. + +import { createHash } from 'node:crypto' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import type { Cache, EvalRequest, EvalResult } from './types.ts' + +/** Recursively sort object keys so key order never perturbs the hash. */ +function sortKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeys) + if (value !== null && typeof value === 'object') { + const sorted: Record = {} + for (const key of Object.keys(value as Record).sort()) { + sorted[key] = sortKeys((value as Record)[key]) + } + return sorted + } + return value +} + +/** Stable, key-sorted JSON of the output-affecting fields of a request. */ +function canonicalRequest(req: EvalRequest): string { + return JSON.stringify( + sortKeys({ + prompt: req.prompt, + systemPrompt: req.systemPrompt ?? null, + appendSystemPrompt: req.appendSystemPrompt ?? null, + model: req.model, + schema: req.schema, + tools: req.tools ?? null, + settings: req.settings ?? null, + }) + ) +} + +/** Hex SHA-256 cache key over a request's output-affecting fields. */ +export function cacheKey(req: EvalRequest): string { + return createHash('sha256').update(canonicalRequest(req)).digest('hex') +} + +/** File-backed response cache: one `/.json` per stored result. */ +export class ResponseCache implements Cache { + constructor(private readonly dir: string) {} + + private path(key: string): string { + return join(this.dir, `${key}.json`) + } + + /** Return the stored result (marked `cached:true`) or `null` on miss/corruption. */ + get(key: string): EvalResult | null { + const p = this.path(key) + if (!existsSync(p)) return null + try { + const result = JSON.parse(readFileSync(p, 'utf8')) as EvalResult + return { ...result, cached: true } + } catch { + return null + } + } + + /** Persist a result under `key`, creating the cache dir on first write. */ + set(key: string, result: EvalResult): void { + mkdirSync(this.dir, { recursive: true }) + writeFileSync(this.path(key), JSON.stringify(result, null, 2)) + } +} diff --git a/src/eval/engine.ts b/src/eval/engine.ts new file mode 100644 index 0000000..8404bb8 --- /dev/null +++ b/src/eval/engine.ts @@ -0,0 +1,228 @@ +// The `claude -p` eval substrate. `runClaude` builds the print-mode argv, spawns +// the CLI through an injectable `Spawn`, parses the JSON envelope, validates the +// forced `structured_output` against the request schema, and returns a typed, +// graded result. It shells out to `claude` ONLY — never `@anthropic-ai/sdk`. +// +// Design notes: +// - The prompt travels on stdin (fed via `Spawn`'s `input`), so argv stays free +// of arbitrarily large prompt text; `claude -p` reads stdin when given no +// positional prompt. +// - Cost/latency/usage are read straight off the envelope — no pricing table. +// - Every failure mode (CLI error, unparseable envelope, schema miss) is a +// graded `ok:false` result, never a thrown exception. An ABSENT CLI is a +// distinct `skipped:true` signal so callers can skip-with-notice. + +import { spawnSync } from 'node:child_process' +import type { + Cache, + Envelope, + EvalRequest, + EvalResult, + EvalUsage, + JsonSchema, + Spawn, +} from './types.ts' +import { cacheKey } from './cache.ts' + +/** Tools disabled by default so a case is single-shot, deterministic, cacheable. */ +const DEFAULT_DISALLOWED_TOOLS = [ + 'Bash', + 'Read', + 'Write', + 'Edit', + 'WebFetch', + 'WebSearch', + 'Task', + 'Glob', + 'Grep', +] + +/** Real spawn: `spawnSync('claude', …)` with the prompt on stdin. */ +export const defaultSpawn: Spawn = (cmd, args, input) => { + const res = spawnSync(cmd, args, { encoding: 'utf8', input, maxBuffer: 64 * 1024 * 1024 }) + return { status: res.status, stdout: res.stdout ?? '', stderr: res.stderr ?? '' } +} + +/** + * Build the pure `claude -p` argv for a request (prompt excluded — it goes on + * stdin). Always forces print mode, JSON output, the request schema, model, + * strict MCP isolation, and inline settings; layers system-prompt and tool-policy + * flags conditionally. + */ +export function buildArgs(req: EvalRequest): string[] { + const args = [ + '--print', + '--output-format', + 'json', + '--json-schema', + JSON.stringify(req.schema), + '--model', + req.model, + '--strict-mcp-config', + '--settings', + req.settings ?? '{}', + ] + if (req.systemPrompt !== undefined) args.push('--system-prompt', req.systemPrompt) + if (req.appendSystemPrompt !== undefined) args.push('--append-system-prompt', req.appendSystemPrompt) + if (req.tools && req.tools.length > 0) { + args.push('--allowedTools', req.tools.join(' ')) + } else { + args.push('--disallowedTools', DEFAULT_DISALLOWED_TOOLS.join(' ')) + } + return args +} + +/** Lift the envelope's snake_case usage block into the normalized shape. */ +function normalizeUsage(envelope: Envelope): EvalUsage { + const u = envelope.usage ?? {} + return { + inputTokens: u.input_tokens ?? 0, + outputTokens: u.output_tokens ?? 0, + cacheReadInputTokens: u.cache_read_input_tokens ?? 0, + cacheCreationInputTokens: u.cache_creation_input_tokens ?? 0, + } +} + +/** + * Shallow structural check: value is an object, every `required` key is present, + * and — when `additionalProperties:false` — no keys beyond `properties` appear. + * Returns an error message or `null`. Intentionally NOT a full JSON-Schema + * validator (no dependency added). + */ +function validateSchema(value: unknown, schema: JsonSchema): string | null { + const wantsObject = schema.type === 'object' || schema.properties !== undefined || schema.required !== undefined + if (!wantsObject) return null + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return 'output is not an object' + } + const obj = value as Record + for (const key of schema.required ?? []) { + if (!(key in obj)) return `output missing required key: ${key}` + } + if (schema.additionalProperties === false && schema.properties) { + const allowed = new Set(Object.keys(schema.properties)) + for (const key of Object.keys(obj)) { + if (!allowed.has(key)) return `output has unexpected key: ${key}` + } + } + return null +} + +/** Assemble a graded result from envelope metrics + an ok/error verdict. */ +function gradedResult( + envelope: Envelope, + req: EvalRequest, + verdict: { ok: true; output: unknown } | { ok: false; error: string } +): EvalResult { + const base = { + cost: envelope.total_cost_usd ?? 0, + usage: normalizeUsage(envelope), + durationMs: envelope.duration_ms ?? 0, + apiDurationMs: envelope.duration_api_ms ?? 0, + numTurns: envelope.num_turns ?? 0, + model: req.model, + cached: false, + } + return verdict.ok + ? { ok: true, output: verdict.output, error: null, ...base } + : { ok: false, output: null, error: verdict.error, ...base } +} + +/** A graded failure carrying zeroed metrics (used when the envelope is unusable). */ +function failureNoMetrics(req: EvalRequest, error: string): EvalResult { + return { + ok: false, + output: null, + error, + cost: 0, + usage: { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 }, + durationMs: 0, + apiDurationMs: 0, + numTurns: 0, + model: req.model, + cached: false, + } +} + +/** + * Parse a `claude --output-format json` envelope into a graded `EvalResult`. + * Never throws: an unparseable envelope, a CLI/model error, a missing/bad + * output, or a schema miss all yield `ok:false` with a descriptive `error`. + */ +export function parseEnvelope(stdout: string, req: EvalRequest): EvalResult { + let envelope: Envelope + try { + envelope = JSON.parse(stdout) as Envelope + } catch { + return failureNoMetrics(req, 'unparseable envelope') + } + + if (envelope.is_error === true || (envelope.subtype !== undefined && envelope.subtype !== 'success')) { + return gradedResult(envelope, req, { + ok: false, + error: `claude error: is_error=${String(envelope.is_error)} subtype=${String(envelope.subtype)}`, + }) + } + + // Prefer the parsed, schema-forced object; fall back to parsing stringified `result`. + let output = envelope.structured_output + if (output === undefined || output === null) { + if (typeof envelope.result !== 'string') { + return gradedResult(envelope, req, { ok: false, error: 'no structured_output and no string result to parse' }) + } + try { + output = JSON.parse(envelope.result) + } catch { + return gradedResult(envelope, req, { ok: false, error: 'unparseable result JSON' }) + } + } + + const schemaError = validateSchema(output, req.schema) + if (schemaError) return gradedResult(envelope, req, { ok: false, error: schemaError }) + + return gradedResult(envelope, req, { ok: true, output }) +} + +export interface RunOptions { + /** Injected subprocess boundary; defaults to the real `claude` spawn. */ + spawn?: Spawn + /** Optional response cache consulted before spawning and written on success. */ + cache?: Cache +} + +/** + * Run one eval request: cache lookup first, else spawn `claude -p`, parse, cache + * (on success), return. An absent CLI (non-zero exit with empty stdout, or a + * spawn throw) surfaces as a `skipped` result — distinct from a graded failure. + */ +export function runClaude(req: EvalRequest, { spawn = defaultSpawn, cache }: RunOptions = {}): EvalResult { + const key = cache ? cacheKey(req) : null + if (cache && key) { + const hit = cache.get(key) + if (hit) return { ...hit, cached: true } + } + + let res + try { + res = spawn('claude', buildArgs(req), req.prompt) + } catch { + return { ...failureNoMetrics(req, 'claude CLI unavailable'), skipped: true } + } + + if (res.status !== 0 && res.stdout.trim() === '') { + return { ...failureNoMetrics(req, 'claude CLI unavailable'), skipped: true } + } + + const result = parseEnvelope(res.stdout, req) + if (cache && key && result.ok) cache.set(key, result) + return result +} + +/** Probe whether the `claude` CLI is invokable via `claude --version`. */ +export function isAvailable(spawn: Spawn = defaultSpawn): boolean { + try { + return spawn('claude', ['--version']).status === 0 + } catch { + return false + } +} diff --git a/src/eval/types.ts b/src/eval/types.ts new file mode 100644 index 0000000..1cb125a --- /dev/null +++ b/src/eval/types.ts @@ -0,0 +1,113 @@ +// Shared types for the prompt-eval engine. The engine shells out to the Claude +// Code CLI in print mode (`claude -p`) — NEVER the Anthropic SDK — and reads +// cost/latency/usage straight off the `--output-format json` envelope. The one +// faked boundary in tests is `Spawn`; everything else is real. + +/** + * A minimal JSON-Schema shape. The engine only performs a SHALLOW structural + * check (`required` keys present, no extras when `additionalProperties:false`), + * so this deliberately does not model the full spec — do NOT add a schema + * validator dependency. The harness's generated schemas satisfy this shape. + */ +export interface JsonSchema { + type?: string + required?: string[] + properties?: Record + additionalProperties?: boolean + items?: JsonSchema + [key: string]: unknown +} + +/** A single-shot eval request: the prompt, the schema it is forced against, and knobs. */ +export interface EvalRequest { + /** User prompt — passed to `claude -p` on stdin. */ + prompt: string + /** Replace the CLI's default system prompt (`--system-prompt`) for isolation. */ + systemPrompt?: string + /** Layer extra context on top of the default system prompt (`--append-system-prompt`). */ + appendSystemPrompt?: string + /** Model id, e.g. `claude-haiku-4-5` (`--model`). */ + model: string + /** Schema forced via `--json-schema` and validated against `structured_output`. */ + schema: JsonSchema + /** Opt into a bounded toolset (`--allowedTools`); omit for a single-shot, tool-free run. */ + tools?: string[] + /** Inline settings JSON (`--settings`); defaults to `'{}'` to isolate from ambient config. */ + settings?: string +} + +/** Normalized token usage lifted from the envelope's snake_case `usage` block. */ +export interface EvalUsage { + inputTokens: number + outputTokens: number + cacheReadInputTokens: number + cacheCreationInputTokens: number +} + +/** + * The graded outcome of one eval invocation. A model/CLI error, an unparseable + * envelope, or a schema-nonconforming output all yield `ok:false` with a + * descriptive `error` — the engine NEVER throws for these. `skipped:true` is a + * distinct signal that the CLI itself was unavailable (not a graded failure). + */ +export interface EvalResult { + ok: boolean + output: unknown | null + error: string | null + cost: number + usage: EvalUsage + durationMs: number + apiDurationMs: number + numTurns: number + model: string + cached: boolean + /** True only when the `claude` CLI was unavailable — callers skip-with-notice. */ + skipped?: boolean +} + +/** Envelope usage block as emitted by `claude --output-format json` (snake_case). */ +export interface EnvelopeUsage { + input_tokens?: number + output_tokens?: number + cache_read_input_tokens?: number + cache_creation_input_tokens?: number + service_tier?: string +} + +/** + * Raw `--output-format json` envelope (verified against installed `claude` + * 2.1.207). Only the fields the engine reads are typed; the rest are ignored. + */ +export interface Envelope { + type?: string + subtype?: string + is_error?: boolean + result?: string + structured_output?: unknown + total_cost_usd?: number + usage?: EnvelopeUsage + duration_ms?: number + duration_api_ms?: number + num_turns?: number + modelUsage?: Record +} + +/** Result of a subprocess spawn — the single injectable boundary tests fake. */ +export interface SpawnResult { + status: number | null + stdout: string + stderr: string +} + +/** + * The injected spawn dependency. Default is a thin `spawnSync('claude', …)` + * wrapper; tests pass a stub returning a canned envelope. `input` is fed to the + * child's stdin (the prompt travels there, keeping argv free of large text). + */ +export type Spawn = (cmd: string, args: string[], input?: string) => SpawnResult + +/** A response cache the engine can consult before spawning. */ +export interface Cache { + get(key: string): EvalResult | null + set(key: string, result: EvalResult): void +} diff --git a/tests/eval/cache.test.ts b/tests/eval/cache.test.ts new file mode 100644 index 0000000..8bab195 --- /dev/null +++ b/tests/eval/cache.test.ts @@ -0,0 +1,130 @@ +// Cache tests: key stability, per-field isolation, file round-trip in a temp +// dir, and the runClaude hit/miss integration (a hit must NOT spawn). + +import assert from 'node:assert/strict' +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'bun:test' +import { cacheKey, ResponseCache } from '../../src/eval/cache.ts' +import { runClaude } from '../../src/eval/engine.ts' +import type { EvalRequest, JsonSchema, Spawn } from '../../src/eval/types.ts' +import { fakeSpawn, successEnvelope, throwingSpawn } from '../helpers/fake-claude.ts' + +const SCHEMA: JsonSchema = { + type: 'object', + required: ['answer'], + properties: { answer: { type: 'number' } }, + additionalProperties: false, +} + +const baseReq = (overrides: Partial = {}): EvalRequest => ({ + prompt: 'What is 6 * 7?', + model: 'claude-haiku-4-5', + schema: SCHEMA, + ...overrides, +}) + +const tempDir = (): string => mkdtempSync(join(tmpdir(), 'eval-cache-')) + +// --- key stability + isolation (acceptance criterion 4) ---------------------- + +test('cacheKey is stable for identical requests', () => { + assert.equal(cacheKey(baseReq()), cacheKey(baseReq())) +}) + +test('cacheKey is insensitive to schema key ordering (canonical JSON)', () => { + const a = baseReq({ schema: { type: 'object', required: ['answer'], additionalProperties: false } }) + const b = baseReq({ schema: { additionalProperties: false, required: ['answer'], type: 'object' } }) + assert.equal(cacheKey(a), cacheKey(b)) +}) + +test('cacheKey changes when any output-affecting field changes', () => { + const base = cacheKey(baseReq()) + const variants: EvalRequest[] = [ + baseReq({ prompt: 'different' }), + baseReq({ model: 'claude-opus-4-8' }), + baseReq({ schema: { type: 'object', required: ['result'] } }), + baseReq({ systemPrompt: 'You are terse.' }), + baseReq({ appendSystemPrompt: 'Context.' }), + baseReq({ tools: ['Read'] }), + baseReq({ settings: '{"x":1}' }), + ] + for (const v of variants) { + assert.notEqual(cacheKey(v), base) + } + // All variants are distinct from each other too. + const keys = new Set(variants.map(cacheKey)) + assert.equal(keys.size, variants.length) +}) + +// --- file round-trip --------------------------------------------------------- + +test('ResponseCache round-trips a result and marks it cached on read', () => { + const cache = new ResponseCache(tempDir()) + const key = cacheKey(baseReq()) + assert.equal(cache.get(key), null) + + const result = runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 42 })) }) + cache.set(key, result) + + const hit = cache.get(key) + assert.ok(hit) + assert.equal(hit.cached, true) + assert.deepEqual(hit.output, { answer: 42 }) + assert.equal(hit.cost, result.cost) +}) + +test('ResponseCaches in separate dirs are isolated', () => { + const a = new ResponseCache(tempDir()) + const b = new ResponseCache(tempDir()) + const key = cacheKey(baseReq()) + a.set(key, runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 1 })) })) + assert.ok(a.get(key)) + assert.equal(b.get(key), null) +}) + +// --- runClaude cache integration --------------------------------------------- + +test('runClaude serves an identical request from cache without spawning', () => { + const cache = new ResponseCache(tempDir()) + let spawnCalls = 0 + const countingSpawn: Spawn = (cmd, args, input) => { + spawnCalls++ + return fakeSpawn(successEnvelope({ answer: 42 }))(cmd, args, input) + } + + const first = runClaude(baseReq(), { spawn: countingSpawn, cache }) + assert.equal(first.ok, true) + assert.equal(first.cached, false) + assert.equal(spawnCalls, 1) + + // Second identical call: cache hit, no further spawn (throwingSpawn proves it). + const second = runClaude(baseReq(), { spawn: throwingSpawn(), cache }) + assert.equal(second.ok, true) + assert.equal(second.cached, true) + assert.deepEqual(second.output, { answer: 42 }) + assert.equal(spawnCalls, 1) +}) + +test('runClaude misses the cache when a keyed field changes', () => { + const cache = new ResponseCache(tempDir()) + runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 42 })) , cache }) + + let spawned = false + const spawn: Spawn = (cmd, args, input) => { + spawned = true + return fakeSpawn(successEnvelope({ answer: 7 }))(cmd, args, input) + } + const result = runClaude(baseReq({ model: 'claude-opus-4-8' }), { spawn, cache }) + assert.equal(spawned, true) + assert.equal(result.cached, false) + assert.deepEqual(result.output, { answer: 7 }) +}) + +test('runClaude does not cache a graded failure', () => { + const cache = new ResponseCache(tempDir()) + // A schema-miss success envelope grades ok:false and must not be cached. + runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ wrong: 1 })), cache }) + assert.equal(cache.get(cacheKey(baseReq())), null) +}) diff --git a/tests/eval/engine.test.ts b/tests/eval/engine.test.ts new file mode 100644 index 0000000..de3c309 --- /dev/null +++ b/tests/eval/engine.test.ts @@ -0,0 +1,181 @@ +// Engine tests: pure argv construction, envelope grading over +// success/fallback/error/schema-miss/unparseable, and the isAvailable/skip +// paths — all through the injected `Spawn` stub, never a real `claude`. + +import assert from 'node:assert/strict' +import { test } from 'bun:test' +import { buildArgs, isAvailable, parseEnvelope, runClaude } from '../../src/eval/engine.ts' +import type { EvalRequest, JsonSchema } from '../../src/eval/types.ts' +import { + errorEnvelope, + fakeSpawn, + rawSpawn, + schemaMismatchEnvelope, + successEnvelope, + throwingSpawn, +} from '../helpers/fake-claude.ts' + +const SCHEMA: JsonSchema = { + type: 'object', + required: ['answer'], + properties: { answer: { type: 'number' } }, + additionalProperties: false, +} + +const baseReq = (overrides: Partial = {}): EvalRequest => ({ + prompt: 'What is 6 * 7?', + model: 'claude-haiku-4-5', + schema: SCHEMA, + ...overrides, +}) + +// --- buildArgs (acceptance criterion 1) -------------------------------------- + +test('buildArgs forces print mode, JSON output, schema, model, strict MCP, and settings', () => { + const args = buildArgs(baseReq()) + assert.ok(args.includes('--print')) + assert.deepEqual(args.slice(args.indexOf('--output-format'), args.indexOf('--output-format') + 2), [ + '--output-format', + 'json', + ]) + assert.deepEqual(args.slice(args.indexOf('--json-schema'), args.indexOf('--json-schema') + 2), [ + '--json-schema', + JSON.stringify(SCHEMA), + ]) + assert.deepEqual(args.slice(args.indexOf('--model'), args.indexOf('--model') + 2), [ + '--model', + 'claude-haiku-4-5', + ]) + assert.ok(args.includes('--strict-mcp-config')) + assert.deepEqual(args.slice(args.indexOf('--settings'), args.indexOf('--settings') + 2), ['--settings', '{}']) +}) + +test('buildArgs disables tools by default and never puts the prompt in argv', () => { + const args = buildArgs(baseReq()) + assert.ok(args.includes('--disallowedTools')) + assert.ok(!args.includes('--allowedTools')) + // The prompt travels on stdin, so it must not appear in argv. + assert.ok(!args.some(a => a.includes('6 * 7'))) +}) + +test('buildArgs opts into an allow-list when tools are provided', () => { + const args = buildArgs(baseReq({ tools: ['Read', 'Grep'] })) + assert.deepEqual(args.slice(args.indexOf('--allowedTools'), args.indexOf('--allowedTools') + 2), [ + '--allowedTools', + 'Read Grep', + ]) + assert.ok(!args.includes('--disallowedTools')) +}) + +test('buildArgs adds system-prompt flags only when set, and passes custom settings', () => { + const bare = buildArgs(baseReq()) + assert.ok(!bare.includes('--system-prompt')) + assert.ok(!bare.includes('--append-system-prompt')) + + const withSys = buildArgs(baseReq({ systemPrompt: 'You are terse.', appendSystemPrompt: 'Context.', settings: '{"x":1}' })) + assert.deepEqual(withSys.slice(withSys.indexOf('--system-prompt'), withSys.indexOf('--system-prompt') + 2), [ + '--system-prompt', + 'You are terse.', + ]) + assert.deepEqual( + withSys.slice(withSys.indexOf('--append-system-prompt'), withSys.indexOf('--append-system-prompt') + 2), + ['--append-system-prompt', 'Context.'] + ) + assert.deepEqual(withSys.slice(withSys.indexOf('--settings'), withSys.indexOf('--settings') + 2), [ + '--settings', + '{"x":1}', + ]) +}) + +// --- success + fallback (acceptance criterion 2) ----------------------------- + +test('a success envelope with structured_output yields ok:true with cost/usage/output', () => { + const envelope = successEnvelope( + { answer: 42 }, + { cost: 0.0463, usage: { inputTokens: 10, outputTokens: 158, cacheReadInputTokens: 3 }, durationMs: 2419, apiDurationMs: 2394, numTurns: 2 } + ) + const result = runClaude(baseReq(), { spawn: fakeSpawn(envelope) }) + assert.equal(result.ok, true) + assert.deepEqual(result.output, { answer: 42 }) + assert.equal(result.error, null) + assert.equal(result.cost, 0.0463) + assert.deepEqual(result.usage, { + inputTokens: 10, + outputTokens: 158, + cacheReadInputTokens: 3, + cacheCreationInputTokens: 0, + }) + assert.equal(result.durationMs, 2419) + assert.equal(result.apiDurationMs, 2394) + assert.equal(result.numTurns, 2) + assert.equal(result.model, 'claude-haiku-4-5') + assert.equal(result.cached, false) +}) + +test('a missing structured_output falls back to JSON.parse(result)', () => { + const envelope = successEnvelope({ answer: 7 }, { structured: false }) + assert.equal(envelope.structured_output, undefined) + const result = runClaude(baseReq(), { spawn: fakeSpawn(envelope) }) + assert.equal(result.ok, true) + assert.deepEqual(result.output, { answer: 7 }) +}) + +// --- graded failures, no throw (acceptance criterion 3) ---------------------- + +test('is_error / non-success subtype grades ok:false without throwing', () => { + const result = runClaude(baseReq(), { spawn: fakeSpawn(errorEnvelope('error_max_turns')) }) + assert.equal(result.ok, false) + assert.equal(result.output, null) + assert.match(result.error ?? '', /error_max_turns/) + // Metrics from the envelope are still surfaced. + assert.equal(result.cost, 0.002) +}) + +test('schema-nonconforming output grades ok:false with a descriptive error', () => { + const result = runClaude(baseReq(), { spawn: fakeSpawn(schemaMismatchEnvelope()) }) + assert.equal(result.ok, false) + assert.match(result.error ?? '', /required key: answer/) +}) + +test('additionalProperties:false rejects an extra key', () => { + const envelope = successEnvelope({ answer: 1, extra: 2 }) + const result = parseEnvelope(JSON.stringify(envelope), baseReq()) + assert.equal(result.ok, false) + assert.match(result.error ?? '', /unexpected key: extra/) +}) + +test('unparseable stdout grades ok:false with zeroed metrics, no throw', () => { + const result = runClaude(baseReq(), { spawn: rawSpawn(0, 'not json at all') }) + assert.equal(result.ok, false) + assert.equal(result.error, 'unparseable envelope') + assert.equal(result.cost, 0) + assert.equal(result.durationMs, 0) +}) + +// --- availability + skip (acceptance criterion 5) ---------------------------- + +test('isAvailable is false when claude --version exits non-zero or throws', () => { + assert.equal(isAvailable(rawSpawn(127, '', 'command not found')), false) + assert.equal(isAvailable(throwingSpawn()), false) + assert.equal(isAvailable(rawSpawn(0, 'claude 2.1.207')), true) +}) + +test('runClaude against an absent CLI returns a skipped result, never throws', () => { + const viaExit = runClaude(baseReq(), { spawn: rawSpawn(127, '', 'not found') }) + assert.equal(viaExit.skipped, true) + assert.equal(viaExit.ok, false) + assert.match(viaExit.error ?? '', /unavailable/) + + const viaThrow = runClaude(baseReq(), { spawn: throwingSpawn() }) + assert.equal(viaThrow.skipped, true) + assert.equal(viaThrow.ok, false) +}) + +test('a non-zero exit that still produced envelope stdout is graded, not skipped', () => { + // The CLI can exit non-zero while emitting an error envelope — that is a + // graded failure, not an availability skip. + const result = runClaude(baseReq(), { spawn: rawSpawn(1, JSON.stringify(errorEnvelope('error_during_execution'))) }) + assert.equal(result.skipped, undefined) + assert.equal(result.ok, false) + assert.match(result.error ?? '', /error_during_execution/) +}) diff --git a/tests/helpers/fake-claude.ts b/tests/helpers/fake-claude.ts new file mode 100644 index 0000000..b3f434c --- /dev/null +++ b/tests/helpers/fake-claude.ts @@ -0,0 +1,82 @@ +// Hermetic stub for the `claude -p` subprocess boundary. Tests inject `fakeSpawn` +// (or a raw variant) so `bun test` never shells out to a real `claude` — the ONLY +// faked seam in the eval engine. Envelope factories mirror the verified real +// `--output-format json` shape (installed claude 2.1.207). + +import type { Envelope, EvalUsage, Spawn } from '../../src/eval/types.ts' + +/** A `Spawn` that ignores args/stdin and emits `envelope` as JSON on stdout, exit 0. */ +export function fakeSpawn(envelope: Envelope): Spawn { + return () => ({ status: 0, stdout: JSON.stringify(envelope), stderr: '' }) +} + +/** A `Spawn` returning a fixed status/stdout/stderr verbatim (for edge cases). */ +export function rawSpawn(status: number | null, stdout: string, stderr = ''): Spawn { + return () => ({ status, stdout, stderr }) +} + +/** A `Spawn` that throws, modelling a missing binary (ENOENT). */ +export function throwingSpawn(): Spawn { + return () => { + throw new Error('spawn claude ENOENT') + } +} + +export interface SuccessOptions { + cost?: number + usage?: Partial + durationMs?: number + apiDurationMs?: number + numTurns?: number + /** When false, omit `structured_output` so the engine falls back to `result`. */ + structured?: boolean +} + +/** A success envelope carrying `output` as both `structured_output` and `result`. */ +export function successEnvelope(output: unknown, opts: SuccessOptions = {}): Envelope { + const { + cost = 0.001, + usage = {}, + durationMs = 1000, + apiDurationMs = 950, + numTurns = 2, + structured = true, + } = opts + const envelope: Envelope = { + type: 'result', + subtype: 'success', + is_error: false, + result: JSON.stringify(output), + total_cost_usd: cost, + duration_ms: durationMs, + duration_api_ms: apiDurationMs, + num_turns: numTurns, + usage: { + input_tokens: usage.inputTokens ?? 10, + output_tokens: usage.outputTokens ?? 20, + cache_read_input_tokens: usage.cacheReadInputTokens ?? 0, + cache_creation_input_tokens: usage.cacheCreationInputTokens ?? 0, + }, + } + if (structured) envelope.structured_output = output + return envelope +} + +/** An error envelope (`is_error:true`, non-success subtype) with metrics present. */ +export function errorEnvelope(subtype = 'error_max_turns'): Envelope { + return { + type: 'result', + subtype, + is_error: true, + total_cost_usd: 0.002, + duration_ms: 500, + duration_api_ms: 480, + num_turns: 1, + usage: { input_tokens: 5, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + } +} + +/** A success envelope whose `structured_output` omits a required key (schema miss). */ +export function schemaMismatchEnvelope(): Envelope { + return successEnvelope({ unexpected: true }) +} From 49c8dfbaf282e3751a3ac7f1463dce8187e0108c Mon Sep 17 00:00:00 2001 From: Christian Schuetz Date: Wed, 15 Jul 2026 00:43:34 -0500 Subject: [PATCH 2/2] =?UTF-8?q?feat(evaluation-system):=20eval=20framework?= =?UTF-8?q?=20=E2=80=94=20cases,=20graders,=20A/B=20+=20matrix=20runner,?= =?UTF-8?q?=20report,=20CLI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grading + orchestration layer on top of the D1 `claude -p` engine (D2). - src/eval/case.ts: EvalCase / Variant types, defineCase, toRequest lowering. - src/eval/grade.ts: Grader/GradeResult, built-in schemaConforms/assert/judge (judge routes through the D1 engine, cached/stubbable), weighted gradeCase. - src/eval/runner.ts: runCase, bounded-parallel runSuite (mapPool), runAB delta record (Δcorrectness/Δcost/Δlatency/Δturns), runMatrix per-model grid; skips with notice when the CLI is unavailable. - src/eval/report.ts: aligned text table + JSON over all three axes, optional clearly-labelled projected-cost column from a static pricing table. - src/eval/cli.ts: `bun src/eval/cli.ts` — suite loading, --ab/--matrix/--filter/ --models/--json/--tolerance; report-only exit 0 except an A/B correctness regression past tolerance → exit 1; absent claude → notice + exit 0. - package.json: `eval` script (kept OUT of `test`; test stays hermetic/offline). - CLAUDE.md + conventions.md: run the eval suite alongside `bun test` when changing harness prompts (docs-only, no generated-artifact churn, no bump). - tests/eval/framework.test.ts: graders, gradeCase, runAB/runMatrix deltas, bounded pool, report text/JSON, CLI exit-code semantics — all via fakeSpawn. --- CLAUDE.md | 2 + package.json | 1 + plugins/strapped/conventions.md | 6 + src/eval/case.ts | 77 ++++++ src/eval/cli.ts | 174 +++++++++++++ src/eval/grade.ts | 144 +++++++++++ src/eval/report.ts | 134 +++++++++++ src/eval/runner.ts | 172 +++++++++++++ tests/eval/framework.test.ts | 415 ++++++++++++++++++++++++++++++++ 9 files changed, 1125 insertions(+) create mode 100644 src/eval/case.ts create mode 100644 src/eval/cli.ts create mode 100644 src/eval/grade.ts create mode 100644 src/eval/report.ts create mode 100644 src/eval/runner.ts create mode 100644 tests/eval/framework.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 1a70ceb..43a13b8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,3 +1,5 @@ # Guidelines +- **Run the prompt-eval suite alongside `bun test` when changing harness prompts.** When you edit an agent prompt — anything under `src/workflows/strapped-run/**` or the step prose in a skill `SKILL.md` — run `bun run eval --suite ` (see `src/eval/` and the "Prompt evaluation suite" section of `plugins/strapped/conventions.md`) and keep it green alongside `bun test`. The eval suite grades a prompt on correctness / cost / latency through `claude -p`; it is the evidence that a prompt change is a real improvement (use `--ab` to compare a baseline vs a candidate and gate on a correctness regression past `--tolerance`). It is a HEAVY, opt-in layer (needs a real `claude` + spends cost), so it is intentionally NOT part of `bun test` — run it yourself for prompt work. `bun test` stays hermetic/offline and remains the gate for everything else. + - **Bump the plugin version with the tool, never by hand.** `claude plugin update` compares `version` in `plugins/strapped/.claude-plugin/plugin.json` only — an unbumped version leaves installed copies silently pinned to a stale commit after a change merges. Bump ONLY via `bun tools/version.ts bump ` (it rewrites the one semver deterministically); do not hand-edit plugin.json. Pick the level by a conventional-commit rule: a breaking change → `major`, a new user-facing feature (skill/workflow/script behavior) → `minor`, a fix or internal change → `patch`. Before committing, run `bun tools/version.ts check` (alias `bun run version:check`) — it fails when a changed *generated artifact* (the committed deployables under `plugins/strapped/`) is not accompanied by a version bump above the base. Prompt/skill/docs-only changes with NO generated-artifact change are at your discretion — the guard exits 0 for them, so bump when the change is user-facing and skip an internal-only tweak. diff --git a/package.json b/package.json index d08f348..2d4e11e 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "lint": "oxlint src tools tests", "version:bump": "bun tools/version.ts bump", "version:check": "bun tools/version.ts check", + "eval": "bun src/eval/cli.ts", "test": "claude plugin validate . --strict && claude plugin validate plugins/strapped --strict && bun run typecheck && bun run lint && bun test" }, "devDependencies": { diff --git a/plugins/strapped/conventions.md b/plugins/strapped/conventions.md index 5542b56..aede133 100644 --- a/plugins/strapped/conventions.md +++ b/plugins/strapped/conventions.md @@ -398,6 +398,12 @@ This limit is why the retired multi-file architecture (stage workflows dispatchi Every command in the **deliverable's repo's** config `validations` must be green before code review and after every fix round, run inside the deliverable's worktree. +### Prompt evaluation suite + +The strapped repo carries a prompt-effectiveness eval suite under `src/eval/` (run via `bun run eval`, not bundled into any plugin deployable). It grades the harness's own agent prompts on three axes — **correctness** (layered graders: schema-conformance, assertion predicates, an optional LLM judge), **cost** (`total_cost_usd` + token usage), and **latency** (`duration_ms` / `num_turns`) — by shelling out to `claude -p` (never the Anthropic SDK) and reading the `--output-format json` envelope. It supports **A/B** (baseline vs candidate prompt at one model, reporting Δcorrectness / Δcost / Δlatency side by side so a human judges the trade — a correctness dip that buys a large cost/latency win is a WIN) and a **model matrix** (`--models opus,sonnet,haiku`). + +It is the **heavy, opt-in test layer for prompt changes**: it needs a real `claude` and spends cost, so it is deliberately kept OUT of `bun test` (which stays hermetic/offline). When you change an agent prompt (`src/workflows/strapped-run/**` or a skill's `SKILL.md` step prose), run the eval suite alongside `bun test` and keep both green — it is the evidence that the prompt change is a real improvement. An absent `claude` CLI is a graceful skip (notice + exit 0), and only `--ab` gates: exit non-zero on a correctness regression past `--tolerance`. + ## Worktrees and branches All repo-scoped values below come from the **deliverable's repo** — `manifest.repos[deliverable.repo]` for the root, and that repo's config for `worktreeRoot`, `validations`, and `provisioning`. diff --git a/src/eval/case.ts b/src/eval/case.ts new file mode 100644 index 0000000..ba073ca --- /dev/null +++ b/src/eval/case.ts @@ -0,0 +1,77 @@ +// The eval CASE abstraction: a single, self-contained prompt-evaluation unit. A +// case carries the prompt, the schema it is forced against, isolation knobs +// (system prompt / tools / settings), the models it targets, its correctness +// graders, and — for A/B — an optional baseline+candidate prompt pair. A SUITE is +// just a directory of modules that export cases. +// +// Design decision (research §"prompts are eval inputs"): a case owns a VERBATIM +// baseline prompt snapshot copied from the live stage plus a candidate variant to +// A/B. Cases never import the live stage builders — prompts are inputs, not code. + +import type { Grader } from './grade.ts' +import type { EvalRequest, JsonSchema } from './types.ts' + +/** One labelled prompt variant for A/B comparison. */ +export interface Variant { + /** Short label shown in the report (e.g. `baseline`, `terser`). */ + label: string + /** The full prompt text for this variant. */ + prompt: string +} + +/** A baseline↔candidate variant pair a case can be A/B'd on. */ +export interface CaseVariants { + baseline: Variant + candidate: Variant +} + +/** A single prompt-evaluation case. */ +export interface EvalCase { + /** Unique id (also a `--filter` target). */ + id: string + /** Tags for `--filter` selection (e.g. `planner`, `reviewer`). */ + tags: string[] + /** Replace the CLI's default system prompt for isolation (`--system-prompt`). */ + systemPrompt?: string + /** Layer strapped context on the real system prompt (`--append-system-prompt`). */ + appendSystemPrompt?: string + /** The user prompt under evaluation. */ + prompt: string + /** The schema forced via `--json-schema` and validated against the output. */ + schema: JsonSchema + /** Models this case targets in a plain/matrix run; the runner falls back to a default. */ + models?: string[] + /** Opt into a bounded toolset; omit for a single-shot, tool-free run. */ + tools?: string[] + /** Inline settings JSON (`--settings`); defaults to `'{}'` in the engine. */ + settings?: string + /** The correctness graders aggregated into this case's score. */ + graders: Grader[] + /** Optional A/B prompt pair — required for a case to appear in `--ab` mode. */ + variants?: CaseVariants +} + +/** + * Identity helper for authoring a case module — validates nothing beyond types, + * but gives suites a single import and a stable authoring surface. + */ +export function defineCase(spec: EvalCase): EvalCase { + return spec +} + +/** + * Lower a case (at one model, optionally with a variant's prompt substituted) + * into the single-shot `EvalRequest` the D1 engine consumes. The prompt override + * is how A/B swaps baseline vs candidate while holding everything else fixed. + */ +export function toRequest(c: EvalCase, model: string, promptOverride?: string): EvalRequest { + return { + prompt: promptOverride ?? c.prompt, + systemPrompt: c.systemPrompt, + appendSystemPrompt: c.appendSystemPrompt, + model, + schema: c.schema, + tools: c.tools, + settings: c.settings, + } +} diff --git a/src/eval/cli.ts b/src/eval/cli.ts new file mode 100644 index 0000000..37702d0 --- /dev/null +++ b/src/eval/cli.ts @@ -0,0 +1,174 @@ +// `bun src/eval/cli.ts` — the eval suite entrypoint (npm: `bun run eval`). Loads +// case modules from a suite directory, runs them in one of three modes, and +// prints a report. This is the HEAVY, opt-in test layer for prompt changes — it +// is deliberately NOT part of `bun test` (that stays hermetic/offline). +// +// Exit code: a plain/matrix run is REPORT-ONLY (always 0). Only `--ab` gates — +// exit 1 iff some case's Δcorrectness dips past `--tolerance` (a regression). An +// absent `claude` CLI is a graceful skip: print a notice, exit 0, never fail. +// +// Flags: --suite --filter --ab --matrix --models a,b +// --json --tolerance +// +// `main` returns { code, output } (does not exit) so tests can drive the exact +// exit-code semantics through the injected spawn; the bottom guard is the only +// place that touches the process. + +import { readdirSync } from 'node:fs' +import { join, resolve } from 'node:path' +import type { EvalCase } from './case.ts' +import { isAvailable } from './engine.ts' +import { formatReport, toJSON, type Report } from './report.ts' +import { + DEFAULT_MODEL, + runAB, + runMatrix, + runSuite, + type ABResult, + type MatrixResult, +} from './runner.ts' +import type { Cache, Spawn } from './types.ts' + +export interface EvalFlags { + suite?: string + filter?: string + ab: boolean + matrix: boolean + models?: string[] + json: boolean + tolerance: number +} + +/** Hand-rolled argv parser (no dependency). Unknown flags are ignored. */ +export function parseArgs(argv: readonly string[]): EvalFlags { + const flags: EvalFlags = { ab: false, matrix: false, json: false, tolerance: 0 } + for (let i = 0; i < argv.length; i++) { + switch (argv[i]) { + case '--suite': + flags.suite = argv[++i] + break + case '--filter': + flags.filter = argv[++i] + break + case '--ab': + flags.ab = true + break + case '--matrix': + flags.matrix = true + break + case '--models': + flags.models = (argv[++i] ?? '').split(',').filter(Boolean) + break + case '--json': + flags.json = true + break + case '--tolerance': + flags.tolerance = Number(argv[++i] ?? '0') + break + default: + break + } + } + return flags +} + +/** Load every case exported by the modules in a suite directory (sorted). */ +export async function loadSuite(dir: string): Promise { + const abs = resolve(dir) + const files = readdirSync(abs) + .filter(f => /\.(ts|mjs|js)$/.test(f) && !f.endsWith('.d.ts')) + .sort() + const cases: EvalCase[] = [] + for (const file of files) { + const mod = (await import(join(abs, file))) as { cases?: unknown; default?: unknown } + const exported = mod.cases ?? mod.default + if (Array.isArray(exported)) cases.push(...(exported as EvalCase[])) + else if (exported) cases.push(exported as EvalCase) + } + return cases +} + +/** A case matches a filter if the filter equals its id or one of its tags. */ +function matchesFilter(c: EvalCase, filter: string): boolean { + return c.id === filter || c.tags.includes(filter) +} + +/** + * A/B gate: exit 1 iff any case regressed correctness past the tolerance. A + * tolerance of `5` means "a 5-percentage-point dip is allowed"; anything worse + * gates. Cost/latency wins never gate — a human judges the trade from the report. + */ +export function abExitCode(results: readonly ABResult[], tolerancePct: number): number { + const tol = tolerancePct / 100 + return results.some(r => r.deltaCorrectness < -tol) ? 1 : 0 +} + +export interface CliDeps { + spawn?: Spawn + cache?: Cache + /** Test override: supply cases directly instead of loading from `--suite`. */ + cases?: EvalCase[] +} + +export interface CliResult { + code: number + output: string +} + +/** + * Parse argv, run the selected mode, and return an exit code + text to print. + * Never touches the process — the bottom guard does that. An absent CLI short + * circuits to a skip notice + exit 0 before any case runs. + */ +export async function main(argv: readonly string[], deps: CliDeps = {}): Promise { + const flags = parseArgs(argv) + const spawn = deps.spawn + + if (!isAvailable(spawn)) { + return { code: 0, output: 'eval: claude CLI unavailable — skipping suite (exit 0)\n' } + } + + let cases = deps.cases ?? (flags.suite ? await loadSuite(flags.suite) : []) + if (flags.filter) cases = cases.filter(c => matchesFilter(c, flags.filter as string)) + + if (cases.length === 0) { + return { code: 0, output: 'eval: no cases matched (nothing to run)\n' } + } + + let report: Report + let code = 0 + + if (flags.ab) { + const results: ABResult[] = [] + for (const c of cases) { + if (!c.variants) continue + results.push(runAB(c, c.variants.baseline, c.variants.candidate, { model: flags.models?.[0] ?? DEFAULT_MODEL, spawn, cache: deps.cache })) + } + report = { mode: 'ab', cases: results } + code = abExitCode(results, flags.tolerance) + } else if (flags.matrix) { + const grids: MatrixResult[] = [] + for (const c of cases) { + grids.push(await runMatrix(c, flags.models ?? c.models ?? [], { spawn, cache: deps.cache })) + } + report = { mode: 'matrix', cases: grids } + } else { + const results = await runSuite(cases, { models: flags.models, spawn, cache: deps.cache }) + report = { mode: 'suite', cases: results } + } + + const output = flags.json ? `${JSON.stringify(toJSON(report), null, 2)}\n` : `${formatReport(report)}\n` + return { code, output } +} + +if (import.meta.main) { + main(process.argv.slice(2)) + .then(({ code, output }) => { + process.stdout.write(output) + process.exit(code) + }) + .catch((e: unknown) => { + process.stderr.write(`eval: ${e instanceof Error ? e.message : String(e)}\n`) + process.exit(2) + }) +} diff --git a/src/eval/grade.ts b/src/eval/grade.ts new file mode 100644 index 0000000..5dd97ff --- /dev/null +++ b/src/eval/grade.ts @@ -0,0 +1,144 @@ +// Correctness graders for the eval framework. A `Grader` inspects one engine +// result (the parsed output + the full envelope-derived `EvalResult`) and returns +// a `GradeResult`. Three built-ins ship here: +// - `schemaConforms()` — intrinsic: did the forced schema validate? (`result.ok`) +// - `assert(name, predicate)` — a pure predicate over the parsed output. +// - `judge(rubric, {model})` — an LLM-judge that ITSELF routes through the D1 +// engine (so it is cached/stubbable/hermetic), scoring fuzzy quality. +// `gradeCase` aggregates a case's graders into a weighted correctness score in +// [0,1]. Graders are synchronous — the D1 engine (`runClaude`) is synchronous — +// and NEVER throw: a thrown predicate becomes a failed `GradeResult`. + +import { runClaude } from './engine.ts' +import type { Cache, EvalRequest, EvalResult, JsonSchema, Spawn } from './types.ts' + +/** The verdict of one grader over an engine result. `score` is in [0,1]. */ +export interface GradeResult { + /** Stable grader name — appears in the report and JSON. */ + name: string + /** Boolean pass/fail (a threshold view of `score`). */ + pass: boolean + /** Continuous quality in [0,1]; `schemaConforms`/`assert` emit 0 or 1. */ + score: number + /** Human-readable reason — why it passed/failed. */ + detail: string + /** Relative weight in the correctness aggregate (default 1). */ + weight?: number +} + +/** + * Runtime context threaded to every grader by `gradeCase`. Only `judge` reads it + * (to route its rubric call through the same injected spawn/cache as the case), + * so `bun test` stays hermetic. Intrinsic graders ignore it. + */ +export interface GradeContext { + spawn?: Spawn + cache?: Cache +} + +/** A grader: inspect the parsed output + full result, return a verdict. */ +export type Grader = (output: unknown, result: EvalResult, ctx: GradeContext) => GradeResult + +/** Intrinsic grader: passes iff the forced schema validated (`result.ok`). */ +export function schemaConforms(opts: { weight?: number } = {}): Grader { + return (_output, result) => ({ + name: 'schemaConforms', + pass: result.ok, + score: result.ok ? 1 : 0, + detail: result.ok ? 'output conforms to the forced schema' : (result.error ?? 'schema nonconforming'), + weight: opts.weight, + }) +} + +/** + * Assertion grader: run a pure predicate over the parsed output. A `false` return + * fails; a thrown predicate fails with the throw message (never propagates). + */ +export function assert(name: string, predicate: (output: unknown) => boolean, opts: { weight?: number } = {}): Grader { + return output => { + try { + const pass = predicate(output) + return { name, pass, score: pass ? 1 : 0, detail: pass ? 'assertion passed' : 'assertion failed', weight: opts.weight } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + return { name, pass: false, score: 0, detail: `assertion threw: ${msg}`, weight: opts.weight } + } + } +} + +/** The schema the judge forces on itself — a single scored verdict. */ +const JUDGE_SCHEMA: JsonSchema = { + type: 'object', + required: ['score', 'pass', 'reason'], + properties: { + score: { type: 'number' }, + pass: { type: 'boolean' }, + reason: { type: 'string' }, + }, + additionalProperties: false, +} + +/** Parsed shape of a judge verdict (post schema-validation). */ +interface JudgeVerdict { + score?: number + pass?: boolean + reason?: string +} + +export interface JudgeOptions { + /** Model the judge itself runs on (`--model`). */ + model: string + /** Grader name in the report (default `judge`). */ + name?: string + /** Weight in the correctness aggregate. */ + weight?: number +} + +/** + * LLM-judge grader. Composes an `EvalRequest` from `rubric` + the output under + * test and a fixed `{score,pass,reason}` schema, then calls the D1 engine through + * the injected spawn/cache — so the judge is itself cacheable and hermetically + * stubbable. A judge engine failure grades the case as a failed rubric, not a crash. + */ +export function judge(rubric: string, opts: JudgeOptions): Grader { + const name = opts.name ?? 'judge' + return (output, _result, ctx) => { + const req: EvalRequest = { + prompt: `${rubric}\n\nOutput under evaluation:\n${JSON.stringify(output, null, 2)}`, + model: opts.model, + schema: JUDGE_SCHEMA, + } + const verdict = runClaude(req, { spawn: ctx.spawn, cache: ctx.cache }) + if (!verdict.ok || verdict.output === null || typeof verdict.output !== 'object') { + return { name, pass: false, score: 0, detail: verdict.error ?? 'judge produced no verdict', weight: opts.weight } + } + const v = verdict.output as JudgeVerdict + const score = typeof v.score === 'number' ? Math.max(0, Math.min(1, v.score)) : 0 + return { name, pass: v.pass === true, score, detail: v.reason ?? '', weight: opts.weight } + } +} + +/** A case's aggregate correctness plus the individual grader verdicts. */ +export interface CaseGrade { + /** Weighted mean grader score in [0,1]. */ + correctness: number + grades: GradeResult[] +} + +/** + * Run every grader over one engine result and aggregate into a weighted + * correctness score. Correctness = Σ(weight·score) / Σweight (weight defaults to + * 1). An empty grader set yields correctness 0. + */ +export function gradeCase(graders: readonly Grader[], output: unknown, result: EvalResult, ctx: GradeContext = {}): CaseGrade { + const grades = graders.map(g => g(output, result, ctx)) + let weightedSum = 0 + let totalWeight = 0 + for (const grade of grades) { + const weight = grade.weight ?? 1 + weightedSum += weight * grade.score + totalWeight += weight + } + const correctness = totalWeight > 0 ? weightedSum / totalWeight : 0 + return { correctness, grades } +} diff --git a/src/eval/report.ts b/src/eval/report.ts new file mode 100644 index 0000000..8c82e19 --- /dev/null +++ b/src/eval/report.ts @@ -0,0 +1,134 @@ +// The eval REPORT: a human-readable aligned text table + a machine JSON object, +// over any of the three runner modes. The report surfaces all THREE axes +// (correctness / cost / latency) side by side so a human judges the trade — a +// correctness dip that buys a large cost/latency win is a WIN, and the tool +// refuses to collapse that into a single number. +// +// Cost columns: the "Cost $" column is the MEASURED envelope cost. A separate, +// clearly-labelled "Proj $" column is an OPTIONAL projection of a model-swap the +// CLI did not actually run, computed from a small static pricing table — never +// mixed with measured cost. + +import type { CaseResult, ABResult, MatrixResult } from './runner.ts' +import type { EvalUsage } from './types.ts' + +/** Static, approximate USD-per-1M-token prices — projection only, NOT measured. */ +export interface ModelPricing { + inputPerMTok: number + outputPerMTok: number +} + +/** A small static pricing table used ONLY to project a model-swap. */ +export const PRICING: Record = { + 'claude-opus-4-8': { inputPerMTok: 15, outputPerMTok: 75 }, + 'claude-sonnet-5': { inputPerMTok: 3, outputPerMTok: 15 }, + 'claude-haiku-4-5': { inputPerMTok: 1, outputPerMTok: 5 }, +} + +/** + * Project what a run's token usage WOULD cost on `model` from the static table. + * Returns `null` for an unpriced model. This is a projection, never a measurement. + */ +export function projectCost(usage: EvalUsage, model: string): number | null { + const p = PRICING[model] + if (!p) return null + return (usage.inputTokens * p.inputPerMTok + usage.outputTokens * p.outputPerMTok) / 1_000_000 +} + +/** A tagged union over the three runner modes the report can render. */ +export type Report = + | { mode: 'suite'; cases: CaseResult[] } + | { mode: 'ab'; cases: ABResult[] } + | { mode: 'matrix'; cases: MatrixResult[] } + +export interface FormatOptions { + /** When set, add a projected-cost column for a model the run did not execute. */ + projectModel?: string +} + +// --- cell formatters (deterministic — the report text is snapshot-tested) ----- + +const pct = (x: number): string => `${(x * 100).toFixed(1)}%` +const usd = (x: number): string => `$${x.toFixed(5)}` +const ms = (x: number): string => `${Math.round(x)}ms` +const signedPct = (x: number): string => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}%` +const signedUsd = (x: number): string => `${x >= 0 ? '+' : '-'}$${Math.abs(x).toFixed(5)}` +const signedMs = (x: number): string => `${x >= 0 ? '+' : ''}${Math.round(x)}ms` +const signedInt = (x: number): string => `${x >= 0 ? '+' : ''}${x}` + +/** Render aligned columns: header, a dashed rule, then rows (left-padded cells). */ +function renderTable(headers: string[], rows: string[][]): string { + const widths = headers.map((h, i) => Math.max(h.length, ...rows.map(r => (r[i] ?? '').length))) + const fmt = (cells: string[]): string => cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join(' ') + const rule = widths.map(w => '-'.repeat(w)).join(' ') + return [fmt(headers), rule, ...rows.map(fmt)].join('\n') +} + +function caseRow(r: CaseResult, projectModel?: string): string[] { + const cells = [ + r.caseId, + r.model, + r.skipped ? 'skipped' : pct(r.correctness), + r.skipped ? '-' : usd(r.result.cost), + r.skipped ? '-' : ms(r.result.durationMs), + r.skipped ? '-' : String(r.result.numTurns), + ] + if (projectModel) { + const proj = r.skipped ? null : projectCost(r.result.usage, projectModel) + cells.push(proj === null ? '-' : usd(proj)) + } + return cells +} + +function suiteHeaders(projectModel?: string): string[] { + const h = ['Case', 'Model', 'Correct', 'Cost $', 'Latency', 'Turns'] + if (projectModel) h.push(`Proj $ (${projectModel})`) + return h +} + +function formatSuite(cases: CaseResult[], projectModel?: string): string { + if (cases.length === 0) return 'no cases matched' + return renderTable(suiteHeaders(projectModel), cases.map(r => caseRow(r, projectModel))) +} + +function formatMatrix(cases: MatrixResult[], projectModel?: string): string { + if (cases.length === 0) return 'no cases matched' + const rows: string[][] = [] + for (const c of cases) for (const r of c.rows) rows.push(caseRow(r, projectModel)) + return renderTable(suiteHeaders(projectModel), rows) +} + +function formatAB(cases: ABResult[]): string { + if (cases.length === 0) return 'no cases matched' + const headers = ['Case', 'Model', 'ΔCorrect', 'ΔCost', 'ΔLatency', 'ΔTurns', 'Base', 'Cand'] + const rows = cases.map(r => [ + r.caseId, + r.model, + signedPct(r.deltaCorrectness), + signedUsd(r.deltaCostUsd), + signedMs(r.deltaLatencyMs), + signedInt(r.deltaTurns), + pct(r.baseline.correctness), + pct(r.candidate.correctness), + ]) + const legend = + 'Δcost/Δlatency/Δturns: positive = candidate cheaper/faster/tighter. ΔCorrect: negative = regression.' + return `${renderTable(headers, rows)}\n\n${legend}` +} + +/** Render a report to aligned human-readable text. */ +export function formatReport(report: Report, opts: FormatOptions = {}): string { + switch (report.mode) { + case 'suite': + return formatSuite(report.cases, opts.projectModel) + case 'matrix': + return formatMatrix(report.cases, opts.projectModel) + case 'ab': + return formatAB(report.cases) + } +} + +/** The machine-comparable JSON view (already a plain serializable object). */ +export function toJSON(report: Report): Report { + return report +} diff --git a/src/eval/runner.ts b/src/eval/runner.ts new file mode 100644 index 0000000..4dc412d --- /dev/null +++ b/src/eval/runner.ts @@ -0,0 +1,172 @@ +// The eval RUNNER: drives cases through the D1 engine and grades them, in three +// modes. +// - `runSuite(cases, …)` — every case × its models, in BOUNDED parallel. +// - `runAB(case, base, cand, {model})` — same case body, two prompt variants, +// one model → a delta record (Δcorrectness / Δcost / Δlatency / Δturns). +// - `runMatrix(case, models)` — one case across a model grid. +// Every mode takes an injected `spawn` + optional `cache`, so `bun test` runs +// fully offline against the `fakeSpawn` stub. A case whose engine call reports the +// CLI is unavailable is marked `skipped` — a notice, never a failure. +// +// NOTE on "parallel": the D1 spawn boundary is synchronous, so the pool bounds +// how many case promises are in flight rather than truly interleaving blocking +// spawns; the shape is correct and the bound is enforced (`mapPool`). Deltas are +// what the performance verdict rests on — the CLI-boot latency offset cancels. + +import { toRequest, type EvalCase, type Variant } from './case.ts' +import { runClaude } from './engine.ts' +import { gradeCase, type GradeContext, type GradeResult } from './grade.ts' +import type { Cache, EvalResult, Spawn } from './types.ts' + +/** Default model when a case/run names none. */ +export const DEFAULT_MODEL = 'claude-haiku-4-5' +/** Default model grid for `--matrix` when none is given. */ +export const DEFAULT_MODELS: readonly string[] = ['claude-opus-4-8', 'claude-sonnet-5', 'claude-haiku-4-5'] + +/** The graded outcome of running one case at one model (one variant). */ +export interface CaseResult { + caseId: string + model: string + /** Variant label when produced by an A/B leg; absent for plain/matrix runs. */ + variantLabel?: string + /** Weighted correctness in [0,1]. */ + correctness: number + grades: GradeResult[] + /** The full engine result — measured cost/latency/turns/usage live here. */ + result: EvalResult + /** True when the `claude` CLI was unavailable — a skip, not a failure. */ + skipped: boolean +} + +export interface CaseRunOptions { + model: string + /** Substitute the case prompt (used by A/B to swap in a variant). */ + prompt?: string + variantLabel?: string + spawn?: Spawn + cache?: Cache +} + +/** Run one case at one model, then grade it. Never throws. */ +export function runCase(c: EvalCase, opts: CaseRunOptions): CaseResult { + const result = runClaude(toRequest(c, opts.model, opts.prompt), { spawn: opts.spawn, cache: opts.cache }) + const base = { caseId: c.id, model: opts.model, ...(opts.variantLabel ? { variantLabel: opts.variantLabel } : {}) } + if (result.skipped === true) { + return { ...base, correctness: 0, grades: [], result, skipped: true } + } + const ctx: GradeContext = { spawn: opts.spawn, cache: opts.cache } + const { correctness, grades } = gradeCase(c.graders, result.output, result, ctx) + return { ...base, correctness, grades, result, skipped: false } +} + +/** + * Bounded-concurrency map: at most `concurrency` workers run at once, results + * preserve input order. No dependency — a fixed pool draining a shared cursor. + */ +export async function mapPool( + items: readonly T[], + concurrency: number, + worker: (item: T, index: number) => Promise +): Promise { + const results = new Array(items.length) + const limit = Math.max(1, Math.min(concurrency, items.length || 1)) + let cursor = 0 + async function drain(): Promise { + for (;;) { + const i = cursor++ + if (i >= items.length) return + results[i] = await worker(items[i] as T, i) + } + } + await Promise.all(Array.from({ length: limit }, () => drain())) + return results +} + +export interface SuiteOptions { + /** Override the models each case runs on. */ + models?: string[] + /** Max cases in flight (default 4). */ + concurrency?: number + cache?: Cache + spawn?: Spawn +} + +/** Run every case × its resolved models in bounded parallel. */ +export function runSuite(cases: readonly EvalCase[], opts: SuiteOptions = {}): Promise { + const jobs: { c: EvalCase; model: string }[] = [] + for (const c of cases) { + const models = opts.models ?? c.models ?? [DEFAULT_MODEL] + for (const model of models) jobs.push({ c, model }) + } + return mapPool(jobs, opts.concurrency ?? 4, job => + Promise.resolve(runCase(job.c, { model: job.model, spawn: opts.spawn, cache: opts.cache })) + ) +} + +/** A baseline-vs-candidate delta record at one fixed model. */ +export interface ABResult { + caseId: string + model: string + baselineLabel: string + candidateLabel: string + baseline: CaseResult + candidate: CaseResult + /** candidate − baseline: positive = candidate more correct. */ + deltaCorrectness: number + /** baseline − candidate cost: positive = candidate CHEAPER (a saving). */ + deltaCostUsd: number + /** baseline − candidate latency: positive = candidate FASTER. */ + deltaLatencyMs: number + /** baseline − candidate turns: positive = candidate converged in FEWER turns. */ + deltaTurns: number +} + +export interface ABOptions { + model?: string + spawn?: Spawn + cache?: Cache +} + +/** + * Run the same case body with two prompt variants at one model and compute the + * trade-off deltas. Cost/latency/turn deltas are baseline−candidate so a cheaper, + * faster, tighter candidate reads as POSITIVE across the board; correctness is + * candidate−baseline so a regression reads as NEGATIVE (what the CLI gate keys on). + */ +export function runAB(c: EvalCase, baseline: Variant, candidate: Variant, opts: ABOptions = {}): ABResult { + const model = opts.model ?? c.models?.[0] ?? DEFAULT_MODEL + const b = runCase(c, { model, prompt: baseline.prompt, variantLabel: baseline.label, spawn: opts.spawn, cache: opts.cache }) + const cand = runCase(c, { model, prompt: candidate.prompt, variantLabel: candidate.label, spawn: opts.spawn, cache: opts.cache }) + return { + caseId: c.id, + model, + baselineLabel: baseline.label, + candidateLabel: candidate.label, + baseline: b, + candidate: cand, + deltaCorrectness: cand.correctness - b.correctness, + deltaCostUsd: b.result.cost - cand.result.cost, + deltaLatencyMs: b.result.durationMs - cand.result.durationMs, + deltaTurns: b.result.numTurns - cand.result.numTurns, + } +} + +/** A per-model grid for one case. */ +export interface MatrixResult { + caseId: string + rows: CaseResult[] +} + +export interface MatrixOptions { + spawn?: Spawn + cache?: Cache + concurrency?: number +} + +/** Run one case across a model grid; one row per model, input order preserved. */ +export function runMatrix(c: EvalCase, models: readonly string[], opts: MatrixOptions = {}): Promise { + const grid = models.length > 0 ? models : DEFAULT_MODELS + return mapPool(grid, opts.concurrency ?? grid.length, model => + Promise.resolve(runCase(c, { model, spawn: opts.spawn, cache: opts.cache })) + ).then(rows => ({ caseId: c.id, rows })) +} diff --git a/tests/eval/framework.test.ts b/tests/eval/framework.test.ts new file mode 100644 index 0000000..682d2f1 --- /dev/null +++ b/tests/eval/framework.test.ts @@ -0,0 +1,415 @@ +// Framework tests: built-in graders + gradeCase aggregation, the runner's +// runCase/runSuite/runAB/runMatrix over canned envelopes, the bounded-parallel +// pool, report text/JSON, and the CLI exit-code semantics (regression vs clean +// vs CLI-absent). Every model call is faked at the `Spawn` boundary — offline, no +// real `claude`. + +import assert from 'node:assert/strict' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'bun:test' +import { defineCase, toRequest, type EvalCase } from '../../src/eval/case.ts' +import { assert as assertGrader, gradeCase, judge, schemaConforms, type Grader } from '../../src/eval/grade.ts' +import { + mapPool, + runAB, + runCase, + runMatrix, + runSuite, + type ABResult, + type CaseResult, +} from '../../src/eval/runner.ts' +import { formatReport, projectCost, toJSON, type Report } from '../../src/eval/report.ts' +import { abExitCode, main, loadSuite, parseArgs } from '../../src/eval/cli.ts' +import type { EvalResult, JsonSchema, Spawn } from '../../src/eval/types.ts' +import { successEnvelope, throwingSpawn } from '../helpers/fake-claude.ts' + +const SCHEMA: JsonSchema = { + type: 'object', + required: ['answer'], + properties: { answer: { type: 'number' } }, + additionalProperties: false, +} + +/** A minimal ok result carrying the given output — for grading graders directly. */ +function okResult(output: unknown): EvalResult { + return { + ok: true, + output, + error: null, + cost: 0.01, + usage: { inputTokens: 10, outputTokens: 20, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 }, + durationMs: 1000, + apiDurationMs: 950, + numTurns: 2, + model: 'claude-haiku-4-5', + cached: false, + } +} + +const baseCase = (overrides: Partial = {}): EvalCase => + defineCase({ + id: 'add', + tags: ['math'], + prompt: 'What is 6 * 7?', + schema: SCHEMA, + graders: [schemaConforms(), assertGrader('is-42', o => (o as { answer?: number }).answer === 42)], + ...overrides, + }) + +// --- graders (acceptance criterion 2) ---------------------------------------- + +test('schemaConforms passes on an ok result and fails on a graded failure', () => { + const pass = schemaConforms()(null, okResult({ answer: 42 }), {}) + assert.equal(pass.name, 'schemaConforms') + assert.equal(pass.pass, true) + assert.equal(pass.score, 1) + + const failResult = { ...okResult(null), ok: false, error: 'missing key' } + const fail = schemaConforms()(null, failResult, {}) + assert.equal(fail.pass, false) + assert.equal(fail.score, 0) + assert.match(fail.detail, /missing key/) +}) + +test('assert grades a predicate, and a thrown predicate fails without propagating', () => { + const pass = assertGrader('is-42', o => (o as { answer: number }).answer === 42)({ answer: 42 }, okResult({ answer: 42 }), {}) + assert.equal(pass.pass, true) + assert.equal(pass.score, 1) + + const fail = assertGrader('is-42', o => (o as { answer: number }).answer === 42)({ answer: 1 }, okResult({ answer: 1 }), {}) + assert.equal(fail.pass, false) + + const threw = assertGrader('boom', () => { + throw new Error('kaboom') + })(null, okResult(null), {}) + assert.equal(threw.pass, false) + assert.match(threw.detail, /kaboom/) +}) + +test('judge routes through the engine (stubbed) and maps the verdict to a GradeResult', () => { + const verdictEnvelope = successEnvelope({ score: 0.8, pass: true, reason: 'clear and complete' }) + const spawn: Spawn = () => ({ status: 0, stdout: JSON.stringify(verdictEnvelope), stderr: '' }) + const grade = judge('Rate the answer 0..1.', { model: 'claude-haiku-4-5' })( + { answer: 42 }, + okResult({ answer: 42 }), + { spawn } + ) + assert.equal(grade.name, 'judge') + assert.equal(grade.pass, true) + assert.equal(grade.score, 0.8) + assert.match(grade.detail, /clear and complete/) +}) + +test('judge clamps an out-of-range score and fails gracefully when the engine errors', () => { + const highSpawn: Spawn = () => ({ + status: 0, + stdout: JSON.stringify(successEnvelope({ score: 5, pass: true, reason: 'over' })), + stderr: '', + }) + const clamped = judge('r', { model: 'm' })({ answer: 1 }, okResult({ answer: 1 }), { spawn: highSpawn }) + assert.equal(clamped.score, 1) + + const failed = judge('r', { model: 'm' })({ answer: 1 }, okResult({ answer: 1 }), { spawn: throwingSpawn() }) + assert.equal(failed.pass, false) + assert.equal(failed.score, 0) +}) + +// --- gradeCase aggregation (acceptance criterion 2) -------------------------- + +test('gradeCase computes a weighted correctness across graders', () => { + // One passing (weight 3) + one failing (weight 1) → 3/4 = 0.75. + const graders: Grader[] = [ + assertGrader('a', () => true, { weight: 3 }), + assertGrader('b', () => false, { weight: 1 }), + ] + const { correctness, grades } = gradeCase(graders, { answer: 42 }, okResult({ answer: 42 })) + assert.equal(grades.length, 2) + assert.equal(correctness, 0.75) +}) + +test('gradeCase with no graders is 0, not NaN', () => { + const { correctness } = gradeCase([], null, okResult(null)) + assert.equal(correctness, 0) +}) + +// --- toRequest (case lowering) ----------------------------------------------- + +test('toRequest lowers a case to an EvalRequest and honours a prompt override', () => { + const c = baseCase({ systemPrompt: 'terse', tools: ['Read'] }) + const req = toRequest(c, 'claude-opus-4-8', 'OVERRIDDEN') + assert.equal(req.prompt, 'OVERRIDDEN') + assert.equal(req.model, 'claude-opus-4-8') + assert.equal(req.systemPrompt, 'terse') + assert.deepEqual(req.tools, ['Read']) + assert.equal(toRequest(c, 'm').prompt, 'What is 6 * 7?') +}) + +// --- runCase end-to-end (acceptance criterion 1) ----------------------------- + +test('runCase runs a case through the engine and surfaces correctness + envelope metrics', () => { + const spawn: Spawn = () => ({ + status: 0, + stdout: JSON.stringify(successEnvelope({ answer: 42 }, { cost: 0.0463, durationMs: 2419, numTurns: 2 })), + stderr: '', + }) + const r = runCase(baseCase(), { model: 'claude-haiku-4-5', spawn }) + assert.equal(r.caseId, 'add') + assert.equal(r.correctness, 1) // both graders pass + assert.equal(r.skipped, false) + assert.equal(r.result.cost, 0.0463) + assert.equal(r.result.durationMs, 2419) + assert.equal(r.result.numTurns, 2) +}) + +test('runCase marks a case skipped (no failure) when the CLI is unavailable', () => { + const r = runCase(baseCase(), { model: 'm', spawn: throwingSpawn() }) + assert.equal(r.skipped, true) + assert.equal(r.grades.length, 0) +}) + +// --- bounded parallel pool (acceptance criterion 4) -------------------------- + +test('mapPool never exceeds the concurrency bound and preserves order', async () => { + let inFlight = 0 + let maxInFlight = 0 + const worker = (n: number): Promise => + new Promise(resolve => { + inFlight++ + maxInFlight = Math.max(maxInFlight, inFlight) + setTimeout(() => { + inFlight-- + resolve(n * 2) + }, 5) + }) + const out = await mapPool([1, 2, 3, 4, 5, 6, 7], 2, worker) + assert.deepEqual(out, [2, 4, 6, 8, 10, 12, 14]) + assert.ok(maxInFlight <= 2, `maxInFlight=${maxInFlight}`) +}) + +test('runSuite fans a case set across models and returns one result per (case,model)', async () => { + const spawn: Spawn = () => ({ status: 0, stdout: JSON.stringify(successEnvelope({ answer: 42 })), stderr: '' }) + const results = await runSuite([baseCase({ id: 'a' }), baseCase({ id: 'b' })], { + models: ['claude-haiku-4-5', 'claude-sonnet-5'], + concurrency: 2, + spawn, + }) + assert.equal(results.length, 4) + assert.deepEqual( + results.map(r => `${r.caseId}:${r.model}`), + ['a:claude-haiku-4-5', 'a:claude-sonnet-5', 'b:claude-haiku-4-5', 'b:claude-sonnet-5'] + ) +}) + +// --- runAB deltas (acceptance criterion 3) ----------------------------------- + +/** A spawn that returns a cheaper/faster/correct envelope for the candidate prompt. */ +const abSpawn: Spawn = (_cmd, _args, input) => { + const isCandidate = (input ?? '').includes('CANDIDATE') + const env = isCandidate + ? successEnvelope({ answer: 42 }, { cost: 0.01, durationMs: 800, numTurns: 1 }) + : successEnvelope({ answer: 42 }, { cost: 0.05, durationMs: 2000, numTurns: 3 }) + return { status: 0, stdout: JSON.stringify(env), stderr: '' } +} + +test('runAB reports Δcorrectness/Δcost/Δlatency/Δturns (candidate cheaper/faster → positive)', () => { + const ab = runAB( + baseCase(), + { label: 'baseline', prompt: 'BASELINE: what is 6*7?' }, + { label: 'candidate', prompt: 'CANDIDATE: what is 6*7?' }, + { model: 'claude-haiku-4-5', spawn: abSpawn } + ) + assert.equal(ab.deltaCorrectness, 0) // both correct + assert.ok(ab.deltaCostUsd > 0, 'candidate cheaper → positive Δcost') + assert.ok(ab.deltaLatencyMs > 0, 'candidate faster → positive Δlatency') + assert.ok(ab.deltaTurns > 0, 'candidate fewer turns → positive Δturns') + assert.equal(ab.baselineLabel, 'baseline') + assert.equal(ab.candidateLabel, 'candidate') +}) + +test('runAB surfaces a negative Δcorrectness when the candidate regresses', () => { + // Candidate returns a wrong answer → its assertion grader fails. + const regressSpawn: Spawn = (_c, _a, input) => { + const answer = (input ?? '').includes('CANDIDATE') ? 0 : 42 + return { status: 0, stdout: JSON.stringify(successEnvelope({ answer })), stderr: '' } + } + const ab = runAB( + baseCase(), + { label: 'base', prompt: 'BASELINE q' }, + { label: 'cand', prompt: 'CANDIDATE q' }, + { model: 'm', spawn: regressSpawn } + ) + assert.ok(ab.deltaCorrectness < 0, 'candidate wrong → negative Δcorrectness') +}) + +// --- runMatrix grid (acceptance criterion 3) --------------------------------- + +test('runMatrix produces one row per model', async () => { + const spawn: Spawn = () => ({ status: 0, stdout: JSON.stringify(successEnvelope({ answer: 42 })), stderr: '' }) + const grid = await runMatrix(baseCase(), ['claude-opus-4-8', 'claude-sonnet-5', 'claude-haiku-4-5'], { spawn }) + assert.equal(grid.rows.length, 3) + assert.deepEqual(grid.rows.map(r => r.model), ['claude-opus-4-8', 'claude-sonnet-5', 'claude-haiku-4-5']) + assert.ok(grid.rows.every(r => r.correctness === 1)) +}) + +// --- report (acceptance criterion 5) ----------------------------------------- + +const sampleCaseResult = (id: string): CaseResult => ({ + caseId: id, + model: 'claude-haiku-4-5', + correctness: 1, + grades: [], + result: okResult({ answer: 42 }), + skipped: false, +}) + +test('formatReport renders a stable suite text table with all three axes', () => { + const report: Report = { mode: 'suite', cases: [sampleCaseResult('add')] } + const text = formatReport(report) + assert.match(text, /Case/) + assert.match(text, /Correct/) + assert.match(text, /Cost \$/) + assert.match(text, /Latency/) + assert.match(text, /100\.0%/) + assert.match(text, /\$0\.01000/) + assert.match(text, /1000ms/) +}) + +test('formatReport suite adds a clearly-labelled projected-cost column on request', () => { + const report: Report = { mode: 'suite', cases: [sampleCaseResult('add')] } + const text = formatReport(report, { projectModel: 'claude-opus-4-8' }) + assert.match(text, /Proj \$ \(claude-opus-4-8\)/) +}) + +test('projectCost uses the static table and returns null for an unpriced model', () => { + const usage = { inputTokens: 1_000_000, outputTokens: 1_000_000, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 } + assert.equal(projectCost(usage, 'claude-haiku-4-5'), 6) // 1 + 5 per Mtok + assert.equal(projectCost(usage, 'no-such-model'), null) +}) + +test('formatReport ab shows the delta columns and a trade-off legend', () => { + const ab: ABResult = runAB( + baseCase(), + { label: 'base', prompt: 'BASELINE q' }, + { label: 'cand', prompt: 'CANDIDATE q' }, + { model: 'm', spawn: abSpawn } + ) + const text = formatReport({ mode: 'ab', cases: [ab] }) + assert.match(text, /ΔCorrect/) + assert.match(text, /ΔCost/) + assert.match(text, /ΔLatency/) + assert.match(text, /candidate cheaper\/faster/) +}) + +test('toJSON returns the machine-comparable report object', () => { + const report: Report = { mode: 'suite', cases: [sampleCaseResult('add')] } + const json = toJSON(report) + assert.equal(json.mode, 'suite') + // Round-trips through JSON.stringify (what --json emits). + const parsed = JSON.parse(JSON.stringify(json)) as { mode: string; cases: unknown[] } + assert.equal(parsed.mode, 'suite') + assert.equal(parsed.cases.length, 1) +}) + +// --- CLI exit-code semantics (acceptance criterion 5) ------------------------ + +test('parseArgs reads every flag', () => { + const f = parseArgs(['--suite', 'd', '--filter', 'planner', '--ab', '--models', 'a,b', '--json', '--tolerance', '5']) + assert.equal(f.suite, 'd') + assert.equal(f.filter, 'planner') + assert.equal(f.ab, true) + assert.deepEqual(f.models, ['a', 'b']) + assert.equal(f.json, true) + assert.equal(f.tolerance, 5) +}) + +test('abExitCode gates only on a regression past tolerance', () => { + const mk = (delta: number): ABResult => ({ + caseId: 'c', + model: 'm', + baselineLabel: 'b', + candidateLabel: 'c', + baseline: sampleCaseResult('c'), + candidate: sampleCaseResult('c'), + deltaCorrectness: delta, + deltaCostUsd: 0, + deltaLatencyMs: 0, + deltaTurns: 0, + }) + // -3pp within a 5% tolerance → 0; -6pp past it → 1; an improvement → 0. + assert.equal(abExitCode([mk(-0.03)], 5), 0) + assert.equal(abExitCode([mk(-0.06)], 5), 1) + assert.equal(abExitCode([mk(0.1)], 5), 0) +}) + +test('main skips-with-notice and exits 0 when claude is unavailable', async () => { + const res = await main(['--suite', '/nonexistent', '--ab'], { spawn: throwingSpawn() }) + assert.equal(res.code, 0) + assert.match(res.output, /unavailable/) +}) + +/** A spawn where `--version` succeeds (available) and runs return a success envelope. */ +const availableSpawn = (envelope: unknown): Spawn => (_cmd, args) => { + if (args.includes('--version')) return { status: 0, stdout: 'claude 2.1.207', stderr: '' } + return { status: 0, stdout: JSON.stringify(envelope), stderr: '' } +} + +test('main runs a plain suite (report-only, exit 0) over injected cases', async () => { + const res = await main([], { spawn: availableSpawn(successEnvelope({ answer: 42 })), cases: [baseCase()] }) + assert.equal(res.code, 0) + assert.match(res.output, /Correct/) + assert.match(res.output, /add/) +}) + +test('main --json emits parseable machine JSON', async () => { + const res = await main(['--json'], { spawn: availableSpawn(successEnvelope({ answer: 42 })), cases: [baseCase()] }) + const parsed = JSON.parse(res.output) as { mode: string } + assert.equal(parsed.mode, 'suite') +}) + +test('main --ab exits 1 on a regression past tolerance and 0 when clean', async () => { + const abCase = (): EvalCase => + baseCase({ + variants: { baseline: { label: 'base', prompt: 'BASELINE q' }, candidate: { label: 'cand', prompt: 'CANDIDATE q' } }, + }) + // Candidate wrong → regression → exit 1. + const regressSpawn: Spawn = (_c, args, input) => { + if (args.includes('--version')) return { status: 0, stdout: 'claude 2.1.207', stderr: '' } + const answer = (input ?? '').includes('CANDIDATE') ? 0 : 42 + return { status: 0, stdout: JSON.stringify(successEnvelope({ answer })), stderr: '' } + } + const regressed = await main(['--ab', '--tolerance', '5'], { spawn: regressSpawn, cases: [abCase()] }) + assert.equal(regressed.code, 1) + + // Both correct → no regression → exit 0. + const cleanSpawn: Spawn = (_c, args) => { + if (args.includes('--version')) return { status: 0, stdout: 'claude 2.1.207', stderr: '' } + return { status: 0, stdout: JSON.stringify(successEnvelope({ answer: 42 })), stderr: '' } + } + const clean = await main(['--ab', '--tolerance', '5'], { spawn: cleanSpawn, cases: [abCase()] }) + assert.equal(clean.code, 0) +}) + +test('main --filter selects by id or tag', async () => { + const res = await main(['--json', '--filter', 'nope'], { + spawn: availableSpawn(successEnvelope({ answer: 42 })), + cases: [baseCase()], + }) + assert.match(res.output, /no cases matched/) +}) + +// --- suite loading (integration: real dynamic import from a temp dir) -------- + +test('loadSuite dynamically imports case modules from a directory', async () => { + const dir = mkdtempSync(join(tmpdir(), 'eval-suite-')) + const casePath = join(import.meta.dir, '..', '..', 'src', 'eval', 'case.ts').replace(/\\/g, '/') + writeFileSync( + join(dir, 'sample.ts'), + `import { defineCase } from '${casePath}'\n` + + `export const cases = [defineCase({ id: 'loaded', tags: ['t'], prompt: 'p', schema: { type: 'object' }, graders: [] })]\n` + ) + const cases = await loadSuite(dir) + assert.equal(cases.length, 1) + assert.equal(cases[0]?.id, 'loaded') +})