From 0f802e55ed37cd9a2b827a9546a2ea7ffe9d0a0b Mon Sep 17 00:00:00 2001 From: Christian Schuetz Date: Wed, 15 Jul 2026 00:31:10 -0500 Subject: [PATCH 1/3] feat(evaluation-system): claude -p eval engine, envelope parsing, response cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundational layer of the prompt-eval suite (D1). Adds src/eval/ with a single-shot Claude Code CLI (`claude -p`) substrate — never the Anthropic SDK: - types.ts: EvalRequest/EvalResult/Envelope + injectable Spawn and Cache seams. - engine.ts: buildArgs (pure print-mode argv: --print --output-format json --json-schema --model --strict-mcp-config --settings + conditional system-prompt/tool-policy flags; prompt on stdin), parseEnvelope (graded, never throws — prefers structured_output, falls back to result, shallow schema check), runClaude (cache-first, spawn, cache-on-success), isAvailable and a skip signal distinct from graded failure for an absent CLI. - cache.ts: content-addressed ResponseCache keyed on a canonical SHA-256 over every output-affecting field. - tests/helpers/fake-claude.ts + tests/eval/*: hermetic engine/cache coverage via the injected Spawn stub, no real claude. Not a plugin deployable — no build.ts entry, no version bump. --- .gitignore | 1 + src/eval/cache.ts | 70 +++++++++++ src/eval/engine.ts | 228 +++++++++++++++++++++++++++++++++++ src/eval/types.ts | 113 +++++++++++++++++ tests/eval/cache.test.ts | 130 ++++++++++++++++++++ tests/eval/engine.test.ts | 181 +++++++++++++++++++++++++++ tests/helpers/fake-claude.ts | 82 +++++++++++++ 7 files changed, 805 insertions(+) create mode 100644 src/eval/cache.ts create mode 100644 src/eval/engine.ts create mode 100644 src/eval/types.ts create mode 100644 tests/eval/cache.test.ts create mode 100644 tests/eval/engine.test.ts create mode 100644 tests/helpers/fake-claude.ts diff --git a/.gitignore b/.gitignore index c2658d7..8c8f03b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules/ +.eval-cache/ diff --git a/src/eval/cache.ts b/src/eval/cache.ts new file mode 100644 index 0000000..811fd55 --- /dev/null +++ b/src/eval/cache.ts @@ -0,0 +1,70 @@ +// Content-addressed response cache for eval results. The key is a SHA-256 over a +// CANONICAL (sorted-key) JSON of every request field that changes the model's +// output — prompt, both system-prompt knobs, model, schema, tool policy, and +// settings. Include every such field or an A/B would collide; a changed prompt +// naturally misses. Results are stored one JSON file per key; no eviction. + +import { createHash } from 'node:crypto' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import type { Cache, EvalRequest, EvalResult } from './types.ts' + +/** Recursively sort object keys so key order never perturbs the hash. */ +function sortKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeys) + if (value !== null && typeof value === 'object') { + const sorted: Record = {} + for (const key of Object.keys(value as Record).sort()) { + sorted[key] = sortKeys((value as Record)[key]) + } + return sorted + } + return value +} + +/** Stable, key-sorted JSON of the output-affecting fields of a request. */ +function canonicalRequest(req: EvalRequest): string { + return JSON.stringify( + sortKeys({ + prompt: req.prompt, + systemPrompt: req.systemPrompt ?? null, + appendSystemPrompt: req.appendSystemPrompt ?? null, + model: req.model, + schema: req.schema, + tools: req.tools ?? null, + settings: req.settings ?? null, + }) + ) +} + +/** Hex SHA-256 cache key over a request's output-affecting fields. */ +export function cacheKey(req: EvalRequest): string { + return createHash('sha256').update(canonicalRequest(req)).digest('hex') +} + +/** File-backed response cache: one `/.json` per stored result. */ +export class ResponseCache implements Cache { + constructor(private readonly dir: string) {} + + private path(key: string): string { + return join(this.dir, `${key}.json`) + } + + /** Return the stored result (marked `cached:true`) or `null` on miss/corruption. */ + get(key: string): EvalResult | null { + const p = this.path(key) + if (!existsSync(p)) return null + try { + const result = JSON.parse(readFileSync(p, 'utf8')) as EvalResult + return { ...result, cached: true } + } catch { + return null + } + } + + /** Persist a result under `key`, creating the cache dir on first write. */ + set(key: string, result: EvalResult): void { + mkdirSync(this.dir, { recursive: true }) + writeFileSync(this.path(key), JSON.stringify(result, null, 2)) + } +} diff --git a/src/eval/engine.ts b/src/eval/engine.ts new file mode 100644 index 0000000..8404bb8 --- /dev/null +++ b/src/eval/engine.ts @@ -0,0 +1,228 @@ +// The `claude -p` eval substrate. `runClaude` builds the print-mode argv, spawns +// the CLI through an injectable `Spawn`, parses the JSON envelope, validates the +// forced `structured_output` against the request schema, and returns a typed, +// graded result. It shells out to `claude` ONLY — never `@anthropic-ai/sdk`. +// +// Design notes: +// - The prompt travels on stdin (fed via `Spawn`'s `input`), so argv stays free +// of arbitrarily large prompt text; `claude -p` reads stdin when given no +// positional prompt. +// - Cost/latency/usage are read straight off the envelope — no pricing table. +// - Every failure mode (CLI error, unparseable envelope, schema miss) is a +// graded `ok:false` result, never a thrown exception. An ABSENT CLI is a +// distinct `skipped:true` signal so callers can skip-with-notice. + +import { spawnSync } from 'node:child_process' +import type { + Cache, + Envelope, + EvalRequest, + EvalResult, + EvalUsage, + JsonSchema, + Spawn, +} from './types.ts' +import { cacheKey } from './cache.ts' + +/** Tools disabled by default so a case is single-shot, deterministic, cacheable. */ +const DEFAULT_DISALLOWED_TOOLS = [ + 'Bash', + 'Read', + 'Write', + 'Edit', + 'WebFetch', + 'WebSearch', + 'Task', + 'Glob', + 'Grep', +] + +/** Real spawn: `spawnSync('claude', …)` with the prompt on stdin. */ +export const defaultSpawn: Spawn = (cmd, args, input) => { + const res = spawnSync(cmd, args, { encoding: 'utf8', input, maxBuffer: 64 * 1024 * 1024 }) + return { status: res.status, stdout: res.stdout ?? '', stderr: res.stderr ?? '' } +} + +/** + * Build the pure `claude -p` argv for a request (prompt excluded — it goes on + * stdin). Always forces print mode, JSON output, the request schema, model, + * strict MCP isolation, and inline settings; layers system-prompt and tool-policy + * flags conditionally. + */ +export function buildArgs(req: EvalRequest): string[] { + const args = [ + '--print', + '--output-format', + 'json', + '--json-schema', + JSON.stringify(req.schema), + '--model', + req.model, + '--strict-mcp-config', + '--settings', + req.settings ?? '{}', + ] + if (req.systemPrompt !== undefined) args.push('--system-prompt', req.systemPrompt) + if (req.appendSystemPrompt !== undefined) args.push('--append-system-prompt', req.appendSystemPrompt) + if (req.tools && req.tools.length > 0) { + args.push('--allowedTools', req.tools.join(' ')) + } else { + args.push('--disallowedTools', DEFAULT_DISALLOWED_TOOLS.join(' ')) + } + return args +} + +/** Lift the envelope's snake_case usage block into the normalized shape. */ +function normalizeUsage(envelope: Envelope): EvalUsage { + const u = envelope.usage ?? {} + return { + inputTokens: u.input_tokens ?? 0, + outputTokens: u.output_tokens ?? 0, + cacheReadInputTokens: u.cache_read_input_tokens ?? 0, + cacheCreationInputTokens: u.cache_creation_input_tokens ?? 0, + } +} + +/** + * Shallow structural check: value is an object, every `required` key is present, + * and — when `additionalProperties:false` — no keys beyond `properties` appear. + * Returns an error message or `null`. Intentionally NOT a full JSON-Schema + * validator (no dependency added). + */ +function validateSchema(value: unknown, schema: JsonSchema): string | null { + const wantsObject = schema.type === 'object' || schema.properties !== undefined || schema.required !== undefined + if (!wantsObject) return null + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return 'output is not an object' + } + const obj = value as Record + for (const key of schema.required ?? []) { + if (!(key in obj)) return `output missing required key: ${key}` + } + if (schema.additionalProperties === false && schema.properties) { + const allowed = new Set(Object.keys(schema.properties)) + for (const key of Object.keys(obj)) { + if (!allowed.has(key)) return `output has unexpected key: ${key}` + } + } + return null +} + +/** Assemble a graded result from envelope metrics + an ok/error verdict. */ +function gradedResult( + envelope: Envelope, + req: EvalRequest, + verdict: { ok: true; output: unknown } | { ok: false; error: string } +): EvalResult { + const base = { + cost: envelope.total_cost_usd ?? 0, + usage: normalizeUsage(envelope), + durationMs: envelope.duration_ms ?? 0, + apiDurationMs: envelope.duration_api_ms ?? 0, + numTurns: envelope.num_turns ?? 0, + model: req.model, + cached: false, + } + return verdict.ok + ? { ok: true, output: verdict.output, error: null, ...base } + : { ok: false, output: null, error: verdict.error, ...base } +} + +/** A graded failure carrying zeroed metrics (used when the envelope is unusable). */ +function failureNoMetrics(req: EvalRequest, error: string): EvalResult { + return { + ok: false, + output: null, + error, + cost: 0, + usage: { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 }, + durationMs: 0, + apiDurationMs: 0, + numTurns: 0, + model: req.model, + cached: false, + } +} + +/** + * Parse a `claude --output-format json` envelope into a graded `EvalResult`. + * Never throws: an unparseable envelope, a CLI/model error, a missing/bad + * output, or a schema miss all yield `ok:false` with a descriptive `error`. + */ +export function parseEnvelope(stdout: string, req: EvalRequest): EvalResult { + let envelope: Envelope + try { + envelope = JSON.parse(stdout) as Envelope + } catch { + return failureNoMetrics(req, 'unparseable envelope') + } + + if (envelope.is_error === true || (envelope.subtype !== undefined && envelope.subtype !== 'success')) { + return gradedResult(envelope, req, { + ok: false, + error: `claude error: is_error=${String(envelope.is_error)} subtype=${String(envelope.subtype)}`, + }) + } + + // Prefer the parsed, schema-forced object; fall back to parsing stringified `result`. + let output = envelope.structured_output + if (output === undefined || output === null) { + if (typeof envelope.result !== 'string') { + return gradedResult(envelope, req, { ok: false, error: 'no structured_output and no string result to parse' }) + } + try { + output = JSON.parse(envelope.result) + } catch { + return gradedResult(envelope, req, { ok: false, error: 'unparseable result JSON' }) + } + } + + const schemaError = validateSchema(output, req.schema) + if (schemaError) return gradedResult(envelope, req, { ok: false, error: schemaError }) + + return gradedResult(envelope, req, { ok: true, output }) +} + +export interface RunOptions { + /** Injected subprocess boundary; defaults to the real `claude` spawn. */ + spawn?: Spawn + /** Optional response cache consulted before spawning and written on success. */ + cache?: Cache +} + +/** + * Run one eval request: cache lookup first, else spawn `claude -p`, parse, cache + * (on success), return. An absent CLI (non-zero exit with empty stdout, or a + * spawn throw) surfaces as a `skipped` result — distinct from a graded failure. + */ +export function runClaude(req: EvalRequest, { spawn = defaultSpawn, cache }: RunOptions = {}): EvalResult { + const key = cache ? cacheKey(req) : null + if (cache && key) { + const hit = cache.get(key) + if (hit) return { ...hit, cached: true } + } + + let res + try { + res = spawn('claude', buildArgs(req), req.prompt) + } catch { + return { ...failureNoMetrics(req, 'claude CLI unavailable'), skipped: true } + } + + if (res.status !== 0 && res.stdout.trim() === '') { + return { ...failureNoMetrics(req, 'claude CLI unavailable'), skipped: true } + } + + const result = parseEnvelope(res.stdout, req) + if (cache && key && result.ok) cache.set(key, result) + return result +} + +/** Probe whether the `claude` CLI is invokable via `claude --version`. */ +export function isAvailable(spawn: Spawn = defaultSpawn): boolean { + try { + return spawn('claude', ['--version']).status === 0 + } catch { + return false + } +} diff --git a/src/eval/types.ts b/src/eval/types.ts new file mode 100644 index 0000000..1cb125a --- /dev/null +++ b/src/eval/types.ts @@ -0,0 +1,113 @@ +// Shared types for the prompt-eval engine. The engine shells out to the Claude +// Code CLI in print mode (`claude -p`) — NEVER the Anthropic SDK — and reads +// cost/latency/usage straight off the `--output-format json` envelope. The one +// faked boundary in tests is `Spawn`; everything else is real. + +/** + * A minimal JSON-Schema shape. The engine only performs a SHALLOW structural + * check (`required` keys present, no extras when `additionalProperties:false`), + * so this deliberately does not model the full spec — do NOT add a schema + * validator dependency. The harness's generated schemas satisfy this shape. + */ +export interface JsonSchema { + type?: string + required?: string[] + properties?: Record + additionalProperties?: boolean + items?: JsonSchema + [key: string]: unknown +} + +/** A single-shot eval request: the prompt, the schema it is forced against, and knobs. */ +export interface EvalRequest { + /** User prompt — passed to `claude -p` on stdin. */ + prompt: string + /** Replace the CLI's default system prompt (`--system-prompt`) for isolation. */ + systemPrompt?: string + /** Layer extra context on top of the default system prompt (`--append-system-prompt`). */ + appendSystemPrompt?: string + /** Model id, e.g. `claude-haiku-4-5` (`--model`). */ + model: string + /** Schema forced via `--json-schema` and validated against `structured_output`. */ + schema: JsonSchema + /** Opt into a bounded toolset (`--allowedTools`); omit for a single-shot, tool-free run. */ + tools?: string[] + /** Inline settings JSON (`--settings`); defaults to `'{}'` to isolate from ambient config. */ + settings?: string +} + +/** Normalized token usage lifted from the envelope's snake_case `usage` block. */ +export interface EvalUsage { + inputTokens: number + outputTokens: number + cacheReadInputTokens: number + cacheCreationInputTokens: number +} + +/** + * The graded outcome of one eval invocation. A model/CLI error, an unparseable + * envelope, or a schema-nonconforming output all yield `ok:false` with a + * descriptive `error` — the engine NEVER throws for these. `skipped:true` is a + * distinct signal that the CLI itself was unavailable (not a graded failure). + */ +export interface EvalResult { + ok: boolean + output: unknown | null + error: string | null + cost: number + usage: EvalUsage + durationMs: number + apiDurationMs: number + numTurns: number + model: string + cached: boolean + /** True only when the `claude` CLI was unavailable — callers skip-with-notice. */ + skipped?: boolean +} + +/** Envelope usage block as emitted by `claude --output-format json` (snake_case). */ +export interface EnvelopeUsage { + input_tokens?: number + output_tokens?: number + cache_read_input_tokens?: number + cache_creation_input_tokens?: number + service_tier?: string +} + +/** + * Raw `--output-format json` envelope (verified against installed `claude` + * 2.1.207). Only the fields the engine reads are typed; the rest are ignored. + */ +export interface Envelope { + type?: string + subtype?: string + is_error?: boolean + result?: string + structured_output?: unknown + total_cost_usd?: number + usage?: EnvelopeUsage + duration_ms?: number + duration_api_ms?: number + num_turns?: number + modelUsage?: Record +} + +/** Result of a subprocess spawn — the single injectable boundary tests fake. */ +export interface SpawnResult { + status: number | null + stdout: string + stderr: string +} + +/** + * The injected spawn dependency. Default is a thin `spawnSync('claude', …)` + * wrapper; tests pass a stub returning a canned envelope. `input` is fed to the + * child's stdin (the prompt travels there, keeping argv free of large text). + */ +export type Spawn = (cmd: string, args: string[], input?: string) => SpawnResult + +/** A response cache the engine can consult before spawning. */ +export interface Cache { + get(key: string): EvalResult | null + set(key: string, result: EvalResult): void +} diff --git a/tests/eval/cache.test.ts b/tests/eval/cache.test.ts new file mode 100644 index 0000000..8bab195 --- /dev/null +++ b/tests/eval/cache.test.ts @@ -0,0 +1,130 @@ +// Cache tests: key stability, per-field isolation, file round-trip in a temp +// dir, and the runClaude hit/miss integration (a hit must NOT spawn). + +import assert from 'node:assert/strict' +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'bun:test' +import { cacheKey, ResponseCache } from '../../src/eval/cache.ts' +import { runClaude } from '../../src/eval/engine.ts' +import type { EvalRequest, JsonSchema, Spawn } from '../../src/eval/types.ts' +import { fakeSpawn, successEnvelope, throwingSpawn } from '../helpers/fake-claude.ts' + +const SCHEMA: JsonSchema = { + type: 'object', + required: ['answer'], + properties: { answer: { type: 'number' } }, + additionalProperties: false, +} + +const baseReq = (overrides: Partial = {}): EvalRequest => ({ + prompt: 'What is 6 * 7?', + model: 'claude-haiku-4-5', + schema: SCHEMA, + ...overrides, +}) + +const tempDir = (): string => mkdtempSync(join(tmpdir(), 'eval-cache-')) + +// --- key stability + isolation (acceptance criterion 4) ---------------------- + +test('cacheKey is stable for identical requests', () => { + assert.equal(cacheKey(baseReq()), cacheKey(baseReq())) +}) + +test('cacheKey is insensitive to schema key ordering (canonical JSON)', () => { + const a = baseReq({ schema: { type: 'object', required: ['answer'], additionalProperties: false } }) + const b = baseReq({ schema: { additionalProperties: false, required: ['answer'], type: 'object' } }) + assert.equal(cacheKey(a), cacheKey(b)) +}) + +test('cacheKey changes when any output-affecting field changes', () => { + const base = cacheKey(baseReq()) + const variants: EvalRequest[] = [ + baseReq({ prompt: 'different' }), + baseReq({ model: 'claude-opus-4-8' }), + baseReq({ schema: { type: 'object', required: ['result'] } }), + baseReq({ systemPrompt: 'You are terse.' }), + baseReq({ appendSystemPrompt: 'Context.' }), + baseReq({ tools: ['Read'] }), + baseReq({ settings: '{"x":1}' }), + ] + for (const v of variants) { + assert.notEqual(cacheKey(v), base) + } + // All variants are distinct from each other too. + const keys = new Set(variants.map(cacheKey)) + assert.equal(keys.size, variants.length) +}) + +// --- file round-trip --------------------------------------------------------- + +test('ResponseCache round-trips a result and marks it cached on read', () => { + const cache = new ResponseCache(tempDir()) + const key = cacheKey(baseReq()) + assert.equal(cache.get(key), null) + + const result = runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 42 })) }) + cache.set(key, result) + + const hit = cache.get(key) + assert.ok(hit) + assert.equal(hit.cached, true) + assert.deepEqual(hit.output, { answer: 42 }) + assert.equal(hit.cost, result.cost) +}) + +test('ResponseCaches in separate dirs are isolated', () => { + const a = new ResponseCache(tempDir()) + const b = new ResponseCache(tempDir()) + const key = cacheKey(baseReq()) + a.set(key, runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 1 })) })) + assert.ok(a.get(key)) + assert.equal(b.get(key), null) +}) + +// --- runClaude cache integration --------------------------------------------- + +test('runClaude serves an identical request from cache without spawning', () => { + const cache = new ResponseCache(tempDir()) + let spawnCalls = 0 + const countingSpawn: Spawn = (cmd, args, input) => { + spawnCalls++ + return fakeSpawn(successEnvelope({ answer: 42 }))(cmd, args, input) + } + + const first = runClaude(baseReq(), { spawn: countingSpawn, cache }) + assert.equal(first.ok, true) + assert.equal(first.cached, false) + assert.equal(spawnCalls, 1) + + // Second identical call: cache hit, no further spawn (throwingSpawn proves it). + const second = runClaude(baseReq(), { spawn: throwingSpawn(), cache }) + assert.equal(second.ok, true) + assert.equal(second.cached, true) + assert.deepEqual(second.output, { answer: 42 }) + assert.equal(spawnCalls, 1) +}) + +test('runClaude misses the cache when a keyed field changes', () => { + const cache = new ResponseCache(tempDir()) + runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 42 })) , cache }) + + let spawned = false + const spawn: Spawn = (cmd, args, input) => { + spawned = true + return fakeSpawn(successEnvelope({ answer: 7 }))(cmd, args, input) + } + const result = runClaude(baseReq({ model: 'claude-opus-4-8' }), { spawn, cache }) + assert.equal(spawned, true) + assert.equal(result.cached, false) + assert.deepEqual(result.output, { answer: 7 }) +}) + +test('runClaude does not cache a graded failure', () => { + const cache = new ResponseCache(tempDir()) + // A schema-miss success envelope grades ok:false and must not be cached. + runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ wrong: 1 })), cache }) + assert.equal(cache.get(cacheKey(baseReq())), null) +}) diff --git a/tests/eval/engine.test.ts b/tests/eval/engine.test.ts new file mode 100644 index 0000000..de3c309 --- /dev/null +++ b/tests/eval/engine.test.ts @@ -0,0 +1,181 @@ +// Engine tests: pure argv construction, envelope grading over +// success/fallback/error/schema-miss/unparseable, and the isAvailable/skip +// paths — all through the injected `Spawn` stub, never a real `claude`. + +import assert from 'node:assert/strict' +import { test } from 'bun:test' +import { buildArgs, isAvailable, parseEnvelope, runClaude } from '../../src/eval/engine.ts' +import type { EvalRequest, JsonSchema } from '../../src/eval/types.ts' +import { + errorEnvelope, + fakeSpawn, + rawSpawn, + schemaMismatchEnvelope, + successEnvelope, + throwingSpawn, +} from '../helpers/fake-claude.ts' + +const SCHEMA: JsonSchema = { + type: 'object', + required: ['answer'], + properties: { answer: { type: 'number' } }, + additionalProperties: false, +} + +const baseReq = (overrides: Partial = {}): EvalRequest => ({ + prompt: 'What is 6 * 7?', + model: 'claude-haiku-4-5', + schema: SCHEMA, + ...overrides, +}) + +// --- buildArgs (acceptance criterion 1) -------------------------------------- + +test('buildArgs forces print mode, JSON output, schema, model, strict MCP, and settings', () => { + const args = buildArgs(baseReq()) + assert.ok(args.includes('--print')) + assert.deepEqual(args.slice(args.indexOf('--output-format'), args.indexOf('--output-format') + 2), [ + '--output-format', + 'json', + ]) + assert.deepEqual(args.slice(args.indexOf('--json-schema'), args.indexOf('--json-schema') + 2), [ + '--json-schema', + JSON.stringify(SCHEMA), + ]) + assert.deepEqual(args.slice(args.indexOf('--model'), args.indexOf('--model') + 2), [ + '--model', + 'claude-haiku-4-5', + ]) + assert.ok(args.includes('--strict-mcp-config')) + assert.deepEqual(args.slice(args.indexOf('--settings'), args.indexOf('--settings') + 2), ['--settings', '{}']) +}) + +test('buildArgs disables tools by default and never puts the prompt in argv', () => { + const args = buildArgs(baseReq()) + assert.ok(args.includes('--disallowedTools')) + assert.ok(!args.includes('--allowedTools')) + // The prompt travels on stdin, so it must not appear in argv. + assert.ok(!args.some(a => a.includes('6 * 7'))) +}) + +test('buildArgs opts into an allow-list when tools are provided', () => { + const args = buildArgs(baseReq({ tools: ['Read', 'Grep'] })) + assert.deepEqual(args.slice(args.indexOf('--allowedTools'), args.indexOf('--allowedTools') + 2), [ + '--allowedTools', + 'Read Grep', + ]) + assert.ok(!args.includes('--disallowedTools')) +}) + +test('buildArgs adds system-prompt flags only when set, and passes custom settings', () => { + const bare = buildArgs(baseReq()) + assert.ok(!bare.includes('--system-prompt')) + assert.ok(!bare.includes('--append-system-prompt')) + + const withSys = buildArgs(baseReq({ systemPrompt: 'You are terse.', appendSystemPrompt: 'Context.', settings: '{"x":1}' })) + assert.deepEqual(withSys.slice(withSys.indexOf('--system-prompt'), withSys.indexOf('--system-prompt') + 2), [ + '--system-prompt', + 'You are terse.', + ]) + assert.deepEqual( + withSys.slice(withSys.indexOf('--append-system-prompt'), withSys.indexOf('--append-system-prompt') + 2), + ['--append-system-prompt', 'Context.'] + ) + assert.deepEqual(withSys.slice(withSys.indexOf('--settings'), withSys.indexOf('--settings') + 2), [ + '--settings', + '{"x":1}', + ]) +}) + +// --- success + fallback (acceptance criterion 2) ----------------------------- + +test('a success envelope with structured_output yields ok:true with cost/usage/output', () => { + const envelope = successEnvelope( + { answer: 42 }, + { cost: 0.0463, usage: { inputTokens: 10, outputTokens: 158, cacheReadInputTokens: 3 }, durationMs: 2419, apiDurationMs: 2394, numTurns: 2 } + ) + const result = runClaude(baseReq(), { spawn: fakeSpawn(envelope) }) + assert.equal(result.ok, true) + assert.deepEqual(result.output, { answer: 42 }) + assert.equal(result.error, null) + assert.equal(result.cost, 0.0463) + assert.deepEqual(result.usage, { + inputTokens: 10, + outputTokens: 158, + cacheReadInputTokens: 3, + cacheCreationInputTokens: 0, + }) + assert.equal(result.durationMs, 2419) + assert.equal(result.apiDurationMs, 2394) + assert.equal(result.numTurns, 2) + assert.equal(result.model, 'claude-haiku-4-5') + assert.equal(result.cached, false) +}) + +test('a missing structured_output falls back to JSON.parse(result)', () => { + const envelope = successEnvelope({ answer: 7 }, { structured: false }) + assert.equal(envelope.structured_output, undefined) + const result = runClaude(baseReq(), { spawn: fakeSpawn(envelope) }) + assert.equal(result.ok, true) + assert.deepEqual(result.output, { answer: 7 }) +}) + +// --- graded failures, no throw (acceptance criterion 3) ---------------------- + +test('is_error / non-success subtype grades ok:false without throwing', () => { + const result = runClaude(baseReq(), { spawn: fakeSpawn(errorEnvelope('error_max_turns')) }) + assert.equal(result.ok, false) + assert.equal(result.output, null) + assert.match(result.error ?? '', /error_max_turns/) + // Metrics from the envelope are still surfaced. + assert.equal(result.cost, 0.002) +}) + +test('schema-nonconforming output grades ok:false with a descriptive error', () => { + const result = runClaude(baseReq(), { spawn: fakeSpawn(schemaMismatchEnvelope()) }) + assert.equal(result.ok, false) + assert.match(result.error ?? '', /required key: answer/) +}) + +test('additionalProperties:false rejects an extra key', () => { + const envelope = successEnvelope({ answer: 1, extra: 2 }) + const result = parseEnvelope(JSON.stringify(envelope), baseReq()) + assert.equal(result.ok, false) + assert.match(result.error ?? '', /unexpected key: extra/) +}) + +test('unparseable stdout grades ok:false with zeroed metrics, no throw', () => { + const result = runClaude(baseReq(), { spawn: rawSpawn(0, 'not json at all') }) + assert.equal(result.ok, false) + assert.equal(result.error, 'unparseable envelope') + assert.equal(result.cost, 0) + assert.equal(result.durationMs, 0) +}) + +// --- availability + skip (acceptance criterion 5) ---------------------------- + +test('isAvailable is false when claude --version exits non-zero or throws', () => { + assert.equal(isAvailable(rawSpawn(127, '', 'command not found')), false) + assert.equal(isAvailable(throwingSpawn()), false) + assert.equal(isAvailable(rawSpawn(0, 'claude 2.1.207')), true) +}) + +test('runClaude against an absent CLI returns a skipped result, never throws', () => { + const viaExit = runClaude(baseReq(), { spawn: rawSpawn(127, '', 'not found') }) + assert.equal(viaExit.skipped, true) + assert.equal(viaExit.ok, false) + assert.match(viaExit.error ?? '', /unavailable/) + + const viaThrow = runClaude(baseReq(), { spawn: throwingSpawn() }) + assert.equal(viaThrow.skipped, true) + assert.equal(viaThrow.ok, false) +}) + +test('a non-zero exit that still produced envelope stdout is graded, not skipped', () => { + // The CLI can exit non-zero while emitting an error envelope — that is a + // graded failure, not an availability skip. + const result = runClaude(baseReq(), { spawn: rawSpawn(1, JSON.stringify(errorEnvelope('error_during_execution'))) }) + assert.equal(result.skipped, undefined) + assert.equal(result.ok, false) + assert.match(result.error ?? '', /error_during_execution/) +}) diff --git a/tests/helpers/fake-claude.ts b/tests/helpers/fake-claude.ts new file mode 100644 index 0000000..b3f434c --- /dev/null +++ b/tests/helpers/fake-claude.ts @@ -0,0 +1,82 @@ +// Hermetic stub for the `claude -p` subprocess boundary. Tests inject `fakeSpawn` +// (or a raw variant) so `bun test` never shells out to a real `claude` — the ONLY +// faked seam in the eval engine. Envelope factories mirror the verified real +// `--output-format json` shape (installed claude 2.1.207). + +import type { Envelope, EvalUsage, Spawn } from '../../src/eval/types.ts' + +/** A `Spawn` that ignores args/stdin and emits `envelope` as JSON on stdout, exit 0. */ +export function fakeSpawn(envelope: Envelope): Spawn { + return () => ({ status: 0, stdout: JSON.stringify(envelope), stderr: '' }) +} + +/** A `Spawn` returning a fixed status/stdout/stderr verbatim (for edge cases). */ +export function rawSpawn(status: number | null, stdout: string, stderr = ''): Spawn { + return () => ({ status, stdout, stderr }) +} + +/** A `Spawn` that throws, modelling a missing binary (ENOENT). */ +export function throwingSpawn(): Spawn { + return () => { + throw new Error('spawn claude ENOENT') + } +} + +export interface SuccessOptions { + cost?: number + usage?: Partial + durationMs?: number + apiDurationMs?: number + numTurns?: number + /** When false, omit `structured_output` so the engine falls back to `result`. */ + structured?: boolean +} + +/** A success envelope carrying `output` as both `structured_output` and `result`. */ +export function successEnvelope(output: unknown, opts: SuccessOptions = {}): Envelope { + const { + cost = 0.001, + usage = {}, + durationMs = 1000, + apiDurationMs = 950, + numTurns = 2, + structured = true, + } = opts + const envelope: Envelope = { + type: 'result', + subtype: 'success', + is_error: false, + result: JSON.stringify(output), + total_cost_usd: cost, + duration_ms: durationMs, + duration_api_ms: apiDurationMs, + num_turns: numTurns, + usage: { + input_tokens: usage.inputTokens ?? 10, + output_tokens: usage.outputTokens ?? 20, + cache_read_input_tokens: usage.cacheReadInputTokens ?? 0, + cache_creation_input_tokens: usage.cacheCreationInputTokens ?? 0, + }, + } + if (structured) envelope.structured_output = output + return envelope +} + +/** An error envelope (`is_error:true`, non-success subtype) with metrics present. */ +export function errorEnvelope(subtype = 'error_max_turns'): Envelope { + return { + type: 'result', + subtype, + is_error: true, + total_cost_usd: 0.002, + duration_ms: 500, + duration_api_ms: 480, + num_turns: 1, + usage: { input_tokens: 5, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + } +} + +/** A success envelope whose `structured_output` omits a required key (schema miss). */ +export function schemaMismatchEnvelope(): Envelope { + return successEnvelope({ unexpected: true }) +} From 49c8dfbaf282e3751a3ac7f1463dce8187e0108c Mon Sep 17 00:00:00 2001 From: Christian Schuetz Date: Wed, 15 Jul 2026 00:43:34 -0500 Subject: [PATCH 2/3] =?UTF-8?q?feat(evaluation-system):=20eval=20framework?= =?UTF-8?q?=20=E2=80=94=20cases,=20graders,=20A/B=20+=20matrix=20runner,?= =?UTF-8?q?=20report,=20CLI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grading + orchestration layer on top of the D1 `claude -p` engine (D2). - src/eval/case.ts: EvalCase / Variant types, defineCase, toRequest lowering. - src/eval/grade.ts: Grader/GradeResult, built-in schemaConforms/assert/judge (judge routes through the D1 engine, cached/stubbable), weighted gradeCase. - src/eval/runner.ts: runCase, bounded-parallel runSuite (mapPool), runAB delta record (Δcorrectness/Δcost/Δlatency/Δturns), runMatrix per-model grid; skips with notice when the CLI is unavailable. - src/eval/report.ts: aligned text table + JSON over all three axes, optional clearly-labelled projected-cost column from a static pricing table. - src/eval/cli.ts: `bun src/eval/cli.ts` — suite loading, --ab/--matrix/--filter/ --models/--json/--tolerance; report-only exit 0 except an A/B correctness regression past tolerance → exit 1; absent claude → notice + exit 0. - package.json: `eval` script (kept OUT of `test`; test stays hermetic/offline). - CLAUDE.md + conventions.md: run the eval suite alongside `bun test` when changing harness prompts (docs-only, no generated-artifact churn, no bump). - tests/eval/framework.test.ts: graders, gradeCase, runAB/runMatrix deltas, bounded pool, report text/JSON, CLI exit-code semantics — all via fakeSpawn. --- CLAUDE.md | 2 + package.json | 1 + plugins/strapped/conventions.md | 6 + src/eval/case.ts | 77 ++++++ src/eval/cli.ts | 174 +++++++++++++ src/eval/grade.ts | 144 +++++++++++ src/eval/report.ts | 134 +++++++++++ src/eval/runner.ts | 172 +++++++++++++ tests/eval/framework.test.ts | 415 ++++++++++++++++++++++++++++++++ 9 files changed, 1125 insertions(+) create mode 100644 src/eval/case.ts create mode 100644 src/eval/cli.ts create mode 100644 src/eval/grade.ts create mode 100644 src/eval/report.ts create mode 100644 src/eval/runner.ts create mode 100644 tests/eval/framework.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index 1a70ceb..43a13b8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,3 +1,5 @@ # Guidelines +- **Run the prompt-eval suite alongside `bun test` when changing harness prompts.** When you edit an agent prompt — anything under `src/workflows/strapped-run/**` or the step prose in a skill `SKILL.md` — run `bun run eval --suite ` (see `src/eval/` and the "Prompt evaluation suite" section of `plugins/strapped/conventions.md`) and keep it green alongside `bun test`. The eval suite grades a prompt on correctness / cost / latency through `claude -p`; it is the evidence that a prompt change is a real improvement (use `--ab` to compare a baseline vs a candidate and gate on a correctness regression past `--tolerance`). It is a HEAVY, opt-in layer (needs a real `claude` + spends cost), so it is intentionally NOT part of `bun test` — run it yourself for prompt work. `bun test` stays hermetic/offline and remains the gate for everything else. + - **Bump the plugin version with the tool, never by hand.** `claude plugin update` compares `version` in `plugins/strapped/.claude-plugin/plugin.json` only — an unbumped version leaves installed copies silently pinned to a stale commit after a change merges. Bump ONLY via `bun tools/version.ts bump ` (it rewrites the one semver deterministically); do not hand-edit plugin.json. Pick the level by a conventional-commit rule: a breaking change → `major`, a new user-facing feature (skill/workflow/script behavior) → `minor`, a fix or internal change → `patch`. Before committing, run `bun tools/version.ts check` (alias `bun run version:check`) — it fails when a changed *generated artifact* (the committed deployables under `plugins/strapped/`) is not accompanied by a version bump above the base. Prompt/skill/docs-only changes with NO generated-artifact change are at your discretion — the guard exits 0 for them, so bump when the change is user-facing and skip an internal-only tweak. diff --git a/package.json b/package.json index d08f348..2d4e11e 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "lint": "oxlint src tools tests", "version:bump": "bun tools/version.ts bump", "version:check": "bun tools/version.ts check", + "eval": "bun src/eval/cli.ts", "test": "claude plugin validate . --strict && claude plugin validate plugins/strapped --strict && bun run typecheck && bun run lint && bun test" }, "devDependencies": { diff --git a/plugins/strapped/conventions.md b/plugins/strapped/conventions.md index 5542b56..aede133 100644 --- a/plugins/strapped/conventions.md +++ b/plugins/strapped/conventions.md @@ -398,6 +398,12 @@ This limit is why the retired multi-file architecture (stage workflows dispatchi Every command in the **deliverable's repo's** config `validations` must be green before code review and after every fix round, run inside the deliverable's worktree. +### Prompt evaluation suite + +The strapped repo carries a prompt-effectiveness eval suite under `src/eval/` (run via `bun run eval`, not bundled into any plugin deployable). It grades the harness's own agent prompts on three axes — **correctness** (layered graders: schema-conformance, assertion predicates, an optional LLM judge), **cost** (`total_cost_usd` + token usage), and **latency** (`duration_ms` / `num_turns`) — by shelling out to `claude -p` (never the Anthropic SDK) and reading the `--output-format json` envelope. It supports **A/B** (baseline vs candidate prompt at one model, reporting Δcorrectness / Δcost / Δlatency side by side so a human judges the trade — a correctness dip that buys a large cost/latency win is a WIN) and a **model matrix** (`--models opus,sonnet,haiku`). + +It is the **heavy, opt-in test layer for prompt changes**: it needs a real `claude` and spends cost, so it is deliberately kept OUT of `bun test` (which stays hermetic/offline). When you change an agent prompt (`src/workflows/strapped-run/**` or a skill's `SKILL.md` step prose), run the eval suite alongside `bun test` and keep both green — it is the evidence that the prompt change is a real improvement. An absent `claude` CLI is a graceful skip (notice + exit 0), and only `--ab` gates: exit non-zero on a correctness regression past `--tolerance`. + ## Worktrees and branches All repo-scoped values below come from the **deliverable's repo** — `manifest.repos[deliverable.repo]` for the root, and that repo's config for `worktreeRoot`, `validations`, and `provisioning`. diff --git a/src/eval/case.ts b/src/eval/case.ts new file mode 100644 index 0000000..ba073ca --- /dev/null +++ b/src/eval/case.ts @@ -0,0 +1,77 @@ +// The eval CASE abstraction: a single, self-contained prompt-evaluation unit. A +// case carries the prompt, the schema it is forced against, isolation knobs +// (system prompt / tools / settings), the models it targets, its correctness +// graders, and — for A/B — an optional baseline+candidate prompt pair. A SUITE is +// just a directory of modules that export cases. +// +// Design decision (research §"prompts are eval inputs"): a case owns a VERBATIM +// baseline prompt snapshot copied from the live stage plus a candidate variant to +// A/B. Cases never import the live stage builders — prompts are inputs, not code. + +import type { Grader } from './grade.ts' +import type { EvalRequest, JsonSchema } from './types.ts' + +/** One labelled prompt variant for A/B comparison. */ +export interface Variant { + /** Short label shown in the report (e.g. `baseline`, `terser`). */ + label: string + /** The full prompt text for this variant. */ + prompt: string +} + +/** A baseline↔candidate variant pair a case can be A/B'd on. */ +export interface CaseVariants { + baseline: Variant + candidate: Variant +} + +/** A single prompt-evaluation case. */ +export interface EvalCase { + /** Unique id (also a `--filter` target). */ + id: string + /** Tags for `--filter` selection (e.g. `planner`, `reviewer`). */ + tags: string[] + /** Replace the CLI's default system prompt for isolation (`--system-prompt`). */ + systemPrompt?: string + /** Layer strapped context on the real system prompt (`--append-system-prompt`). */ + appendSystemPrompt?: string + /** The user prompt under evaluation. */ + prompt: string + /** The schema forced via `--json-schema` and validated against the output. */ + schema: JsonSchema + /** Models this case targets in a plain/matrix run; the runner falls back to a default. */ + models?: string[] + /** Opt into a bounded toolset; omit for a single-shot, tool-free run. */ + tools?: string[] + /** Inline settings JSON (`--settings`); defaults to `'{}'` in the engine. */ + settings?: string + /** The correctness graders aggregated into this case's score. */ + graders: Grader[] + /** Optional A/B prompt pair — required for a case to appear in `--ab` mode. */ + variants?: CaseVariants +} + +/** + * Identity helper for authoring a case module — validates nothing beyond types, + * but gives suites a single import and a stable authoring surface. + */ +export function defineCase(spec: EvalCase): EvalCase { + return spec +} + +/** + * Lower a case (at one model, optionally with a variant's prompt substituted) + * into the single-shot `EvalRequest` the D1 engine consumes. The prompt override + * is how A/B swaps baseline vs candidate while holding everything else fixed. + */ +export function toRequest(c: EvalCase, model: string, promptOverride?: string): EvalRequest { + return { + prompt: promptOverride ?? c.prompt, + systemPrompt: c.systemPrompt, + appendSystemPrompt: c.appendSystemPrompt, + model, + schema: c.schema, + tools: c.tools, + settings: c.settings, + } +} diff --git a/src/eval/cli.ts b/src/eval/cli.ts new file mode 100644 index 0000000..37702d0 --- /dev/null +++ b/src/eval/cli.ts @@ -0,0 +1,174 @@ +// `bun src/eval/cli.ts` — the eval suite entrypoint (npm: `bun run eval`). Loads +// case modules from a suite directory, runs them in one of three modes, and +// prints a report. This is the HEAVY, opt-in test layer for prompt changes — it +// is deliberately NOT part of `bun test` (that stays hermetic/offline). +// +// Exit code: a plain/matrix run is REPORT-ONLY (always 0). Only `--ab` gates — +// exit 1 iff some case's Δcorrectness dips past `--tolerance` (a regression). An +// absent `claude` CLI is a graceful skip: print a notice, exit 0, never fail. +// +// Flags: --suite --filter --ab --matrix --models a,b +// --json --tolerance +// +// `main` returns { code, output } (does not exit) so tests can drive the exact +// exit-code semantics through the injected spawn; the bottom guard is the only +// place that touches the process. + +import { readdirSync } from 'node:fs' +import { join, resolve } from 'node:path' +import type { EvalCase } from './case.ts' +import { isAvailable } from './engine.ts' +import { formatReport, toJSON, type Report } from './report.ts' +import { + DEFAULT_MODEL, + runAB, + runMatrix, + runSuite, + type ABResult, + type MatrixResult, +} from './runner.ts' +import type { Cache, Spawn } from './types.ts' + +export interface EvalFlags { + suite?: string + filter?: string + ab: boolean + matrix: boolean + models?: string[] + json: boolean + tolerance: number +} + +/** Hand-rolled argv parser (no dependency). Unknown flags are ignored. */ +export function parseArgs(argv: readonly string[]): EvalFlags { + const flags: EvalFlags = { ab: false, matrix: false, json: false, tolerance: 0 } + for (let i = 0; i < argv.length; i++) { + switch (argv[i]) { + case '--suite': + flags.suite = argv[++i] + break + case '--filter': + flags.filter = argv[++i] + break + case '--ab': + flags.ab = true + break + case '--matrix': + flags.matrix = true + break + case '--models': + flags.models = (argv[++i] ?? '').split(',').filter(Boolean) + break + case '--json': + flags.json = true + break + case '--tolerance': + flags.tolerance = Number(argv[++i] ?? '0') + break + default: + break + } + } + return flags +} + +/** Load every case exported by the modules in a suite directory (sorted). */ +export async function loadSuite(dir: string): Promise { + const abs = resolve(dir) + const files = readdirSync(abs) + .filter(f => /\.(ts|mjs|js)$/.test(f) && !f.endsWith('.d.ts')) + .sort() + const cases: EvalCase[] = [] + for (const file of files) { + const mod = (await import(join(abs, file))) as { cases?: unknown; default?: unknown } + const exported = mod.cases ?? mod.default + if (Array.isArray(exported)) cases.push(...(exported as EvalCase[])) + else if (exported) cases.push(exported as EvalCase) + } + return cases +} + +/** A case matches a filter if the filter equals its id or one of its tags. */ +function matchesFilter(c: EvalCase, filter: string): boolean { + return c.id === filter || c.tags.includes(filter) +} + +/** + * A/B gate: exit 1 iff any case regressed correctness past the tolerance. A + * tolerance of `5` means "a 5-percentage-point dip is allowed"; anything worse + * gates. Cost/latency wins never gate — a human judges the trade from the report. + */ +export function abExitCode(results: readonly ABResult[], tolerancePct: number): number { + const tol = tolerancePct / 100 + return results.some(r => r.deltaCorrectness < -tol) ? 1 : 0 +} + +export interface CliDeps { + spawn?: Spawn + cache?: Cache + /** Test override: supply cases directly instead of loading from `--suite`. */ + cases?: EvalCase[] +} + +export interface CliResult { + code: number + output: string +} + +/** + * Parse argv, run the selected mode, and return an exit code + text to print. + * Never touches the process — the bottom guard does that. An absent CLI short + * circuits to a skip notice + exit 0 before any case runs. + */ +export async function main(argv: readonly string[], deps: CliDeps = {}): Promise { + const flags = parseArgs(argv) + const spawn = deps.spawn + + if (!isAvailable(spawn)) { + return { code: 0, output: 'eval: claude CLI unavailable — skipping suite (exit 0)\n' } + } + + let cases = deps.cases ?? (flags.suite ? await loadSuite(flags.suite) : []) + if (flags.filter) cases = cases.filter(c => matchesFilter(c, flags.filter as string)) + + if (cases.length === 0) { + return { code: 0, output: 'eval: no cases matched (nothing to run)\n' } + } + + let report: Report + let code = 0 + + if (flags.ab) { + const results: ABResult[] = [] + for (const c of cases) { + if (!c.variants) continue + results.push(runAB(c, c.variants.baseline, c.variants.candidate, { model: flags.models?.[0] ?? DEFAULT_MODEL, spawn, cache: deps.cache })) + } + report = { mode: 'ab', cases: results } + code = abExitCode(results, flags.tolerance) + } else if (flags.matrix) { + const grids: MatrixResult[] = [] + for (const c of cases) { + grids.push(await runMatrix(c, flags.models ?? c.models ?? [], { spawn, cache: deps.cache })) + } + report = { mode: 'matrix', cases: grids } + } else { + const results = await runSuite(cases, { models: flags.models, spawn, cache: deps.cache }) + report = { mode: 'suite', cases: results } + } + + const output = flags.json ? `${JSON.stringify(toJSON(report), null, 2)}\n` : `${formatReport(report)}\n` + return { code, output } +} + +if (import.meta.main) { + main(process.argv.slice(2)) + .then(({ code, output }) => { + process.stdout.write(output) + process.exit(code) + }) + .catch((e: unknown) => { + process.stderr.write(`eval: ${e instanceof Error ? e.message : String(e)}\n`) + process.exit(2) + }) +} diff --git a/src/eval/grade.ts b/src/eval/grade.ts new file mode 100644 index 0000000..5dd97ff --- /dev/null +++ b/src/eval/grade.ts @@ -0,0 +1,144 @@ +// Correctness graders for the eval framework. A `Grader` inspects one engine +// result (the parsed output + the full envelope-derived `EvalResult`) and returns +// a `GradeResult`. Three built-ins ship here: +// - `schemaConforms()` — intrinsic: did the forced schema validate? (`result.ok`) +// - `assert(name, predicate)` — a pure predicate over the parsed output. +// - `judge(rubric, {model})` — an LLM-judge that ITSELF routes through the D1 +// engine (so it is cached/stubbable/hermetic), scoring fuzzy quality. +// `gradeCase` aggregates a case's graders into a weighted correctness score in +// [0,1]. Graders are synchronous — the D1 engine (`runClaude`) is synchronous — +// and NEVER throw: a thrown predicate becomes a failed `GradeResult`. + +import { runClaude } from './engine.ts' +import type { Cache, EvalRequest, EvalResult, JsonSchema, Spawn } from './types.ts' + +/** The verdict of one grader over an engine result. `score` is in [0,1]. */ +export interface GradeResult { + /** Stable grader name — appears in the report and JSON. */ + name: string + /** Boolean pass/fail (a threshold view of `score`). */ + pass: boolean + /** Continuous quality in [0,1]; `schemaConforms`/`assert` emit 0 or 1. */ + score: number + /** Human-readable reason — why it passed/failed. */ + detail: string + /** Relative weight in the correctness aggregate (default 1). */ + weight?: number +} + +/** + * Runtime context threaded to every grader by `gradeCase`. Only `judge` reads it + * (to route its rubric call through the same injected spawn/cache as the case), + * so `bun test` stays hermetic. Intrinsic graders ignore it. + */ +export interface GradeContext { + spawn?: Spawn + cache?: Cache +} + +/** A grader: inspect the parsed output + full result, return a verdict. */ +export type Grader = (output: unknown, result: EvalResult, ctx: GradeContext) => GradeResult + +/** Intrinsic grader: passes iff the forced schema validated (`result.ok`). */ +export function schemaConforms(opts: { weight?: number } = {}): Grader { + return (_output, result) => ({ + name: 'schemaConforms', + pass: result.ok, + score: result.ok ? 1 : 0, + detail: result.ok ? 'output conforms to the forced schema' : (result.error ?? 'schema nonconforming'), + weight: opts.weight, + }) +} + +/** + * Assertion grader: run a pure predicate over the parsed output. A `false` return + * fails; a thrown predicate fails with the throw message (never propagates). + */ +export function assert(name: string, predicate: (output: unknown) => boolean, opts: { weight?: number } = {}): Grader { + return output => { + try { + const pass = predicate(output) + return { name, pass, score: pass ? 1 : 0, detail: pass ? 'assertion passed' : 'assertion failed', weight: opts.weight } + } catch (e) { + const msg = e instanceof Error ? e.message : String(e) + return { name, pass: false, score: 0, detail: `assertion threw: ${msg}`, weight: opts.weight } + } + } +} + +/** The schema the judge forces on itself — a single scored verdict. */ +const JUDGE_SCHEMA: JsonSchema = { + type: 'object', + required: ['score', 'pass', 'reason'], + properties: { + score: { type: 'number' }, + pass: { type: 'boolean' }, + reason: { type: 'string' }, + }, + additionalProperties: false, +} + +/** Parsed shape of a judge verdict (post schema-validation). */ +interface JudgeVerdict { + score?: number + pass?: boolean + reason?: string +} + +export interface JudgeOptions { + /** Model the judge itself runs on (`--model`). */ + model: string + /** Grader name in the report (default `judge`). */ + name?: string + /** Weight in the correctness aggregate. */ + weight?: number +} + +/** + * LLM-judge grader. Composes an `EvalRequest` from `rubric` + the output under + * test and a fixed `{score,pass,reason}` schema, then calls the D1 engine through + * the injected spawn/cache — so the judge is itself cacheable and hermetically + * stubbable. A judge engine failure grades the case as a failed rubric, not a crash. + */ +export function judge(rubric: string, opts: JudgeOptions): Grader { + const name = opts.name ?? 'judge' + return (output, _result, ctx) => { + const req: EvalRequest = { + prompt: `${rubric}\n\nOutput under evaluation:\n${JSON.stringify(output, null, 2)}`, + model: opts.model, + schema: JUDGE_SCHEMA, + } + const verdict = runClaude(req, { spawn: ctx.spawn, cache: ctx.cache }) + if (!verdict.ok || verdict.output === null || typeof verdict.output !== 'object') { + return { name, pass: false, score: 0, detail: verdict.error ?? 'judge produced no verdict', weight: opts.weight } + } + const v = verdict.output as JudgeVerdict + const score = typeof v.score === 'number' ? Math.max(0, Math.min(1, v.score)) : 0 + return { name, pass: v.pass === true, score, detail: v.reason ?? '', weight: opts.weight } + } +} + +/** A case's aggregate correctness plus the individual grader verdicts. */ +export interface CaseGrade { + /** Weighted mean grader score in [0,1]. */ + correctness: number + grades: GradeResult[] +} + +/** + * Run every grader over one engine result and aggregate into a weighted + * correctness score. Correctness = Σ(weight·score) / Σweight (weight defaults to + * 1). An empty grader set yields correctness 0. + */ +export function gradeCase(graders: readonly Grader[], output: unknown, result: EvalResult, ctx: GradeContext = {}): CaseGrade { + const grades = graders.map(g => g(output, result, ctx)) + let weightedSum = 0 + let totalWeight = 0 + for (const grade of grades) { + const weight = grade.weight ?? 1 + weightedSum += weight * grade.score + totalWeight += weight + } + const correctness = totalWeight > 0 ? weightedSum / totalWeight : 0 + return { correctness, grades } +} diff --git a/src/eval/report.ts b/src/eval/report.ts new file mode 100644 index 0000000..8c82e19 --- /dev/null +++ b/src/eval/report.ts @@ -0,0 +1,134 @@ +// The eval REPORT: a human-readable aligned text table + a machine JSON object, +// over any of the three runner modes. The report surfaces all THREE axes +// (correctness / cost / latency) side by side so a human judges the trade — a +// correctness dip that buys a large cost/latency win is a WIN, and the tool +// refuses to collapse that into a single number. +// +// Cost columns: the "Cost $" column is the MEASURED envelope cost. A separate, +// clearly-labelled "Proj $" column is an OPTIONAL projection of a model-swap the +// CLI did not actually run, computed from a small static pricing table — never +// mixed with measured cost. + +import type { CaseResult, ABResult, MatrixResult } from './runner.ts' +import type { EvalUsage } from './types.ts' + +/** Static, approximate USD-per-1M-token prices — projection only, NOT measured. */ +export interface ModelPricing { + inputPerMTok: number + outputPerMTok: number +} + +/** A small static pricing table used ONLY to project a model-swap. */ +export const PRICING: Record = { + 'claude-opus-4-8': { inputPerMTok: 15, outputPerMTok: 75 }, + 'claude-sonnet-5': { inputPerMTok: 3, outputPerMTok: 15 }, + 'claude-haiku-4-5': { inputPerMTok: 1, outputPerMTok: 5 }, +} + +/** + * Project what a run's token usage WOULD cost on `model` from the static table. + * Returns `null` for an unpriced model. This is a projection, never a measurement. + */ +export function projectCost(usage: EvalUsage, model: string): number | null { + const p = PRICING[model] + if (!p) return null + return (usage.inputTokens * p.inputPerMTok + usage.outputTokens * p.outputPerMTok) / 1_000_000 +} + +/** A tagged union over the three runner modes the report can render. */ +export type Report = + | { mode: 'suite'; cases: CaseResult[] } + | { mode: 'ab'; cases: ABResult[] } + | { mode: 'matrix'; cases: MatrixResult[] } + +export interface FormatOptions { + /** When set, add a projected-cost column for a model the run did not execute. */ + projectModel?: string +} + +// --- cell formatters (deterministic — the report text is snapshot-tested) ----- + +const pct = (x: number): string => `${(x * 100).toFixed(1)}%` +const usd = (x: number): string => `$${x.toFixed(5)}` +const ms = (x: number): string => `${Math.round(x)}ms` +const signedPct = (x: number): string => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}%` +const signedUsd = (x: number): string => `${x >= 0 ? '+' : '-'}$${Math.abs(x).toFixed(5)}` +const signedMs = (x: number): string => `${x >= 0 ? '+' : ''}${Math.round(x)}ms` +const signedInt = (x: number): string => `${x >= 0 ? '+' : ''}${x}` + +/** Render aligned columns: header, a dashed rule, then rows (left-padded cells). */ +function renderTable(headers: string[], rows: string[][]): string { + const widths = headers.map((h, i) => Math.max(h.length, ...rows.map(r => (r[i] ?? '').length))) + const fmt = (cells: string[]): string => cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join(' ') + const rule = widths.map(w => '-'.repeat(w)).join(' ') + return [fmt(headers), rule, ...rows.map(fmt)].join('\n') +} + +function caseRow(r: CaseResult, projectModel?: string): string[] { + const cells = [ + r.caseId, + r.model, + r.skipped ? 'skipped' : pct(r.correctness), + r.skipped ? '-' : usd(r.result.cost), + r.skipped ? '-' : ms(r.result.durationMs), + r.skipped ? '-' : String(r.result.numTurns), + ] + if (projectModel) { + const proj = r.skipped ? null : projectCost(r.result.usage, projectModel) + cells.push(proj === null ? '-' : usd(proj)) + } + return cells +} + +function suiteHeaders(projectModel?: string): string[] { + const h = ['Case', 'Model', 'Correct', 'Cost $', 'Latency', 'Turns'] + if (projectModel) h.push(`Proj $ (${projectModel})`) + return h +} + +function formatSuite(cases: CaseResult[], projectModel?: string): string { + if (cases.length === 0) return 'no cases matched' + return renderTable(suiteHeaders(projectModel), cases.map(r => caseRow(r, projectModel))) +} + +function formatMatrix(cases: MatrixResult[], projectModel?: string): string { + if (cases.length === 0) return 'no cases matched' + const rows: string[][] = [] + for (const c of cases) for (const r of c.rows) rows.push(caseRow(r, projectModel)) + return renderTable(suiteHeaders(projectModel), rows) +} + +function formatAB(cases: ABResult[]): string { + if (cases.length === 0) return 'no cases matched' + const headers = ['Case', 'Model', 'ΔCorrect', 'ΔCost', 'ΔLatency', 'ΔTurns', 'Base', 'Cand'] + const rows = cases.map(r => [ + r.caseId, + r.model, + signedPct(r.deltaCorrectness), + signedUsd(r.deltaCostUsd), + signedMs(r.deltaLatencyMs), + signedInt(r.deltaTurns), + pct(r.baseline.correctness), + pct(r.candidate.correctness), + ]) + const legend = + 'Δcost/Δlatency/Δturns: positive = candidate cheaper/faster/tighter. ΔCorrect: negative = regression.' + return `${renderTable(headers, rows)}\n\n${legend}` +} + +/** Render a report to aligned human-readable text. */ +export function formatReport(report: Report, opts: FormatOptions = {}): string { + switch (report.mode) { + case 'suite': + return formatSuite(report.cases, opts.projectModel) + case 'matrix': + return formatMatrix(report.cases, opts.projectModel) + case 'ab': + return formatAB(report.cases) + } +} + +/** The machine-comparable JSON view (already a plain serializable object). */ +export function toJSON(report: Report): Report { + return report +} diff --git a/src/eval/runner.ts b/src/eval/runner.ts new file mode 100644 index 0000000..4dc412d --- /dev/null +++ b/src/eval/runner.ts @@ -0,0 +1,172 @@ +// The eval RUNNER: drives cases through the D1 engine and grades them, in three +// modes. +// - `runSuite(cases, …)` — every case × its models, in BOUNDED parallel. +// - `runAB(case, base, cand, {model})` — same case body, two prompt variants, +// one model → a delta record (Δcorrectness / Δcost / Δlatency / Δturns). +// - `runMatrix(case, models)` — one case across a model grid. +// Every mode takes an injected `spawn` + optional `cache`, so `bun test` runs +// fully offline against the `fakeSpawn` stub. A case whose engine call reports the +// CLI is unavailable is marked `skipped` — a notice, never a failure. +// +// NOTE on "parallel": the D1 spawn boundary is synchronous, so the pool bounds +// how many case promises are in flight rather than truly interleaving blocking +// spawns; the shape is correct and the bound is enforced (`mapPool`). Deltas are +// what the performance verdict rests on — the CLI-boot latency offset cancels. + +import { toRequest, type EvalCase, type Variant } from './case.ts' +import { runClaude } from './engine.ts' +import { gradeCase, type GradeContext, type GradeResult } from './grade.ts' +import type { Cache, EvalResult, Spawn } from './types.ts' + +/** Default model when a case/run names none. */ +export const DEFAULT_MODEL = 'claude-haiku-4-5' +/** Default model grid for `--matrix` when none is given. */ +export const DEFAULT_MODELS: readonly string[] = ['claude-opus-4-8', 'claude-sonnet-5', 'claude-haiku-4-5'] + +/** The graded outcome of running one case at one model (one variant). */ +export interface CaseResult { + caseId: string + model: string + /** Variant label when produced by an A/B leg; absent for plain/matrix runs. */ + variantLabel?: string + /** Weighted correctness in [0,1]. */ + correctness: number + grades: GradeResult[] + /** The full engine result — measured cost/latency/turns/usage live here. */ + result: EvalResult + /** True when the `claude` CLI was unavailable — a skip, not a failure. */ + skipped: boolean +} + +export interface CaseRunOptions { + model: string + /** Substitute the case prompt (used by A/B to swap in a variant). */ + prompt?: string + variantLabel?: string + spawn?: Spawn + cache?: Cache +} + +/** Run one case at one model, then grade it. Never throws. */ +export function runCase(c: EvalCase, opts: CaseRunOptions): CaseResult { + const result = runClaude(toRequest(c, opts.model, opts.prompt), { spawn: opts.spawn, cache: opts.cache }) + const base = { caseId: c.id, model: opts.model, ...(opts.variantLabel ? { variantLabel: opts.variantLabel } : {}) } + if (result.skipped === true) { + return { ...base, correctness: 0, grades: [], result, skipped: true } + } + const ctx: GradeContext = { spawn: opts.spawn, cache: opts.cache } + const { correctness, grades } = gradeCase(c.graders, result.output, result, ctx) + return { ...base, correctness, grades, result, skipped: false } +} + +/** + * Bounded-concurrency map: at most `concurrency` workers run at once, results + * preserve input order. No dependency — a fixed pool draining a shared cursor. + */ +export async function mapPool( + items: readonly T[], + concurrency: number, + worker: (item: T, index: number) => Promise +): Promise { + const results = new Array(items.length) + const limit = Math.max(1, Math.min(concurrency, items.length || 1)) + let cursor = 0 + async function drain(): Promise { + for (;;) { + const i = cursor++ + if (i >= items.length) return + results[i] = await worker(items[i] as T, i) + } + } + await Promise.all(Array.from({ length: limit }, () => drain())) + return results +} + +export interface SuiteOptions { + /** Override the models each case runs on. */ + models?: string[] + /** Max cases in flight (default 4). */ + concurrency?: number + cache?: Cache + spawn?: Spawn +} + +/** Run every case × its resolved models in bounded parallel. */ +export function runSuite(cases: readonly EvalCase[], opts: SuiteOptions = {}): Promise { + const jobs: { c: EvalCase; model: string }[] = [] + for (const c of cases) { + const models = opts.models ?? c.models ?? [DEFAULT_MODEL] + for (const model of models) jobs.push({ c, model }) + } + return mapPool(jobs, opts.concurrency ?? 4, job => + Promise.resolve(runCase(job.c, { model: job.model, spawn: opts.spawn, cache: opts.cache })) + ) +} + +/** A baseline-vs-candidate delta record at one fixed model. */ +export interface ABResult { + caseId: string + model: string + baselineLabel: string + candidateLabel: string + baseline: CaseResult + candidate: CaseResult + /** candidate − baseline: positive = candidate more correct. */ + deltaCorrectness: number + /** baseline − candidate cost: positive = candidate CHEAPER (a saving). */ + deltaCostUsd: number + /** baseline − candidate latency: positive = candidate FASTER. */ + deltaLatencyMs: number + /** baseline − candidate turns: positive = candidate converged in FEWER turns. */ + deltaTurns: number +} + +export interface ABOptions { + model?: string + spawn?: Spawn + cache?: Cache +} + +/** + * Run the same case body with two prompt variants at one model and compute the + * trade-off deltas. Cost/latency/turn deltas are baseline−candidate so a cheaper, + * faster, tighter candidate reads as POSITIVE across the board; correctness is + * candidate−baseline so a regression reads as NEGATIVE (what the CLI gate keys on). + */ +export function runAB(c: EvalCase, baseline: Variant, candidate: Variant, opts: ABOptions = {}): ABResult { + const model = opts.model ?? c.models?.[0] ?? DEFAULT_MODEL + const b = runCase(c, { model, prompt: baseline.prompt, variantLabel: baseline.label, spawn: opts.spawn, cache: opts.cache }) + const cand = runCase(c, { model, prompt: candidate.prompt, variantLabel: candidate.label, spawn: opts.spawn, cache: opts.cache }) + return { + caseId: c.id, + model, + baselineLabel: baseline.label, + candidateLabel: candidate.label, + baseline: b, + candidate: cand, + deltaCorrectness: cand.correctness - b.correctness, + deltaCostUsd: b.result.cost - cand.result.cost, + deltaLatencyMs: b.result.durationMs - cand.result.durationMs, + deltaTurns: b.result.numTurns - cand.result.numTurns, + } +} + +/** A per-model grid for one case. */ +export interface MatrixResult { + caseId: string + rows: CaseResult[] +} + +export interface MatrixOptions { + spawn?: Spawn + cache?: Cache + concurrency?: number +} + +/** Run one case across a model grid; one row per model, input order preserved. */ +export function runMatrix(c: EvalCase, models: readonly string[], opts: MatrixOptions = {}): Promise { + const grid = models.length > 0 ? models : DEFAULT_MODELS + return mapPool(grid, opts.concurrency ?? grid.length, model => + Promise.resolve(runCase(c, { model, spawn: opts.spawn, cache: opts.cache })) + ).then(rows => ({ caseId: c.id, rows })) +} diff --git a/tests/eval/framework.test.ts b/tests/eval/framework.test.ts new file mode 100644 index 0000000..682d2f1 --- /dev/null +++ b/tests/eval/framework.test.ts @@ -0,0 +1,415 @@ +// Framework tests: built-in graders + gradeCase aggregation, the runner's +// runCase/runSuite/runAB/runMatrix over canned envelopes, the bounded-parallel +// pool, report text/JSON, and the CLI exit-code semantics (regression vs clean +// vs CLI-absent). Every model call is faked at the `Spawn` boundary — offline, no +// real `claude`. + +import assert from 'node:assert/strict' +import { mkdtempSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'bun:test' +import { defineCase, toRequest, type EvalCase } from '../../src/eval/case.ts' +import { assert as assertGrader, gradeCase, judge, schemaConforms, type Grader } from '../../src/eval/grade.ts' +import { + mapPool, + runAB, + runCase, + runMatrix, + runSuite, + type ABResult, + type CaseResult, +} from '../../src/eval/runner.ts' +import { formatReport, projectCost, toJSON, type Report } from '../../src/eval/report.ts' +import { abExitCode, main, loadSuite, parseArgs } from '../../src/eval/cli.ts' +import type { EvalResult, JsonSchema, Spawn } from '../../src/eval/types.ts' +import { successEnvelope, throwingSpawn } from '../helpers/fake-claude.ts' + +const SCHEMA: JsonSchema = { + type: 'object', + required: ['answer'], + properties: { answer: { type: 'number' } }, + additionalProperties: false, +} + +/** A minimal ok result carrying the given output — for grading graders directly. */ +function okResult(output: unknown): EvalResult { + return { + ok: true, + output, + error: null, + cost: 0.01, + usage: { inputTokens: 10, outputTokens: 20, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 }, + durationMs: 1000, + apiDurationMs: 950, + numTurns: 2, + model: 'claude-haiku-4-5', + cached: false, + } +} + +const baseCase = (overrides: Partial = {}): EvalCase => + defineCase({ + id: 'add', + tags: ['math'], + prompt: 'What is 6 * 7?', + schema: SCHEMA, + graders: [schemaConforms(), assertGrader('is-42', o => (o as { answer?: number }).answer === 42)], + ...overrides, + }) + +// --- graders (acceptance criterion 2) ---------------------------------------- + +test('schemaConforms passes on an ok result and fails on a graded failure', () => { + const pass = schemaConforms()(null, okResult({ answer: 42 }), {}) + assert.equal(pass.name, 'schemaConforms') + assert.equal(pass.pass, true) + assert.equal(pass.score, 1) + + const failResult = { ...okResult(null), ok: false, error: 'missing key' } + const fail = schemaConforms()(null, failResult, {}) + assert.equal(fail.pass, false) + assert.equal(fail.score, 0) + assert.match(fail.detail, /missing key/) +}) + +test('assert grades a predicate, and a thrown predicate fails without propagating', () => { + const pass = assertGrader('is-42', o => (o as { answer: number }).answer === 42)({ answer: 42 }, okResult({ answer: 42 }), {}) + assert.equal(pass.pass, true) + assert.equal(pass.score, 1) + + const fail = assertGrader('is-42', o => (o as { answer: number }).answer === 42)({ answer: 1 }, okResult({ answer: 1 }), {}) + assert.equal(fail.pass, false) + + const threw = assertGrader('boom', () => { + throw new Error('kaboom') + })(null, okResult(null), {}) + assert.equal(threw.pass, false) + assert.match(threw.detail, /kaboom/) +}) + +test('judge routes through the engine (stubbed) and maps the verdict to a GradeResult', () => { + const verdictEnvelope = successEnvelope({ score: 0.8, pass: true, reason: 'clear and complete' }) + const spawn: Spawn = () => ({ status: 0, stdout: JSON.stringify(verdictEnvelope), stderr: '' }) + const grade = judge('Rate the answer 0..1.', { model: 'claude-haiku-4-5' })( + { answer: 42 }, + okResult({ answer: 42 }), + { spawn } + ) + assert.equal(grade.name, 'judge') + assert.equal(grade.pass, true) + assert.equal(grade.score, 0.8) + assert.match(grade.detail, /clear and complete/) +}) + +test('judge clamps an out-of-range score and fails gracefully when the engine errors', () => { + const highSpawn: Spawn = () => ({ + status: 0, + stdout: JSON.stringify(successEnvelope({ score: 5, pass: true, reason: 'over' })), + stderr: '', + }) + const clamped = judge('r', { model: 'm' })({ answer: 1 }, okResult({ answer: 1 }), { spawn: highSpawn }) + assert.equal(clamped.score, 1) + + const failed = judge('r', { model: 'm' })({ answer: 1 }, okResult({ answer: 1 }), { spawn: throwingSpawn() }) + assert.equal(failed.pass, false) + assert.equal(failed.score, 0) +}) + +// --- gradeCase aggregation (acceptance criterion 2) -------------------------- + +test('gradeCase computes a weighted correctness across graders', () => { + // One passing (weight 3) + one failing (weight 1) → 3/4 = 0.75. + const graders: Grader[] = [ + assertGrader('a', () => true, { weight: 3 }), + assertGrader('b', () => false, { weight: 1 }), + ] + const { correctness, grades } = gradeCase(graders, { answer: 42 }, okResult({ answer: 42 })) + assert.equal(grades.length, 2) + assert.equal(correctness, 0.75) +}) + +test('gradeCase with no graders is 0, not NaN', () => { + const { correctness } = gradeCase([], null, okResult(null)) + assert.equal(correctness, 0) +}) + +// --- toRequest (case lowering) ----------------------------------------------- + +test('toRequest lowers a case to an EvalRequest and honours a prompt override', () => { + const c = baseCase({ systemPrompt: 'terse', tools: ['Read'] }) + const req = toRequest(c, 'claude-opus-4-8', 'OVERRIDDEN') + assert.equal(req.prompt, 'OVERRIDDEN') + assert.equal(req.model, 'claude-opus-4-8') + assert.equal(req.systemPrompt, 'terse') + assert.deepEqual(req.tools, ['Read']) + assert.equal(toRequest(c, 'm').prompt, 'What is 6 * 7?') +}) + +// --- runCase end-to-end (acceptance criterion 1) ----------------------------- + +test('runCase runs a case through the engine and surfaces correctness + envelope metrics', () => { + const spawn: Spawn = () => ({ + status: 0, + stdout: JSON.stringify(successEnvelope({ answer: 42 }, { cost: 0.0463, durationMs: 2419, numTurns: 2 })), + stderr: '', + }) + const r = runCase(baseCase(), { model: 'claude-haiku-4-5', spawn }) + assert.equal(r.caseId, 'add') + assert.equal(r.correctness, 1) // both graders pass + assert.equal(r.skipped, false) + assert.equal(r.result.cost, 0.0463) + assert.equal(r.result.durationMs, 2419) + assert.equal(r.result.numTurns, 2) +}) + +test('runCase marks a case skipped (no failure) when the CLI is unavailable', () => { + const r = runCase(baseCase(), { model: 'm', spawn: throwingSpawn() }) + assert.equal(r.skipped, true) + assert.equal(r.grades.length, 0) +}) + +// --- bounded parallel pool (acceptance criterion 4) -------------------------- + +test('mapPool never exceeds the concurrency bound and preserves order', async () => { + let inFlight = 0 + let maxInFlight = 0 + const worker = (n: number): Promise => + new Promise(resolve => { + inFlight++ + maxInFlight = Math.max(maxInFlight, inFlight) + setTimeout(() => { + inFlight-- + resolve(n * 2) + }, 5) + }) + const out = await mapPool([1, 2, 3, 4, 5, 6, 7], 2, worker) + assert.deepEqual(out, [2, 4, 6, 8, 10, 12, 14]) + assert.ok(maxInFlight <= 2, `maxInFlight=${maxInFlight}`) +}) + +test('runSuite fans a case set across models and returns one result per (case,model)', async () => { + const spawn: Spawn = () => ({ status: 0, stdout: JSON.stringify(successEnvelope({ answer: 42 })), stderr: '' }) + const results = await runSuite([baseCase({ id: 'a' }), baseCase({ id: 'b' })], { + models: ['claude-haiku-4-5', 'claude-sonnet-5'], + concurrency: 2, + spawn, + }) + assert.equal(results.length, 4) + assert.deepEqual( + results.map(r => `${r.caseId}:${r.model}`), + ['a:claude-haiku-4-5', 'a:claude-sonnet-5', 'b:claude-haiku-4-5', 'b:claude-sonnet-5'] + ) +}) + +// --- runAB deltas (acceptance criterion 3) ----------------------------------- + +/** A spawn that returns a cheaper/faster/correct envelope for the candidate prompt. */ +const abSpawn: Spawn = (_cmd, _args, input) => { + const isCandidate = (input ?? '').includes('CANDIDATE') + const env = isCandidate + ? successEnvelope({ answer: 42 }, { cost: 0.01, durationMs: 800, numTurns: 1 }) + : successEnvelope({ answer: 42 }, { cost: 0.05, durationMs: 2000, numTurns: 3 }) + return { status: 0, stdout: JSON.stringify(env), stderr: '' } +} + +test('runAB reports Δcorrectness/Δcost/Δlatency/Δturns (candidate cheaper/faster → positive)', () => { + const ab = runAB( + baseCase(), + { label: 'baseline', prompt: 'BASELINE: what is 6*7?' }, + { label: 'candidate', prompt: 'CANDIDATE: what is 6*7?' }, + { model: 'claude-haiku-4-5', spawn: abSpawn } + ) + assert.equal(ab.deltaCorrectness, 0) // both correct + assert.ok(ab.deltaCostUsd > 0, 'candidate cheaper → positive Δcost') + assert.ok(ab.deltaLatencyMs > 0, 'candidate faster → positive Δlatency') + assert.ok(ab.deltaTurns > 0, 'candidate fewer turns → positive Δturns') + assert.equal(ab.baselineLabel, 'baseline') + assert.equal(ab.candidateLabel, 'candidate') +}) + +test('runAB surfaces a negative Δcorrectness when the candidate regresses', () => { + // Candidate returns a wrong answer → its assertion grader fails. + const regressSpawn: Spawn = (_c, _a, input) => { + const answer = (input ?? '').includes('CANDIDATE') ? 0 : 42 + return { status: 0, stdout: JSON.stringify(successEnvelope({ answer })), stderr: '' } + } + const ab = runAB( + baseCase(), + { label: 'base', prompt: 'BASELINE q' }, + { label: 'cand', prompt: 'CANDIDATE q' }, + { model: 'm', spawn: regressSpawn } + ) + assert.ok(ab.deltaCorrectness < 0, 'candidate wrong → negative Δcorrectness') +}) + +// --- runMatrix grid (acceptance criterion 3) --------------------------------- + +test('runMatrix produces one row per model', async () => { + const spawn: Spawn = () => ({ status: 0, stdout: JSON.stringify(successEnvelope({ answer: 42 })), stderr: '' }) + const grid = await runMatrix(baseCase(), ['claude-opus-4-8', 'claude-sonnet-5', 'claude-haiku-4-5'], { spawn }) + assert.equal(grid.rows.length, 3) + assert.deepEqual(grid.rows.map(r => r.model), ['claude-opus-4-8', 'claude-sonnet-5', 'claude-haiku-4-5']) + assert.ok(grid.rows.every(r => r.correctness === 1)) +}) + +// --- report (acceptance criterion 5) ----------------------------------------- + +const sampleCaseResult = (id: string): CaseResult => ({ + caseId: id, + model: 'claude-haiku-4-5', + correctness: 1, + grades: [], + result: okResult({ answer: 42 }), + skipped: false, +}) + +test('formatReport renders a stable suite text table with all three axes', () => { + const report: Report = { mode: 'suite', cases: [sampleCaseResult('add')] } + const text = formatReport(report) + assert.match(text, /Case/) + assert.match(text, /Correct/) + assert.match(text, /Cost \$/) + assert.match(text, /Latency/) + assert.match(text, /100\.0%/) + assert.match(text, /\$0\.01000/) + assert.match(text, /1000ms/) +}) + +test('formatReport suite adds a clearly-labelled projected-cost column on request', () => { + const report: Report = { mode: 'suite', cases: [sampleCaseResult('add')] } + const text = formatReport(report, { projectModel: 'claude-opus-4-8' }) + assert.match(text, /Proj \$ \(claude-opus-4-8\)/) +}) + +test('projectCost uses the static table and returns null for an unpriced model', () => { + const usage = { inputTokens: 1_000_000, outputTokens: 1_000_000, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 } + assert.equal(projectCost(usage, 'claude-haiku-4-5'), 6) // 1 + 5 per Mtok + assert.equal(projectCost(usage, 'no-such-model'), null) +}) + +test('formatReport ab shows the delta columns and a trade-off legend', () => { + const ab: ABResult = runAB( + baseCase(), + { label: 'base', prompt: 'BASELINE q' }, + { label: 'cand', prompt: 'CANDIDATE q' }, + { model: 'm', spawn: abSpawn } + ) + const text = formatReport({ mode: 'ab', cases: [ab] }) + assert.match(text, /ΔCorrect/) + assert.match(text, /ΔCost/) + assert.match(text, /ΔLatency/) + assert.match(text, /candidate cheaper\/faster/) +}) + +test('toJSON returns the machine-comparable report object', () => { + const report: Report = { mode: 'suite', cases: [sampleCaseResult('add')] } + const json = toJSON(report) + assert.equal(json.mode, 'suite') + // Round-trips through JSON.stringify (what --json emits). + const parsed = JSON.parse(JSON.stringify(json)) as { mode: string; cases: unknown[] } + assert.equal(parsed.mode, 'suite') + assert.equal(parsed.cases.length, 1) +}) + +// --- CLI exit-code semantics (acceptance criterion 5) ------------------------ + +test('parseArgs reads every flag', () => { + const f = parseArgs(['--suite', 'd', '--filter', 'planner', '--ab', '--models', 'a,b', '--json', '--tolerance', '5']) + assert.equal(f.suite, 'd') + assert.equal(f.filter, 'planner') + assert.equal(f.ab, true) + assert.deepEqual(f.models, ['a', 'b']) + assert.equal(f.json, true) + assert.equal(f.tolerance, 5) +}) + +test('abExitCode gates only on a regression past tolerance', () => { + const mk = (delta: number): ABResult => ({ + caseId: 'c', + model: 'm', + baselineLabel: 'b', + candidateLabel: 'c', + baseline: sampleCaseResult('c'), + candidate: sampleCaseResult('c'), + deltaCorrectness: delta, + deltaCostUsd: 0, + deltaLatencyMs: 0, + deltaTurns: 0, + }) + // -3pp within a 5% tolerance → 0; -6pp past it → 1; an improvement → 0. + assert.equal(abExitCode([mk(-0.03)], 5), 0) + assert.equal(abExitCode([mk(-0.06)], 5), 1) + assert.equal(abExitCode([mk(0.1)], 5), 0) +}) + +test('main skips-with-notice and exits 0 when claude is unavailable', async () => { + const res = await main(['--suite', '/nonexistent', '--ab'], { spawn: throwingSpawn() }) + assert.equal(res.code, 0) + assert.match(res.output, /unavailable/) +}) + +/** A spawn where `--version` succeeds (available) and runs return a success envelope. */ +const availableSpawn = (envelope: unknown): Spawn => (_cmd, args) => { + if (args.includes('--version')) return { status: 0, stdout: 'claude 2.1.207', stderr: '' } + return { status: 0, stdout: JSON.stringify(envelope), stderr: '' } +} + +test('main runs a plain suite (report-only, exit 0) over injected cases', async () => { + const res = await main([], { spawn: availableSpawn(successEnvelope({ answer: 42 })), cases: [baseCase()] }) + assert.equal(res.code, 0) + assert.match(res.output, /Correct/) + assert.match(res.output, /add/) +}) + +test('main --json emits parseable machine JSON', async () => { + const res = await main(['--json'], { spawn: availableSpawn(successEnvelope({ answer: 42 })), cases: [baseCase()] }) + const parsed = JSON.parse(res.output) as { mode: string } + assert.equal(parsed.mode, 'suite') +}) + +test('main --ab exits 1 on a regression past tolerance and 0 when clean', async () => { + const abCase = (): EvalCase => + baseCase({ + variants: { baseline: { label: 'base', prompt: 'BASELINE q' }, candidate: { label: 'cand', prompt: 'CANDIDATE q' } }, + }) + // Candidate wrong → regression → exit 1. + const regressSpawn: Spawn = (_c, args, input) => { + if (args.includes('--version')) return { status: 0, stdout: 'claude 2.1.207', stderr: '' } + const answer = (input ?? '').includes('CANDIDATE') ? 0 : 42 + return { status: 0, stdout: JSON.stringify(successEnvelope({ answer })), stderr: '' } + } + const regressed = await main(['--ab', '--tolerance', '5'], { spawn: regressSpawn, cases: [abCase()] }) + assert.equal(regressed.code, 1) + + // Both correct → no regression → exit 0. + const cleanSpawn: Spawn = (_c, args) => { + if (args.includes('--version')) return { status: 0, stdout: 'claude 2.1.207', stderr: '' } + return { status: 0, stdout: JSON.stringify(successEnvelope({ answer: 42 })), stderr: '' } + } + const clean = await main(['--ab', '--tolerance', '5'], { spawn: cleanSpawn, cases: [abCase()] }) + assert.equal(clean.code, 0) +}) + +test('main --filter selects by id or tag', async () => { + const res = await main(['--json', '--filter', 'nope'], { + spawn: availableSpawn(successEnvelope({ answer: 42 })), + cases: [baseCase()], + }) + assert.match(res.output, /no cases matched/) +}) + +// --- suite loading (integration: real dynamic import from a temp dir) -------- + +test('loadSuite dynamically imports case modules from a directory', async () => { + const dir = mkdtempSync(join(tmpdir(), 'eval-suite-')) + const casePath = join(import.meta.dir, '..', '..', 'src', 'eval', 'case.ts').replace(/\\/g, '/') + writeFileSync( + join(dir, 'sample.ts'), + `import { defineCase } from '${casePath}'\n` + + `export const cases = [defineCase({ id: 'loaded', tags: ['t'], prompt: 'p', schema: { type: 'object' }, graders: [] })]\n` + ) + const cases = await loadSuite(dir) + assert.equal(cases.length, 1) + assert.equal(cases[0]?.id, 'loaded') +}) From 4d7471dd5d3fa55f5caaea818521aaa39a9d674b Mon Sep 17 00:00:00 2001 From: Christian Schuetz Date: Wed, 15 Jul 2026 00:59:31 -0500 Subject: [PATCH 3/3] test(evaluation-system): add harness prompt eval suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Modular eval cases exercising the harness's real agent prompts against their real forced schemas from schemas.generated.ts: planner, reviewer, refuter, and implementer. Each case carries a baseline prompt snapshot copied verbatim from its live stage (holes filled with inline fixtures), imports the matching schema constant, and grades correctness with pure assert/schemaConforms predicates. Fixtures embed a compact source-plan ask, a seeded-gap plan (an AC with no covering test + a dropped docs requirement), and an obviously-weak finding. index.ts aggregates the cases as the CLI --suite export; a hermetic smoke test (tests/eval/harness-suite.test.ts) proves every case is well-formed, ids/tags are unique, the suite loads through the D2 loader, and each case's graders discriminate a good canned structured_output from a bad one via fakeSpawn — no real claude. Entirely under src/eval/ + tests/, no plugin deployable churn. Implements D3 of the evaluation-system run. --- src/eval/suites/harness/fixtures/context.ts | 66 ++++++++ .../harness/fixtures/seeded-gap-plan.ts | 72 ++++++++ .../suites/harness/fixtures/source-plan.ts | 22 +++ .../suites/harness/fixtures/weak-finding.ts | 21 +++ src/eval/suites/harness/implementer.case.ts | 69 ++++++++ src/eval/suites/harness/index.ts | 25 +++ src/eval/suites/harness/planner.case.ts | 75 +++++++++ src/eval/suites/harness/refuter.case.ts | 48 ++++++ src/eval/suites/harness/reviewer.case.ts | 84 ++++++++++ tests/eval/harness-suite.test.ts | 158 ++++++++++++++++++ 10 files changed, 640 insertions(+) create mode 100644 src/eval/suites/harness/fixtures/context.ts create mode 100644 src/eval/suites/harness/fixtures/seeded-gap-plan.ts create mode 100644 src/eval/suites/harness/fixtures/source-plan.ts create mode 100644 src/eval/suites/harness/fixtures/weak-finding.ts create mode 100644 src/eval/suites/harness/implementer.case.ts create mode 100644 src/eval/suites/harness/index.ts create mode 100644 src/eval/suites/harness/planner.case.ts create mode 100644 src/eval/suites/harness/refuter.case.ts create mode 100644 src/eval/suites/harness/reviewer.case.ts create mode 100644 tests/eval/harness-suite.test.ts diff --git a/src/eval/suites/harness/fixtures/context.ts b/src/eval/suites/harness/fixtures/context.ts new file mode 100644 index 0000000..c5ac447 --- /dev/null +++ b/src/eval/suites/harness/fixtures/context.ts @@ -0,0 +1,66 @@ +// Shared fixture constants for the harness eval suite. Each case fills the +// runtime `${...}` holes of a live stage prompt with these concrete values, so a +// snapshot reads as a real agent invocation against a small, deterministic +// context (fidelity note in the D3 plan). Kept tiny so cases run fast and cache +// well. + +import type { JsonSchema } from '../../../types.ts' + +/** + * Adapt a generated schema constant to the engine's structural `JsonSchema` + * shape. The generated schemas use JSON-Schema `type: ["string","null"]` unions + * that don't match `JsonSchema`'s shallow `type?: string`, so a direct + * `X as JsonSchema` would trip TypeScript's "may be a mistake" narrowing guard + * and a double `as unknown as JsonSchema` is banned repo-wide + * (`tests/no-type-workarounds.test.ts`). Widening to `object` first lets a single + * assertion narrow cleanly. The engine only reads top-level `type`/`properties`/ + * `required`/`additionalProperties`, all present and correct on every schema. + */ +export function asSchema(schema: object): JsonSchema { + return schema as JsonSchema +} + +/** Run slug the fixture prompts are keyed to. */ +export const FIXTURE_SLUG = 'eval-fixture' +/** Scaffolded output directory the plan/review prompts reference. */ +export const FIXTURE_DIR = 'plans/runs/eval-fixture' +/** Conventions file every stage prompt points at. */ +export const FIXTURE_CONVENTIONS = 'plugins/strapped/conventions.md' +/** State script path the planner/coordinator prompts shell out to. */ +export const FIXTURE_STATE_SCRIPT = 'scripts/state.mjs' +/** Deterministic seed + effective budgets recorded in the manifest prompt. */ +export const FIXTURE_SEED = 1 +export const FIXTURE_PLAN_ROUNDS = 2 +export const FIXTURE_CODE_ROUNDS = 2 +export const FIXTURE_CONFIDENCE_MIN = 70 + +/** `repoList(a.repos)` output for a single-repo fixture run. */ +export const FIXTURE_REPOS = '- strapped → /home/user/strapped' + +/** + * The strapped operating context layered on the real agent prompt via + * `--append-system-prompt` (fidelity note). Deliberately compact — it names the + * harness role and the one hard rule an agent must respect, approximating the + * session-start context injection without pulling in the whole thing. + */ +export const STRAPPED_CONTEXT = `You are an agent inside the strapped orchestration harness. You return a single +schema-forced JSON completion — no tool use in this evaluation context. Honor the +stage contract exactly: produce the structured output the prompt asks for, and +never invent state you cannot verify.` + +/** + * The two plan-review lenses, snapshotted verbatim from `PLAN_LENSES` in + * `src/workflows/strapped-run/review-loop.ts`. The reviewer case runs the + * completeness lens (`a`, which catches an acceptance-criterion with no test); + * the soundness lens (`b`) is snapshotted here so the suite carries both. D4 may + * compact these strings — this is the baseline copy. + */ +export const PLAN_LENSES = { + a: 'completeness: is every element of the original ask covered by some deliverable? Hunt for missing requirements, unhandled edge cases, acceptance criteria without tests, and parts of the ask that silently disappeared', + b: 'soundness: wrong assumptions about the codebase, DAG dependency errors (missing or backwards deps, undeclared cross-deliverable coupling), deliverables that mix unrelated themes or whose estimated meaningful diff (excluding generated code, dependency bumps, and fixtures) exceeds ~1,000 lines and should be split, deliverables/chains that should be CONSOLIDATED (fragments of one theme, or a linear chain whose combined meaningful diff — excluding generated code, dependency bumps, and fixtures — is under the ~1,000-line threshold and could be a single deliverable/PR), planned work that is dead, duplicated, or superseded within the plan (steps or files a later step obviates, two deliverables doing the same work, or acceptance criteria/tests no remaining step produces), and steps that cannot work as written', +} as const + +/** A fixture assigned-rule block (`ruleBlock(rules)` output) for the reviewer prompt. */ +export const FIXTURE_RULE_BLOCK = '- CM-1 (CLAUDE.md): every new user-facing feature ships with a test that exercises it' +/** The rule ids the reviewer must return a checklist verdict for. */ +export const FIXTURE_RULE_IDS = 'CM-1' diff --git a/src/eval/suites/harness/fixtures/seeded-gap-plan.ts b/src/eval/suites/harness/fixtures/seeded-gap-plan.ts new file mode 100644 index 0000000..b074658 --- /dev/null +++ b/src/eval/suites/harness/fixtures/seeded-gap-plan.ts @@ -0,0 +1,72 @@ +// A fixture implementation plan for the reviewer case, carrying a DELIBERATELY +// SEEDED GAP: the JSON output mode (requirement 3 of the source ask, and AC2 +// below) has NO covering test, and the ask's requirement 4 (the "Dry run" docs +// section) is silently dropped — no deliverable produces it. A completeness +// reviewer should surface both as blocking gaps and mark AC2 a violation. +// +// The plan is embedded inline in the reviewer prompt because a single-shot eval +// has no filesystem to read the artifact files from (fidelity note in the D3 plan). + +export const SEEDED_GAP_PLAN = `=== manifest.md === +--- +status: in-review +seed: 1 +budgets: { plan_rounds: 2, code_rounds: 2, confidence_min: 70 } +repos: + - name: strapped + root: /home/user/strapped + config: plugins/strapped/conventions.md +deliverables: + - id: D1 + file: deliverables/D1-dry-run-resolver.md + repo: strapped + deps: [] +--- +Theme: add an offline \`--dry-run\` mode to the CLI. One deliverable extracts a +pure DAG resolver and wires the flag. + +DAG: + D1 + +=== deliverables/D1-dry-run-resolver.md === +--- +id: D1 +title: Dry-run resolver + --dry-run flag +deps: [] +repo: strapped +status: pending +branch: strapped/eval-fixture/D1-dry-run-resolver +base: main +worktree: null +pr: null +review_rounds_used: 0 +feedback_rounds_used: 0 +parked_reason: null +estimated_diff_lines: 300 +--- +## Context +Extract the wave-planning logic from the implement stage into a pure, +side-effect-free resolver, then add a \`--dry-run\` flag that prints the wave plan +without spawning agents or mutating state. + +## Files to touch +- src/workflows/strapped-run/resolver.ts — NEW pure resolver. +- src/scripts/run.ts — wire the \`--dry-run\` flag. + +## Implementation steps +1. Move the topological wave-planning code into \`resolver.ts\` as a pure function. +2. Add \`--dry-run\` to the run command: resolve, print the wave plan, exit without + side effects. +3. Add a \`--json\` sub-mode that emits the wave plan as a machine-readable object. + +## Acceptance criteria +1. \`--dry-run\` prints the wave plan and performs no state mutation, spawn, or + worktree creation. +2. \`--dry-run --json\` emits the wave plan as a machine-readable JSON object. + +## Tests +- resolver.test.ts — the pure resolver returns the correct topological wave order. +- dry-run.test.ts — \`--dry-run\` performs no state mutation (asserts no files change). + +## Out of scope +- Changing the live implement-stage dispatch behavior.` diff --git a/src/eval/suites/harness/fixtures/source-plan.ts b/src/eval/suites/harness/fixtures/source-plan.ts new file mode 100644 index 0000000..7638da7 --- /dev/null +++ b/src/eval/suites/harness/fixtures/source-plan.ts @@ -0,0 +1,22 @@ +// A compact but realistic source-plan ask used by the planner case (and reused +// as the "original ask" the reviewer/refuter check the fixture plan against). +// Small enough to run fast and cache well; concrete enough that a planner must +// split it into more than one deliverable. + +export const FIXTURE_SOURCE_PLAN = `Add an offline-friendly \`--dry-run\` mode to the strapped CLI. + +Requirements: +1. A new \`--dry-run\` flag on the \`run\` command that resolves the DAG and prints + the wave plan (which deliverables would dispatch, in what order) WITHOUT + spawning any agent, mutating any state file, or creating any worktree. +2. The dry-run planner must reuse the existing DAG-resolution code path — no + forked copy of the topological-sort logic. +3. A JSON output mode (\`--dry-run --json\`) emitting the same wave plan as a + machine-readable object for CI consumption. +4. Documentation: a short "Dry run" section in the CLI reference explaining the + flag and its guarantees (no side effects). + +Constraints: +- The wave-planning logic lives in the implement stage today; extracting a pure, + side-effect-free resolver is expected and should be independently testable. +- Every new user-facing behavior ships with a test that exercises it.` diff --git a/src/eval/suites/harness/fixtures/weak-finding.ts b/src/eval/suites/harness/fixtures/weak-finding.ts new file mode 100644 index 0000000..aba8bdc --- /dev/null +++ b/src/eval/suites/harness/fixtures/weak-finding.ts @@ -0,0 +1,21 @@ +// A fixture review finding for the refuter case that is CLEARLY NOT a real issue: +// it claims the plan never states where the resolver lives, but the fixture plan's +// "Files to touch" section names `resolver.ts` explicitly, and the ask itself +// calls for exactly that extraction. A skeptical refuter reading the inline plan +// should return `verdict: 'refuted'` with low confidence that the gap is real. + +export interface FixtureFinding { + severity: 'blocking' | 'concern' | 'suggestion' + location: string + what: string + why: string + evidence: string +} + +export const WEAK_FINDING: FixtureFinding = { + severity: 'blocking', + location: 'deliverables/D1-dry-run-resolver.md', + what: 'The plan never says which file the new resolver should live in, so an implementer cannot know where to put it.', + why: 'Without a named file the implementer has to guess, risking a scattered change.', + evidence: 'I could not find a target path for the resolver anywhere in the deliverable.', +} diff --git a/src/eval/suites/harness/implementer.case.ts b/src/eval/suites/harness/implementer.case.ts new file mode 100644 index 0000000..027589d --- /dev/null +++ b/src/eval/suites/harness/implementer.case.ts @@ -0,0 +1,69 @@ +// Harness eval case: the IMPLEMENTER prompt. +// +// The `prompt` is a baseline snapshot copied verbatim from the non-addendum +// branch of `implementPrompt` in `src/workflows/strapped-run/stages/implement.ts`, +// with the runtime `${item.*}`/`${cfg.*}` holes filled by fixture WaveItem values. +// A single-shot, tool-free eval cannot actually implement + validate on disk, so +// this case grades the SHAPE of the returned `ImplementResult` (a valid status +// enum, a boolean `validations_green`, and status/green/blocker consistency) — +// the contract every implementer completion must honor. +// D4 may compact this text and A/B the candidate against this baseline. + +import { defineCase } from '../../case.ts' +import { assert, schemaConforms } from '../../grade.ts' +import { IMPLEMENT_SCHEMA } from '../../../workflows/strapped-run/schemas.generated.ts' +import { asSchema, FIXTURE_DIR, FIXTURE_SLUG, STRAPPED_CONTEXT } from './fixtures/context.ts' + +/** Parsed implementer output shape (mirrors `ImplementResult`). */ +interface ImplementOutput { + status?: unknown + validations_green?: unknown + blocker?: unknown +} + +// Fixture WaveItem values the implement prompt interpolates. +const ITEM_ID = 'D1' +const ITEM_WORKTREE = '/home/user/strapped__worktrees/eval-fixture/D1-dry-run-resolver' +const ITEM_BRANCH = 'strapped/eval-fixture/D1-dry-run-resolver' +const ITEM_BASE = 'main' +const ITEM_REPO = 'strapped' +const ITEM_REPO_ROOT = '/home/user/strapped' +const ITEM_PLAN_FILE = `${FIXTURE_DIR}/deliverables/D1-dry-run-resolver.md` +const ITEM_VALIDATIONS = ['bun run typecheck', 'bun run lint', 'bun test'] + +const IMPLEMENTER_PROMPT = `You are the implementation agent for deliverable ${ITEM_ID} of strapped run "${FIXTURE_SLUG}". You have fresh context — everything you need is in the files below. + +Work EXCLUSIVELY inside the worktree: ${ITEM_WORKTREE} (branch ${ITEM_BRANCH}, based on ${ITEM_BASE}). This deliverable targets repo "${ITEM_REPO}" — never touch ${ITEM_REPO_ROOT} directly. + +1. Read your deliverable plan in full: ${ITEM_PLAN_FILE} +2. Read the shared research digest: ${FIXTURE_DIR}/research.md +3. Read the project guidelines: every CLAUDE.md that applies (repo root at minimum). + +Implement exactly what the plan specifies — its acceptance criteria are the contract. Write the tests the plan names (integration-style, public interfaces). Stay in scope: anything under "Out of scope" is off limits; note side-discoveries in your summary instead of fixing them. + +Before finishing, ALL validations must pass inside the worktree: +${ITEM_VALIDATIONS.map(v => `- ${v}`).join('\n')} + +Commit your work on ${ITEM_BRANCH} with a Conventional-Commits message (\`(${FIXTURE_SLUG}): \` — scope is the run slug, no \`${ITEM_ID}:\` title prefix; reference ${ITEM_ID} in the body). If validations pass, commit and return status "implemented" with validations_green true. If you hit a blocker you cannot resolve (missing dependency, contradictory plan, validation failure you cannot fix), commit what is safe, return status "blocked" with the blocker described — do NOT loop indefinitely.` + +export const implementerCase = defineCase({ + id: 'implementer', + tags: ['implementer'], + appendSystemPrompt: STRAPPED_CONTEXT, + prompt: IMPLEMENTER_PROMPT, + schema: asSchema(IMPLEMENT_SCHEMA), + graders: [ + schemaConforms(), + assert('valid-status', o => { + const s = (o as ImplementOutput).status + return s === 'implemented' || s === 'blocked' + }), + assert('validations-green-boolean', o => typeof (o as ImplementOutput).validations_green === 'boolean'), + // Discriminator: an "implemented" verdict is only coherent when validations + // actually went green — a self-inconsistent envelope must not pass. + assert('implemented-implies-green', o => { + const out = o as ImplementOutput + return out.status !== 'implemented' || out.validations_green === true + }), + ], +}) diff --git a/src/eval/suites/harness/index.ts b/src/eval/suites/harness/index.ts new file mode 100644 index 0000000..5728203 --- /dev/null +++ b/src/eval/suites/harness/index.ts @@ -0,0 +1,25 @@ +// The harness prompt-eval SUITE: modular cases that exercise the strapped +// harness's REAL agent prompts (baseline snapshots) against their REAL forced +// schemas from `schemas.generated.ts`. Each case is one file so covering a new +// harness feature is a single-file add. The D2 CLI loads the `cases` export +// below when pointed at this directory. +// +// Run it live (needs a real `claude`; NOT part of `bun test`): +// bun run eval --suite src/eval/suites/harness +// bun run eval --suite src/eval/suites/harness --filter reviewer # by tag/id +// bun run eval --suite src/eval/suites/harness --matrix # model grid +// bun run eval --suite src/eval/suites/harness --json > report.json +// A/B (`--ab`) compares a case's baseline prompt against a candidate variant; +// D4 adds the candidates when it compacts the live stage prompts. +// +// `tests/eval/harness-suite.test.ts` proves every case is well-formed and its +// graders discriminate good from bad canned outputs — fully hermetic, no `claude`. + +import type { EvalCase } from '../../case.ts' +import { plannerCase } from './planner.case.ts' +import { reviewerCase } from './reviewer.case.ts' +import { refuterCase } from './refuter.case.ts' +import { implementerCase } from './implementer.case.ts' + +/** The suite the CLI `--suite` loads (`cases` is the magic export name). */ +export const cases: EvalCase[] = [plannerCase, reviewerCase, refuterCase, implementerCase] diff --git a/src/eval/suites/harness/planner.case.ts b/src/eval/suites/harness/planner.case.ts new file mode 100644 index 0000000..35b7531 --- /dev/null +++ b/src/eval/suites/harness/planner.case.ts @@ -0,0 +1,75 @@ +// Harness eval case: the PLANNER prompt. +// +// The `prompt` below is a baseline snapshot copied verbatim from the planner +// `agent(…)` call in `src/workflows/strapped-run/stages/plan.ts`, with +// the runtime `${...}` holes filled by fixture constants. Do NOT import the stage +// module — its prompt is interpolated at runtime; this is an eval INPUT. +// D4 may compact this text and A/B the candidate against this baseline. + +import { defineCase } from '../../case.ts' +import { assert, schemaConforms } from '../../grade.ts' +import { PLAN_SCHEMA } from '../../../workflows/strapped-run/schemas.generated.ts' +import { + asSchema, + FIXTURE_CONFIDENCE_MIN, + FIXTURE_CONVENTIONS, + FIXTURE_CODE_ROUNDS, + FIXTURE_DIR, + FIXTURE_PLAN_ROUNDS, + FIXTURE_REPOS, + FIXTURE_SEED, + FIXTURE_SLUG, + FIXTURE_STATE_SCRIPT, + STRAPPED_CONTEXT, +} from './fixtures/context.ts' +import { FIXTURE_SOURCE_PLAN } from './fixtures/source-plan.ts' + +/** Parsed planner output shape (mirrors `PlanResult`). */ +interface PlanOutput { + deliverables?: Array<{ id?: unknown; file?: unknown; title?: unknown; deps?: unknown }> + summary?: unknown +} + +const nonEmptyString = (v: unknown): boolean => typeof v === 'string' && v.trim().length > 0 + +const PLANNER_PROMPT = `You are the planning agent for strapped run "${FIXTURE_SLUG}". Produce a complete, reviewable implementation plan from a large source plan document. + +Source plan (the original ask): ${FIXTURE_SOURCE_PLAN} +Target repos (the run state is keyed by the run slug, not by any repo; the work spans these repos — an unordered set): +${FIXTURE_REPOS} +Output directory (already scaffolded): ${FIXTURE_DIR} +Conventions you MUST follow for every file format: ${FIXTURE_CONVENTIONS} + +Procedure: +1. Read the source plan in full, then research each target repo's codebase thoroughly: architecture, the modules the ask touches, existing utilities to reuse, test patterns. +2. Write ${FIXTURE_DIR}/research.md — a distilled digest (~300 lines max): architecture notes, key files with one-line roles, library/API findings, decisions with rationale, known pitfalls. This is the only research context implementers will ever see. +3. Split the work into deliverables by discrete theme, forming a DAG: independent work has no deps, dependent work lists its parent deliverable ids. Keep one coherent theme in a single deliverable so a reviewer can grasp the whole change in one PR — split a theme into multiple deliverables only when its estimated meaningful diff (excluding generated code, dependency/lockfile bumps, generated clients/schemas, vendored code, and large fixtures) exceeds ~1,000 changed lines. Prefer a few cohesive, independently-shippable nodes over many fragments that scatter one theme across PRs. Assign each deliverable to exactly one target repo. +4. Write one self-contained file per deliverable at ${FIXTURE_DIR}/deliverables/-.md per the conventions (frontmatter: id, title, deps, repo: , status: pending, branch: strapped/${FIXTURE_SLUG}/-, base, worktree: null, pr: null, review_rounds_used: 0, feedback_rounds_used: 0, parked_reason: null, estimated_diff_lines; body: Context slice from your research, Files to touch, Implementation steps, Acceptance criteria, Tests, Out of scope). Set base per the cross-repo base rule: a deliverable's base is a parent branch WITHIN THE SAME repo, otherwise that repo's main (roots, and any cross-repo child, base on their own repo's main — you can never branch across repos). A fresh implementer seeded with ONLY this file plus research.md must be able to do the work. +5. Cross-repo deps are ordering-only, NEVER a code dependency: a cross-repo child bases on its own repo's main and does not have its parent's unmerged code. Reject or restructure any plan where a cross-repo child has a true code dependency on its parent — either require the shared change to merge to the parent repo's main first, or keep both sides in the same repo/chain. +6. Write ${FIXTURE_DIR}/manifest.md per the conventions (status: in-review, seed: ${FIXTURE_SEED}, budgets — record the EFFECTIVE budgets of this run: plan_rounds: ${FIXTURE_PLAN_ROUNDS}, code_rounds: ${FIXTURE_CODE_ROUNDS}, confidence_min: ${FIXTURE_CONFIDENCE_MIN} — the repos: map listing every target repo above per the conventions — name, root, config path (repos: is an unordered set, no repo is special); the deliverables list with ids/files/repos/deps, theme summary, ASCII DAG sketch). +7. After all plan artifacts are written, run \`node ${FIXTURE_STATE_SCRIPT} commit ${FIXTURE_DIR}\` via Bash so the run's state root is git-backed from birth (it git-inits the state root if absent and commits the artifacts). Best-effort: proceed even if it reports an error. + +Return the deliverable list and a one-paragraph summary.` + +export const plannerCase = defineCase({ + id: 'planner', + tags: ['planner'], + appendSystemPrompt: STRAPPED_CONTEXT, + prompt: PLANNER_PROMPT, + schema: asSchema(PLAN_SCHEMA), + graders: [ + schemaConforms(), + // Discriminator: the ask has independently-testable pieces (a pure resolver, + // the flag, the JSON mode, docs) → a real planner returns more than one. + assert('at-least-two-deliverables', o => { + const ds = (o as PlanOutput).deliverables + return Array.isArray(ds) && ds.length >= 2 + }), + assert('deliverables-well-formed', o => { + const ds = (o as PlanOutput).deliverables + if (!Array.isArray(ds) || ds.length === 0) return false + return ds.every(d => nonEmptyString(d.id) && nonEmptyString(d.file) && nonEmptyString(d.title) && Array.isArray(d.deps)) + }), + assert('non-empty-summary', o => nonEmptyString((o as PlanOutput).summary)), + ], +}) diff --git a/src/eval/suites/harness/refuter.case.ts b/src/eval/suites/harness/refuter.case.ts new file mode 100644 index 0000000..ebb3484 --- /dev/null +++ b/src/eval/suites/harness/refuter.case.ts @@ -0,0 +1,48 @@ +// Harness eval case: the REFUTER prompt. +// +// The `prompt` is a baseline snapshot copied verbatim from `refutePrompt` in +// `src/workflows/strapped-run/review-loop.ts`, with the runtime `${...}` holes +// filled by fixture constants and the fixture plan embedded inline (a single-shot +// eval has no filesystem to read the artifact files from). The finding under +// scrutiny is deliberately weak — the plan names the resolver's file explicitly — +// so a skeptical refuter should return `verdict: 'refuted'`. +// D4 may compact this text and A/B the candidate against this baseline. + +import { defineCase } from '../../case.ts' +import { assert, schemaConforms } from '../../grade.ts' +import { REFUTE_SCHEMA } from '../../../workflows/strapped-run/schemas.generated.ts' +import { asSchema, FIXTURE_DIR, FIXTURE_REPOS, STRAPPED_CONTEXT } from './fixtures/context.ts' +import { FIXTURE_SOURCE_PLAN } from './fixtures/source-plan.ts' +import { SEEDED_GAP_PLAN } from './fixtures/seeded-gap-plan.ts' +import { WEAK_FINDING } from './fixtures/weak-finding.ts' + +/** Parsed refuter output shape (mirrors `RefuteResult`). */ +interface RefuteOutput { + verdict?: unknown +} + +const REFUTER_PROMPT = `You are a skeptical verifier with fresh context. A plan reviewer claims the following gap in the implementation plan at ${FIXTURE_DIR} (original ask: ${FIXTURE_SOURCE_PLAN}). Target repos you may explore to check the claim: +${FIXTURE_REPOS} + +Claim [${WEAK_FINDING.severity}] at ${WEAK_FINDING.location}: ${WEAK_FINDING.what} +Why: ${WEAK_FINDING.why} +Evidence: ${WEAK_FINDING.evidence} + +Your stance: this is NOT a real gap unless the documents prove otherwise. Read the ask and the plan files yourself — the claimed-missing item may be covered elsewhere in the plan, the assumption may actually hold in the codebase, or the claim may misread the ask. Return your verdict, a corrected confidence (0-100) that the gap is real, and one line of evidence. + +--- Plan under review (inlined for this single-shot eval; the files above are provided here verbatim) --- +${SEEDED_GAP_PLAN}` + +export const refuterCase = defineCase({ + id: 'refuter', + tags: ['refuter'], + appendSystemPrompt: STRAPPED_CONTEXT, + prompt: REFUTER_PROMPT, + schema: asSchema(REFUTE_SCHEMA), + graders: [ + schemaConforms(), + // Discriminator: the finding is not real (the plan's "Files to touch" names + // resolver.ts), so the correct verdict is `refuted`. + assert('verdict-refuted', o => (o as RefuteOutput).verdict === 'refuted'), + ], +}) diff --git a/src/eval/suites/harness/reviewer.case.ts b/src/eval/suites/harness/reviewer.case.ts new file mode 100644 index 0000000..605da91 --- /dev/null +++ b/src/eval/suites/harness/reviewer.case.ts @@ -0,0 +1,84 @@ +// Harness eval case: the plan REVIEWER prompt. +// +// The `prompt` is a baseline snapshot copied verbatim from `reviewerPrompt` in +// `src/workflows/strapped-run/review-loop.ts` (the completeness lens `a`, which is +// the one that catches an acceptance-criterion with no test), with the runtime +// `${...}` holes filled by fixture constants. Both `PLAN_LENSES` are snapshotted +// in `fixtures/context.ts`. The fixture plan under review is embedded inline +// because a single-shot eval has no filesystem to read the artifact files from. +// D4 may compact this text and A/B the candidate against this baseline. + +import { defineCase } from '../../case.ts' +import { assert, schemaConforms } from '../../grade.ts' +import { FINDINGS_SCHEMA } from '../../../workflows/strapped-run/schemas.generated.ts' +import { + asSchema, + FIXTURE_CONVENTIONS, + FIXTURE_CONFIDENCE_MIN, + FIXTURE_DIR, + FIXTURE_REPOS, + FIXTURE_RULE_BLOCK, + FIXTURE_RULE_IDS, + PLAN_LENSES, + STRAPPED_CONTEXT, +} from './fixtures/context.ts' +import { FIXTURE_SOURCE_PLAN } from './fixtures/source-plan.ts' +import { SEEDED_GAP_PLAN } from './fixtures/seeded-gap-plan.ts' + +/** Parsed reviewer output shape (mirrors `FindingsResult`). */ +interface FindingsOutput { + findings?: Array<{ severity?: unknown; what?: unknown }> + ac_checklist?: Array<{ id?: unknown; verdict?: unknown }> +} + +const REVIEWER_PROMPT = `You are an adversarial plan reviewer with fresh context. Your job is to find real gaps between a produced implementation plan and the original ask, before any code is written. + +Original ask: ${FIXTURE_SOURCE_PLAN} +Plan under review, in ${FIXTURE_DIR}: manifest.md, research.md, and every file in deliverables/. +Conventions the plan must follow: ${FIXTURE_CONVENTIONS} +Target repos (explore any of these as needed to check the plan's claims against reality): +${FIXTURE_REPOS} + +Read the original ask first, then the whole plan, then verify claims against the actual codebase(s) where they matter. + +Your lens (your main hunting ground beyond the rules): ${PLAN_LENSES.a}. +Your assigned guideline rules — you are the ONLY reviewer checking the plan against these, so check every one explicitly (does the plan instruct or imply work that would violate the rule?): +${FIXTURE_RULE_BLOCK} + +Known findings from earlier rounds — do NOT re-report unless the revision failed to address them: +(none — first round) + +Enumerated AC checklist — you and the other reviewer BOTH return this every round (it is NOT partitioned like the guideline rules): read EVERY artifact file's \`## Acceptance criteria\` section, enumerate each item in order as AC1..ACn across the whole artifact, and return one ac_checklist entry per item ({ id: "AC", verdict: pass|violation|na, evidence: one line }). An item the plan fails to satisfy, or that no step/test covers, is a BLOCKING finding carrying full guideline-rule weight — enumerating and checking these items is as load-bearing as the rule checklist. If no file has a \`## Acceptance criteria\` section, return \`ac_checklist: []\`. + +Severity: "blocking" = the plan as written produces wrong or missing work; "concern" = likely gap needing a fix or an explicit justification; "suggestion" = optional polish (never drives revision). Stable key format ":". Confidence under ${FIXTURE_CONFIDENCE_MIN} will be dropped. + +You MUST return a rule_checklist verdict (pass/violation/na + one line of evidence) for every assigned rule (${FIXTURE_RULE_IDS}), the ac_checklist covering every AC item, plus your findings. Round: 1. + +--- Artifact under review (inlined for this single-shot eval; the files above are provided here verbatim) --- +${SEEDED_GAP_PLAN}` + +export const reviewerCase = defineCase({ + id: 'reviewer', + tags: ['reviewer'], + appendSystemPrompt: STRAPPED_CONTEXT, + prompt: REVIEWER_PROMPT, + schema: asSchema(FINDINGS_SCHEMA), + graders: [ + schemaConforms(), + // Discriminator: the seeded gap is an AC (the --dry-run --json mode) with no + // covering test, plus the dropped docs requirement. A completeness reviewer + // must surface it — either as a gating finding mentioning the missing + // test/JSON coverage, or as an `ac_checklist` violation. + assert('surfaces-seeded-gap', o => { + const out = o as FindingsOutput + const findings = Array.isArray(out.findings) ? out.findings : [] + const ac = Array.isArray(out.ac_checklist) ? out.ac_checklist : [] + const gapInFindings = findings.some(f => { + const what = typeof f.what === 'string' ? f.what.toLowerCase() : '' + return f.severity !== 'suggestion' && (what.includes('test') || what.includes('json') || what.includes('doc')) + }) + const gapInAc = ac.some(a => a.verdict === 'violation') + return gapInFindings || gapInAc + }), + ], +}) diff --git a/tests/eval/harness-suite.test.ts b/tests/eval/harness-suite.test.ts new file mode 100644 index 0000000..f706e98 --- /dev/null +++ b/tests/eval/harness-suite.test.ts @@ -0,0 +1,158 @@ +// Harness eval SUITE smoke test — fully hermetic (no real `claude`). +// +// Proves the eval CONTENT is sound without spending a cent: +// 1. every case is well-formed (id/tags/prompt/schema/graders present), +// 2. ids and primary tags are unique across the suite, +// 3. each case's graders return the expected verdict against a canned +// `structured_output` — a GOOD output grades to correctness 1, a BAD one +// (the discriminator's failure mode) grades strictly below 1 — driven +// through the D2 runner (`runCase`) with the output fed via `fakeSpawn`, +// 4. the suite `index` loads through the D2 CLI loader (`loadSuite`). + +import assert from 'node:assert/strict' +import { join } from 'node:path' +import { test } from 'bun:test' +import { runCase } from '../../src/eval/runner.ts' +import { loadSuite } from '../../src/eval/cli.ts' +import type { EvalCase } from '../../src/eval/case.ts' +import { cases } from '../../src/eval/suites/harness/index.ts' +import { plannerCase } from '../../src/eval/suites/harness/planner.case.ts' +import { reviewerCase } from '../../src/eval/suites/harness/reviewer.case.ts' +import { refuterCase } from '../../src/eval/suites/harness/refuter.case.ts' +import { implementerCase } from '../../src/eval/suites/harness/implementer.case.ts' +import { fakeSpawn, successEnvelope } from '../helpers/fake-claude.ts' + +const SUITE_DIR = join(import.meta.dir, '..', '..', 'src', 'eval', 'suites', 'harness') + +/** A canned good/bad structured_output pair for one case's graders. */ +interface Fixture { + case: EvalCase + good: unknown + bad: unknown +} + +const FIXTURES: Fixture[] = [ + { + case: plannerCase, + // Three well-formed deliverables + a summary → every planner grader passes. + good: { + deliverables: [ + { id: 'D1', file: 'deliverables/D1-resolver.md', title: 'Pure dry-run resolver', deps: [] }, + { id: 'D2', file: 'deliverables/D2-flag.md', title: 'Wire --dry-run flag', deps: ['D1'] }, + { id: 'D3', file: 'deliverables/D3-json-and-docs.md', title: 'JSON mode + docs', deps: ['D2'] }, + ], + summary: 'Extract a pure resolver, then layer the --dry-run flag, JSON mode, and docs on top.', + }, + // Schema-valid but empty: fails at-least-two-deliverables + non-empty-summary. + bad: { deliverables: [], summary: '' }, + }, + { + case: reviewerCase, + // A blocking finding naming the missing test + an AC violation → gap surfaced. + good: { + findings: [ + { + id: 'g1', + key: 'gap:deliverables/D1-dry-run-resolver.md', + rule: null, + severity: 'blocking', + location: 'deliverables/D1-dry-run-resolver.md', + what: 'The --dry-run --json mode (AC2) has no covering test.', + why: 'A user-facing behavior ships untested, violating the ask.', + evidence: 'Tests section lists resolver.test.ts and dry-run.test.ts only.', + confidence: 90, + recommendation: 'Add a test asserting the JSON wave-plan output shape.', + }, + ], + rule_checklist: [{ rule: 'CM-1', verdict: 'violation', evidence: 'AC2 has no test' }], + ac_checklist: [ + { id: 'AC1', verdict: 'pass', evidence: 'covered by dry-run.test.ts' }, + { id: 'AC2', verdict: 'violation', evidence: 'no test covers the JSON mode' }, + ], + }, + // Reviewer waved everything through → the seeded gap is NOT surfaced. + bad: { + findings: [], + rule_checklist: [{ rule: 'CM-1', verdict: 'pass', evidence: 'looks fine' }], + ac_checklist: [ + { id: 'AC1', verdict: 'pass', evidence: 'ok' }, + { id: 'AC2', verdict: 'pass', evidence: 'ok' }, + ], + }, + }, + { + case: refuterCase, + good: { verdict: 'refuted', confidence: 15, evidence: 'The deliverable names resolver.ts in Files to touch.' }, + bad: { verdict: 'confirmed', confidence: 90, evidence: 'No target file found.' }, + }, + { + case: implementerCase, + // Honest single-shot outcome: blocked, validations not green, blocker set. + good: { status: 'blocked', summary: 'Cannot implement in a single-shot eval.', validations_green: false, blocker: 'no filesystem/tool access to write code or run validations' }, + // Self-inconsistent: claims implemented while validations are red. + bad: { status: 'implemented', summary: 'done', validations_green: false, blocker: null }, + }, +] + +// --- acceptance criterion 1: every case is well-formed ----------------------- + +test('every harness case is well-formed (id/tags/prompt/schema/graders)', () => { + assert.equal(cases.length, 4) + for (const c of cases) { + assert.ok(typeof c.id === 'string' && c.id.length > 0, `case id: ${c.id}`) + assert.ok(Array.isArray(c.tags) && c.tags.length > 0, `case ${c.id} has tags`) + assert.ok(typeof c.prompt === 'string' && c.prompt.trim().length > 0, `case ${c.id} has a prompt`) + assert.ok(c.schema && typeof c.schema === 'object', `case ${c.id} has a schema`) + assert.equal(c.schema.type, 'object', `case ${c.id} schema is an object schema`) + assert.ok(c.schema.properties && typeof c.schema.properties === 'object', `case ${c.id} schema has properties`) + assert.ok(Array.isArray(c.graders) && c.graders.length >= 1, `case ${c.id} has ≥1 grader`) + } +}) + +test('the four required harness cases are present by id', () => { + const ids = cases.map(c => c.id).sort() + assert.deepEqual(ids, ['implementer', 'planner', 'refuter', 'reviewer']) +}) + +// --- acceptance criterion: ids + tags unique --------------------------------- + +test('case ids and primary tags are unique across the suite', () => { + const ids = cases.map(c => c.id) + assert.equal(new Set(ids).size, ids.length, 'ids are unique') + const tags = cases.flatMap(c => c.tags) + assert.equal(new Set(tags).size, tags.length, 'no tag is shared between cases') + for (const expected of ['planner', 'reviewer', 'refuter', 'implementer']) { + assert.equal(cases.filter(c => c.tags.includes(expected)).length, 1, `exactly one case tagged ${expected}`) + } +}) + +// --- acceptance criterion 2: graders discriminate good from bad -------------- + +for (const fx of FIXTURES) { + test(`${fx.case.id} case: graders pass a good output and fail a bad one (canned, hermetic)`, () => { + const good = runCase(fx.case, { model: 'claude-haiku-4-5', spawn: fakeSpawn(successEnvelope(fx.good)) }) + assert.equal(good.skipped, false) + assert.equal(good.result.ok, true, `${fx.case.id} good output conforms to its forced schema`) + assert.equal(good.correctness, 1, `${fx.case.id} good output grades to full correctness`) + + const bad = runCase(fx.case, { model: 'claude-haiku-4-5', spawn: fakeSpawn(successEnvelope(fx.bad)) }) + assert.ok(bad.correctness < 1, `${fx.case.id} bad output grades below full correctness (was ${bad.correctness})`) + // At least one named grader must have flipped — proving the discriminator bites. + assert.ok(bad.grades.some(g => !g.pass), `${fx.case.id} bad output trips a grader`) + }) +} + +// --- acceptance criterion 3/4: the suite loads through the D2 CLI loader ------ + +test('loadSuite loads exactly the four harness cases from the suite directory', async () => { + const loaded = await loadSuite(SUITE_DIR) + assert.equal(loaded.length, 4) + assert.deepEqual(loaded.map(c => c.id).sort(), ['implementer', 'planner', 'refuter', 'reviewer']) +}) + +test('the imported case objects are identical to the suite export', () => { + assert.equal(cases.includes(plannerCase), true) + assert.equal(cases.includes(reviewerCase), true) + assert.equal(cases.includes(refuterCase), true) + assert.equal(cases.includes(implementerCase), true) +})