diff --git a/.gitignore b/.gitignore
index c2658d7..8c8f03b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,2 @@
 node_modules/
+.eval-cache/
diff --git a/CLAUDE.md b/CLAUDE.md
index 1a70ceb..43a13b8 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -1,3 +1,5 @@
 # Guidelines
 
+- **Run the prompt-eval suite alongside `bun test` when changing harness prompts.** When you edit an agent prompt — anything under `src/workflows/strapped-run/**` or the step prose in a skill `SKILL.md` — run `bun run eval --suite <dir>` (see `src/eval/` and the "Prompt evaluation suite" section of `plugins/strapped/conventions.md`) and keep it green alongside `bun test`. The eval suite grades a prompt on correctness / cost / latency through `claude -p`; it is the evidence that a prompt change is a real improvement (use `--ab` to compare a baseline vs a candidate and gate on a correctness regression past `--tolerance`). It is a HEAVY, opt-in layer (needs a real `claude` + spends cost), so it is intentionally NOT part of `bun test` — run it yourself for prompt work. `bun test` stays hermetic/offline and remains the gate for everything else.
+
 - **Bump the plugin version with the tool, never by hand.** `claude plugin update` compares `version` in `plugins/strapped/.claude-plugin/plugin.json` only — an unbumped version leaves installed copies silently pinned to a stale commit after a change merges. Bump ONLY via `bun tools/version.ts bump <major|minor|patch>` (it rewrites the one semver deterministically); do not hand-edit plugin.json. Pick the level by a conventional-commit rule: a breaking change → `major`, a new user-facing feature (skill/workflow/script behavior) → `minor`, a fix or internal change → `patch`. Before committing, run `bun tools/version.ts check` (alias `bun run version:check`) — it fails when a changed *generated artifact* (the committed deployables under `plugins/strapped/`) is not accompanied by a version bump above the base. Prompt/skill/docs-only changes with NO generated-artifact change are at your discretion — the guard exits 0 for them, so bump when the change is user-facing and skip an internal-only tweak.
diff --git a/package.json b/package.json
index d08f348..2d4e11e 100644
--- a/package.json
+++ b/package.json
@@ -8,6 +8,7 @@
     "lint": "oxlint src tools tests",
     "version:bump": "bun tools/version.ts bump",
     "version:check": "bun tools/version.ts check",
+    "eval": "bun src/eval/cli.ts",
     "test": "claude plugin validate . --strict && claude plugin validate plugins/strapped --strict && bun run typecheck && bun run lint && bun test"
   },
   "devDependencies": {
diff --git a/plugins/strapped/conventions.md b/plugins/strapped/conventions.md
index 5542b56..aede133 100644
--- a/plugins/strapped/conventions.md
+++ b/plugins/strapped/conventions.md
@@ -398,6 +398,12 @@ This limit is why the retired multi-file architecture (stage workflows dispatchi
 
 Every command in the **deliverable's repo's** config `validations` must be green before code review and after every fix round, run inside the deliverable's worktree.
 
+### Prompt evaluation suite
+
+The strapped repo carries a prompt-effectiveness eval suite under `src/eval/` (run via `bun run eval`, not bundled into any plugin deployable). It grades the harness's own agent prompts on three axes — **correctness** (layered graders: schema-conformance, assertion predicates, an optional LLM judge), **cost** (`total_cost_usd` + token usage), and **latency** (`duration_ms` / `num_turns`) — by shelling out to `claude -p` (never the Anthropic SDK) and reading the `--output-format json` envelope. It supports **A/B** (baseline vs candidate prompt at one model, reporting Δcorrectness / Δcost / Δlatency side by side so a human judges the trade — a correctness dip that buys a large cost/latency win is a WIN) and a **model matrix** (`--models opus,sonnet,haiku`).
+
+It is the **heavy, opt-in test layer for prompt changes**: it needs a real `claude` and spends cost, so it is deliberately kept OUT of `bun test` (which stays hermetic/offline). When you change an agent prompt (`src/workflows/strapped-run/**` or a skill's `SKILL.md` step prose), run the eval suite alongside `bun test` and keep both green — it is the evidence that the prompt change is a real improvement. An absent `claude` CLI is a graceful skip (notice + exit 0), and only `--ab` gates: exit non-zero on a correctness regression past `--tolerance`.
+
 ## Worktrees and branches
 
 All repo-scoped values below come from the **deliverable's repo** — `manifest.repos[deliverable.repo]` for the root, and that repo's config for `worktreeRoot`, `validations`, and `provisioning`.
diff --git a/src/eval/cache.ts b/src/eval/cache.ts
new file mode 100644
index 0000000..811fd55
--- /dev/null
+++ b/src/eval/cache.ts
@@ -0,0 +1,70 @@
+// Content-addressed response cache for eval results. The key is a SHA-256 over a
+// CANONICAL (sorted-key) JSON of every request field that changes the model's
+// output — prompt, both system-prompt knobs, model, schema, tool policy, and
+// settings. Include every such field or an A/B would collide; a changed prompt
+// naturally misses. Results are stored one JSON file per key; no eviction.
+
+import { createHash } from 'node:crypto'
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import type { Cache, EvalRequest, EvalResult } from './types.ts'
+
+/** Recursively sort object keys so key order never perturbs the hash. */
+function sortKeys(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(sortKeys)
+  if (value !== null && typeof value === 'object') {
+    const sorted: Record<string, unknown> = {}
+    for (const key of Object.keys(value as Record<string, unknown>).sort()) {
+      sorted[key] = sortKeys((value as Record<string, unknown>)[key])
+    }
+    return sorted
+  }
+  return value
+}
+
+/** Stable, key-sorted JSON of the output-affecting fields of a request. */
+function canonicalRequest(req: EvalRequest): string {
+  return JSON.stringify(
+    sortKeys({
+      prompt: req.prompt,
+      systemPrompt: req.systemPrompt ?? null,
+      appendSystemPrompt: req.appendSystemPrompt ?? null,
+      model: req.model,
+      schema: req.schema,
+      tools: req.tools ?? null,
+      settings: req.settings ?? null,
+    })
+  )
+}
+
+/** Hex SHA-256 cache key over a request's output-affecting fields. */
+export function cacheKey(req: EvalRequest): string {
+  return createHash('sha256').update(canonicalRequest(req)).digest('hex')
+}
+
+/** File-backed response cache: one `<dir>/<key>.json` per stored result. */
+export class ResponseCache implements Cache {
+  constructor(private readonly dir: string) {}
+
+  private path(key: string): string {
+    return join(this.dir, `${key}.json`)
+  }
+
+  /** Return the stored result (marked `cached:true`) or `null` on miss/corruption. */
+  get(key: string): EvalResult | null {
+    const p = this.path(key)
+    if (!existsSync(p)) return null
+    try {
+      const result = JSON.parse(readFileSync(p, 'utf8')) as EvalResult
+      return { ...result, cached: true }
+    } catch {
+      return null
+    }
+  }
+
+  /** Persist a result under `key`, creating the cache dir on first write. */
+  set(key: string, result: EvalResult): void {
+    mkdirSync(this.dir, { recursive: true })
+    writeFileSync(this.path(key), JSON.stringify(result, null, 2))
+  }
+}
diff --git a/src/eval/case.ts b/src/eval/case.ts
new file mode 100644
index 0000000..ba073ca
--- /dev/null
+++ b/src/eval/case.ts
@@ -0,0 +1,77 @@
+// The eval CASE abstraction: a single, self-contained prompt-evaluation unit. A
+// case carries the prompt, the schema it is forced against, isolation knobs
+// (system prompt / tools / settings), the models it targets, its correctness
+// graders, and — for A/B — an optional baseline+candidate prompt pair. A SUITE is
+// just a directory of modules that export cases.
+//
+// Design decision (research §"prompts are eval inputs"): a case owns a VERBATIM
+// baseline prompt snapshot copied from the live stage plus a candidate variant to
+// A/B. Cases never import the live stage builders — prompts are inputs, not code.
+
+import type { Grader } from './grade.ts'
+import type { EvalRequest, JsonSchema } from './types.ts'
+
+/** One labelled prompt variant for A/B comparison. */
+export interface Variant {
+  /** Short label shown in the report (e.g. `baseline`, `terser`). */
+  label: string
+  /** The full prompt text for this variant. */
+  prompt: string
+}
+
+/** A baseline↔candidate variant pair a case can be A/B'd on. */
+export interface CaseVariants {
+  baseline: Variant
+  candidate: Variant
+}
+
+/** A single prompt-evaluation case. */
+export interface EvalCase {
+  /** Unique id (also a `--filter` target). */
+  id: string
+  /** Tags for `--filter` selection (e.g. `planner`, `reviewer`). */
+  tags: string[]
+  /** Replace the CLI's default system prompt for isolation (`--system-prompt`). */
+  systemPrompt?: string
+  /** Layer strapped context on the real system prompt (`--append-system-prompt`). */
+  appendSystemPrompt?: string
+  /** The user prompt under evaluation. */
+  prompt: string
+  /** The schema forced via `--json-schema` and validated against the output. */
+  schema: JsonSchema
+  /** Models this case targets in a plain/matrix run; the runner falls back to a default. */
+  models?: string[]
+  /** Opt into a bounded toolset; omit for a single-shot, tool-free run. */
+  tools?: string[]
+  /** Inline settings JSON (`--settings`); defaults to `'{}'` in the engine. */
+  settings?: string
+  /** The correctness graders aggregated into this case's score. */
+  graders: Grader[]
+  /** Optional A/B prompt pair — required for a case to appear in `--ab` mode. */
+  variants?: CaseVariants
+}
+
+/**
+ * Identity helper for authoring a case module — validates nothing beyond types,
+ * but gives suites a single import and a stable authoring surface.
+ */
+export function defineCase(spec: EvalCase): EvalCase {
+  return spec
+}
+
+/**
+ * Lower a case (at one model, optionally with a variant's prompt substituted)
+ * into the single-shot `EvalRequest` the D1 engine consumes. The prompt override
+ * is how A/B swaps baseline vs candidate while holding everything else fixed.
+ */
+export function toRequest(c: EvalCase, model: string, promptOverride?: string): EvalRequest {
+  return {
+    prompt: promptOverride ?? c.prompt,
+    systemPrompt: c.systemPrompt,
+    appendSystemPrompt: c.appendSystemPrompt,
+    model,
+    schema: c.schema,
+    tools: c.tools,
+    settings: c.settings,
+  }
+}
diff --git a/src/eval/cli.ts b/src/eval/cli.ts
new file mode 100644
index 0000000..37702d0
--- /dev/null
+++ b/src/eval/cli.ts
@@ -0,0 +1,174 @@
+// `bun src/eval/cli.ts` — the eval suite entrypoint (npm: `bun run eval`). Loads
+// case modules from a suite directory, runs them in one of three modes, and
+// prints a report. This is the HEAVY, opt-in test layer for prompt changes — it
+// is deliberately NOT part of `bun test` (that stays hermetic/offline).
+//
+// Exit code: a plain/matrix run is REPORT-ONLY (always 0). Only `--ab` gates —
+// exit 1 iff some case's Δcorrectness dips past `--tolerance` (a regression). An
+// absent `claude` CLI is a graceful skip: print a notice, exit 0, never fail.
+//
+// Flags: --suite <dir>  --filter <tag|id>  --ab  --matrix  --models a,b
+//        --json  --tolerance <pct>
+//
+// `main` returns { code, output } (does not exit) so tests can drive the exact
+// exit-code semantics through the injected spawn; the bottom guard is the only
+// place that touches the process.
+
+import { readdirSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import type { EvalCase } from './case.ts'
+import { isAvailable } from './engine.ts'
+import { formatReport, toJSON, type Report } from './report.ts'
+import {
+  DEFAULT_MODEL,
+  runAB,
+  runMatrix,
+  runSuite,
+  type ABResult,
+  type MatrixResult,
+} from './runner.ts'
+import type { Cache, Spawn } from './types.ts'
+
+export interface EvalFlags {
+  suite?: string
+  filter?: string
+  ab: boolean
+  matrix: boolean
+  models?: string[]
+  json: boolean
+  tolerance: number
+}
+
+/** Hand-rolled argv parser (no dependency). Unknown flags are ignored. */
+export function parseArgs(argv: readonly string[]): EvalFlags {
+  const flags: EvalFlags = { ab: false, matrix: false, json: false, tolerance: 0 }
+  for (let i = 0; i < argv.length; i++) {
+    switch (argv[i]) {
+      case '--suite':
+        flags.suite = argv[++i]
+        break
+      case '--filter':
+        flags.filter = argv[++i]
+        break
+      case '--ab':
+        flags.ab = true
+        break
+      case '--matrix':
+        flags.matrix = true
+        break
+      case '--models':
+        flags.models = (argv[++i] ?? '').split(',').filter(Boolean)
+        break
+      case '--json':
+        flags.json = true
+        break
+      case '--tolerance':
+        flags.tolerance = Number(argv[++i] ?? '0')
+        break
+      default:
+        break
+    }
+  }
+  return flags
+}
+
+/** Load every case exported by the modules in a suite directory (sorted). */
+export async function loadSuite(dir: string): Promise<EvalCase[]> {
+  const abs = resolve(dir)
+  const files = readdirSync(abs)
+    .filter(f => /\.(ts|mjs|js)$/.test(f) && !f.endsWith('.d.ts'))
+    .sort()
+  const cases: EvalCase[] = []
+  for (const file of files) {
+    const mod = (await import(join(abs, file))) as { cases?: unknown; default?: unknown }
+    const exported = mod.cases ?? mod.default
+    if (Array.isArray(exported)) cases.push(...(exported as EvalCase[]))
+    else if (exported) cases.push(exported as EvalCase)
+  }
+  return cases
+}
+
+/** A case matches a filter if the filter equals its id or one of its tags. */
+function matchesFilter(c: EvalCase, filter: string): boolean {
+  return c.id === filter || c.tags.includes(filter)
+}
+
+/**
+ * A/B gate: exit 1 iff any case regressed correctness past the tolerance. A
+ * tolerance of `5` means "a 5-percentage-point dip is allowed"; anything worse
+ * gates. Cost/latency wins never gate — a human judges the trade from the report.
+ */
+export function abExitCode(results: readonly ABResult[], tolerancePct: number): number {
+  const tol = tolerancePct / 100
+  return results.some(r => r.deltaCorrectness < -tol) ? 1 : 0
+}
+
+export interface CliDeps {
+  spawn?: Spawn
+  cache?: Cache
+  /** Test override: supply cases directly instead of loading from `--suite`. */
+  cases?: EvalCase[]
+}
+
+export interface CliResult {
+  code: number
+  output: string
+}
+
+/**
+ * Parse argv, run the selected mode, and return an exit code + text to print.
+ * Never touches the process — the bottom guard does that. An absent CLI short
+ * circuits to a skip notice + exit 0 before any case runs.
+ */
+export async function main(argv: readonly string[], deps: CliDeps = {}): Promise<CliResult> {
+  const flags = parseArgs(argv)
+  const spawn = deps.spawn
+
+  if (!isAvailable(spawn)) {
+    return { code: 0, output: 'eval: claude CLI unavailable — skipping suite (exit 0)\n' }
+  }
+
+  let cases = deps.cases ?? (flags.suite ? await loadSuite(flags.suite) : [])
+  if (flags.filter) cases = cases.filter(c => matchesFilter(c, flags.filter as string))
+
+  if (cases.length === 0) {
+    return { code: 0, output: 'eval: no cases matched (nothing to run)\n' }
+  }
+
+  let report: Report
+  let code = 0
+
+  if (flags.ab) {
+    const results: ABResult[] = []
+    for (const c of cases) {
+      if (!c.variants) continue
+      results.push(runAB(c, c.variants.baseline, c.variants.candidate, { model: flags.models?.[0] ?? DEFAULT_MODEL, spawn, cache: deps.cache }))
+    }
+    report = { mode: 'ab', cases: results }
+    code = abExitCode(results, flags.tolerance)
+  } else if (flags.matrix) {
+    const grids: MatrixResult[] = []
+    for (const c of cases) {
+      grids.push(await runMatrix(c, flags.models ?? c.models ?? [], { spawn, cache: deps.cache }))
+    }
+    report = { mode: 'matrix', cases: grids }
+  } else {
+    const results = await runSuite(cases, { models: flags.models, spawn, cache: deps.cache })
+    report = { mode: 'suite', cases: results }
+  }
+
+  const output = flags.json ? `${JSON.stringify(toJSON(report), null, 2)}\n` : `${formatReport(report)}\n`
+  return { code, output }
+}
+
+if (import.meta.main) {
+  main(process.argv.slice(2))
+    .then(({ code, output }) => {
+      process.stdout.write(output)
+      process.exit(code)
+    })
+    .catch((e: unknown) => {
+      process.stderr.write(`eval: ${e instanceof Error ? e.message : String(e)}\n`)
+      process.exit(2)
+    })
+}
diff --git a/src/eval/engine.ts b/src/eval/engine.ts
new file mode 100644
index 0000000..8404bb8
--- /dev/null
+++ b/src/eval/engine.ts
@@ -0,0 +1,228 @@
+// The `claude -p` eval substrate. `runClaude` builds the print-mode argv, spawns
+// the CLI through an injectable `Spawn`, parses the JSON envelope, validates the
+// forced `structured_output` against the request schema, and returns a typed,
+// graded result. It shells out to `claude` ONLY — never `@anthropic-ai/sdk`.
+//
+// Design notes:
+//  - The prompt travels on stdin (fed via `Spawn`'s `input`), so argv stays free
+//    of arbitrarily large prompt text; `claude -p` reads stdin when given no
+//    positional prompt.
+//  - Cost/latency/usage are read straight off the envelope — no pricing table.
+//  - Every failure mode (CLI error, unparseable envelope, schema miss) is a
+//    graded `ok:false` result, never a thrown exception. An ABSENT CLI is a
+//    distinct `skipped:true` signal so callers can skip-with-notice.
+
+import { spawnSync } from 'node:child_process'
+import type {
+  Cache,
+  Envelope,
+  EvalRequest,
+  EvalResult,
+  EvalUsage,
+  JsonSchema,
+  Spawn,
+} from './types.ts'
+import { cacheKey } from './cache.ts'
+
+/** Tools disabled by default so a case is single-shot, deterministic, cacheable. */
+const DEFAULT_DISALLOWED_TOOLS = [
+  'Bash',
+  'Read',
+  'Write',
+  'Edit',
+  'WebFetch',
+  'WebSearch',
+  'Task',
+  'Glob',
+  'Grep',
+]
+
+/** Real spawn: `spawnSync('claude', …)` with the prompt on stdin. */
+export const defaultSpawn: Spawn = (cmd, args, input) => {
+  const res = spawnSync(cmd, args, { encoding: 'utf8', input, maxBuffer: 64 * 1024 * 1024 })
+  return { status: res.status, stdout: res.stdout ?? '', stderr: res.stderr ?? '' }
+}
+
+/**
+ * Build the pure `claude -p` argv for a request (prompt excluded — it goes on
+ * stdin). Always forces print mode, JSON output, the request schema, model,
+ * strict MCP isolation, and inline settings; layers system-prompt and tool-policy
+ * flags conditionally.
+ */
+export function buildArgs(req: EvalRequest): string[] {
+  const args = [
+    '--print',
+    '--output-format',
+    'json',
+    '--json-schema',
+    JSON.stringify(req.schema),
+    '--model',
+    req.model,
+    '--strict-mcp-config',
+    '--settings',
+    req.settings ?? '{}',
+  ]
+  if (req.systemPrompt !== undefined) args.push('--system-prompt', req.systemPrompt)
+  if (req.appendSystemPrompt !== undefined) args.push('--append-system-prompt', req.appendSystemPrompt)
+  if (req.tools && req.tools.length > 0) {
+    args.push('--allowedTools', req.tools.join(' '))
+  } else {
+    args.push('--disallowedTools', DEFAULT_DISALLOWED_TOOLS.join(' '))
+  }
+  return args
+}
+
+/** Lift the envelope's snake_case usage block into the normalized shape. */
+function normalizeUsage(envelope: Envelope): EvalUsage {
+  const u = envelope.usage ?? {}
+  return {
+    inputTokens: u.input_tokens ?? 0,
+    outputTokens: u.output_tokens ?? 0,
+    cacheReadInputTokens: u.cache_read_input_tokens ?? 0,
+    cacheCreationInputTokens: u.cache_creation_input_tokens ?? 0,
+  }
+}
+
+/**
+ * Shallow structural check: value is an object, every `required` key is present,
+ * and — when `additionalProperties:false` — no keys beyond `properties` appear.
+ * Returns an error message or `null`. Intentionally NOT a full JSON-Schema
+ * validator (no dependency added).
+ */
+function validateSchema(value: unknown, schema: JsonSchema): string | null {
+  const wantsObject = schema.type === 'object' || schema.properties !== undefined || schema.required !== undefined
+  if (!wantsObject) return null
+  if (typeof value !== 'object' || value === null || Array.isArray(value)) {
+    return 'output is not an object'
+  }
+  const obj = value as Record<string, unknown>
+  for (const key of schema.required ?? []) {
+    if (!(key in obj)) return `output missing required key: ${key}`
+  }
+  if (schema.additionalProperties === false && schema.properties) {
+    const allowed = new Set(Object.keys(schema.properties))
+    for (const key of Object.keys(obj)) {
+      if (!allowed.has(key)) return `output has unexpected key: ${key}`
+    }
+  }
+  return null
+}
+
+/** Assemble a graded result from envelope metrics + an ok/error verdict. */
+function gradedResult(
+  envelope: Envelope,
+  req: EvalRequest,
+  verdict: { ok: true; output: unknown } | { ok: false; error: string }
+): EvalResult {
+  const base = {
+    cost: envelope.total_cost_usd ?? 0,
+    usage: normalizeUsage(envelope),
+    durationMs: envelope.duration_ms ?? 0,
+    apiDurationMs: envelope.duration_api_ms ?? 0,
+    numTurns: envelope.num_turns ?? 0,
+    model: req.model,
+    cached: false,
+  }
+  return verdict.ok
+    ? { ok: true, output: verdict.output, error: null, ...base }
+    : { ok: false, output: null, error: verdict.error, ...base }
+}
+
+/** A graded failure carrying zeroed metrics (used when the envelope is unusable). */
+function failureNoMetrics(req: EvalRequest, error: string): EvalResult {
+  return {
+    ok: false,
+    output: null,
+    error,
+    cost: 0,
+    usage: { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 },
+    durationMs: 0,
+    apiDurationMs: 0,
+    numTurns: 0,
+    model: req.model,
+    cached: false,
+  }
+}
+
+/**
+ * Parse a `claude --output-format json` envelope into a graded `EvalResult`.
+ * Never throws: an unparseable envelope, a CLI/model error, a missing/bad
+ * output, or a schema miss all yield `ok:false` with a descriptive `error`.
+ */
+export function parseEnvelope(stdout: string, req: EvalRequest): EvalResult {
+  let envelope: Envelope
+  try {
+    envelope = JSON.parse(stdout) as Envelope
+  } catch {
+    return failureNoMetrics(req, 'unparseable envelope')
+  }
+
+  if (envelope.is_error === true || (envelope.subtype !== undefined && envelope.subtype !== 'success')) {
+    return gradedResult(envelope, req, {
+      ok: false,
+      error: `claude error: is_error=${String(envelope.is_error)} subtype=${String(envelope.subtype)}`,
+    })
+  }
+
+  // Prefer the parsed, schema-forced object; fall back to parsing stringified `result`.
+  let output = envelope.structured_output
+  if (output === undefined || output === null) {
+    if (typeof envelope.result !== 'string') {
+      return gradedResult(envelope, req, { ok: false, error: 'no structured_output and no string result to parse' })
+    }
+    try {
+      output = JSON.parse(envelope.result)
+    } catch {
+      return gradedResult(envelope, req, { ok: false, error: 'unparseable result JSON' })
+    }
+  }
+
+  const schemaError = validateSchema(output, req.schema)
+  if (schemaError) return gradedResult(envelope, req, { ok: false, error: schemaError })
+
+  return gradedResult(envelope, req, { ok: true, output })
+}
+
+export interface RunOptions {
+  /** Injected subprocess boundary; defaults to the real `claude` spawn. */
+  spawn?: Spawn
+  /** Optional response cache consulted before spawning and written on success. */
+  cache?: Cache
+}
+
+/**
+ * Run one eval request: cache lookup first, else spawn `claude -p`, parse, cache
+ * (on success), return. An absent CLI (non-zero exit with empty stdout, or a
+ * spawn throw) surfaces as a `skipped` result — distinct from a graded failure.
+ */
+export function runClaude(req: EvalRequest, { spawn = defaultSpawn, cache }: RunOptions = {}): EvalResult {
+  const key = cache ? cacheKey(req) : null
+  if (cache && key) {
+    const hit = cache.get(key)
+    if (hit) return { ...hit, cached: true }
+  }
+
+  let res
+  try {
+    res = spawn('claude', buildArgs(req), req.prompt)
+  } catch {
+    return { ...failureNoMetrics(req, 'claude CLI unavailable'), skipped: true }
+  }
+
+  if (res.status !== 0 && res.stdout.trim() === '') {
+    return { ...failureNoMetrics(req, 'claude CLI unavailable'), skipped: true }
+  }
+
+  const result = parseEnvelope(res.stdout, req)
+  if (cache && key && result.ok) cache.set(key, result)
+  return result
+}
+
+/** Probe whether the `claude` CLI is invokable via `claude --version`. */
+export function isAvailable(spawn: Spawn = defaultSpawn): boolean {
+  try {
+    return spawn('claude', ['--version']).status === 0
+  } catch {
+    return false
+  }
+}
diff --git a/src/eval/grade.ts b/src/eval/grade.ts
new file mode 100644
index 0000000..5dd97ff
--- /dev/null
+++ b/src/eval/grade.ts
@@ -0,0 +1,144 @@
+// Correctness graders for the eval framework. A `Grader` inspects one engine
+// result (the parsed output + the full envelope-derived `EvalResult`) and returns
+// a `GradeResult`. Three built-ins ship here:
+//   - `schemaConforms()` — intrinsic: did the forced schema validate? (`result.ok`)
+//   - `assert(name, predicate)` — a pure predicate over the parsed output.
+//   - `judge(rubric, {model})` — an LLM-judge that ITSELF routes through the D1
+//     engine (so it is cached/stubbable/hermetic), scoring fuzzy quality.
+// `gradeCase` aggregates a case's graders into a weighted correctness score in
+// [0,1]. Graders are synchronous — the D1 engine (`runClaude`) is synchronous —
+// and NEVER throw: a thrown predicate becomes a failed `GradeResult`.
+
+import { runClaude } from './engine.ts'
+import type { Cache, EvalRequest, EvalResult, JsonSchema, Spawn } from './types.ts'
+
+/** The verdict of one grader over an engine result. `score` is in [0,1]. */
+export interface GradeResult {
+  /** Stable grader name — appears in the report and JSON. */
+  name: string
+  /** Boolean pass/fail (a threshold view of `score`). */
+  pass: boolean
+  /** Continuous quality in [0,1]; `schemaConforms`/`assert` emit 0 or 1. */
+  score: number
+  /** Human-readable reason — why it passed/failed. */
+  detail: string
+  /** Relative weight in the correctness aggregate (default 1). */
+  weight?: number
+}
+
+/**
+ * Runtime context threaded to every grader by `gradeCase`. Only `judge` reads it
+ * (to route its rubric call through the same injected spawn/cache as the case),
+ * so `bun test` stays hermetic. Intrinsic graders ignore it.
+ */
+export interface GradeContext {
+  spawn?: Spawn
+  cache?: Cache
+}
+
+/** A grader: inspect the parsed output + full result, return a verdict. */
+export type Grader = (output: unknown, result: EvalResult, ctx: GradeContext) => GradeResult
+
+/** Intrinsic grader: passes iff the forced schema validated (`result.ok`). */
+export function schemaConforms(opts: { weight?: number } = {}): Grader {
+  return (_output, result) => ({
+    name: 'schemaConforms',
+    pass: result.ok,
+    score: result.ok ? 1 : 0,
+    detail: result.ok ? 'output conforms to the forced schema' : (result.error ?? 'schema nonconforming'),
+    weight: opts.weight,
+  })
+}
+
+/**
+ * Assertion grader: run a pure predicate over the parsed output. A `false` return
+ * fails; a thrown predicate fails with the throw message (never propagates).
+ */
+export function assert(name: string, predicate: (output: unknown) => boolean, opts: { weight?: number } = {}): Grader {
+  return output => {
+    try {
+      const pass = predicate(output)
+      return { name, pass, score: pass ? 1 : 0, detail: pass ? 'assertion passed' : 'assertion failed', weight: opts.weight }
+    } catch (e) {
+      const msg = e instanceof Error ? e.message : String(e)
+      return { name, pass: false, score: 0, detail: `assertion threw: ${msg}`, weight: opts.weight }
+    }
+  }
+}
+
+/** The schema the judge forces on itself — a single scored verdict. */
+const JUDGE_SCHEMA: JsonSchema = {
+  type: 'object',
+  required: ['score', 'pass', 'reason'],
+  properties: {
+    score: { type: 'number' },
+    pass: { type: 'boolean' },
+    reason: { type: 'string' },
+  },
+  additionalProperties: false,
+}
+
+/** Parsed shape of a judge verdict (post schema-validation). */
+interface JudgeVerdict {
+  score?: number
+  pass?: boolean
+  reason?: string
+}
+
+export interface JudgeOptions {
+  /** Model the judge itself runs on (`--model`). */
+  model: string
+  /** Grader name in the report (default `judge`). */
+  name?: string
+  /** Weight in the correctness aggregate. */
+  weight?: number
+}
+
+/**
+ * LLM-judge grader. Composes an `EvalRequest` from `rubric` + the output under
+ * test and a fixed `{score,pass,reason}` schema, then calls the D1 engine through
+ * the injected spawn/cache — so the judge is itself cacheable and hermetically
+ * stubbable. A judge engine failure grades the case as a failed rubric, not a crash.
+ */
+export function judge(rubric: string, opts: JudgeOptions): Grader {
+  const name = opts.name ?? 'judge'
+  return (output, _result, ctx) => {
+    const req: EvalRequest = {
+      prompt: `${rubric}\n\nOutput under evaluation:\n${JSON.stringify(output, null, 2)}`,
+      model: opts.model,
+      schema: JUDGE_SCHEMA,
+    }
+    const verdict = runClaude(req, { spawn: ctx.spawn, cache: ctx.cache })
+    if (!verdict.ok || verdict.output === null || typeof verdict.output !== 'object') {
+      return { name, pass: false, score: 0, detail: verdict.error ?? 'judge produced no verdict', weight: opts.weight }
+    }
+    const v = verdict.output as JudgeVerdict
+    const score = typeof v.score === 'number' ? Math.max(0, Math.min(1, v.score)) : 0
+    return { name, pass: v.pass === true, score, detail: v.reason ?? '', weight: opts.weight }
+  }
+}
+
+/** A case's aggregate correctness plus the individual grader verdicts. */
+export interface CaseGrade {
+  /** Weighted mean grader score in [0,1]. */
+  correctness: number
+  grades: GradeResult[]
+}
+
+/**
+ * Run every grader over one engine result and aggregate into a weighted
+ * correctness score. Correctness = Σ(weight·score) / Σweight (weight defaults to
+ * 1). An empty grader set yields correctness 0.
+ */
+export function gradeCase(graders: readonly Grader[], output: unknown, result: EvalResult, ctx: GradeContext = {}): CaseGrade {
+  const grades = graders.map(g => g(output, result, ctx))
+  let weightedSum = 0
+  let totalWeight = 0
+  for (const grade of grades) {
+    const weight = grade.weight ?? 1
+    weightedSum += weight * grade.score
+    totalWeight += weight
+  }
+  const correctness = totalWeight > 0 ? weightedSum / totalWeight : 0
+  return { correctness, grades }
+}
diff --git a/src/eval/report.ts b/src/eval/report.ts
new file mode 100644
index 0000000..8c82e19
--- /dev/null
+++ b/src/eval/report.ts
@@ -0,0 +1,134 @@
+// The eval REPORT: a human-readable aligned text table + a machine JSON object,
+// over any of the three runner modes. The report surfaces all THREE axes
+// (correctness / cost / latency) side by side so a human judges the trade — a
+// correctness dip that buys a large cost/latency win is a WIN, and the tool
+// refuses to collapse that into a single number.
+//
+// Cost columns: the "Cost $" column is the MEASURED envelope cost. A separate,
+// clearly-labelled "Proj $" column is an OPTIONAL projection of a model-swap the
+// CLI did not actually run, computed from a small static pricing table — never
+// mixed with measured cost.
+
+import type { CaseResult, ABResult, MatrixResult } from './runner.ts'
+import type { EvalUsage } from './types.ts'
+
+/** Static, approximate USD-per-1M-token prices — projection only, NOT measured. */
+export interface ModelPricing {
+  inputPerMTok: number
+  outputPerMTok: number
+}
+
+/** A small static pricing table used ONLY to project a model-swap. */
+export const PRICING: Record<string, ModelPricing> = {
+  'claude-opus-4-8': { inputPerMTok: 15, outputPerMTok: 75 },
+  'claude-sonnet-5': { inputPerMTok: 3, outputPerMTok: 15 },
+  'claude-haiku-4-5': { inputPerMTok: 1, outputPerMTok: 5 },
+}
+
+/**
+ * Project what a run's token usage WOULD cost on `model` from the static table.
+ * Returns `null` for an unpriced model. This is a projection, never a measurement.
+ */
+export function projectCost(usage: EvalUsage, model: string): number | null {
+  const p = PRICING[model]
+  if (!p) return null
+  return (usage.inputTokens * p.inputPerMTok + usage.outputTokens * p.outputPerMTok) / 1_000_000
+}
+
+/** A tagged union over the three runner modes the report can render. */
+export type Report =
+  | { mode: 'suite'; cases: CaseResult[] }
+  | { mode: 'ab'; cases: ABResult[] }
+  | { mode: 'matrix'; cases: MatrixResult[] }
+
+export interface FormatOptions {
+  /** When set, add a projected-cost column for a model the run did not execute. */
+  projectModel?: string
+}
+
+// --- cell formatters (deterministic — the report text is snapshot-tested) -----
+
+const pct = (x: number): string => `${(x * 100).toFixed(1)}%`
+const usd = (x: number): string => `$${x.toFixed(5)}`
+const ms = (x: number): string => `${Math.round(x)}ms`
+const signedPct = (x: number): string => `${x >= 0 ? '+' : ''}${(x * 100).toFixed(1)}%`
+const signedUsd = (x: number): string => `${x >= 0 ? '+' : '-'}$${Math.abs(x).toFixed(5)}`
+const signedMs = (x: number): string => `${x >= 0 ? '+' : ''}${Math.round(x)}ms`
+const signedInt = (x: number): string => `${x >= 0 ? '+' : ''}${x}`
+
+/** Render aligned columns: header, a dashed rule, then rows (left-padded cells). */
+function renderTable(headers: string[], rows: string[][]): string {
+  const widths = headers.map((h, i) => Math.max(h.length, ...rows.map(r => (r[i] ?? '').length)))
+  const fmt = (cells: string[]): string => cells.map((c, i) => c.padEnd(widths[i] ?? 0)).join('  ')
+  const rule = widths.map(w => '-'.repeat(w)).join('  ')
+  return [fmt(headers), rule, ...rows.map(fmt)].join('\n')
+}
+
+function caseRow(r: CaseResult, projectModel?: string): string[] {
+  const cells = [
+    r.caseId,
+    r.model,
+    r.skipped ? 'skipped' : pct(r.correctness),
+    r.skipped ? '-' : usd(r.result.cost),
+    r.skipped ? '-' : ms(r.result.durationMs),
+    r.skipped ? '-' : String(r.result.numTurns),
+  ]
+  if (projectModel) {
+    const proj = r.skipped ? null : projectCost(r.result.usage, projectModel)
+    cells.push(proj === null ? '-' : usd(proj))
+  }
+  return cells
+}
+
+function suiteHeaders(projectModel?: string): string[] {
+  const h = ['Case', 'Model', 'Correct', 'Cost $', 'Latency', 'Turns']
+  if (projectModel) h.push(`Proj $ (${projectModel})`)
+  return h
+}
+
+function formatSuite(cases: CaseResult[], projectModel?: string): string {
+  if (cases.length === 0) return 'no cases matched'
+  return renderTable(suiteHeaders(projectModel), cases.map(r => caseRow(r, projectModel)))
+}
+
+function formatMatrix(cases: MatrixResult[], projectModel?: string): string {
+  if (cases.length === 0) return 'no cases matched'
+  const rows: string[][] = []
+  for (const c of cases) for (const r of c.rows) rows.push(caseRow(r, projectModel))
+  return renderTable(suiteHeaders(projectModel), rows)
+}
+
+function formatAB(cases: ABResult[]): string {
+  if (cases.length === 0) return 'no cases matched'
+  const headers = ['Case', 'Model', 'ΔCorrect', 'ΔCost', 'ΔLatency', 'ΔTurns', 'Base', 'Cand']
+  const rows = cases.map(r => [
+    r.caseId,
+    r.model,
+    signedPct(r.deltaCorrectness),
+    signedUsd(r.deltaCostUsd),
+    signedMs(r.deltaLatencyMs),
+    signedInt(r.deltaTurns),
+    pct(r.baseline.correctness),
+    pct(r.candidate.correctness),
+  ])
+  const legend =
+    'Δcost/Δlatency/Δturns: positive = candidate cheaper/faster/tighter. ΔCorrect: negative = regression.'
+  return `${renderTable(headers, rows)}\n\n${legend}`
+}
+
+/** Render a report to aligned human-readable text. */
+export function formatReport(report: Report, opts: FormatOptions = {}): string {
+  switch (report.mode) {
+    case 'suite':
+      return formatSuite(report.cases, opts.projectModel)
+    case 'matrix':
+      return formatMatrix(report.cases, opts.projectModel)
+    case 'ab':
+      return formatAB(report.cases)
+  }
+}
+
+/** The machine-comparable JSON view (already a plain serializable object). */
+export function toJSON(report: Report): Report {
+  return report
+}
diff --git a/src/eval/runner.ts b/src/eval/runner.ts
new file mode 100644
index 0000000..4dc412d
--- /dev/null
+++ b/src/eval/runner.ts
@@ -0,0 +1,172 @@
+// The eval RUNNER: drives cases through the D1 engine and grades them, in three
+// modes.
+//   - `runSuite(cases, …)`  — every case × its models, in BOUNDED parallel.
+//   - `runAB(case, base, cand, {model})` — same case body, two prompt variants,
+//     one model → a delta record (Δcorrectness / Δcost / Δlatency / Δturns).
+//   - `runMatrix(case, models)` — one case across a model grid.
+// Every mode takes an injected `spawn` + optional `cache`, so `bun test` runs
+// fully offline against the `fakeSpawn` stub. A case whose engine call reports the
+// CLI is unavailable is marked `skipped` — a notice, never a failure.
+//
+// NOTE on "parallel": the D1 spawn boundary is synchronous, so the pool bounds
+// how many case promises are in flight rather than truly interleaving blocking
+// spawns; the shape is correct and the bound is enforced (`mapPool`). Deltas are
+// what the performance verdict rests on — the CLI-boot latency offset cancels.
+
+import { toRequest, type EvalCase, type Variant } from './case.ts'
+import { runClaude } from './engine.ts'
+import { gradeCase, type GradeContext, type GradeResult } from './grade.ts'
+import type { Cache, EvalResult, Spawn } from './types.ts'
+
+/** Default model when a case/run names none. */
+export const DEFAULT_MODEL = 'claude-haiku-4-5'
+/** Default model grid for `--matrix` when none is given. */
+export const DEFAULT_MODELS: readonly string[] = ['claude-opus-4-8', 'claude-sonnet-5', 'claude-haiku-4-5']
+
+/** The graded outcome of running one case at one model (one variant). */
+export interface CaseResult {
+  caseId: string
+  model: string
+  /** Variant label when produced by an A/B leg; absent for plain/matrix runs. */
+  variantLabel?: string
+  /** Weighted correctness in [0,1]. */
+  correctness: number
+  grades: GradeResult[]
+  /** The full engine result — measured cost/latency/turns/usage live here. */
+  result: EvalResult
+  /** True when the `claude` CLI was unavailable — a skip, not a failure. */
+  skipped: boolean
+}
+
+export interface CaseRunOptions {
+  model: string
+  /** Substitute the case prompt (used by A/B to swap in a variant). */
+  prompt?: string
+  variantLabel?: string
+  spawn?: Spawn
+  cache?: Cache
+}
+
+/** Run one case at one model, then grade it. Never throws. */
+export function runCase(c: EvalCase, opts: CaseRunOptions): CaseResult {
+  const result = runClaude(toRequest(c, opts.model, opts.prompt), { spawn: opts.spawn, cache: opts.cache })
+  const base = { caseId: c.id, model: opts.model, ...(opts.variantLabel ? { variantLabel: opts.variantLabel } : {}) }
+  if (result.skipped === true) {
+    return { ...base, correctness: 0, grades: [], result, skipped: true }
+  }
+  const ctx: GradeContext = { spawn: opts.spawn, cache: opts.cache }
+  const { correctness, grades } = gradeCase(c.graders, result.output, result, ctx)
+  return { ...base, correctness, grades, result, skipped: false }
+}
+
+/**
+ * Bounded-concurrency map: at most `concurrency` workers run at once, results
+ * preserve input order. No dependency — a fixed pool draining a shared cursor.
+ */
+export async function mapPool<T, R>(
+  items: readonly T[],
+  concurrency: number,
+  worker: (item: T, index: number) => Promise<R>
+): Promise<R[]> {
+  const results = new Array<R>(items.length)
+  const limit = Math.max(1, Math.min(concurrency, items.length || 1))
+  let cursor = 0
+  async function drain(): Promise<void> {
+    for (;;) {
+      const i = cursor++
+      if (i >= items.length) return
+      results[i] = await worker(items[i] as T, i)
+    }
+  }
+  await Promise.all(Array.from({ length: limit }, () => drain()))
+  return results
+}
+
+export interface SuiteOptions {
+  /** Override the models each case runs on. */
+  models?: string[]
+  /** Max cases in flight (default 4). */
+  concurrency?: number
+  cache?: Cache
+  spawn?: Spawn
+}
+
+/** Run every case × its resolved models in bounded parallel. */
+export function runSuite(cases: readonly EvalCase[], opts: SuiteOptions = {}): Promise<CaseResult[]> {
+  const jobs: { c: EvalCase; model: string }[] = []
+  for (const c of cases) {
+    const models = opts.models ?? c.models ?? [DEFAULT_MODEL]
+    for (const model of models) jobs.push({ c, model })
+  }
+  return mapPool(jobs, opts.concurrency ?? 4, job =>
+    Promise.resolve(runCase(job.c, { model: job.model, spawn: opts.spawn, cache: opts.cache }))
+  )
+}
+
+/** A baseline-vs-candidate delta record at one fixed model. */
+export interface ABResult {
+  caseId: string
+  model: string
+  baselineLabel: string
+  candidateLabel: string
+  baseline: CaseResult
+  candidate: CaseResult
+  /** candidate − baseline: positive = candidate more correct. */
+  deltaCorrectness: number
+  /** baseline − candidate cost: positive = candidate CHEAPER (a saving). */
+  deltaCostUsd: number
+  /** baseline − candidate latency: positive = candidate FASTER. */
+  deltaLatencyMs: number
+  /** baseline − candidate turns: positive = candidate converged in FEWER turns. */
+  deltaTurns: number
+}
+
+export interface ABOptions {
+  model?: string
+  spawn?: Spawn
+  cache?: Cache
+}
+
+/**
+ * Run the same case body with two prompt variants at one model and compute the
+ * trade-off deltas. Cost/latency/turn deltas are baseline−candidate so a cheaper,
+ * faster, tighter candidate reads as POSITIVE across the board; correctness is
+ * candidate−baseline so a regression reads as NEGATIVE (what the CLI gate keys on).
+ */
+export function runAB(c: EvalCase, baseline: Variant, candidate: Variant, opts: ABOptions = {}): ABResult {
+  const model = opts.model ?? c.models?.[0] ?? DEFAULT_MODEL
+  const b = runCase(c, { model, prompt: baseline.prompt, variantLabel: baseline.label, spawn: opts.spawn, cache: opts.cache })
+  const cand = runCase(c, { model, prompt: candidate.prompt, variantLabel: candidate.label, spawn: opts.spawn, cache: opts.cache })
+  return {
+    caseId: c.id,
+    model,
+    baselineLabel: baseline.label,
+    candidateLabel: candidate.label,
+    baseline: b,
+    candidate: cand,
+    deltaCorrectness: cand.correctness - b.correctness,
+    deltaCostUsd: b.result.cost - cand.result.cost,
+    deltaLatencyMs: b.result.durationMs - cand.result.durationMs,
+    deltaTurns: b.result.numTurns - cand.result.numTurns,
+  }
+}
+
+/** A per-model grid for one case. */
+export interface MatrixResult {
+  caseId: string
+  rows: CaseResult[]
+}
+
+export interface MatrixOptions {
+  spawn?: Spawn
+  cache?: Cache
+  concurrency?: number
+}
+
+/** Run one case across a model grid; one row per model, input order preserved. */
+export function runMatrix(c: EvalCase, models: readonly string[], opts: MatrixOptions = {}): Promise<MatrixResult> {
+  const grid = models.length > 0 ? models : DEFAULT_MODELS
+  return mapPool(grid, opts.concurrency ?? grid.length, model =>
+    Promise.resolve(runCase(c, { model, spawn: opts.spawn, cache: opts.cache }))
+  ).then(rows => ({ caseId: c.id, rows }))
+}
diff --git a/src/eval/types.ts b/src/eval/types.ts
new file mode 100644
index 0000000..1cb125a
--- /dev/null
+++ b/src/eval/types.ts
@@ -0,0 +1,113 @@
+// Shared types for the prompt-eval engine. The engine shells out to the Claude
+// Code CLI in print mode (`claude -p`) — NEVER the Anthropic SDK — and reads
+// cost/latency/usage straight off the `--output-format json` envelope. The one
+// faked boundary in tests is `Spawn`; everything else is real.
+
+/**
+ * A minimal JSON-Schema shape. The engine only performs a SHALLOW structural
+ * check (`required` keys present, no extras when `additionalProperties:false`),
+ * so this deliberately does not model the full spec — do NOT add a schema
+ * validator dependency. The harness's generated schemas satisfy this shape.
+ */
+export interface JsonSchema {
+  type?: string
+  required?: string[]
+  properties?: Record<string, JsonSchema>
+  additionalProperties?: boolean
+  items?: JsonSchema
+  [key: string]: unknown
+}
+
+/** A single-shot eval request: the prompt, the schema it is forced against, and knobs. */
+export interface EvalRequest {
+  /** User prompt — passed to `claude -p` on stdin. */
+  prompt: string
+  /** Replace the CLI's default system prompt (`--system-prompt`) for isolation. */
+  systemPrompt?: string
+  /** Layer extra context on top of the default system prompt (`--append-system-prompt`). */
+  appendSystemPrompt?: string
+  /** Model id, e.g. `claude-haiku-4-5` (`--model`). */
+  model: string
+  /** Schema forced via `--json-schema` and validated against `structured_output`. */
+  schema: JsonSchema
+  /** Opt into a bounded toolset (`--allowedTools`); omit for a single-shot, tool-free run. */
+  tools?: string[]
+  /** Inline settings JSON (`--settings`); defaults to `'{}'` to isolate from ambient config. */
+  settings?: string
+}
+
+/** Normalized token usage lifted from the envelope's snake_case `usage` block. */
+export interface EvalUsage {
+  inputTokens: number
+  outputTokens: number
+  cacheReadInputTokens: number
+  cacheCreationInputTokens: number
+}
+
+/**
+ * The graded outcome of one eval invocation. A model/CLI error, an unparseable
+ * envelope, or a schema-nonconforming output all yield `ok:false` with a
+ * descriptive `error` — the engine NEVER throws for these. `skipped:true` is a
+ * distinct signal that the CLI itself was unavailable (not a graded failure).
+ */
+export interface EvalResult {
+  ok: boolean
+  output: unknown | null
+  error: string | null
+  cost: number
+  usage: EvalUsage
+  durationMs: number
+  apiDurationMs: number
+  numTurns: number
+  model: string
+  cached: boolean
+  /** True only when the `claude` CLI was unavailable — callers skip-with-notice. */
+  skipped?: boolean
+}
+
+/** Envelope usage block as emitted by `claude --output-format json` (snake_case). */
+export interface EnvelopeUsage {
+  input_tokens?: number
+  output_tokens?: number
+  cache_read_input_tokens?: number
+  cache_creation_input_tokens?: number
+  service_tier?: string
+}
+
+/**
+ * Raw `--output-format json` envelope (verified against installed `claude`
+ * 2.1.207). Only the fields the engine reads are typed; the rest are ignored.
+ */
+export interface Envelope {
+  type?: string
+  subtype?: string
+  is_error?: boolean
+  result?: string
+  structured_output?: unknown
+  total_cost_usd?: number
+  usage?: EnvelopeUsage
+  duration_ms?: number
+  duration_api_ms?: number
+  num_turns?: number
+  modelUsage?: Record<string, unknown>
+}
+
+/** Result of a subprocess spawn — the single injectable boundary tests fake. */
+export interface SpawnResult {
+  status: number | null
+  stdout: string
+  stderr: string
+}
+
+/**
+ * The injected spawn dependency. Default is a thin `spawnSync('claude', …)`
+ * wrapper; tests pass a stub returning a canned envelope. `input` is fed to the
+ * child's stdin (the prompt travels there, keeping argv free of large text).
+ */
+export type Spawn = (cmd: string, args: string[], input?: string) => SpawnResult
+
+/** A response cache the engine can consult before spawning. */
+export interface Cache {
+  get(key: string): EvalResult | null
+  set(key: string, result: EvalResult): void
+}
diff --git a/tests/eval/cache.test.ts b/tests/eval/cache.test.ts
new file mode 100644
index 0000000..8bab195
--- /dev/null
+++ b/tests/eval/cache.test.ts
@@ -0,0 +1,130 @@
+// Cache tests: key stability, per-field isolation, file round-trip in a temp
+// dir, and the runClaude hit/miss integration (a hit must NOT spawn).
+
+import assert from 'node:assert/strict'
+import { mkdtempSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test } from 'bun:test'
+import { cacheKey, ResponseCache } from '../../src/eval/cache.ts'
+import { runClaude } from '../../src/eval/engine.ts'
+import type { EvalRequest, JsonSchema, Spawn } from '../../src/eval/types.ts'
+import { fakeSpawn, successEnvelope, throwingSpawn } from '../helpers/fake-claude.ts'
+
+const SCHEMA: JsonSchema = {
+  type: 'object',
+  required: ['answer'],
+  properties: { answer: { type: 'number' } },
+  additionalProperties: false,
+}
+
+const baseReq = (overrides: Partial<EvalRequest> = {}): EvalRequest => ({
+  prompt: 'What is 6 * 7?',
+  model: 'claude-haiku-4-5',
+  schema: SCHEMA,
+  ...overrides,
+})
+
+const tempDir = (): string => mkdtempSync(join(tmpdir(), 'eval-cache-'))
+
+// --- key stability + isolation (acceptance criterion 4) ----------------------
+
+test('cacheKey is stable for identical requests', () => {
+  assert.equal(cacheKey(baseReq()), cacheKey(baseReq()))
+})
+
+test('cacheKey is insensitive to schema key ordering (canonical JSON)', () => {
+  const a = baseReq({ schema: { type: 'object', required: ['answer'], additionalProperties: false } })
+  const b = baseReq({ schema: { additionalProperties: false, required: ['answer'], type: 'object' } })
+  assert.equal(cacheKey(a), cacheKey(b))
+})
+
+test('cacheKey changes when any output-affecting field changes', () => {
+  const base = cacheKey(baseReq())
+  const variants: EvalRequest[] = [
+    baseReq({ prompt: 'different' }),
+    baseReq({ model: 'claude-opus-4-8' }),
+    baseReq({ schema: { type: 'object', required: ['result'] } }),
+    baseReq({ systemPrompt: 'You are terse.' }),
+    baseReq({ appendSystemPrompt: 'Context.' }),
+    baseReq({ tools: ['Read'] }),
+    baseReq({ settings: '{"x":1}' }),
+  ]
+  for (const v of variants) {
+    assert.notEqual(cacheKey(v), base)
+  }
+  // All variants are distinct from each other too.
+  const keys = new Set(variants.map(cacheKey))
+  assert.equal(keys.size, variants.length)
+})
+
+// --- file round-trip ---------------------------------------------------------
+
+test('ResponseCache round-trips a result and marks it cached on read', () => {
+  const cache = new ResponseCache(tempDir())
+  const key = cacheKey(baseReq())
+  assert.equal(cache.get(key), null)
+
+  const result = runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 42 })) })
+  cache.set(key, result)
+
+  const hit = cache.get(key)
+  assert.ok(hit)
+  assert.equal(hit.cached, true)
+  assert.deepEqual(hit.output, { answer: 42 })
+  assert.equal(hit.cost, result.cost)
+})
+
+test('ResponseCaches in separate dirs are isolated', () => {
+  const a = new ResponseCache(tempDir())
+  const b = new ResponseCache(tempDir())
+  const key = cacheKey(baseReq())
+  a.set(key, runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 1 })) }))
+  assert.ok(a.get(key))
+  assert.equal(b.get(key), null)
+})
+
+// --- runClaude cache integration ---------------------------------------------
+
+test('runClaude serves an identical request from cache without spawning', () => {
+  const cache = new ResponseCache(tempDir())
+  let spawnCalls = 0
+  const countingSpawn: Spawn = (cmd, args, input) => {
+    spawnCalls++
+    return fakeSpawn(successEnvelope({ answer: 42 }))(cmd, args, input)
+  }
+
+  const first = runClaude(baseReq(), { spawn: countingSpawn, cache })
+  assert.equal(first.ok, true)
+  assert.equal(first.cached, false)
+  assert.equal(spawnCalls, 1)
+
+  // Second identical call: cache hit, no further spawn (throwingSpawn proves it).
+  const second = runClaude(baseReq(), { spawn: throwingSpawn(), cache })
+  assert.equal(second.ok, true)
+  assert.equal(second.cached, true)
+  assert.deepEqual(second.output, { answer: 42 })
+  assert.equal(spawnCalls, 1)
+})
+
+test('runClaude misses the cache when a keyed field changes', () => {
+  const cache = new ResponseCache(tempDir())
+  runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 42 })) , cache })
+
+  let spawned = false
+  const spawn: Spawn = (cmd, args, input) => {
+    spawned = true
+    return fakeSpawn(successEnvelope({ answer: 7 }))(cmd, args, input)
+  }
+  const result = runClaude(baseReq({ model: 'claude-opus-4-8' }), { spawn, cache })
+  assert.equal(spawned, true)
+  assert.equal(result.cached, false)
+  assert.deepEqual(result.output, { answer: 7 })
+})
+
+test('runClaude does not cache a graded failure', () => {
+  const cache = new ResponseCache(tempDir())
+  // A schema-miss success envelope grades ok:false and must not be cached.
+  runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ wrong: 1 })), cache })
+  assert.equal(cache.get(cacheKey(baseReq())), null)
+})
diff --git a/tests/eval/engine.test.ts b/tests/eval/engine.test.ts
new file mode 100644
index 0000000..de3c309
--- /dev/null
+++ b/tests/eval/engine.test.ts
@@ -0,0 +1,181 @@
+// Engine tests: pure argv construction, envelope grading over
+// success/fallback/error/schema-miss/unparseable, and the isAvailable/skip
+// paths — all through the injected `Spawn` stub, never a real `claude`.
+
+import assert from 'node:assert/strict'
+import { test } from 'bun:test'
+import { buildArgs, isAvailable, parseEnvelope, runClaude } from '../../src/eval/engine.ts'
+import type { EvalRequest, JsonSchema } from '../../src/eval/types.ts'
+import {
+  errorEnvelope,
+  fakeSpawn,
+  rawSpawn,
+  schemaMismatchEnvelope,
+  successEnvelope,
+  throwingSpawn,
+} from '../helpers/fake-claude.ts'
+
+const SCHEMA: JsonSchema = {
+  type: 'object',
+  required: ['answer'],
+  properties: { answer: { type: 'number' } },
+  additionalProperties: false,
+}
+
+const baseReq = (overrides: Partial<EvalRequest> = {}): EvalRequest => ({
+  prompt: 'What is 6 * 7?',
+  model: 'claude-haiku-4-5',
+  schema: SCHEMA,
+  ...overrides,
+})
+
+// --- buildArgs (acceptance criterion 1) --------------------------------------
+
+test('buildArgs forces print mode, JSON output, schema, model, strict MCP, and settings', () => {
+  const args = buildArgs(baseReq())
+  assert.ok(args.includes('--print'))
+  assert.deepEqual(args.slice(args.indexOf('--output-format'), args.indexOf('--output-format') + 2), [
+    '--output-format',
+    'json',
+  ])
+  assert.deepEqual(args.slice(args.indexOf('--json-schema'), args.indexOf('--json-schema') + 2), [
+    '--json-schema',
+    JSON.stringify(SCHEMA),
+  ])
+  assert.deepEqual(args.slice(args.indexOf('--model'), args.indexOf('--model') + 2), [
+    '--model',
+    'claude-haiku-4-5',
+  ])
+  assert.ok(args.includes('--strict-mcp-config'))
+  assert.deepEqual(args.slice(args.indexOf('--settings'), args.indexOf('--settings') + 2), ['--settings', '{}'])
+})
+
+test('buildArgs disables tools by default and never puts the prompt in argv', () => {
+  const args = buildArgs(baseReq())
+  assert.ok(args.includes('--disallowedTools'))
+  assert.ok(!args.includes('--allowedTools'))
+  // The prompt travels on stdin, so it must not appear in argv.
+  assert.ok(!args.some(a => a.includes('6 * 7')))
+})
+
+test('buildArgs opts into an allow-list when tools are provided', () => {
+  const args = buildArgs(baseReq({ tools: ['Read', 'Grep'] }))
+  assert.deepEqual(args.slice(args.indexOf('--allowedTools'), args.indexOf('--allowedTools') + 2), [
+    '--allowedTools',
+    'Read Grep',
+  ])
+  assert.ok(!args.includes('--disallowedTools'))
+})
+
+test('buildArgs adds system-prompt flags only when set, and passes custom settings', () => {
+  const bare = buildArgs(baseReq())
+  assert.ok(!bare.includes('--system-prompt'))
+  assert.ok(!bare.includes('--append-system-prompt'))
+
+  const withSys = buildArgs(baseReq({ systemPrompt: 'You are terse.', appendSystemPrompt: 'Context.', settings: '{"x":1}' }))
+  assert.deepEqual(withSys.slice(withSys.indexOf('--system-prompt'), withSys.indexOf('--system-prompt') + 2), [
+    '--system-prompt',
+    'You are terse.',
+  ])
+  assert.deepEqual(
+    withSys.slice(withSys.indexOf('--append-system-prompt'), withSys.indexOf('--append-system-prompt') + 2),
+    ['--append-system-prompt', 'Context.']
+  )
+  assert.deepEqual(withSys.slice(withSys.indexOf('--settings'), withSys.indexOf('--settings') + 2), [
+    '--settings',
+    '{"x":1}',
+  ])
+})
+
+// --- success + fallback (acceptance criterion 2) -----------------------------
+
+test('a success envelope with structured_output yields ok:true with cost/usage/output', () => {
+  const envelope = successEnvelope(
+    { answer: 42 },
+    { cost: 0.0463, usage: { inputTokens: 10, outputTokens: 158, cacheReadInputTokens: 3 }, durationMs: 2419, apiDurationMs: 2394, numTurns: 2 }
+  )
+  const result = runClaude(baseReq(), { spawn: fakeSpawn(envelope) })
+  assert.equal(result.ok, true)
+  assert.deepEqual(result.output, { answer: 42 })
+  assert.equal(result.error, null)
+  assert.equal(result.cost, 0.0463)
+  assert.deepEqual(result.usage, {
+    inputTokens: 10,
+    outputTokens: 158,
+    cacheReadInputTokens: 3,
+    cacheCreationInputTokens: 0,
+  })
+  assert.equal(result.durationMs, 2419)
+  assert.equal(result.apiDurationMs, 2394)
+  assert.equal(result.numTurns, 2)
+  assert.equal(result.model, 'claude-haiku-4-5')
+  assert.equal(result.cached, false)
+})
+
+test('a missing structured_output falls back to JSON.parse(result)', () => {
+  const envelope = successEnvelope({ answer: 7 }, { structured: false })
+  assert.equal(envelope.structured_output, undefined)
+  const result = runClaude(baseReq(), { spawn: fakeSpawn(envelope) })
+  assert.equal(result.ok, true)
+  assert.deepEqual(result.output, { answer: 7 })
+})
+
+// --- graded failures, no throw (acceptance criterion 3) ----------------------
+
+test('is_error / non-success subtype grades ok:false without throwing', () => {
+  const result = runClaude(baseReq(), { spawn: fakeSpawn(errorEnvelope('error_max_turns')) })
+  assert.equal(result.ok, false)
+  assert.equal(result.output, null)
+  assert.match(result.error ?? '', /error_max_turns/)
+  // Metrics from the envelope are still surfaced.
+  assert.equal(result.cost, 0.002)
+})
+
+test('schema-nonconforming output grades ok:false with a descriptive error', () => {
+  const result = runClaude(baseReq(), { spawn: fakeSpawn(schemaMismatchEnvelope()) })
+  assert.equal(result.ok, false)
+  assert.match(result.error ?? '', /required key: answer/)
+})
+
+test('additionalProperties:false rejects an extra key', () => {
+  const envelope = successEnvelope({ answer: 1, extra: 2 })
+  const result = parseEnvelope(JSON.stringify(envelope), baseReq())
+  assert.equal(result.ok, false)
+  assert.match(result.error ?? '', /unexpected key: extra/)
+})
+
+test('unparseable stdout grades ok:false with zeroed metrics, no throw', () => {
+  const result = runClaude(baseReq(), { spawn: rawSpawn(0, 'not json at all') })
+  assert.equal(result.ok, false)
+  assert.equal(result.error, 'unparseable envelope')
+  assert.equal(result.cost, 0)
+  assert.equal(result.durationMs, 0)
+})
+
+// --- availability + skip (acceptance criterion 5) ----------------------------
+
+test('isAvailable is false when claude --version exits non-zero or throws', () => {
+  assert.equal(isAvailable(rawSpawn(127, '', 'command not found')), false)
+  assert.equal(isAvailable(throwingSpawn()), false)
+  assert.equal(isAvailable(rawSpawn(0, 'claude 2.1.207')), true)
+})
+
+test('runClaude against an absent CLI returns a skipped result, never throws', () => {
+  const viaExit = runClaude(baseReq(), { spawn: rawSpawn(127, '', 'not found') })
+  assert.equal(viaExit.skipped, true)
+  assert.equal(viaExit.ok, false)
+  assert.match(viaExit.error ?? '', /unavailable/)
+
+  const viaThrow = runClaude(baseReq(), { spawn: throwingSpawn() })
+  assert.equal(viaThrow.skipped, true)
+  assert.equal(viaThrow.ok, false)
+})
+
+test('a non-zero exit that still produced envelope stdout is graded, not skipped', () => {
+  // The CLI can exit non-zero while emitting an error envelope — that is a
+  // graded failure, not an availability skip.
+  const result = runClaude(baseReq(), { spawn: rawSpawn(1, JSON.stringify(errorEnvelope('error_during_execution'))) })
+  assert.equal(result.skipped, undefined)
+  assert.equal(result.ok, false)
+  assert.match(result.error ?? '', /error_during_execution/)
+})
diff --git a/tests/eval/framework.test.ts b/tests/eval/framework.test.ts
new file mode 100644
index 0000000..682d2f1
--- /dev/null
+++ b/tests/eval/framework.test.ts
@@ -0,0 +1,415 @@
+// Framework tests: built-in graders + gradeCase aggregation, the runner's
+// runCase/runSuite/runAB/runMatrix over canned envelopes, the bounded-parallel
+// pool, report text/JSON, and the CLI exit-code semantics (regression vs clean
+// vs CLI-absent). Every model call is faked at the `Spawn` boundary — offline, no
+// real `claude`.
+
+import assert from 'node:assert/strict'
+import { mkdtempSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test } from 'bun:test'
+import { defineCase, toRequest, type EvalCase } from '../../src/eval/case.ts'
+import { assert as assertGrader, gradeCase, judge, schemaConforms, type Grader } from '../../src/eval/grade.ts'
+import {
+  mapPool,
+  runAB,
+  runCase,
+  runMatrix,
+  runSuite,
+  type ABResult,
+  type CaseResult,
+} from '../../src/eval/runner.ts'
+import { formatReport, projectCost, toJSON, type Report } from '../../src/eval/report.ts'
+import { abExitCode, main, loadSuite, parseArgs } from '../../src/eval/cli.ts'
+import type { EvalResult, JsonSchema, Spawn } from '../../src/eval/types.ts'
+import { successEnvelope, throwingSpawn } from '../helpers/fake-claude.ts'
+
+const SCHEMA: JsonSchema = {
+  type: 'object',
+  required: ['answer'],
+  properties: { answer: { type: 'number' } },
+  additionalProperties: false,
+}
+
+/** A minimal ok result carrying the given output — for grading graders directly. */
+function okResult(output: unknown): EvalResult {
+  return {
+    ok: true,
+    output,
+    error: null,
+    cost: 0.01,
+    usage: { inputTokens: 10, outputTokens: 20, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 },
+    durationMs: 1000,
+    apiDurationMs: 950,
+    numTurns: 2,
+    model: 'claude-haiku-4-5',
+    cached: false,
+  }
+}
+
+const baseCase = (overrides: Partial<EvalCase> = {}): EvalCase =>
+  defineCase({
+    id: 'add',
+    tags: ['math'],
+    prompt: 'What is 6 * 7?',
+    schema: SCHEMA,
+    graders: [schemaConforms(), assertGrader('is-42', o => (o as { answer?: number }).answer === 42)],
+    ...overrides,
+  })
+
+// --- graders (acceptance criterion 2) ----------------------------------------
+
+test('schemaConforms passes on an ok result and fails on a graded failure', () => {
+  const pass = schemaConforms()(null, okResult({ answer: 42 }), {})
+  assert.equal(pass.name, 'schemaConforms')
+  assert.equal(pass.pass, true)
+  assert.equal(pass.score, 1)
+
+  const failResult = { ...okResult(null), ok: false, error: 'missing key' }
+  const fail = schemaConforms()(null, failResult, {})
+  assert.equal(fail.pass, false)
+  assert.equal(fail.score, 0)
+  assert.match(fail.detail, /missing key/)
+})
+
+test('assert grades a predicate, and a thrown predicate fails without propagating', () => {
+  const pass = assertGrader('is-42', o => (o as { answer: number }).answer === 42)({ answer: 42 }, okResult({ answer: 42 }), {})
+  assert.equal(pass.pass, true)
+  assert.equal(pass.score, 1)
+
+  const fail = assertGrader('is-42', o => (o as { answer: number }).answer === 42)({ answer: 1 }, okResult({ answer: 1 }), {})
+  assert.equal(fail.pass, false)
+
+  const threw = assertGrader('boom', () => {
+    throw new Error('kaboom')
+  })(null, okResult(null), {})
+  assert.equal(threw.pass, false)
+  assert.match(threw.detail, /kaboom/)
+})
+
+test('judge routes through the engine (stubbed) and maps the verdict to a GradeResult', () => {
+  const verdictEnvelope = successEnvelope({ score: 0.8, pass: true, reason: 'clear and complete' })
+  const spawn: Spawn = () => ({ status: 0, stdout: JSON.stringify(verdictEnvelope), stderr: '' })
+  const grade = judge('Rate the answer 0..1.', { model: 'claude-haiku-4-5' })(
+    { answer: 42 },
+    okResult({ answer: 42 }),
+    { spawn }
+  )
+  assert.equal(grade.name, 'judge')
+  assert.equal(grade.pass, true)
+  assert.equal(grade.score, 0.8)
+  assert.match(grade.detail, /clear and complete/)
+})
+
+test('judge clamps an out-of-range score and fails gracefully when the engine errors', () => {
+  const highSpawn: Spawn = () => ({
+    status: 0,
+    stdout: JSON.stringify(successEnvelope({ score: 5, pass: true, reason: 'over' })),
+    stderr: '',
+  })
+  const clamped = judge('r', { model: 'm' })({ answer: 1 }, okResult({ answer: 1 }), { spawn: highSpawn })
+  assert.equal(clamped.score, 1)
+
+  const failed = judge('r', { model: 'm' })({ answer: 1 }, okResult({ answer: 1 }), { spawn: throwingSpawn() })
+  assert.equal(failed.pass, false)
+  assert.equal(failed.score, 0)
+})
+
+// --- gradeCase aggregation (acceptance criterion 2) --------------------------
+
+test('gradeCase computes a weighted correctness across graders', () => {
+  // One passing (weight 3) + one failing (weight 1) → 3/4 = 0.75.
+  const graders: Grader[] = [
+    assertGrader('a', () => true, { weight: 3 }),
+    assertGrader('b', () => false, { weight: 1 }),
+  ]
+  const { correctness, grades } = gradeCase(graders, { answer: 42 }, okResult({ answer: 42 }))
+  assert.equal(grades.length, 2)
+  assert.equal(correctness, 0.75)
+})
+
+test('gradeCase with no graders is 0, not NaN', () => {
+  const { correctness } = gradeCase([], null, okResult(null))
+  assert.equal(correctness, 0)
+})
+
+// --- toRequest (case lowering) -----------------------------------------------
+
+test('toRequest lowers a case to an EvalRequest and honours a prompt override', () => {
+  const c = baseCase({ systemPrompt: 'terse', tools: ['Read'] })
+  const req = toRequest(c, 'claude-opus-4-8', 'OVERRIDDEN')
+  assert.equal(req.prompt, 'OVERRIDDEN')
+  assert.equal(req.model, 'claude-opus-4-8')
+  assert.equal(req.systemPrompt, 'terse')
+  assert.deepEqual(req.tools, ['Read'])
+  assert.equal(toRequest(c, 'm').prompt, 'What is 6 * 7?')
+})
+
+// --- runCase end-to-end (acceptance criterion 1) -----------------------------
+
+test('runCase runs a case through the engine and surfaces correctness + envelope metrics', () => {
+  const spawn: Spawn = () => ({
+    status: 0,
+    stdout: JSON.stringify(successEnvelope({ answer: 42 }, { cost: 0.0463, durationMs: 2419, numTurns: 2 })),
+    stderr: '',
+  })
+  const r = runCase(baseCase(), { model: 'claude-haiku-4-5', spawn })
+  assert.equal(r.caseId, 'add')
+  assert.equal(r.correctness, 1) // both graders pass
+  assert.equal(r.skipped, false)
+  assert.equal(r.result.cost, 0.0463)
+  assert.equal(r.result.durationMs, 2419)
+  assert.equal(r.result.numTurns, 2)
+})
+
+test('runCase marks a case skipped (no failure) when the CLI is unavailable', () => {
+  const r = runCase(baseCase(), { model: 'm', spawn: throwingSpawn() })
+  assert.equal(r.skipped, true)
+  assert.equal(r.grades.length, 0)
+})
+
+// --- bounded parallel pool (acceptance criterion 4) --------------------------
+
+test('mapPool never exceeds the concurrency bound and preserves order', async () => {
+  let inFlight = 0
+  let maxInFlight = 0
+  const worker = (n: number): Promise<number> =>
+    new Promise(resolve => {
+      inFlight++
+      maxInFlight = Math.max(maxInFlight, inFlight)
+      setTimeout(() => {
+        inFlight--
+        resolve(n * 2)
+      }, 5)
+    })
+  const out = await mapPool([1, 2, 3, 4, 5, 6, 7], 2, worker)
+  assert.deepEqual(out, [2, 4, 6, 8, 10, 12, 14])
+  assert.ok(maxInFlight <= 2, `maxInFlight=${maxInFlight}`)
+})
+
+test('runSuite fans a case set across models and returns one result per (case,model)', async () => {
+  const spawn: Spawn = () => ({ status: 0, stdout: JSON.stringify(successEnvelope({ answer: 42 })), stderr: '' })
+  const results = await runSuite([baseCase({ id: 'a' }), baseCase({ id: 'b' })], {
+    models: ['claude-haiku-4-5', 'claude-sonnet-5'],
+    concurrency: 2,
+    spawn,
+  })
+  assert.equal(results.length, 4)
+  assert.deepEqual(
+    results.map(r => `${r.caseId}:${r.model}`),
+    ['a:claude-haiku-4-5', 'a:claude-sonnet-5', 'b:claude-haiku-4-5', 'b:claude-sonnet-5']
+  )
+})
+
+// --- runAB deltas (acceptance criterion 3) -----------------------------------
+
+/** A spawn that returns a cheaper/faster/correct envelope for the candidate prompt. */
+const abSpawn: Spawn = (_cmd, _args, input) => {
+  const isCandidate = (input ?? '').includes('CANDIDATE')
+  const env = isCandidate
+    ? successEnvelope({ answer: 42 }, { cost: 0.01, durationMs: 800, numTurns: 1 })
+    : successEnvelope({ answer: 42 }, { cost: 0.05, durationMs: 2000, numTurns: 3 })
+  return { status: 0, stdout: JSON.stringify(env), stderr: '' }
+}
+
+test('runAB reports Δcorrectness/Δcost/Δlatency/Δturns (candidate cheaper/faster → positive)', () => {
+  const ab = runAB(
+    baseCase(),
+    { label: 'baseline', prompt: 'BASELINE: what is 6*7?' },
+    { label: 'candidate', prompt: 'CANDIDATE: what is 6*7?' },
+    { model: 'claude-haiku-4-5', spawn: abSpawn }
+  )
+  assert.equal(ab.deltaCorrectness, 0) // both correct
+  assert.ok(ab.deltaCostUsd > 0, 'candidate cheaper → positive Δcost')
+  assert.ok(ab.deltaLatencyMs > 0, 'candidate faster → positive Δlatency')
+  assert.ok(ab.deltaTurns > 0, 'candidate fewer turns → positive Δturns')
+  assert.equal(ab.baselineLabel, 'baseline')
+  assert.equal(ab.candidateLabel, 'candidate')
+})
+
+test('runAB surfaces a negative Δcorrectness when the candidate regresses', () => {
+  // Candidate returns a wrong answer → its assertion grader fails.
+  const regressSpawn: Spawn = (_c, _a, input) => {
+    const answer = (input ?? '').includes('CANDIDATE') ? 0 : 42
+    return { status: 0, stdout: JSON.stringify(successEnvelope({ answer })), stderr: '' }
+  }
+  const ab = runAB(
+    baseCase(),
+    { label: 'base', prompt: 'BASELINE q' },
+    { label: 'cand', prompt: 'CANDIDATE q' },
+    { model: 'm', spawn: regressSpawn }
+  )
+  assert.ok(ab.deltaCorrectness < 0, 'candidate wrong → negative Δcorrectness')
+})
+
+// --- runMatrix grid (acceptance criterion 3) ---------------------------------
+
+test('runMatrix produces one row per model', async () => {
+  const spawn: Spawn = () => ({ status: 0, stdout: JSON.stringify(successEnvelope({ answer: 42 })), stderr: '' })
+  const grid = await runMatrix(baseCase(), ['claude-opus-4-8', 'claude-sonnet-5', 'claude-haiku-4-5'], { spawn })
+  assert.equal(grid.rows.length, 3)
+  assert.deepEqual(grid.rows.map(r => r.model), ['claude-opus-4-8', 'claude-sonnet-5', 'claude-haiku-4-5'])
+  assert.ok(grid.rows.every(r => r.correctness === 1))
+})
+
+// --- report (acceptance criterion 5) -----------------------------------------
+
+const sampleCaseResult = (id: string): CaseResult => ({
+  caseId: id,
+  model: 'claude-haiku-4-5',
+  correctness: 1,
+  grades: [],
+  result: okResult({ answer: 42 }),
+  skipped: false,
+})
+
+test('formatReport renders a stable suite text table with all three axes', () => {
+  const report: Report = { mode: 'suite', cases: [sampleCaseResult('add')] }
+  const text = formatReport(report)
+  assert.match(text, /Case/)
+  assert.match(text, /Correct/)
+  assert.match(text, /Cost \$/)
+  assert.match(text, /Latency/)
+  assert.match(text, /100\.0%/)
+  assert.match(text, /\$0\.01000/)
+  assert.match(text, /1000ms/)
+})
+
+test('formatReport suite adds a clearly-labelled projected-cost column on request', () => {
+  const report: Report = { mode: 'suite', cases: [sampleCaseResult('add')] }
+  const text = formatReport(report, { projectModel: 'claude-opus-4-8' })
+  assert.match(text, /Proj \$ \(claude-opus-4-8\)/)
+})
+
+test('projectCost uses the static table and returns null for an unpriced model', () => {
+  const usage = { inputTokens: 1_000_000, outputTokens: 1_000_000, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 }
+  assert.equal(projectCost(usage, 'claude-haiku-4-5'), 6) // 1 + 5 per Mtok
+  assert.equal(projectCost(usage, 'no-such-model'), null)
+})
+
+test('formatReport ab shows the delta columns and a trade-off legend', () => {
+  const ab: ABResult = runAB(
+    baseCase(),
+    { label: 'base', prompt: 'BASELINE q' },
+    { label: 'cand', prompt: 'CANDIDATE q' },
+    { model: 'm', spawn: abSpawn }
+  )
+  const text = formatReport({ mode: 'ab', cases: [ab] })
+  assert.match(text, /ΔCorrect/)
+  assert.match(text, /ΔCost/)
+  assert.match(text, /ΔLatency/)
+  assert.match(text, /candidate cheaper\/faster/)
+})
+
+test('toJSON returns the machine-comparable report object', () => {
+  const report: Report = { mode: 'suite', cases: [sampleCaseResult('add')] }
+  const json = toJSON(report)
+  assert.equal(json.mode, 'suite')
+  // Round-trips through JSON.stringify (what --json emits).
+  const parsed = JSON.parse(JSON.stringify(json)) as { mode: string; cases: unknown[] }
+  assert.equal(parsed.mode, 'suite')
+  assert.equal(parsed.cases.length, 1)
+})
+
+// --- CLI exit-code semantics (acceptance criterion 5) ------------------------
+
+test('parseArgs reads every flag', () => {
+  const f = parseArgs(['--suite', 'd', '--filter', 'planner', '--ab', '--models', 'a,b', '--json', '--tolerance', '5'])
+  assert.equal(f.suite, 'd')
+  assert.equal(f.filter, 'planner')
+  assert.equal(f.ab, true)
+  assert.deepEqual(f.models, ['a', 'b'])
+  assert.equal(f.json, true)
+  assert.equal(f.tolerance, 5)
+})
+
+test('abExitCode gates only on a regression past tolerance', () => {
+  const mk = (delta: number): ABResult => ({
+    caseId: 'c',
+    model: 'm',
+    baselineLabel: 'b',
+    candidateLabel: 'c',
+    baseline: sampleCaseResult('c'),
+    candidate: sampleCaseResult('c'),
+    deltaCorrectness: delta,
+    deltaCostUsd: 0,
+    deltaLatencyMs: 0,
+    deltaTurns: 0,
+  })
+  // -3pp within a 5% tolerance → 0; -6pp past it → 1; an improvement → 0.
+  assert.equal(abExitCode([mk(-0.03)], 5), 0)
+  assert.equal(abExitCode([mk(-0.06)], 5), 1)
+  assert.equal(abExitCode([mk(0.1)], 5), 0)
+})
+
+test('main skips-with-notice and exits 0 when claude is unavailable', async () => {
+  const res = await main(['--suite', '/nonexistent', '--ab'], { spawn: throwingSpawn() })
+  assert.equal(res.code, 0)
+  assert.match(res.output, /unavailable/)
+})
+
+/** A spawn where `--version` succeeds (available) and runs return a success envelope. */
+const availableSpawn = (envelope: unknown): Spawn => (_cmd, args) => {
+  if (args.includes('--version')) return { status: 0, stdout: 'claude 2.1.207', stderr: '' }
+  return { status: 0, stdout: JSON.stringify(envelope), stderr: '' }
+}
+
+test('main runs a plain suite (report-only, exit 0) over injected cases', async () => {
+  const res = await main([], { spawn: availableSpawn(successEnvelope({ answer: 42 })), cases: [baseCase()] })
+  assert.equal(res.code, 0)
+  assert.match(res.output, /Correct/)
+  assert.match(res.output, /add/)
+})
+
+test('main --json emits parseable machine JSON', async () => {
+  const res = await main(['--json'], { spawn: availableSpawn(successEnvelope({ answer: 42 })), cases: [baseCase()] })
+  const parsed = JSON.parse(res.output) as { mode: string }
+  assert.equal(parsed.mode, 'suite')
+})
+
+test('main --ab exits 1 on a regression past tolerance and 0 when clean', async () => {
+  const abCase = (): EvalCase =>
+    baseCase({
+      variants: { baseline: { label: 'base', prompt: 'BASELINE q' }, candidate: { label: 'cand', prompt: 'CANDIDATE q' } },
+    })
+  // Candidate wrong → regression → exit 1.
+  const regressSpawn: Spawn = (_c, args, input) => {
+    if (args.includes('--version')) return { status: 0, stdout: 'claude 2.1.207', stderr: '' }
+    const answer = (input ?? '').includes('CANDIDATE') ? 0 : 42
+    return { status: 0, stdout: JSON.stringify(successEnvelope({ answer })), stderr: '' }
+  }
+  const regressed = await main(['--ab', '--tolerance', '5'], { spawn: regressSpawn, cases: [abCase()] })
+  assert.equal(regressed.code, 1)
+
+  // Both correct → no regression → exit 0.
+  const cleanSpawn: Spawn = (_c, args) => {
+    if (args.includes('--version')) return { status: 0, stdout: 'claude 2.1.207', stderr: '' }
+    return { status: 0, stdout: JSON.stringify(successEnvelope({ answer: 42 })), stderr: '' }
+  }
+  const clean = await main(['--ab', '--tolerance', '5'], { spawn: cleanSpawn, cases: [abCase()] })
+  assert.equal(clean.code, 0)
+})
+
+test('main --filter selects by id or tag', async () => {
+  const res = await main(['--json', '--filter', 'nope'], {
+    spawn: availableSpawn(successEnvelope({ answer: 42 })),
+    cases: [baseCase()],
+  })
+  assert.match(res.output, /no cases matched/)
+})
+
+// --- suite loading (integration: real dynamic import from a temp dir) --------
+
+test('loadSuite dynamically imports case modules from a directory', async () => {
+  const dir = mkdtempSync(join(tmpdir(), 'eval-suite-'))
+  const casePath = join(import.meta.dir, '..', '..', 'src', 'eval', 'case.ts').replace(/\\/g, '/')
+  writeFileSync(
+    join(dir, 'sample.ts'),
+    `import { defineCase } from '${casePath}'\n` +
+      `export const cases = [defineCase({ id: 'loaded', tags: ['t'], prompt: 'p', schema: { type: 'object' }, graders: [] })]\n`
+  )
+  const cases = await loadSuite(dir)
+  assert.equal(cases.length, 1)
+  assert.equal(cases[0]?.id, 'loaded')
+})
diff --git a/tests/helpers/fake-claude.ts b/tests/helpers/fake-claude.ts
new file mode 100644
index 0000000..b3f434c
--- /dev/null
+++ b/tests/helpers/fake-claude.ts
@@ -0,0 +1,82 @@
+// Hermetic stub for the `claude -p` subprocess boundary. Tests inject `fakeSpawn`
+// (or a raw variant) so `bun test` never shells out to a real `claude` — the ONLY
+// faked seam in the eval engine. Envelope factories mirror the verified real
+// `--output-format json` shape (installed claude 2.1.207).
+
+import type { Envelope, EvalUsage, Spawn } from '../../src/eval/types.ts'
+
+/** A `Spawn` that ignores args/stdin and emits `envelope` as JSON on stdout, exit 0. */
+export function fakeSpawn(envelope: Envelope): Spawn {
+  return () => ({ status: 0, stdout: JSON.stringify(envelope), stderr: '' })
+}
+
+/** A `Spawn` returning a fixed status/stdout/stderr verbatim (for edge cases). */
+export function rawSpawn(status: number | null, stdout: string, stderr = ''): Spawn {
+  return () => ({ status, stdout, stderr })
+}
+
+/** A `Spawn` that throws, modelling a missing binary (ENOENT). */
+export function throwingSpawn(): Spawn {
+  return () => {
+    throw new Error('spawn claude ENOENT')
+  }
+}
+
+export interface SuccessOptions {
+  cost?: number
+  usage?: Partial<EvalUsage>
+  durationMs?: number
+  apiDurationMs?: number
+  numTurns?: number
+  /** When false, omit `structured_output` so the engine falls back to `result`. */
+  structured?: boolean
+}
+
+/** A success envelope carrying `output` as both `structured_output` and `result`. */
+export function successEnvelope(output: unknown, opts: SuccessOptions = {}): Envelope {
+  const {
+    cost = 0.001,
+    usage = {},
+    durationMs = 1000,
+    apiDurationMs = 950,
+    numTurns = 2,
+    structured = true,
+  } = opts
+  const envelope: Envelope = {
+    type: 'result',
+    subtype: 'success',
+    is_error: false,
+    result: JSON.stringify(output),
+    total_cost_usd: cost,
+    duration_ms: durationMs,
+    duration_api_ms: apiDurationMs,
+    num_turns: numTurns,
+    usage: {
+      input_tokens: usage.inputTokens ?? 10,
+      output_tokens: usage.outputTokens ?? 20,
+      cache_read_input_tokens: usage.cacheReadInputTokens ?? 0,
+      cache_creation_input_tokens: usage.cacheCreationInputTokens ?? 0,
+    },
+  }
+  if (structured) envelope.structured_output = output
+  return envelope
+}
+
+/** An error envelope (`is_error:true`, non-success subtype) with metrics present. */
+export function errorEnvelope(subtype = 'error_max_turns'): Envelope {
+  return {
+    type: 'result',
+    subtype,
+    is_error: true,
+    total_cost_usd: 0.002,
+    duration_ms: 500,
+    duration_api_ms: 480,
+    num_turns: 1,
+    usage: { input_tokens: 5, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 },
+  }
+}
+
+/** A success envelope whose `structured_output` omits a required key (schema miss). */
+export function schemaMismatchEnvelope(): Envelope {
+  return successEnvelope({ unexpected: true })
+}