feat(evaluation-system): add eval framework with graders, runner, report

.gitignore

@@ -1 +1,2 @@
 node_modules/
+.eval-cache/

CLAUDE.md

@@ -1,3 +1,5 @@
 # Guidelines
 
+- **Run the prompt-eval suite alongside `bun test` when changing harness prompts.** When you edit an agent prompt — anything under `src/workflows/strapped-run/**` or the step prose in a skill `SKILL.md` — run `bun run eval --suite <dir>` (see `src/eval/` and the "Prompt evaluation suite" section of `plugins/strapped/conventions.md`) and keep it green alongside `bun test`. The eval suite grades a prompt on correctness / cost / latency through `claude -p`; it is the evidence that a prompt change is a real improvement (use `--ab` to compare a baseline vs a candidate and gate on a correctness regression past `--tolerance`). It is a HEAVY, opt-in layer (needs a real `claude` + spends cost), so it is intentionally NOT part of `bun test` — run it yourself for prompt work. `bun test` stays hermetic/offline and remains the gate for everything else.
+
 - **Bump the plugin version with the tool, never by hand.** `claude plugin update` compares `version` in `plugins/strapped/.claude-plugin/plugin.json` only — an unbumped version leaves installed copies silently pinned to a stale commit after a change merges. Bump ONLY via `bun tools/version.ts bump <major|minor|patch>` (it rewrites the one semver deterministically); do not hand-edit plugin.json. Pick the level by a conventional-commit rule: a breaking change → `major`, a new user-facing feature (skill/workflow/script behavior) → `minor`, a fix or internal change → `patch`. Before committing, run `bun tools/version.ts check` (alias `bun run version:check`) — it fails when a changed *generated artifact* (the committed deployables under `plugins/strapped/`) is not accompanied by a version bump above the base. Prompt/skill/docs-only changes with NO generated-artifact change are at your discretion — the guard exits 0 for them, so bump when the change is user-facing and skip an internal-only tweak.

package.json

@@ -8,6 +8,7 @@
     "lint": "oxlint src tools tests",
     "version:bump": "bun tools/version.ts bump",
     "version:check": "bun tools/version.ts check",
+    "eval": "bun src/eval/cli.ts",
     "test": "claude plugin validate . --strict && claude plugin validate plugins/strapped --strict && bun run typecheck && bun run lint && bun test"
   },
   "devDependencies": {

plugins/strapped/conventions.md

@@ -398,6 +398,12 @@ This limit is why the retired multi-file architecture (stage workflows dispatchi
 
 Every command in the **deliverable's repo's** config `validations` must be green before code review and after every fix round, run inside the deliverable's worktree.
 
+### Prompt evaluation suite
+
+The strapped repo carries a prompt-effectiveness eval suite under `src/eval/` (run via `bun run eval`, not bundled into any plugin deployable). It grades the harness's own agent prompts on three axes — **correctness** (layered graders: schema-conformance, assertion predicates, an optional LLM judge), **cost** (`total_cost_usd` + token usage), and **latency** (`duration_ms` / `num_turns`) — by shelling out to `claude -p` (never the Anthropic SDK) and reading the `--output-format json` envelope. It supports **A/B** (baseline vs candidate prompt at one model, reporting Δcorrectness / Δcost / Δlatency side by side so a human judges the trade — a correctness dip that buys a large cost/latency win is a WIN) and a **model matrix** (`--models opus,sonnet,haiku`).
+
+It is the **heavy, opt-in test layer for prompt changes**: it needs a real `claude` and spends cost, so it is deliberately kept OUT of `bun test` (which stays hermetic/offline). When you change an agent prompt (`src/workflows/strapped-run/**` or a skill's `SKILL.md` step prose), run the eval suite alongside `bun test` and keep both green — it is the evidence that the prompt change is a real improvement. An absent `claude` CLI is a graceful skip (notice + exit 0), and only `--ab` gates: exit non-zero on a correctness regression past `--tolerance`.
+
 ## Worktrees and branches
 
 All repo-scoped values below come from the **deliverable's repo** — `manifest.repos[deliverable.repo]` for the root, and that repo's config for `worktreeRoot`, `validations`, and `provisioning`.

src/eval/cache.ts

@@ -0,0 +1,70 @@
+// Content-addressed response cache for eval results. The key is a SHA-256 over a
+// CANONICAL (sorted-key) JSON of every request field that changes the model's
+// output — prompt, both system-prompt knobs, model, schema, tool policy, and
+// settings. Include every such field or an A/B would collide; a changed prompt
+// naturally misses. Results are stored one JSON file per key; no eviction.
+
+import { createHash } from 'node:crypto'
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { join } from 'node:path'
+import type { Cache, EvalRequest, EvalResult } from './types.ts'
+
+/** Recursively sort object keys so key order never perturbs the hash. */
+function sortKeys(value: unknown): unknown {
+  if (Array.isArray(value)) return value.map(sortKeys)
+  if (value !== null && typeof value === 'object') {
+    const sorted: Record<string, unknown> = {}
+    for (const key of Object.keys(value as Record<string, unknown>).sort()) {
+      sorted[key] = sortKeys((value as Record<string, unknown>)[key])
+    }
+    return sorted
+  }
+  return value
+}
+
+/** Stable, key-sorted JSON of the output-affecting fields of a request. */
+function canonicalRequest(req: EvalRequest): string {
+  return JSON.stringify(
+    sortKeys({
+      prompt: req.prompt,
+      systemPrompt: req.systemPrompt ?? null,
+      appendSystemPrompt: req.appendSystemPrompt ?? null,
+      model: req.model,
+      schema: req.schema,
+      tools: req.tools ?? null,
+      settings: req.settings ?? null,
+    })
+  )
+}
+
+/** Hex SHA-256 cache key over a request's output-affecting fields. */
+export function cacheKey(req: EvalRequest): string {
+  return createHash('sha256').update(canonicalRequest(req)).digest('hex')
+}
+
+/** File-backed response cache: one `<dir>/<key>.json` per stored result. */
+export class ResponseCache implements Cache {
+  constructor(private readonly dir: string) {}
+
+  private path(key: string): string {
+    return join(this.dir, `${key}.json`)
+  }
+
+  /** Return the stored result (marked `cached:true`) or `null` on miss/corruption. */
+  get(key: string): EvalResult | null {
+    const p = this.path(key)
+    if (!existsSync(p)) return null
+    try {
+      const result = JSON.parse(readFileSync(p, 'utf8')) as EvalResult
+      return { ...result, cached: true }
+    } catch {
+      return null
+    }
+  }
+
+  /** Persist a result under `key`, creating the cache dir on first write. */
+  set(key: string, result: EvalResult): void {
+    mkdirSync(this.dir, { recursive: true })
+    writeFileSync(this.path(key), JSON.stringify(result, null, 2))
+  }
+}

src/eval/case.ts

@@ -0,0 +1,77 @@
+// The eval CASE abstraction: a single, self-contained prompt-evaluation unit. A
+// case carries the prompt, the schema it is forced against, isolation knobs
+// (system prompt / tools / settings), the models it targets, its correctness
+// graders, and — for A/B — an optional baseline+candidate prompt pair. A SUITE is
+// just a directory of modules that export cases.
+//
+// Design decision (research §"prompts are eval inputs"): a case owns a VERBATIM
+// baseline prompt snapshot copied from the live stage plus a candidate variant to
+// A/B. Cases never import the live stage builders — prompts are inputs, not code.
+
+import type { Grader } from './grade.ts'
+import type { EvalRequest, JsonSchema } from './types.ts'
+
+/** One labelled prompt variant for A/B comparison. */
+export interface Variant {
+  /** Short label shown in the report (e.g. `baseline`, `terser`). */
+  label: string
+  /** The full prompt text for this variant. */
+  prompt: string
+}
+
+/** A baseline↔candidate variant pair a case can be A/B'd on. */
+export interface CaseVariants {
+  baseline: Variant
+  candidate: Variant
+}
+
+/** A single prompt-evaluation case. */
+export interface EvalCase {
+  /** Unique id (also a `--filter` target). */
+  id: string
+  /** Tags for `--filter` selection (e.g. `planner`, `reviewer`). */
+  tags: string[]
+  /** Replace the CLI's default system prompt for isolation (`--system-prompt`). */
+  systemPrompt?: string
+  /** Layer strapped context on the real system prompt (`--append-system-prompt`). */
+  appendSystemPrompt?: string
+  /** The user prompt under evaluation. */
+  prompt: string
+  /** The schema forced via `--json-schema` and validated against the output. */
+  schema: JsonSchema
+  /** Models this case targets in a plain/matrix run; the runner falls back to a default. */
+  models?: string[]
+  /** Opt into a bounded toolset; omit for a single-shot, tool-free run. */
+  tools?: string[]
+  /** Inline settings JSON (`--settings`); defaults to `'{}'` in the engine. */
+  settings?: string
+  /** The correctness graders aggregated into this case's score. */
+  graders: Grader[]
+  /** Optional A/B prompt pair — required for a case to appear in `--ab` mode. */
+  variants?: CaseVariants
+}
+
+/**
+ * Identity helper for authoring a case module — validates nothing beyond types,
+ * but gives suites a single import and a stable authoring surface.
+ */
+export function defineCase(spec: EvalCase): EvalCase {
+  return spec
+}
+
+/**
+ * Lower a case (at one model, optionally with a variant's prompt substituted)
+ * into the single-shot `EvalRequest` the D1 engine consumes. The prompt override
+ * is how A/B swaps baseline vs candidate while holding everything else fixed.
+ */
+export function toRequest(c: EvalCase, model: string, promptOverride?: string): EvalRequest {
+  return {
+    prompt: promptOverride ?? c.prompt,
+    systemPrompt: c.systemPrompt,
+    appendSystemPrompt: c.appendSystemPrompt,
+    model,
+    schema: c.schema,
+    tools: c.tools,
+    settings: c.settings,
+  }
+}

src/eval/cli.ts

@@ -0,0 +1,174 @@
+// `bun src/eval/cli.ts` — the eval suite entrypoint (npm: `bun run eval`). Loads
+// case modules from a suite directory, runs them in one of three modes, and
+// prints a report. This is the HEAVY, opt-in test layer for prompt changes — it
+// is deliberately NOT part of `bun test` (that stays hermetic/offline).
+//
+// Exit code: a plain/matrix run is REPORT-ONLY (always 0). Only `--ab` gates —
+// exit 1 iff some case's Δcorrectness dips past `--tolerance` (a regression). An
+// absent `claude` CLI is a graceful skip: print a notice, exit 0, never fail.
+//
+// Flags: --suite <dir>  --filter <tag|id>  --ab  --matrix  --models a,b
+//        --json  --tolerance <pct>
+//
+// `main` returns { code, output } (does not exit) so tests can drive the exact
+// exit-code semantics through the injected spawn; the bottom guard is the only
+// place that touches the process.
+
+import { readdirSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import type { EvalCase } from './case.ts'
+import { isAvailable } from './engine.ts'
+import { formatReport, toJSON, type Report } from './report.ts'
+import {
+  DEFAULT_MODEL,
+  runAB,
+  runMatrix,
+  runSuite,
+  type ABResult,
+  type MatrixResult,
+} from './runner.ts'
+import type { Cache, Spawn } from './types.ts'
+
+export interface EvalFlags {
+  suite?: string
+  filter?: string
+  ab: boolean
+  matrix: boolean
+  models?: string[]
+  json: boolean
+  tolerance: number
+}
+
+/** Hand-rolled argv parser (no dependency). Unknown flags are ignored. */
+export function parseArgs(argv: readonly string[]): EvalFlags {
+  const flags: EvalFlags = { ab: false, matrix: false, json: false, tolerance: 0 }
+  for (let i = 0; i < argv.length; i++) {
+    switch (argv[i]) {
+      case '--suite':
+        flags.suite = argv[++i]
+        break
+      case '--filter':
+        flags.filter = argv[++i]
+        break
+      case '--ab':
+        flags.ab = true
+        break
+      case '--matrix':
+        flags.matrix = true
+        break
+      case '--models':
+        flags.models = (argv[++i] ?? '').split(',').filter(Boolean)
+        break
+      case '--json':
+        flags.json = true
+        break
+      case '--tolerance':
+        flags.tolerance = Number(argv[++i] ?? '0')
+        break
+      default:
+        break
+    }
+  }
+  return flags
+}
+
+/** Load every case exported by the modules in a suite directory (sorted). */
+export async function loadSuite(dir: string): Promise<EvalCase[]> {
+  const abs = resolve(dir)
+  const files = readdirSync(abs)
+    .filter(f => /\.(ts|mjs|js)$/.test(f) && !f.endsWith('.d.ts'))
+    .sort()
+  const cases: EvalCase[] = []
+  for (const file of files) {
+    const mod = (await import(join(abs, file))) as { cases?: unknown; default?: unknown }
+    const exported = mod.cases ?? mod.default
+    if (Array.isArray(exported)) cases.push(...(exported as EvalCase[]))
+    else if (exported) cases.push(exported as EvalCase)
+  }
+  return cases
+}
+
+/** A case matches a filter if the filter equals its id or one of its tags. */
+function matchesFilter(c: EvalCase, filter: string): boolean {
+  return c.id === filter || c.tags.includes(filter)
+}
+
+/**
+ * A/B gate: exit 1 iff any case regressed correctness past the tolerance. A
+ * tolerance of `5` means "a 5-percentage-point dip is allowed"; anything worse
+ * gates. Cost/latency wins never gate — a human judges the trade from the report.
+ */
+export function abExitCode(results: readonly ABResult[], tolerancePct: number): number {
+  const tol = tolerancePct / 100
+  return results.some(r => r.deltaCorrectness < -tol) ? 1 : 0
+}
+
+export interface CliDeps {
+  spawn?: Spawn
+  cache?: Cache
+  /** Test override: supply cases directly instead of loading from `--suite`. */
+  cases?: EvalCase[]
+}
+
+export interface CliResult {
+  code: number
+  output: string
+}
+
+/**
+ * Parse argv, run the selected mode, and return an exit code + text to print.
+ * Never touches the process — the bottom guard does that. An absent CLI short
+ * circuits to a skip notice + exit 0 before any case runs.
+ */
+export async function main(argv: readonly string[], deps: CliDeps = {}): Promise<CliResult> {
+  const flags = parseArgs(argv)
+  const spawn = deps.spawn
+
+  if (!isAvailable(spawn)) {
+    return { code: 0, output: 'eval: claude CLI unavailable — skipping suite (exit 0)\n' }
+  }
+
+  let cases = deps.cases ?? (flags.suite ? await loadSuite(flags.suite) : [])
+  if (flags.filter) cases = cases.filter(c => matchesFilter(c, flags.filter as string))
+
+  if (cases.length === 0) {
+    return { code: 0, output: 'eval: no cases matched (nothing to run)\n' }
+  }
+
+  let report: Report
+  let code = 0
+
+  if (flags.ab) {
+    const results: ABResult[] = []
+    for (const c of cases) {
+      if (!c.variants) continue
+      results.push(runAB(c, c.variants.baseline, c.variants.candidate, { model: flags.models?.[0] ?? DEFAULT_MODEL, spawn, cache: deps.cache }))
+    }
+    report = { mode: 'ab', cases: results }
+    code = abExitCode(results, flags.tolerance)
+  } else if (flags.matrix) {
+    const grids: MatrixResult[] = []
+    for (const c of cases) {
+      grids.push(await runMatrix(c, flags.models ?? c.models ?? [], { spawn, cache: deps.cache }))
+    }
+    report = { mode: 'matrix', cases: grids }
+  } else {
+    const results = await runSuite(cases, { models: flags.models, spawn, cache: deps.cache })
+    report = { mode: 'suite', cases: results }
+  }
+
+  const output = flags.json ? `${JSON.stringify(toJSON(report), null, 2)}\n` : `${formatReport(report)}\n`
+  return { code, output }
+}
+
+if (import.meta.main) {
+  main(process.argv.slice(2))
+    .then(({ code, output }) => {
+      process.stdout.write(output)
+      process.exit(code)
+    })
+    .catch((e: unknown) => {
+      process.stderr.write(`eval: ${e instanceof Error ? e.message : String(e)}\n`)
+      process.exit(2)
+    })
+}