From 0f802e55ed37cd9a2b827a9546a2ea7ffe9d0a0b Mon Sep 17 00:00:00 2001 From: Christian Schuetz Date: Wed, 15 Jul 2026 00:31:10 -0500 Subject: [PATCH] feat(evaluation-system): claude -p eval engine, envelope parsing, response cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Foundational layer of the prompt-eval suite (D1). Adds src/eval/ with a single-shot Claude Code CLI (`claude -p`) substrate — never the Anthropic SDK: - types.ts: EvalRequest/EvalResult/Envelope + injectable Spawn and Cache seams. - engine.ts: buildArgs (pure print-mode argv: --print --output-format json --json-schema --model --strict-mcp-config --settings + conditional system-prompt/tool-policy flags; prompt on stdin), parseEnvelope (graded, never throws — prefers structured_output, falls back to result, shallow schema check), runClaude (cache-first, spawn, cache-on-success), isAvailable and a skip signal distinct from graded failure for an absent CLI. - cache.ts: content-addressed ResponseCache keyed on a canonical SHA-256 over every output-affecting field. - tests/helpers/fake-claude.ts + tests/eval/*: hermetic engine/cache coverage via the injected Spawn stub, no real claude. Not a plugin deployable — no build.ts entry, no version bump. --- .gitignore | 1 + src/eval/cache.ts | 70 +++++++++++ src/eval/engine.ts | 228 +++++++++++++++++++++++++++++++++++ src/eval/types.ts | 113 +++++++++++++++++ tests/eval/cache.test.ts | 130 ++++++++++++++++++++ tests/eval/engine.test.ts | 181 +++++++++++++++++++++++++++ tests/helpers/fake-claude.ts | 82 +++++++++++++ 7 files changed, 805 insertions(+) create mode 100644 src/eval/cache.ts create mode 100644 src/eval/engine.ts create mode 100644 src/eval/types.ts create mode 100644 tests/eval/cache.test.ts create mode 100644 tests/eval/engine.test.ts create mode 100644 tests/helpers/fake-claude.ts diff --git a/.gitignore b/.gitignore index c2658d7..8c8f03b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ node_modules/ +.eval-cache/ diff --git a/src/eval/cache.ts b/src/eval/cache.ts new file mode 100644 index 0000000..811fd55 --- /dev/null +++ b/src/eval/cache.ts @@ -0,0 +1,70 @@ +// Content-addressed response cache for eval results. The key is a SHA-256 over a +// CANONICAL (sorted-key) JSON of every request field that changes the model's +// output — prompt, both system-prompt knobs, model, schema, tool policy, and +// settings. Include every such field or an A/B would collide; a changed prompt +// naturally misses. Results are stored one JSON file per key; no eviction. + +import { createHash } from 'node:crypto' +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs' +import { join } from 'node:path' +import type { Cache, EvalRequest, EvalResult } from './types.ts' + +/** Recursively sort object keys so key order never perturbs the hash. */ +function sortKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeys) + if (value !== null && typeof value === 'object') { + const sorted: Record = {} + for (const key of Object.keys(value as Record).sort()) { + sorted[key] = sortKeys((value as Record)[key]) + } + return sorted + } + return value +} + +/** Stable, key-sorted JSON of the output-affecting fields of a request. */ +function canonicalRequest(req: EvalRequest): string { + return JSON.stringify( + sortKeys({ + prompt: req.prompt, + systemPrompt: req.systemPrompt ?? null, + appendSystemPrompt: req.appendSystemPrompt ?? null, + model: req.model, + schema: req.schema, + tools: req.tools ?? null, + settings: req.settings ?? null, + }) + ) +} + +/** Hex SHA-256 cache key over a request's output-affecting fields. */ +export function cacheKey(req: EvalRequest): string { + return createHash('sha256').update(canonicalRequest(req)).digest('hex') +} + +/** File-backed response cache: one `/.json` per stored result. */ +export class ResponseCache implements Cache { + constructor(private readonly dir: string) {} + + private path(key: string): string { + return join(this.dir, `${key}.json`) + } + + /** Return the stored result (marked `cached:true`) or `null` on miss/corruption. */ + get(key: string): EvalResult | null { + const p = this.path(key) + if (!existsSync(p)) return null + try { + const result = JSON.parse(readFileSync(p, 'utf8')) as EvalResult + return { ...result, cached: true } + } catch { + return null + } + } + + /** Persist a result under `key`, creating the cache dir on first write. */ + set(key: string, result: EvalResult): void { + mkdirSync(this.dir, { recursive: true }) + writeFileSync(this.path(key), JSON.stringify(result, null, 2)) + } +} diff --git a/src/eval/engine.ts b/src/eval/engine.ts new file mode 100644 index 0000000..8404bb8 --- /dev/null +++ b/src/eval/engine.ts @@ -0,0 +1,228 @@ +// The `claude -p` eval substrate. `runClaude` builds the print-mode argv, spawns +// the CLI through an injectable `Spawn`, parses the JSON envelope, validates the +// forced `structured_output` against the request schema, and returns a typed, +// graded result. It shells out to `claude` ONLY — never `@anthropic-ai/sdk`. +// +// Design notes: +// - The prompt travels on stdin (fed via `Spawn`'s `input`), so argv stays free +// of arbitrarily large prompt text; `claude -p` reads stdin when given no +// positional prompt. +// - Cost/latency/usage are read straight off the envelope — no pricing table. +// - Every failure mode (CLI error, unparseable envelope, schema miss) is a +// graded `ok:false` result, never a thrown exception. An ABSENT CLI is a +// distinct `skipped:true` signal so callers can skip-with-notice. + +import { spawnSync } from 'node:child_process' +import type { + Cache, + Envelope, + EvalRequest, + EvalResult, + EvalUsage, + JsonSchema, + Spawn, +} from './types.ts' +import { cacheKey } from './cache.ts' + +/** Tools disabled by default so a case is single-shot, deterministic, cacheable. */ +const DEFAULT_DISALLOWED_TOOLS = [ + 'Bash', + 'Read', + 'Write', + 'Edit', + 'WebFetch', + 'WebSearch', + 'Task', + 'Glob', + 'Grep', +] + +/** Real spawn: `spawnSync('claude', …)` with the prompt on stdin. */ +export const defaultSpawn: Spawn = (cmd, args, input) => { + const res = spawnSync(cmd, args, { encoding: 'utf8', input, maxBuffer: 64 * 1024 * 1024 }) + return { status: res.status, stdout: res.stdout ?? '', stderr: res.stderr ?? '' } +} + +/** + * Build the pure `claude -p` argv for a request (prompt excluded — it goes on + * stdin). Always forces print mode, JSON output, the request schema, model, + * strict MCP isolation, and inline settings; layers system-prompt and tool-policy + * flags conditionally. + */ +export function buildArgs(req: EvalRequest): string[] { + const args = [ + '--print', + '--output-format', + 'json', + '--json-schema', + JSON.stringify(req.schema), + '--model', + req.model, + '--strict-mcp-config', + '--settings', + req.settings ?? '{}', + ] + if (req.systemPrompt !== undefined) args.push('--system-prompt', req.systemPrompt) + if (req.appendSystemPrompt !== undefined) args.push('--append-system-prompt', req.appendSystemPrompt) + if (req.tools && req.tools.length > 0) { + args.push('--allowedTools', req.tools.join(' ')) + } else { + args.push('--disallowedTools', DEFAULT_DISALLOWED_TOOLS.join(' ')) + } + return args +} + +/** Lift the envelope's snake_case usage block into the normalized shape. */ +function normalizeUsage(envelope: Envelope): EvalUsage { + const u = envelope.usage ?? {} + return { + inputTokens: u.input_tokens ?? 0, + outputTokens: u.output_tokens ?? 0, + cacheReadInputTokens: u.cache_read_input_tokens ?? 0, + cacheCreationInputTokens: u.cache_creation_input_tokens ?? 0, + } +} + +/** + * Shallow structural check: value is an object, every `required` key is present, + * and — when `additionalProperties:false` — no keys beyond `properties` appear. + * Returns an error message or `null`. Intentionally NOT a full JSON-Schema + * validator (no dependency added). + */ +function validateSchema(value: unknown, schema: JsonSchema): string | null { + const wantsObject = schema.type === 'object' || schema.properties !== undefined || schema.required !== undefined + if (!wantsObject) return null + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + return 'output is not an object' + } + const obj = value as Record + for (const key of schema.required ?? []) { + if (!(key in obj)) return `output missing required key: ${key}` + } + if (schema.additionalProperties === false && schema.properties) { + const allowed = new Set(Object.keys(schema.properties)) + for (const key of Object.keys(obj)) { + if (!allowed.has(key)) return `output has unexpected key: ${key}` + } + } + return null +} + +/** Assemble a graded result from envelope metrics + an ok/error verdict. */ +function gradedResult( + envelope: Envelope, + req: EvalRequest, + verdict: { ok: true; output: unknown } | { ok: false; error: string } +): EvalResult { + const base = { + cost: envelope.total_cost_usd ?? 0, + usage: normalizeUsage(envelope), + durationMs: envelope.duration_ms ?? 0, + apiDurationMs: envelope.duration_api_ms ?? 0, + numTurns: envelope.num_turns ?? 0, + model: req.model, + cached: false, + } + return verdict.ok + ? { ok: true, output: verdict.output, error: null, ...base } + : { ok: false, output: null, error: verdict.error, ...base } +} + +/** A graded failure carrying zeroed metrics (used when the envelope is unusable). */ +function failureNoMetrics(req: EvalRequest, error: string): EvalResult { + return { + ok: false, + output: null, + error, + cost: 0, + usage: { inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0 }, + durationMs: 0, + apiDurationMs: 0, + numTurns: 0, + model: req.model, + cached: false, + } +} + +/** + * Parse a `claude --output-format json` envelope into a graded `EvalResult`. + * Never throws: an unparseable envelope, a CLI/model error, a missing/bad + * output, or a schema miss all yield `ok:false` with a descriptive `error`. + */ +export function parseEnvelope(stdout: string, req: EvalRequest): EvalResult { + let envelope: Envelope + try { + envelope = JSON.parse(stdout) as Envelope + } catch { + return failureNoMetrics(req, 'unparseable envelope') + } + + if (envelope.is_error === true || (envelope.subtype !== undefined && envelope.subtype !== 'success')) { + return gradedResult(envelope, req, { + ok: false, + error: `claude error: is_error=${String(envelope.is_error)} subtype=${String(envelope.subtype)}`, + }) + } + + // Prefer the parsed, schema-forced object; fall back to parsing stringified `result`. + let output = envelope.structured_output + if (output === undefined || output === null) { + if (typeof envelope.result !== 'string') { + return gradedResult(envelope, req, { ok: false, error: 'no structured_output and no string result to parse' }) + } + try { + output = JSON.parse(envelope.result) + } catch { + return gradedResult(envelope, req, { ok: false, error: 'unparseable result JSON' }) + } + } + + const schemaError = validateSchema(output, req.schema) + if (schemaError) return gradedResult(envelope, req, { ok: false, error: schemaError }) + + return gradedResult(envelope, req, { ok: true, output }) +} + +export interface RunOptions { + /** Injected subprocess boundary; defaults to the real `claude` spawn. */ + spawn?: Spawn + /** Optional response cache consulted before spawning and written on success. */ + cache?: Cache +} + +/** + * Run one eval request: cache lookup first, else spawn `claude -p`, parse, cache + * (on success), return. An absent CLI (non-zero exit with empty stdout, or a + * spawn throw) surfaces as a `skipped` result — distinct from a graded failure. + */ +export function runClaude(req: EvalRequest, { spawn = defaultSpawn, cache }: RunOptions = {}): EvalResult { + const key = cache ? cacheKey(req) : null + if (cache && key) { + const hit = cache.get(key) + if (hit) return { ...hit, cached: true } + } + + let res + try { + res = spawn('claude', buildArgs(req), req.prompt) + } catch { + return { ...failureNoMetrics(req, 'claude CLI unavailable'), skipped: true } + } + + if (res.status !== 0 && res.stdout.trim() === '') { + return { ...failureNoMetrics(req, 'claude CLI unavailable'), skipped: true } + } + + const result = parseEnvelope(res.stdout, req) + if (cache && key && result.ok) cache.set(key, result) + return result +} + +/** Probe whether the `claude` CLI is invokable via `claude --version`. */ +export function isAvailable(spawn: Spawn = defaultSpawn): boolean { + try { + return spawn('claude', ['--version']).status === 0 + } catch { + return false + } +} diff --git a/src/eval/types.ts b/src/eval/types.ts new file mode 100644 index 0000000..1cb125a --- /dev/null +++ b/src/eval/types.ts @@ -0,0 +1,113 @@ +// Shared types for the prompt-eval engine. The engine shells out to the Claude +// Code CLI in print mode (`claude -p`) — NEVER the Anthropic SDK — and reads +// cost/latency/usage straight off the `--output-format json` envelope. The one +// faked boundary in tests is `Spawn`; everything else is real. + +/** + * A minimal JSON-Schema shape. The engine only performs a SHALLOW structural + * check (`required` keys present, no extras when `additionalProperties:false`), + * so this deliberately does not model the full spec — do NOT add a schema + * validator dependency. The harness's generated schemas satisfy this shape. + */ +export interface JsonSchema { + type?: string + required?: string[] + properties?: Record + additionalProperties?: boolean + items?: JsonSchema + [key: string]: unknown +} + +/** A single-shot eval request: the prompt, the schema it is forced against, and knobs. */ +export interface EvalRequest { + /** User prompt — passed to `claude -p` on stdin. */ + prompt: string + /** Replace the CLI's default system prompt (`--system-prompt`) for isolation. */ + systemPrompt?: string + /** Layer extra context on top of the default system prompt (`--append-system-prompt`). */ + appendSystemPrompt?: string + /** Model id, e.g. `claude-haiku-4-5` (`--model`). */ + model: string + /** Schema forced via `--json-schema` and validated against `structured_output`. */ + schema: JsonSchema + /** Opt into a bounded toolset (`--allowedTools`); omit for a single-shot, tool-free run. */ + tools?: string[] + /** Inline settings JSON (`--settings`); defaults to `'{}'` to isolate from ambient config. */ + settings?: string +} + +/** Normalized token usage lifted from the envelope's snake_case `usage` block. */ +export interface EvalUsage { + inputTokens: number + outputTokens: number + cacheReadInputTokens: number + cacheCreationInputTokens: number +} + +/** + * The graded outcome of one eval invocation. A model/CLI error, an unparseable + * envelope, or a schema-nonconforming output all yield `ok:false` with a + * descriptive `error` — the engine NEVER throws for these. `skipped:true` is a + * distinct signal that the CLI itself was unavailable (not a graded failure). + */ +export interface EvalResult { + ok: boolean + output: unknown | null + error: string | null + cost: number + usage: EvalUsage + durationMs: number + apiDurationMs: number + numTurns: number + model: string + cached: boolean + /** True only when the `claude` CLI was unavailable — callers skip-with-notice. */ + skipped?: boolean +} + +/** Envelope usage block as emitted by `claude --output-format json` (snake_case). */ +export interface EnvelopeUsage { + input_tokens?: number + output_tokens?: number + cache_read_input_tokens?: number + cache_creation_input_tokens?: number + service_tier?: string +} + +/** + * Raw `--output-format json` envelope (verified against installed `claude` + * 2.1.207). Only the fields the engine reads are typed; the rest are ignored. + */ +export interface Envelope { + type?: string + subtype?: string + is_error?: boolean + result?: string + structured_output?: unknown + total_cost_usd?: number + usage?: EnvelopeUsage + duration_ms?: number + duration_api_ms?: number + num_turns?: number + modelUsage?: Record +} + +/** Result of a subprocess spawn — the single injectable boundary tests fake. */ +export interface SpawnResult { + status: number | null + stdout: string + stderr: string +} + +/** + * The injected spawn dependency. Default is a thin `spawnSync('claude', …)` + * wrapper; tests pass a stub returning a canned envelope. `input` is fed to the + * child's stdin (the prompt travels there, keeping argv free of large text). + */ +export type Spawn = (cmd: string, args: string[], input?: string) => SpawnResult + +/** A response cache the engine can consult before spawning. */ +export interface Cache { + get(key: string): EvalResult | null + set(key: string, result: EvalResult): void +} diff --git a/tests/eval/cache.test.ts b/tests/eval/cache.test.ts new file mode 100644 index 0000000..8bab195 --- /dev/null +++ b/tests/eval/cache.test.ts @@ -0,0 +1,130 @@ +// Cache tests: key stability, per-field isolation, file round-trip in a temp +// dir, and the runClaude hit/miss integration (a hit must NOT spawn). + +import assert from 'node:assert/strict' +import { mkdtempSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { test } from 'bun:test' +import { cacheKey, ResponseCache } from '../../src/eval/cache.ts' +import { runClaude } from '../../src/eval/engine.ts' +import type { EvalRequest, JsonSchema, Spawn } from '../../src/eval/types.ts' +import { fakeSpawn, successEnvelope, throwingSpawn } from '../helpers/fake-claude.ts' + +const SCHEMA: JsonSchema = { + type: 'object', + required: ['answer'], + properties: { answer: { type: 'number' } }, + additionalProperties: false, +} + +const baseReq = (overrides: Partial = {}): EvalRequest => ({ + prompt: 'What is 6 * 7?', + model: 'claude-haiku-4-5', + schema: SCHEMA, + ...overrides, +}) + +const tempDir = (): string => mkdtempSync(join(tmpdir(), 'eval-cache-')) + +// --- key stability + isolation (acceptance criterion 4) ---------------------- + +test('cacheKey is stable for identical requests', () => { + assert.equal(cacheKey(baseReq()), cacheKey(baseReq())) +}) + +test('cacheKey is insensitive to schema key ordering (canonical JSON)', () => { + const a = baseReq({ schema: { type: 'object', required: ['answer'], additionalProperties: false } }) + const b = baseReq({ schema: { additionalProperties: false, required: ['answer'], type: 'object' } }) + assert.equal(cacheKey(a), cacheKey(b)) +}) + +test('cacheKey changes when any output-affecting field changes', () => { + const base = cacheKey(baseReq()) + const variants: EvalRequest[] = [ + baseReq({ prompt: 'different' }), + baseReq({ model: 'claude-opus-4-8' }), + baseReq({ schema: { type: 'object', required: ['result'] } }), + baseReq({ systemPrompt: 'You are terse.' }), + baseReq({ appendSystemPrompt: 'Context.' }), + baseReq({ tools: ['Read'] }), + baseReq({ settings: '{"x":1}' }), + ] + for (const v of variants) { + assert.notEqual(cacheKey(v), base) + } + // All variants are distinct from each other too. + const keys = new Set(variants.map(cacheKey)) + assert.equal(keys.size, variants.length) +}) + +// --- file round-trip --------------------------------------------------------- + +test('ResponseCache round-trips a result and marks it cached on read', () => { + const cache = new ResponseCache(tempDir()) + const key = cacheKey(baseReq()) + assert.equal(cache.get(key), null) + + const result = runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 42 })) }) + cache.set(key, result) + + const hit = cache.get(key) + assert.ok(hit) + assert.equal(hit.cached, true) + assert.deepEqual(hit.output, { answer: 42 }) + assert.equal(hit.cost, result.cost) +}) + +test('ResponseCaches in separate dirs are isolated', () => { + const a = new ResponseCache(tempDir()) + const b = new ResponseCache(tempDir()) + const key = cacheKey(baseReq()) + a.set(key, runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 1 })) })) + assert.ok(a.get(key)) + assert.equal(b.get(key), null) +}) + +// --- runClaude cache integration --------------------------------------------- + +test('runClaude serves an identical request from cache without spawning', () => { + const cache = new ResponseCache(tempDir()) + let spawnCalls = 0 + const countingSpawn: Spawn = (cmd, args, input) => { + spawnCalls++ + return fakeSpawn(successEnvelope({ answer: 42 }))(cmd, args, input) + } + + const first = runClaude(baseReq(), { spawn: countingSpawn, cache }) + assert.equal(first.ok, true) + assert.equal(first.cached, false) + assert.equal(spawnCalls, 1) + + // Second identical call: cache hit, no further spawn (throwingSpawn proves it). + const second = runClaude(baseReq(), { spawn: throwingSpawn(), cache }) + assert.equal(second.ok, true) + assert.equal(second.cached, true) + assert.deepEqual(second.output, { answer: 42 }) + assert.equal(spawnCalls, 1) +}) + +test('runClaude misses the cache when a keyed field changes', () => { + const cache = new ResponseCache(tempDir()) + runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ answer: 42 })) , cache }) + + let spawned = false + const spawn: Spawn = (cmd, args, input) => { + spawned = true + return fakeSpawn(successEnvelope({ answer: 7 }))(cmd, args, input) + } + const result = runClaude(baseReq({ model: 'claude-opus-4-8' }), { spawn, cache }) + assert.equal(spawned, true) + assert.equal(result.cached, false) + assert.deepEqual(result.output, { answer: 7 }) +}) + +test('runClaude does not cache a graded failure', () => { + const cache = new ResponseCache(tempDir()) + // A schema-miss success envelope grades ok:false and must not be cached. + runClaude(baseReq(), { spawn: fakeSpawn(successEnvelope({ wrong: 1 })), cache }) + assert.equal(cache.get(cacheKey(baseReq())), null) +}) diff --git a/tests/eval/engine.test.ts b/tests/eval/engine.test.ts new file mode 100644 index 0000000..de3c309 --- /dev/null +++ b/tests/eval/engine.test.ts @@ -0,0 +1,181 @@ +// Engine tests: pure argv construction, envelope grading over +// success/fallback/error/schema-miss/unparseable, and the isAvailable/skip +// paths — all through the injected `Spawn` stub, never a real `claude`. + +import assert from 'node:assert/strict' +import { test } from 'bun:test' +import { buildArgs, isAvailable, parseEnvelope, runClaude } from '../../src/eval/engine.ts' +import type { EvalRequest, JsonSchema } from '../../src/eval/types.ts' +import { + errorEnvelope, + fakeSpawn, + rawSpawn, + schemaMismatchEnvelope, + successEnvelope, + throwingSpawn, +} from '../helpers/fake-claude.ts' + +const SCHEMA: JsonSchema = { + type: 'object', + required: ['answer'], + properties: { answer: { type: 'number' } }, + additionalProperties: false, +} + +const baseReq = (overrides: Partial = {}): EvalRequest => ({ + prompt: 'What is 6 * 7?', + model: 'claude-haiku-4-5', + schema: SCHEMA, + ...overrides, +}) + +// --- buildArgs (acceptance criterion 1) -------------------------------------- + +test('buildArgs forces print mode, JSON output, schema, model, strict MCP, and settings', () => { + const args = buildArgs(baseReq()) + assert.ok(args.includes('--print')) + assert.deepEqual(args.slice(args.indexOf('--output-format'), args.indexOf('--output-format') + 2), [ + '--output-format', + 'json', + ]) + assert.deepEqual(args.slice(args.indexOf('--json-schema'), args.indexOf('--json-schema') + 2), [ + '--json-schema', + JSON.stringify(SCHEMA), + ]) + assert.deepEqual(args.slice(args.indexOf('--model'), args.indexOf('--model') + 2), [ + '--model', + 'claude-haiku-4-5', + ]) + assert.ok(args.includes('--strict-mcp-config')) + assert.deepEqual(args.slice(args.indexOf('--settings'), args.indexOf('--settings') + 2), ['--settings', '{}']) +}) + +test('buildArgs disables tools by default and never puts the prompt in argv', () => { + const args = buildArgs(baseReq()) + assert.ok(args.includes('--disallowedTools')) + assert.ok(!args.includes('--allowedTools')) + // The prompt travels on stdin, so it must not appear in argv. + assert.ok(!args.some(a => a.includes('6 * 7'))) +}) + +test('buildArgs opts into an allow-list when tools are provided', () => { + const args = buildArgs(baseReq({ tools: ['Read', 'Grep'] })) + assert.deepEqual(args.slice(args.indexOf('--allowedTools'), args.indexOf('--allowedTools') + 2), [ + '--allowedTools', + 'Read Grep', + ]) + assert.ok(!args.includes('--disallowedTools')) +}) + +test('buildArgs adds system-prompt flags only when set, and passes custom settings', () => { + const bare = buildArgs(baseReq()) + assert.ok(!bare.includes('--system-prompt')) + assert.ok(!bare.includes('--append-system-prompt')) + + const withSys = buildArgs(baseReq({ systemPrompt: 'You are terse.', appendSystemPrompt: 'Context.', settings: '{"x":1}' })) + assert.deepEqual(withSys.slice(withSys.indexOf('--system-prompt'), withSys.indexOf('--system-prompt') + 2), [ + '--system-prompt', + 'You are terse.', + ]) + assert.deepEqual( + withSys.slice(withSys.indexOf('--append-system-prompt'), withSys.indexOf('--append-system-prompt') + 2), + ['--append-system-prompt', 'Context.'] + ) + assert.deepEqual(withSys.slice(withSys.indexOf('--settings'), withSys.indexOf('--settings') + 2), [ + '--settings', + '{"x":1}', + ]) +}) + +// --- success + fallback (acceptance criterion 2) ----------------------------- + +test('a success envelope with structured_output yields ok:true with cost/usage/output', () => { + const envelope = successEnvelope( + { answer: 42 }, + { cost: 0.0463, usage: { inputTokens: 10, outputTokens: 158, cacheReadInputTokens: 3 }, durationMs: 2419, apiDurationMs: 2394, numTurns: 2 } + ) + const result = runClaude(baseReq(), { spawn: fakeSpawn(envelope) }) + assert.equal(result.ok, true) + assert.deepEqual(result.output, { answer: 42 }) + assert.equal(result.error, null) + assert.equal(result.cost, 0.0463) + assert.deepEqual(result.usage, { + inputTokens: 10, + outputTokens: 158, + cacheReadInputTokens: 3, + cacheCreationInputTokens: 0, + }) + assert.equal(result.durationMs, 2419) + assert.equal(result.apiDurationMs, 2394) + assert.equal(result.numTurns, 2) + assert.equal(result.model, 'claude-haiku-4-5') + assert.equal(result.cached, false) +}) + +test('a missing structured_output falls back to JSON.parse(result)', () => { + const envelope = successEnvelope({ answer: 7 }, { structured: false }) + assert.equal(envelope.structured_output, undefined) + const result = runClaude(baseReq(), { spawn: fakeSpawn(envelope) }) + assert.equal(result.ok, true) + assert.deepEqual(result.output, { answer: 7 }) +}) + +// --- graded failures, no throw (acceptance criterion 3) ---------------------- + +test('is_error / non-success subtype grades ok:false without throwing', () => { + const result = runClaude(baseReq(), { spawn: fakeSpawn(errorEnvelope('error_max_turns')) }) + assert.equal(result.ok, false) + assert.equal(result.output, null) + assert.match(result.error ?? '', /error_max_turns/) + // Metrics from the envelope are still surfaced. + assert.equal(result.cost, 0.002) +}) + +test('schema-nonconforming output grades ok:false with a descriptive error', () => { + const result = runClaude(baseReq(), { spawn: fakeSpawn(schemaMismatchEnvelope()) }) + assert.equal(result.ok, false) + assert.match(result.error ?? '', /required key: answer/) +}) + +test('additionalProperties:false rejects an extra key', () => { + const envelope = successEnvelope({ answer: 1, extra: 2 }) + const result = parseEnvelope(JSON.stringify(envelope), baseReq()) + assert.equal(result.ok, false) + assert.match(result.error ?? '', /unexpected key: extra/) +}) + +test('unparseable stdout grades ok:false with zeroed metrics, no throw', () => { + const result = runClaude(baseReq(), { spawn: rawSpawn(0, 'not json at all') }) + assert.equal(result.ok, false) + assert.equal(result.error, 'unparseable envelope') + assert.equal(result.cost, 0) + assert.equal(result.durationMs, 0) +}) + +// --- availability + skip (acceptance criterion 5) ---------------------------- + +test('isAvailable is false when claude --version exits non-zero or throws', () => { + assert.equal(isAvailable(rawSpawn(127, '', 'command not found')), false) + assert.equal(isAvailable(throwingSpawn()), false) + assert.equal(isAvailable(rawSpawn(0, 'claude 2.1.207')), true) +}) + +test('runClaude against an absent CLI returns a skipped result, never throws', () => { + const viaExit = runClaude(baseReq(), { spawn: rawSpawn(127, '', 'not found') }) + assert.equal(viaExit.skipped, true) + assert.equal(viaExit.ok, false) + assert.match(viaExit.error ?? '', /unavailable/) + + const viaThrow = runClaude(baseReq(), { spawn: throwingSpawn() }) + assert.equal(viaThrow.skipped, true) + assert.equal(viaThrow.ok, false) +}) + +test('a non-zero exit that still produced envelope stdout is graded, not skipped', () => { + // The CLI can exit non-zero while emitting an error envelope — that is a + // graded failure, not an availability skip. + const result = runClaude(baseReq(), { spawn: rawSpawn(1, JSON.stringify(errorEnvelope('error_during_execution'))) }) + assert.equal(result.skipped, undefined) + assert.equal(result.ok, false) + assert.match(result.error ?? '', /error_during_execution/) +}) diff --git a/tests/helpers/fake-claude.ts b/tests/helpers/fake-claude.ts new file mode 100644 index 0000000..b3f434c --- /dev/null +++ b/tests/helpers/fake-claude.ts @@ -0,0 +1,82 @@ +// Hermetic stub for the `claude -p` subprocess boundary. Tests inject `fakeSpawn` +// (or a raw variant) so `bun test` never shells out to a real `claude` — the ONLY +// faked seam in the eval engine. Envelope factories mirror the verified real +// `--output-format json` shape (installed claude 2.1.207). + +import type { Envelope, EvalUsage, Spawn } from '../../src/eval/types.ts' + +/** A `Spawn` that ignores args/stdin and emits `envelope` as JSON on stdout, exit 0. */ +export function fakeSpawn(envelope: Envelope): Spawn { + return () => ({ status: 0, stdout: JSON.stringify(envelope), stderr: '' }) +} + +/** A `Spawn` returning a fixed status/stdout/stderr verbatim (for edge cases). */ +export function rawSpawn(status: number | null, stdout: string, stderr = ''): Spawn { + return () => ({ status, stdout, stderr }) +} + +/** A `Spawn` that throws, modelling a missing binary (ENOENT). */ +export function throwingSpawn(): Spawn { + return () => { + throw new Error('spawn claude ENOENT') + } +} + +export interface SuccessOptions { + cost?: number + usage?: Partial + durationMs?: number + apiDurationMs?: number + numTurns?: number + /** When false, omit `structured_output` so the engine falls back to `result`. */ + structured?: boolean +} + +/** A success envelope carrying `output` as both `structured_output` and `result`. */ +export function successEnvelope(output: unknown, opts: SuccessOptions = {}): Envelope { + const { + cost = 0.001, + usage = {}, + durationMs = 1000, + apiDurationMs = 950, + numTurns = 2, + structured = true, + } = opts + const envelope: Envelope = { + type: 'result', + subtype: 'success', + is_error: false, + result: JSON.stringify(output), + total_cost_usd: cost, + duration_ms: durationMs, + duration_api_ms: apiDurationMs, + num_turns: numTurns, + usage: { + input_tokens: usage.inputTokens ?? 10, + output_tokens: usage.outputTokens ?? 20, + cache_read_input_tokens: usage.cacheReadInputTokens ?? 0, + cache_creation_input_tokens: usage.cacheCreationInputTokens ?? 0, + }, + } + if (structured) envelope.structured_output = output + return envelope +} + +/** An error envelope (`is_error:true`, non-success subtype) with metrics present. */ +export function errorEnvelope(subtype = 'error_max_turns'): Envelope { + return { + type: 'result', + subtype, + is_error: true, + total_cost_usd: 0.002, + duration_ms: 500, + duration_api_ms: 480, + num_turns: 1, + usage: { input_tokens: 5, output_tokens: 0, cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + } +} + +/** A success envelope whose `structured_output` omits a required key (schema miss). */ +export function schemaMismatchEnvelope(): Envelope { + return successEnvelope({ unexpected: true }) +}