From a716b13e8bfbd47dadc96247dd02c027764ccb0c Mon Sep 17 00:00:00 2001 From: Yusuke Wada Date: Mon, 7 Sep 2026 15:25:50 +0900 Subject: [PATCH] feat(batch): say what differed on a failed step --- docs/agent-dx-log.md | 17 +++++++++++ src/commands/batch/batch.test.ts | 14 ++++++++- src/commands/batch/batch.ts | 49 +++++++++++++++++++++++--------- src/commands/batch/index.test.ts | 1 + src/commands/batch/index.ts | 1 + 5 files changed, 67 insertions(+), 15 deletions(-) diff --git a/docs/agent-dx-log.md b/docs/agent-dx-log.md index 6b4f1b1..a697ba2 100644 --- a/docs/agent-dx-log.md +++ b/docs/agent-dx-log.md @@ -3,6 +3,23 @@ How measurements from [honojs/agent-dx](https://github.com/honojs/agent-dx) changed Hono CLI. Newest first. +## 2026-09-07: Say what differed — the diff joins the failed step + +**Experiment**: the cost-trio measurement. `--status-only` cut a +40-item fixture from 472k to 210k; but the dominant cost is the +number of fix-verify laps, with a huge variance (164k-980k in one +condition). + +**Findings**: each lap starts with the agent comparing the actual +body against the expected one by eye — the exact failure class that +brought `expect` back. The comparison is deterministic; its result +should be, too. + +**Changes**: a failed step now carries `diff` — one line per +mismatch, like `status: expected 200, got 404` and `body.name: +expected "Alice", got "Bob"`. In `--compact` output the diff is most +of what remains: fix what it names, rerun. + ## 2026-09-07: The loop works — now make it cheap **Experiment**: `next.5` re-measurements. The spec-in-the-request diff --git a/src/commands/batch/batch.test.ts b/src/commands/batch/batch.test.ts index e799bfa..46241de 100644 --- a/src/commands/batch/batch.test.ts +++ b/src/commands/batch/batch.test.ts @@ -1,7 +1,7 @@ import { Hono } from 'hono' import { describe, it, expect } from 'vitest' import { CliError } from '../../utils/output.js' -import { getByPath, interpolate, matchesSubset, parseBatch, runBatch } from './batch.js' +import { getByPath, interpolate, matchesSubset, parseBatch, runBatch, subsetDiff } from './batch.js' describe('parseBatch', () => { it('parses one step per line and skips empty lines', () => { @@ -71,6 +71,17 @@ describe('matchesSubset', () => { }) }) +describe('subsetDiff', () => { + it('says what differed, one line per mismatch', () => { + expect(subsetDiff({ name: 'Bob', id: 1 }, { name: 'Alice' })).toEqual([ + 'body.name: expected "Alice", got "Bob"', + ]) + expect(subsetDiff({ a: {} }, { a: { b: 1 } })).toEqual(['body.a.b: missing']) + expect(subsetDiff([1, 2, 3], [1, 2])).toEqual(['body: expected length 2, got 3']) + expect(subsetDiff('text', { a: 1 })).toEqual(['body: expected an object, got "text"']) + }) +}) + describe('getByPath', () => { it('walks dot paths including array indexes', () => { expect(getByPath({ items: [{ id: 3 }] }, '.items.0.id')).toBe(3) @@ -148,6 +159,7 @@ describe('runBatch', () => { it('fails a step on an expect.status mismatch and suggests --trace on 404', async () => { const result = await runBatch(crudApp(), parseBatch('{"path":"/nope","expect":{"status":200}}')) expect(result.steps[0].pass).toBe(false) + expect(result.steps[0].diff).toEqual(['status: expected 200, got 404']) expect(result.steps[0].suggestions).toEqual([ 'See which routes matched: hono request /nope --trace', ]) diff --git a/src/commands/batch/batch.ts b/src/commands/batch/batch.ts index b93a3f9..e804f39 100644 --- a/src/commands/batch/batch.ts +++ b/src/commands/batch/batch.ts @@ -22,6 +22,7 @@ export interface StepResult { body: unknown pass: boolean expect?: StepExpect + diff?: string[] saved?: Record error?: string suggestions?: string[] @@ -103,30 +104,46 @@ const isValidExpect = (value: unknown): value is StepExpect => { return Object.keys(expect).every((key) => key === 'status' || key === 'body') } +const shortValue = (value: unknown): string => { + const text = JSON.stringify(value) ?? 'undefined' + return text.length > 80 ? `${text.slice(0, 77)}...` : text +} + /** * Deep partial match, like `toMatchObject`: declared fields must * match, extra fields in the actual value are ignored. Arrays match - * by index and length. + * by index and length. Returns one line per mismatch — agents miss + * differences when they compare by eye, so the comparison says what + * differed. */ -export const matchesSubset = (actual: unknown, expected: unknown): boolean => { +export const subsetDiff = (actual: unknown, expected: unknown, path = 'body'): string[] => { if (Array.isArray(expected)) { - return ( - Array.isArray(actual) && - actual.length === expected.length && - expected.every((item, i) => matchesSubset(actual[i], item)) - ) + if (!Array.isArray(actual)) { + return [`${path}: expected an array, got ${shortValue(actual)}`] + } + if (actual.length !== expected.length) { + return [`${path}: expected length ${expected.length}, got ${actual.length}`] + } + return expected.flatMap((item, i) => subsetDiff(actual[i], item, `${path}.${i}`)) } if (typeof expected === 'object' && expected !== null) { if (typeof actual !== 'object' || actual === null || Array.isArray(actual)) { - return false + return [`${path}: expected an object, got ${shortValue(actual)}`] } - return Object.entries(expected).every(([key, value]) => - matchesSubset((actual as Record)[key], value) + return Object.entries(expected).flatMap(([key, value]) => + key in (actual as Record) + ? subsetDiff((actual as Record)[key], value, `${path}.${key}`) + : [`${path}.${key}: missing`] ) } return actual === expected + ? [] + : [`${path}: expected ${shortValue(expected)}, got ${shortValue(actual)}`] } +export const matchesSubset = (actual: unknown, expected: unknown): boolean => + subsetDiff(actual, expected).length === 0 + const isStringRecord = (value: unknown): value is Record => typeof value === 'object' && value !== null && @@ -242,11 +259,15 @@ export const runBatch = async ( } if (result.expect !== undefined) { - const statusOk = - result.expect.status === undefined || response.status === result.expect.status - const bodyOk = result.expect.body === undefined || matchesSubset(body, result.expect.body) - if (!statusOk || !bodyOk) { + const diff = [ + ...(result.expect.status !== undefined && response.status !== result.expect.status + ? [`status: expected ${result.expect.status}, got ${response.status}`] + : []), + ...(result.expect.body === undefined ? [] : subsetDiff(body, result.expect.body)), + ] + if (diff.length > 0) { result.pass = false + result.diff = diff if (response.status === 404) { result.suggestions = [`See which routes matched: hono request ${path} --trace`] } diff --git a/src/commands/batch/index.test.ts b/src/commands/batch/index.test.ts index 914240d..14d139a 100644 --- a/src/commands/batch/index.test.ts +++ b/src/commands/batch/index.test.ts @@ -86,6 +86,7 @@ describe('batchCommand', () => { body: { ok: 1 }, pass: false, expect: { status: 404 }, + diff: ['status: expected 404, got 200'], }, ], summary: { total: 2, passed: 1, failed: 1 }, diff --git a/src/commands/batch/index.ts b/src/commands/batch/index.ts index 56ace29..d950825 100644 --- a/src/commands/batch/index.ts +++ b/src/commands/batch/index.ts @@ -24,6 +24,7 @@ EOF`, 'Declare the acceptance criteria in "expect": {"status":201} and/or {"body":{...}} (a deep partial match — declared fields must match, extra response fields are ignored). Turn the spec into batch lines and rerun until "failed" is 0 — comparing a spec table by eye misses lines.', 'A shared header from -H goes to every step. Prefer a heredoc over writing a file: the lines live in your context.', '--compact prints only the failed steps and the summary — use it when you only need the failed: 0 loop.', + 'A failed step carries "diff": one line per mismatch (e.g. "body.name: expected \'Alice\', got \'Bob\'"). Fix what the diff names — no need to compare the bodies yourself.', 'hono snapshot prints the current behavior of an app in this format — capture before a refactor, rerun after.', ], }