feat(batch): say what differed on a failed step

docs/agent-dx-log.md

@@ -3,6 +3,23 @@
 How measurements from [honojs/agent-dx](https://github.com/honojs/agent-dx)
 changed Hono CLI. Newest first.
 
+## 2026-09-07: Say what differed — the diff joins the failed step
+
+**Experiment**: the cost-trio measurement. `--status-only` cut a
+40-item fixture from 472k to 210k; but the dominant cost is the
+number of fix-verify laps, with a huge variance (164k-980k in one
+condition).
+
+**Findings**: each lap starts with the agent comparing the actual
+body against the expected one by eye — the exact failure class that
+brought `expect` back. The comparison is deterministic; its result
+should be, too.
+
+**Changes**: a failed step now carries `diff` — one line per
+mismatch, like `status: expected 200, got 404` and `body.name:
+expected "Alice", got "Bob"`. In `--compact` output the diff is most
+of what remains: fix what it names, rerun.
+
 ## 2026-09-07: The loop works — now make it cheap
 
 **Experiment**: `next.5` re-measurements. The spec-in-the-request

src/commands/batch/batch.test.ts

@@ -1,7 +1,7 @@
 import { Hono } from 'hono'
 import { describe, it, expect } from 'vitest'
 import { CliError } from '../../utils/output.js'
-import { getByPath, interpolate, matchesSubset, parseBatch, runBatch } from './batch.js'
+import { getByPath, interpolate, matchesSubset, parseBatch, runBatch, subsetDiff } from './batch.js'
 
 describe('parseBatch', () => {
   it('parses one step per line and skips empty lines', () => {
@@ -71,6 +71,17 @@ describe('matchesSubset', () => {
   })
 })
 
+describe('subsetDiff', () => {
+  it('says what differed, one line per mismatch', () => {
+    expect(subsetDiff({ name: 'Bob', id: 1 }, { name: 'Alice' })).toEqual([
+      'body.name: expected "Alice", got "Bob"',
+    ])
+    expect(subsetDiff({ a: {} }, { a: { b: 1 } })).toEqual(['body.a.b: missing'])
+    expect(subsetDiff([1, 2, 3], [1, 2])).toEqual(['body: expected length 2, got 3'])
+    expect(subsetDiff('text', { a: 1 })).toEqual(['body: expected an object, got "text"'])
+  })
+})
+
 describe('getByPath', () => {
   it('walks dot paths including array indexes', () => {
     expect(getByPath({ items: [{ id: 3 }] }, '.items.0.id')).toBe(3)
@@ -148,6 +159,7 @@ describe('runBatch', () => {
   it('fails a step on an expect.status mismatch and suggests --trace on 404', async () => {
     const result = await runBatch(crudApp(), parseBatch('{"path":"/nope","expect":{"status":200}}'))
     expect(result.steps[0].pass).toBe(false)
+    expect(result.steps[0].diff).toEqual(['status: expected 200, got 404'])
     expect(result.steps[0].suggestions).toEqual([
       'See which routes matched: hono request /nope --trace',
     ])

src/commands/batch/batch.ts

@@ -22,6 +22,7 @@ export interface StepResult {
   body: unknown
   pass: boolean
   expect?: StepExpect
+  diff?: string[]
   saved?: Record<string, unknown>
   error?: string
   suggestions?: string[]
@@ -103,30 +104,46 @@ const isValidExpect = (value: unknown): value is StepExpect => {
   return Object.keys(expect).every((key) => key === 'status' || key === 'body')
 }
 
+const shortValue = (value: unknown): string => {
+  const text = JSON.stringify(value) ?? 'undefined'
+  return text.length > 80 ? `${text.slice(0, 77)}...` : text
+}
+
 /**
  * Deep partial match, like `toMatchObject`: declared fields must
  * match, extra fields in the actual value are ignored. Arrays match
- * by index and length.
+ * by index and length. Returns one line per mismatch — agents miss
+ * differences when they compare by eye, so the comparison says what
+ * differed.
  */
-export const matchesSubset = (actual: unknown, expected: unknown): boolean => {
+export const subsetDiff = (actual: unknown, expected: unknown, path = 'body'): string[] => {
   if (Array.isArray(expected)) {
-    return (
-      Array.isArray(actual) &&
-      actual.length === expected.length &&
-      expected.every((item, i) => matchesSubset(actual[i], item))
-    )
+    if (!Array.isArray(actual)) {
+      return [`${path}: expected an array, got ${shortValue(actual)}`]
+    }
+    if (actual.length !== expected.length) {
+      return [`${path}: expected length ${expected.length}, got ${actual.length}`]
+    }
+    return expected.flatMap((item, i) => subsetDiff(actual[i], item, `${path}.${i}`))
   }
   if (typeof expected === 'object' && expected !== null) {
     if (typeof actual !== 'object' || actual === null || Array.isArray(actual)) {
-      return false
+      return [`${path}: expected an object, got ${shortValue(actual)}`]
     }
-    return Object.entries(expected).every(([key, value]) =>
-      matchesSubset((actual as Record<string, unknown>)[key], value)
+    return Object.entries(expected).flatMap(([key, value]) =>
+      key in (actual as Record<string, unknown>)
+        ? subsetDiff((actual as Record<string, unknown>)[key], value, `${path}.${key}`)
+        : [`${path}.${key}: missing`]
     )
   }
   return actual === expected
+    ? []
+    : [`${path}: expected ${shortValue(expected)}, got ${shortValue(actual)}`]
 }
 
+export const matchesSubset = (actual: unknown, expected: unknown): boolean =>
+  subsetDiff(actual, expected).length === 0
+
 const isStringRecord = (value: unknown): value is Record<string, string> =>
   typeof value === 'object' &&
   value !== null &&
@@ -242,11 +259,15 @@ export const runBatch = async (
     }
 
     if (result.expect !== undefined) {
-      const statusOk =
-        result.expect.status === undefined || response.status === result.expect.status
-      const bodyOk = result.expect.body === undefined || matchesSubset(body, result.expect.body)
-      if (!statusOk || !bodyOk) {
+      const diff = [
+        ...(result.expect.status !== undefined && response.status !== result.expect.status
+          ? [`status: expected ${result.expect.status}, got ${response.status}`]
+          : []),
+        ...(result.expect.body === undefined ? [] : subsetDiff(body, result.expect.body)),
+      ]
+      if (diff.length > 0) {
         result.pass = false
+        result.diff = diff
         if (response.status === 404) {
           result.suggestions = [`See which routes matched: hono request ${path} --trace`]
         }

src/commands/batch/index.test.ts

@@ -86,6 +86,7 @@ describe('batchCommand', () => {
             body: { ok: 1 },
             pass: false,
             expect: { status: 404 },
+            diff: ['status: expected 404, got 200'],
           },
         ],
         summary: { total: 2, passed: 1, failed: 1 },

src/commands/batch/index.ts

@@ -24,6 +24,7 @@ EOF`,
     'Declare the acceptance criteria in "expect": {"status":201} and/or {"body":{...}} (a deep partial match — declared fields must match, extra response fields are ignored). Turn the spec into batch lines and rerun until "failed" is 0 — comparing a spec table by eye misses lines.',
     'A shared header from -H goes to every step. Prefer a heredoc over writing a file: the lines live in your context.',
     '--compact prints only the failed steps and the summary — use it when you only need the failed: 0 loop.',
+    'A failed step carries "diff": one line per mismatch (e.g. "body.name: expected \'Alice\', got \'Bob\'"). Fix what the diff names — no need to compare the bodies yourself.',
     'hono snapshot prints the current behavior of an app in this format — capture before a refactor, rerun after.',
   ],
 }