bench: evaluate agent continuation with --json-envelope

bench/RESULTS.md

@@ -265,3 +265,59 @@ one sample lost outright. Reported as median-of-3, loss included.
 
 **Tool-warm scoreboard (Opus, all verified correct):** drift −67%, clean
 extract −65%, live web −23% (median of 3). These are the site numbers.
+
+## JSON-envelope continuation (Sonnet 5, n=3 per condition)
+
+### Method
+
+A deterministic 120-row HTML fixture was generated from an independent fixed seed.
+Each run used budget 600, while ground truth remained in the parent process.
+Conformance explicitly taught continuation; Adoption compared the same natural task
+with the full Skill versus a runtime-only continuation-guidance ablation.
+Raw session records stayed outside the repository.
+
+### Preliminary stopped attempts
+
+Three earlier formal matrix invocations were stopped by gradability gates and excluded from the reported sample.
+They exposed fixture-metadata preflights, compound or interpreter-wrapped ax commands,
+and terminal responses containing text outside the required JSON object.
+The methodology was then changed to require standalone ax calls, prohibit all non-ax
+fixture access, enforce a JSON-only response, and tighten access auditing before the
+final matrix was collected. A post-run review further replaced permissive non-ax
+handling with a conservative allowlist; regrading the retained runs did not change
+their gradability or published scores. These are adaptive, exploratory results rather
+than a preregistered confirmatory evaluation.
+
+### Per-run results
+
+| Run | Condition        | Exact | Protocol | Envelope | Pages | Turns | Duration (s) |  Input | Output | Cache create | Cache read |   Cost |
+| --: | ---------------- | :---: | -------- | :------: | ----: | ----: | -----------: | -----: | -----: | -----------: | ---------: | -----: |
+|   1 | Conformance      |  no   | pass     |    —     |     5 |     8 |        170.5 |  57346 |   9746 |            0 |     314752 | $0.413 |
+|   2 | Adoption guided  |  no   | fail     |    no    |     3 |     6 |        370.2 |  14794 |  23254 |            0 |     290048 | $0.480 |
+|   3 | Adoption ablated |  no   | fail     |    no    |     6 |    10 |        630.9 |  72592 |  32273 |            0 |     503680 | $0.853 |
+|   4 | Conformance      |  yes  | pass     |    —     |     5 |     8 |        167.6 |  56400 |   7907 |            0 |     360832 | $0.396 |
+|   5 | Adoption guided  |  no   | pass     |   yes    |     5 |     7 |        199.8 |  20118 |  10369 |            0 |     358784 | $0.324 |
+|   6 | Adoption ablated |  no   | fail     |    no    |     3 |     5 |        747.2 |  86911 |  43130 |            0 |     289280 | $0.994 |
+|   7 | Conformance      |  yes  | pass     |    —     |     5 |     8 |        153.6 |  18545 |   7222 |            0 |     398464 | $0.284 |
+|   8 | Adoption guided  |  no   | pass     |   yes    |     5 |     8 |        413.9 |  33677 |  22772 |            0 |     496384 | $0.592 |
+|   9 | Adoption ablated |  no   | fail     |    no    |     6 |     8 |        609.2 | 102120 |  29857 |            0 |     462464 | $0.893 |
+
+### Aggregate
+
+| Condition        | Exact | Protocol pass | Envelope adoption | Median turns | Median duration (s) | Median input | Median output | Median cost |
+| ---------------- | ----: | ------------: | ----------------: | -----------: | ------------------: | -----------: | ------------: | ----------: |
+| Conformance      |   2/3 |           3/3 |                 — |            8 |               167.6 |        56400 |          7907 |      $0.396 |
+| Adoption guided  |   0/3 |           2/3 |               2/3 |            7 |               370.2 |        20118 |         22772 |      $0.480 |
+| Adoption ablated |   0/3 |           0/3 |               0/3 |            8 |               630.9 |        86911 |         32273 |      $0.893 |
+
+### Observed failure modes
+
+Across 4 Adoption runs without envelope adoption, 4 used `offset-without-envelope`; 4 failed protocol grading.
+Among 7 non-exact answers, 4 were schema-invalid. Among the 3 schema-valid non-exact answers, 0 had missing records, 3 had unexpected records, 0 had duplicate records, 0 had ordering errors, and 0 had field mismatches. Categories can overlap.
+
+### Interpretation and limitations
+
+Answer correctness, continuation-protocol correctness, and active envelope adoption
+are reported separately. Each condition has only three runs on one deterministic
+fixture and one model, so differences are descriptive rather than general claims.
+Claude Code event structure may also change across CLI versions.

bench/continuation/audit.test.ts

@@ -0,0 +1,205 @@
+import { expect, test } from 'bun:test'
+import { auditToolContext } from './audit'
+import type { AdoptionGrade } from './grade'
+import type { NormalizedToolCall } from './stream'
+import type { AxInvocation } from './telemetry'
+
+function tool(name: string, input: Record<string, unknown>): NormalizedToolCall {
+  return {
+    ordinal: 1,
+    name,
+    input,
+    resultText: '',
+    isError: false,
+  }
+}
+
+function bash(command: string): NormalizedToolCall {
+  return tool('Bash', { command, description: 'synthetic' })
+}
+
+function invocation(argv: string[], start = 1): AxInvocation {
+  return {
+    argv,
+    cwd: '/external/run/agent',
+    stdout: '',
+    stderr: '',
+    exitCode: 0,
+    startedAtMs: start,
+    endedAtMs: start + 1,
+  }
+}
+
+test('adoption grade can record direct fixture access', () => {
+  const strategy: AdoptionGrade['alternativeStrategy'] = 'direct-fixture-read'
+  expect(strategy).toBe('direct-fixture-read')
+})
+
+test.each([
+  tool('Read', { file_path: '/external/run/agent/continuation.html' }),
+  tool('Grep', { pattern: 'incident', path: 'continuation.html' }),
+  tool('Glob', { pattern: '**/continuation.html', path: '.' }),
+  tool('Grep', { pattern: 'incident', path: '.' }),
+  tool('Grep', { pattern: 'incident' }),
+  tool('Glob', { pattern: '**/*.html', path: '.' }),
+])('%s directly accessing the fixture is a violation', (call) => {
+  expect(auditToolContext([call], [])).toEqual({
+    status: 'violation',
+    issues: ['direct fixture access'],
+    alternativeStrategy: 'direct-fixture-read',
+  })
+})
+
+test.each([
+  'cat continuation.html',
+  'grep incident continuation.html',
+  'python3 -c "open(\\"continuation.html\\")"',
+  'f=continuation.html; cat "$f"',
+  'cat *.html',
+  'rg incident .',
+  'ax continuation.html .incident --json-envelope; cat continuation.html',
+])('%s is a direct fixture read violation', (command) => {
+  expect(auditToolContext([bash(command)], [])).toEqual({
+    status: 'violation',
+    issues: ['direct fixture access'],
+    alternativeStrategy: 'direct-fixture-read',
+  })
+})
+
+test.each([
+  tool('Grep', { pattern: 'incident', path: '/external/run/agent' }),
+  bash('rg incident /external/run/agent'),
+])('treats the known Agent directory as fixture-targeting', (call) => {
+  expect(auditToolContext([call], [], 'continuation.html', '/external/run/agent')).toEqual({
+    status: 'violation',
+    issues: ['direct fixture access'],
+    alternativeStrategy: 'direct-fixture-read',
+  })
+})
+
+test('accepts a simple absolute shim command when telemetry matches', () => {
+  const argv = ['continuation.html', '.incident', '--json-envelope']
+  expect(
+    auditToolContext(
+      [bash('/external/run/bin/ax continuation.html .incident --json-envelope')],
+      [invocation(argv)]
+    )
+  ).toEqual({
+    status: 'pass',
+    issues: [],
+    alternativeStrategy: null,
+  })
+})
+
+test.each([
+  {
+    name: 'semicolon',
+    command:
+      'ax continuation.html .incident --json-envelope; ' +
+      'ax continuation.html .incident --json-envelope --offset 29',
+    invocations: [
+      invocation(['continuation.html', '.incident', '--json-envelope'], 1),
+      invocation(['continuation.html', '.incident', '--json-envelope', '--offset', '29'], 3),
+    ],
+  },
+  {
+    name: 'and',
+    command: 'ax continuation.html .incident --json-envelope && which ax',
+    invocations: [invocation(['continuation.html', '.incident', '--json-envelope'])],
+  },
+])('$name-batched fixture ax command is not gradable', ({ command, invocations }) => {
+  expect(auditToolContext([bash(command)], [...invocations])).toEqual({
+    status: 'not_gradable',
+    issues: ['unsupported fixture-targeting Bash command'],
+    alternativeStrategy: null,
+  })
+})
+
+test.each([
+  'wrapper continuation.html',
+  'ax continuation.html .incident --json-envelope | cat',
+  'ax continuation.html .incident --json-envelope > /tmp/envelope.json',
+  "python3 <<'PY'\nimport subprocess\nsubprocess.run(['ax', 'continuation.html'])\nPY",
+  '$(command -v ax) continuation.html .incident --json-envelope',
+  'ax continuation.html "unterminated',
+])('%s is not gradable', (command) => {
+  expect(auditToolContext([bash(command)], []).status).toBe('not_gradable')
+})
+
+test('ignores unsupported non-fixture discovery commands', () => {
+  expect(auditToolContext([bash('command -v ax && ax --help | head -20')], [])).toEqual({
+    status: 'pass',
+    issues: [],
+    alternativeStrategy: null,
+  })
+})
+
+test.each([
+  'python3 -c \'import glob; print(open(glob.glob("*.html")[0]).read())\'',
+  'fixture-reader',
+])('%s is not gradable without a proven-safe command shape', (command) => {
+  expect(auditToolContext([bash(command)], [])).toEqual({
+    status: 'not_gradable',
+    issues: ['unverifiable non-ax Bash command'],
+    alternativeStrategy: null,
+  })
+})
+
+test('allows a non-file echo with arithmetic expansion', () => {
+  expect(auditToolContext([bash('echo "Verification: $((1+1))"')], [])).toEqual({
+    status: 'pass',
+    issues: [],
+    alternativeStrategy: null,
+  })
+})
+
+test('rejects fixture metadata inspection outside ax', () => {
+  expect(auditToolContext([bash('ls -la continuation.html && which ax')], [])).toEqual({
+    status: 'violation',
+    issues: ['fixture access outside ax'],
+    alternativeStrategy: null,
+  })
+})
+
+test('allows an unresolved ax path that never reached the shim', () => {
+  const call = bash('./bin/ax continuation.html .incident --json-envelope')
+  call.isError = true
+  call.resultText = 'Exit code 127\n(eval):1: no such file or directory: ./bin/ax'
+  expect(auditToolContext([call], [])).toEqual({
+    status: 'pass',
+    issues: [],
+    alternativeStrategy: null,
+  })
+})
+
+test.each([
+  {
+    name: 'missing telemetry',
+    calls: [bash('ax continuation.html .incident --json-envelope')],
+    invocations: [],
+  },
+  {
+    name: 'extra telemetry',
+    calls: [bash('ax continuation.html .incident --json-envelope')],
+    invocations: [
+      invocation(['continuation.html', '.incident', '--json-envelope'], 1),
+      invocation(['continuation.html', '.incident', '--json-envelope', '--offset', '29'], 3),
+    ],
+  },
+  {
+    name: 'argv mismatch',
+    calls: [bash('ax continuation.html .incident --json-envelope')],
+    invocations: [invocation(['continuation.html', '.other', '--json-envelope'])],
+  },
+  {
+    name: 'stream omits a fixture invocation',
+    calls: [bash('ax --help')],
+    invocations: [invocation(['continuation.html', '.incident', '--json-envelope'])],
+  },
+])('$name is not gradable', ({ calls, invocations }) => {
+  expect(auditToolContext([...calls], [...invocations])).toEqual({
+    status: 'not_gradable',
+    issues: ['fixture ax telemetry mismatch'],
+    alternativeStrategy: null,
+  })
+})