test(term-fidelity): cursor-convention normalization + divergence discriminator

tests/term-fidelity/harness.ts

@@ -34,6 +34,9 @@ export interface FidelityHarness {
   relayVersions: Record<string, string>
   telemetry: TelemetryRecord[]
   mainLogs: string[]
+  // Renderer console lines tagged [terminal]/[diag], captured to trace which
+  // reflow/scroll/repair path fires during a divergence.
+  rendererConsole: string[]
   currentWorkload: string | null
   // Playwright retry index (0 = first attempt). Divergence + telemetry bundles
   // are written under `attempt-<n>/` so a retry never overwrites the prior
@@ -124,6 +127,13 @@ async function installActivityProbe(page: Page): Promise<void> {
     type Activity = { lastOutputAt: number; chunks: number; bytes: number }
     type Probe = {
       activity: Record<string, Activity>
+      // The full concatenated raw byte stream the renderer received from the
+      // broker IPC (post main-process dedup, pre renderer buffer store / echo
+      // router). Replaying THIS through a fresh headless xterm and diffing it
+      // against the broker snapshot discriminates delivery loss from
+      // renderer-internal (echo-router / predictive-echo / reconciler)
+      // divergence. Read-only tap; no production code patched.
+      raw: Record<string, string>
       unsubscribe: () => void
     }
     type PearWindow = Window & {
@@ -138,15 +148,17 @@ async function installActivityProbe(page: Page): Promise<void> {
     const win = window as unknown as PearWindow
     win.__termFidelityProbe?.unsubscribe()
     const activity: Record<string, Activity> = {}
+    const raw: Record<string, string> = {}
     const unsubscribe = win.pear.broker.onPtyChunk((_projectId, name, chunk) => {
       const previous = activity[name] || { lastOutputAt: 0, chunks: 0, bytes: 0 }
       activity[name] = {
         lastOutputAt: Date.now(),
         chunks: previous.chunks + 1,
         bytes: previous.bytes + new TextEncoder().encode(chunk).byteLength
       }
+      raw[name] = (raw[name] || '') + chunk
     })
-    win.__termFidelityProbe = { activity, unsubscribe }
+    win.__termFidelityProbe = { activity, raw, unsubscribe }
   })
 }
 
@@ -228,6 +240,7 @@ export async function launchFidelityHarness(
 
     const telemetry: TelemetryRecord[] = []
     const mainLogs: string[] = []
+    const rendererConsole: string[] = []
     const harnessState = { currentWorkload: null as string | null }
     const recordMain = (source: 'main:stdout' | 'main:stderr', value: Buffer | string): void => {
       const text = value.toString()
@@ -263,8 +276,20 @@ export async function launchFidelityHarness(
     }
 
     const page = await electronApp.firstWindow({ timeout: 60_000 })
+    // Enable the renderer's per-chunk [diag:pty-append] byte diagnostic
+    // (localStorage-gated) so it survives the reload below. Read-only.
+    await page.addInitScript(() => {
+      try {
+        localStorage.setItem('PEAR_DIAG_PTY', '1')
+      } catch {
+        /* ignore */
+      }
+    })
     page.on('console', (message) => {
       const line = message.text()
+      if (line.includes('[terminal]') || line.includes('[diag')) {
+        rendererConsole.push(`${new Date().toISOString()} [${harnessState.currentWorkload || 'setup'}] ${line}`)
+      }
       if (line.includes(RECONCILER_REPAIR_LINE)) {
         telemetry.push({
           at: new Date().toISOString(),
@@ -326,6 +351,7 @@ export async function launchFidelityHarness(
       telemetry,
       mainLogs,
       attempt,
+      rendererConsole,
       get currentWorkload() {
         return harnessState.currentWorkload
       },
@@ -366,3 +392,14 @@ export async function getActivity(
     return probe?.activity[name] || { lastOutputAt: 0, chunks: 0, bytes: 0 }
   }, agentName)
 }
+
+// The full raw byte stream the renderer received for `agentName` (post main
+// dedup, pre renderer processing). Empty string if nothing captured.
+export async function getRawStream(page: Page, agentName: string): Promise<string> {
+  return await page.evaluate((name) => {
+    const probe = (window as Window & {
+      __termFidelityProbe?: { raw?: Record<string, string> }
+    }).__termFidelityProbe
+    return probe?.raw?.[name] || ''
+  }, agentName)
+}

tests/term-fidelity/oracle.ts

@@ -2,7 +2,7 @@ import { Terminal as HeadlessTerminal } from '@xterm/headless'
 import { mkdir, readFile, writeFile } from 'node:fs/promises'
 import { join } from 'node:path'
 import type { Page } from 'playwright'
-import { getActivity, type FidelityHarness } from './harness'
+import { getActivity, getRawStream, type FidelityHarness } from './harness'
 import { deriveByteAccounting } from './byte-accounting'
 
 export const QUIET_WINDOW_MS = 1_500
@@ -164,6 +164,62 @@ export async function readRendererGrid(page: Page, agentName: string): Promise<R
   }, agentName)
 }
 
+// Reach the live xterm runtime the same way readRendererGrid does, call
+// scrollToBottom(), and report whether that alone re-pins the viewport. If it
+// does, a divergence is purely a scroll-position artifact (content/baseY are
+// correct and a re-pin fixes it).
+export async function probeScrollToBottom(
+  page: Page,
+  agentName: string
+): Promise<{ before: [number, number]; after: [number, number]; atBottomAfter: boolean }> {
+  return await page.evaluate((name) => {
+    const container = document.querySelector<HTMLElement>(
+      `[data-testid="terminal-instance"][data-agent-name="${CSS.escape(name)}"]`
+    )
+    if (!container) throw new Error(`Terminal container not found for ${name}`)
+    const fiberKey = Object.keys(container).find((key) => key.startsWith('__reactFiber$'))
+    if (!fiberKey) throw new Error(`React fiber not found for ${name}`)
+    type Runtime = {
+      term: {
+        scrollToBottom(): void
+        buffer: { active: { baseY: number; viewportY: number } }
+      }
+    }
+    const looksLikeRuntime = (value: unknown): value is Runtime => {
+      const c = value as Partial<Runtime> | null
+      return Boolean(
+        c && typeof c === 'object' && c.term &&
+        typeof (c.term as Runtime['term']).scrollToBottom === 'function' &&
+        (c.term as Runtime['term']).buffer?.active
+      )
+    }
+    let runtime: Runtime | null = null
+    let fiber: { memoizedState?: unknown; return?: unknown } | null =
+      (container as unknown as Record<string, { memoizedState?: unknown; return?: unknown }>)[fiberKey]
+    while (fiber && !runtime) {
+      let hook = fiber.memoizedState as { memoizedState?: unknown; next?: unknown } | null
+      const seen = new Set<unknown>()
+      while (hook && typeof hook === 'object' && !seen.has(hook)) {
+        seen.add(hook)
+        const state = hook.memoizedState
+        const current = state && typeof state === 'object'
+          ? (state as { current?: unknown }).current
+          : undefined
+        if (looksLikeRuntime(current)) { runtime = current; break }
+        hook = hook.next as typeof hook
+      }
+      fiber = (fiber.return as typeof fiber) || null
+    }
+    if (!runtime) throw new Error(`Live xterm runtime not found for ${name}`)
+    const b = runtime.term.buffer.active
+    const before: [number, number] = [b.viewportY, b.baseY]
+    runtime.term.scrollToBottom()
+    const a = runtime.term.buffer.active
+    const after: [number, number] = [a.viewportY, a.baseY]
+    return { before, after, atBottomAfter: a.viewportY === a.baseY }
+  }, agentName)
+}
+
 export async function readBrokerSnapshot(
   connectionPath: string,
   agentName: string
@@ -242,6 +298,50 @@ async function brokerCellGrid(snapshot: BrokerSnapshot): Promise<Cell[][]> {
   }
 }
 
+// Paint an arbitrary raw byte stream into a fresh headless xterm at the given
+// dims and read back its cell grid + cursor. Used to replay the exact bytes the
+// renderer received (getRawStream) so a clean-emulator render of the delivered
+// bytes can be compared against both the broker oracle and the live renderer.
+async function replayRawToGrid(
+  raw: string,
+  rows: number,
+  cols: number
+): Promise<{ lines: string[]; cells: Cell[][]; cursor: [number, number] }> {
+  const term = new HeadlessTerminal({ rows, cols, scrollback: 0, allowProposedApi: true })
+  try {
+    await writeHeadless(term, raw)
+    const buffer = term.buffer.active
+    const lines: string[] = []
+    const cells: Cell[][] = []
+    for (let row = 0; row < rows; row += 1) {
+      const line = buffer.getLine(buffer.baseY + row)
+      lines.push(line ? line.translateToString(true, 0, cols) : '')
+      const rowCells: Cell[] = []
+      for (let col = 0; col < cols; col += 1) {
+        const cell = line?.getCell(col)
+        rowCells.push({ chars: cell?.getChars() || '', width: cell?.getWidth() ?? 1 })
+      }
+      cells.push(rowCells)
+    }
+    return { lines, cells, cursor: [buffer.cursorY, buffer.cursorX] }
+  } finally {
+    term.dispose()
+  }
+}
+
+function countCellDiffs(a: Cell[][], b: Cell[][], rows: number, cols: number): number {
+  let total = 0
+  for (let row = 0; row < rows; row += 1) {
+    for (let col = 0; col < cols; col += 1) {
+      const ca = a[row]?.[col]
+      const cb = b[row]?.[col]
+      if (normalizedCellChars(ca) === normalizedCellChars(cb) && ca?.width === cb?.width) continue
+      total += 1
+    }
+  }
+  return total
+}
+
 function visibleCell(cell: Cell | undefined): string {
   if (!cell) return '<missing>'
   const chars = cell.chars === '' || cell.chars === ' ' ? '<blank>' : cell.chars
@@ -415,9 +515,25 @@ async function writeDivergenceBundle(
       broker: { rows: broker.rows, cols: broker.cols }
     },
     rendererBufferType: renderer.bufferType,
+    // Cursor convention normalization. The renderer cursor comes from xterm's
+    // buffer API, which is 0-indexed [row, col]. The broker plain-snapshot
+    // cursor is 1-indexed [row, col] — it mirrors the raw CUP coordinates (a
+    // stream ending in ESC[40;3H reports broker cursor [40, 3] while xterm
+    // reports [39, 2] for the SAME cell). Comparing the raw values makes a
+    // faithful render look like an off-by-one bug (this misread #403). Record
+    // the raw values WITH their conventions, plus both normalized to a common
+    // 0-indexed base and a computed match: a `false` here on an otherwise
+    // content-matching bundle is a REAL cursor divergence, not the convention.
     cursor: {
-      renderer: renderer.cursor,
-      broker: broker.cursor
+      rendererRaw0Indexed: renderer.cursor,
+      brokerRaw1Indexed: broker.cursor,
+      normalized0Indexed: {
+        renderer: renderer.cursor,
+        broker: [broker.cursor[0] - 1, broker.cursor[1] - 1] as [number, number]
+      },
+      match:
+        renderer.cursor[0] === broker.cursor[0] - 1 &&
+        renderer.cursor[1] === broker.cursor[1] - 1
     },
     // Self-documenting client-vs-broker byte accounting. Replaces the former
     // bare `brokerOffset` + `quiet.activity.bytes` pair, whose exact-2.0 reading
@@ -485,8 +601,51 @@ export async function captureCheckpoint(
       quiet,
       options
     )
+
+    // Replay the exact bytes the renderer received through a clean headless
+    // emulator to locate where the divergence was introduced:
+    //   rawVsBroker == 0 → delivery is byte-clean; the divergence is
+    //                      renderer-internal (echo-router / predictive-echo /
+    //                      reconciler write path).
+    //   rawVsBroker  > 0 → bytes were lost/reordered BEFORE the renderer (main
+    //                      IPC / dedup) — a delivery vector.
+    // rawVsRenderer shows how far the live grid drifted from a faithful render
+    // of the delivered bytes. The scroll probe distinguishes a stranded
+    // viewport (content correct, just scrolled) from real content corruption.
+    let discriminator = 'raw-stream unavailable'
+    try {
+      const raw = await getRawStream(harness.page, agentName)
+      const rawReplay = await replayRawToGrid(raw, broker.rows, broker.cols)
+      const brokerCells = await brokerCellGrid(broker)
+      const rawVsBroker = countCellDiffs(rawReplay.cells, brokerCells, broker.rows, broker.cols)
+      const rawVsRenderer = countCellDiffs(rawReplay.cells, renderer.cells, broker.rows, broker.cols)
+      let scrollProbe = 'skipped'
+      try {
+        const probe = await probeScrollToBottom(harness.page, agentName)
+        scrollProbe =
+          `viewport[vY,baseY]before=[${probe.before}] after=[${probe.after}] ` +
+          `atBottomAfterScroll=${probe.atBottomAfter}`
+      } catch (probeError) {
+        scrollProbe = `probe failed: ${probeError instanceof Error ? probeError.message : String(probeError)}`
+      }
+      discriminator =
+        `rawBytes=${raw.length} rawReplayCursor=[${rawReplay.cursor}] ` +
+        `rendererViewport=[vY=${renderer.viewportY},baseY=${renderer.baseY}] ` +
+        `rawVsBroker=${rawVsBroker} rawVsRenderer=${rawVsRenderer} ` +
+        `verdict=${rawVsBroker === 0 ? 'RENDERER-INTERNAL' : 'DELIVERY-LOSS'} ${scrollProbe}`
+      await Promise.all([
+        writeFile(join(artifactDir, 'raw.bin'), raw),
+        writeFile(join(artifactDir, 'raw-replay.txt'), rawReplay.lines.join('\n')),
+        writeFile(join(artifactDir, 'discriminator.txt'), `${discriminator}\n`),
+        writeFile(join(artifactDir, 'renderer-console.log'), `${harness.rendererConsole.join('\n')}\n`)
+      ])
+    } catch (error) {
+      discriminator = `raw forensics failed: ${error instanceof Error ? error.message : String(error)}`
+    }
+
     throw new Error(
       `${harness.cli}/${workload} diverged in ${differingCells} cells; artifacts: ${artifactDir}\n` +
+      `[discriminator] ${discriminator}\n` +
       differences.slice(0, 20).join('\n')
     )
   }
@@ -506,6 +665,42 @@ export async function writeTelemetryArtifact(
   )
   await mkdir(artifactDir, { recursive: true })
   const screenshot = await harness.page.screenshot({ animations: 'disabled', type: 'png' })
+
+  // A reconciler repair fired mid-workload, so the LIVE grid was already
+  // repainted from the broker snapshot before we got here — the pre-repair grid
+  // is gone. But the cumulative raw stream survives: replaying every delivered
+  // byte through a clean emulator and diffing against the final broker snapshot
+  // still discriminates the class. rawVsBroker==0 ⇒ delivery was byte-clean
+  // over the whole session, so the transient divergence the reconciler repaired
+  // was RENDERER-INTERNAL (the live grid drifted from a faithful render of the
+  // same bytes — a creation vector in the renderer write path). rawVsBroker>0 ⇒
+  // bytes were lost/reordered before the renderer and the repair masked it.
+  let discriminator = 'raw-stream unavailable'
+  try {
+    const raw = await getRawStream(harness.page, agentName)
+    const broker = await readBrokerSnapshot(harness.connectionPath, agentName)
+    const renderer = await readRendererGrid(harness.page, agentName)
+    const rawReplay = await replayRawToGrid(raw, broker.rows, broker.cols)
+    const brokerCells = await brokerCellGrid(broker)
+    const rawVsBroker = countCellDiffs(rawReplay.cells, brokerCells, broker.rows, broker.cols)
+    const rawVsRenderer = countCellDiffs(rawReplay.cells, renderer.cells, broker.rows, broker.cols)
+    discriminator =
+      `rawBytes=${raw.length} bufferType=${renderer.bufferType} ` +
+      `rawVsBroker=${rawVsBroker} rawVsRenderer=${rawVsRenderer} ` +
+      `verdict=${rawVsBroker === 0 ? 'RENDERER-INTERNAL' : 'DELIVERY-LOSS'} ` +
+      `(post-repair grid==broker by construction; this is the session-cumulative delivery check)`
+    await Promise.all([
+      writeFile(join(artifactDir, 'raw.bin'), raw),
+      writeFile(join(artifactDir, 'raw-replay.txt'), rawReplay.lines.join('\n')),
+      writeFile(join(artifactDir, 'broker.txt'), broker.screen),
+      writeFile(join(artifactDir, 'renderer.txt'), renderer.lines.join('\n')),
+      writeFile(join(artifactDir, 'discriminator.txt'), `${discriminator}\n`),
+      writeFile(join(artifactDir, 'renderer-console.log'), `${harness.rendererConsole.join('\n')}\n`)
+    ])
+  } catch (error) {
+    discriminator = `raw forensics failed: ${error instanceof Error ? error.message : String(error)}`
+  }
+
   await Promise.all([
     writeFile(join(artifactDir, 'screen.png'), screenshot),
     writeFile(join(artifactDir, 'meta.json'), `${JSON.stringify({
@@ -515,7 +710,8 @@ export async function writeTelemetryArtifact(
       instanceName: harness.instanceName,
       apiPort: harness.apiPort,
       relayVersions: { broker: harness.brokerVersion, ...harness.relayVersions },
-      reconcilerTelemetryLines: harness.telemetry
+      reconcilerTelemetryLines: harness.telemetry,
+      discriminator
     }, null, 2)}\n`)
   ])
   return artifactDir