From 4c25f60f477a31569a6e372aad6b182180133945 Mon Sep 17 00:00:00 2001 From: Khaliq Date: Sat, 18 Jul 2026 02:33:49 +0200 Subject: [PATCH] test(term-fidelity): harden opencode readiness + isolate opencode data dir MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The opencode leg of the real-agent matrix intermittently failed at startup with "opencode startup did not reach TF_OPENCODE_READY" (a blank/idle home screen). Direct probes localized it: opencode's OpenTUI takes several seconds to accept input after launch (MCP init, fs watcher / location services, and a scan of the user's global skill + config trees), while the broker injects the readiness task once at spawn. An injection that lands before opencode's input is live is silently dropped, and opencode then idles at its home screen until the 90s readiness deadline. Standalone, opencode renders and completes the readiness model turn reliably (replies in 2-5s), so this is purely an injection/boot race, not a renderer or model-backend fault. Two opencode-scoped harness changes (no other CLI's path is touched): 1. Boot re-nudge: once the home screen is up and the marker is still absent past a grace window, re-submit the idempotent readiness prompt on an interval until the marker appears. It runs strictly before any workload and stops the instant the marker shows, so it cannot bleed into or mask the typing-during-stream observation that follows. 2. XDG_DATA_HOME isolation: give opencode a fresh, empty data dir under the harness run root (auth.json copied across; the model cache in XDG_CACHE_HOME is untouched). Independent hygiene win: the matrix was otherwise writing test sessions into the operator's real, shared ~/.local/share/opencode/opencode.db. Not a #417 fix — this only makes the harness reach a live-input opencode deterministically so the divergence hunt has a trustworthy denominator. Verification: - Forced-drop (broker delivers an empty task, i.e. the injection is fully dropped): the re-nudge alone drives opencode to readiness and the typing-during-stream workload passes. - lsof: the harness opencode holds only the isolated empty DB and never opens the shared one. - Happy path: normal runs still reach readiness and pass with no spurious nudge. Co-Authored-By: Claude Opus 4.8 --- tests/term-fidelity/harness.ts | 30 +++++++++++++++++-- tests/term-fidelity/workloads.ts | 49 ++++++++++++++++++++++++++++++-- 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/tests/term-fidelity/harness.ts b/tests/term-fidelity/harness.ts index 83c26944..de1eeb50 100644 --- a/tests/term-fidelity/harness.ts +++ b/tests/term-fidelity/harness.ts @@ -1,6 +1,6 @@ import { _electron as electron, type ElectronApplication, type Page } from 'playwright' import { HarnessDriverClient } from '@agent-relay/harness-driver' -import { mkdir, mkdtemp, readFile, realpath, rm } from 'node:fs/promises' +import { copyFile, mkdir, mkdtemp, readFile, realpath, rm } from 'node:fs/promises' import { createServer } from 'node:net' import { homedir } from 'node:os' import { basename, join, resolve } from 'node:path' @@ -208,6 +208,29 @@ export async function launchFidelityHarness( const instanceName = `term-fidelity-${cli}-${process.pid}-${basename(runRoot).slice(-6)}` if (instanceName === 'pear') throw new Error('Refusing to use the live broker instance name') + // OpenCode keeps its session history in a single SQLite DB under + // XDG_DATA_HOME/opencode. By default that resolves to the user's real, + // shared ~/.local/share/opencode/opencode.db, which the live app's + // ai-history sync opens read/write. When that sync's WAL activity coincides + // with the harness agent's boot write, OpenCode's first paint blocks and it + // renders an empty frame that never reaches readiness — an intermittent + // TF_OPENCODE_READY timeout unrelated to the renderer under test. Give + // OpenCode an isolated, empty data dir so its boot never contends with the + // shared DB; auth lives in the same dir, so copy the real auth.json across. + // The model cache stays in XDG_CACHE_HOME and is untouched. + const opencodeDataHome = cli === 'opencode' ? join(runRoot, 'xdg-data') : null + if (opencodeDataHome) { + await mkdir(join(opencodeDataHome, 'opencode'), { recursive: true }) + const sourceDataHome = process.env.XDG_DATA_HOME || join(homedir(), '.local', 'share') + await copyFile( + join(sourceDataHome, 'opencode', 'auth.json'), + join(opencodeDataHome, 'opencode', 'auth.json') + ).catch(() => { + // No auth.json (env-key auth, or never logged in): OpenCode falls back to + // its other credential sources. A fresh empty data dir is still correct. + }) + } + let broker: HarnessDriverClient | null = null let electronApp: ElectronApplication | null = null try { @@ -221,7 +244,10 @@ export async function launchFidelityHarness( OPENCODE_CONFIG_CONTENT: JSON.stringify({ autoupdate: false, permission: { bash: 'ask', external_directory: 'ask' } - }) + }), + // Isolate OpenCode's session DB (see opencodeDataHome above). Only set + // for the OpenCode matrix leg so other CLIs' data dirs are unaffected. + ...(opencodeDataHome ? { XDG_DATA_HOME: opencodeDataHome } : {}) } broker = await spawnBrokerWithoutInheritedIdentity({ cwd: projectRoot, diff --git a/tests/term-fidelity/workloads.ts b/tests/term-fidelity/workloads.ts index f13a042e..773000bc 100644 --- a/tests/term-fidelity/workloads.ts +++ b/tests/term-fidelity/workloads.ts @@ -11,6 +11,19 @@ import { const MARKER_TIMEOUT_MS = 5 * 60_000 const AUTOMATIC_PERMISSION_MODE = /(?:bypass permissions|always approve|auto[- ]?approve|auto mode|plan mode|don'?t ask|yolo mode|full access \(current\)).*(?:on|enabled)?/iu +// OpenCode's OpenTUI can take several seconds to accept input after launch +// (MCP init, filesystem watcher / location services, and a scan of the user's +// global skill + config trees). The broker injects the readiness task once at +// spawn; an injection that lands before OpenCode's input is live is silently +// dropped, and OpenCode then idles at its home screen until the readiness +// deadline. These bound an OpenCode-only boot re-nudge that re-submits the +// readiness prompt once the home screen is up and the marker still hasn't +// appeared. HOME_HINT matches only OpenCode's idle home screen, so the nudge +// never fires mid-response; GRACE first lets the broker's own injection land. +const OPENCODE_HOME_HINT = /ask anything|opencode zen|ctrl\+p/iu +const READINESS_NUDGE_GRACE_MS = 6_000 +const READINESS_NUDGE_INTERVAL_MS = 8_000 + interface SpawnedAgent { name: string terminal: Locator @@ -93,10 +106,16 @@ async function acceptWorkspaceTrustIfShown( harness: FidelityHarness, terminal: Locator, agentName: string, - marker: string + marker: string, + // OpenCode only: the readiness prompt to re-submit if the broker's spawn-time + // injection was dropped by a not-yet-live OpenTUI input. Leave undefined for + // CLIs whose readiness injection is reliable so their paths are untouched. + readinessNudge?: string ): Promise { const deadline = Date.now() + 90_000 + const startedAt = Date.now() let acceptedTrust = false + let lastNudgeAt = 0 let lastScreen = '' while (Date.now() < deadline) { try { @@ -116,6 +135,21 @@ async function acceptWorkspaceTrustIfShown( ) { throw new Error(`${harness.cli} is not authenticated:\n${lastScreen}`) } + // Boot re-nudge (see READINESS_NUDGE_* above): only once the home screen + // is up, the marker is still absent, the broker's own injection has had + // its grace window, and the last nudge has drained. Re-submitting the + // idempotent readiness prompt lets a dropped spawn-time injection self- + // heal. This runs strictly before any workload and stops the instant the + // marker appears, so it cannot bleed into the workload it precedes. + if ( + readinessNudge && + OPENCODE_HOME_HINT.test(lastScreen) && + Date.now() - startedAt >= READINESS_NUDGE_GRACE_MS && + Date.now() - lastNudgeAt >= READINESS_NUDGE_INTERVAL_MS + ) { + await submitPrompt(terminal, readinessNudge) + lastNudgeAt = Date.now() + } } catch (error) { if (error instanceof Error && error.message.includes('not authenticated')) throw error } @@ -152,6 +186,7 @@ export async function spawnRealAgent(harness: FidelityHarness): Promise { const api = (window as unknown as Window & { pear: { @@ -180,7 +215,7 @@ export async function spawnRealAgent(harness: FidelityHarness): Promise