Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 28 additions & 2 deletions tests/term-fidelity/harness.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import { _electron as electron, type ElectronApplication, type Page } from 'playwright'
import { HarnessDriverClient } from '@agent-relay/harness-driver'
import { mkdir, mkdtemp, readFile, realpath, rm } from 'node:fs/promises'
import { copyFile, mkdir, mkdtemp, readFile, realpath, rm } from 'node:fs/promises'
import { createServer } from 'node:net'
import { homedir } from 'node:os'
import { basename, join, resolve } from 'node:path'
Expand Down Expand Up @@ -208,6 +208,29 @@ export async function launchFidelityHarness(
const instanceName = `term-fidelity-${cli}-${process.pid}-${basename(runRoot).slice(-6)}`
if (instanceName === 'pear') throw new Error('Refusing to use the live broker instance name')

// OpenCode keeps its session history in a single SQLite DB under
// XDG_DATA_HOME/opencode. By default that resolves to the user's real,
// shared ~/.local/share/opencode/opencode.db, which the live app's
// ai-history sync opens read/write. When that sync's WAL activity coincides
// with the harness agent's boot write, OpenCode's first paint blocks and it
// renders an empty frame that never reaches readiness — an intermittent
// TF_OPENCODE_READY timeout unrelated to the renderer under test. Give
// OpenCode an isolated, empty data dir so its boot never contends with the
// shared DB; auth lives in the same dir, so copy the real auth.json across.
// The model cache stays in XDG_CACHE_HOME and is untouched.
const opencodeDataHome = cli === 'opencode' ? join(runRoot, 'xdg-data') : null
if (opencodeDataHome) {
await mkdir(join(opencodeDataHome, 'opencode'), { recursive: true })
const sourceDataHome = process.env.XDG_DATA_HOME || join(homedir(), '.local', 'share')
await copyFile(
join(sourceDataHome, 'opencode', 'auth.json'),
join(opencodeDataHome, 'opencode', 'auth.json')
).catch(() => {
// No auth.json (env-key auth, or never logged in): OpenCode falls back to
// its other credential sources. A fresh empty data dir is still correct.
})
}

let broker: HarnessDriverClient | null = null
let electronApp: ElectronApplication | null = null
try {
Expand All @@ -221,7 +244,10 @@ export async function launchFidelityHarness(
OPENCODE_CONFIG_CONTENT: JSON.stringify({
autoupdate: false,
permission: { bash: 'ask', external_directory: 'ask' }
})
}),
// Isolate OpenCode's session DB (see opencodeDataHome above). Only set
// for the OpenCode matrix leg so other CLIs' data dirs are unaffected.
...(opencodeDataHome ? { XDG_DATA_HOME: opencodeDataHome } : {})
}
broker = await spawnBrokerWithoutInheritedIdentity({
cwd: projectRoot,
Expand Down
49 changes: 46 additions & 3 deletions tests/term-fidelity/workloads.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,19 @@ import {
const MARKER_TIMEOUT_MS = 5 * 60_000
const AUTOMATIC_PERMISSION_MODE = /(?:bypass permissions|always approve|auto[- ]?approve|auto mode|plan mode|don'?t ask|yolo mode|full access \(current\)).*(?:on|enabled)?/iu

// OpenCode's OpenTUI can take several seconds to accept input after launch
// (MCP init, filesystem watcher / location services, and a scan of the user's
// global skill + config trees). The broker injects the readiness task once at
// spawn; an injection that lands before OpenCode's input is live is silently
// dropped, and OpenCode then idles at its home screen until the readiness
// deadline. These bound an OpenCode-only boot re-nudge that re-submits the
// readiness prompt once the home screen is up and the marker still hasn't
// appeared. HOME_HINT matches only OpenCode's idle home screen, so the nudge
// never fires mid-response; GRACE first lets the broker's own injection land.
const OPENCODE_HOME_HINT = /ask anything|opencode zen|ctrl\+p/iu
const READINESS_NUDGE_GRACE_MS = 6_000
const READINESS_NUDGE_INTERVAL_MS = 8_000

interface SpawnedAgent {
name: string
terminal: Locator
Expand Down Expand Up @@ -93,10 +106,16 @@ async function acceptWorkspaceTrustIfShown(
harness: FidelityHarness,
terminal: Locator,
agentName: string,
marker: string
marker: string,
// OpenCode only: the readiness prompt to re-submit if the broker's spawn-time
// injection was dropped by a not-yet-live OpenTUI input. Leave undefined for
// CLIs whose readiness injection is reliable so their paths are untouched.
readinessNudge?: string
): Promise<void> {
const deadline = Date.now() + 90_000
const startedAt = Date.now()
let acceptedTrust = false
let lastNudgeAt = 0
let lastScreen = ''
while (Date.now() < deadline) {
try {
Expand All @@ -116,6 +135,21 @@ async function acceptWorkspaceTrustIfShown(
) {
throw new Error(`${harness.cli} is not authenticated:\n${lastScreen}`)
}
// Boot re-nudge (see READINESS_NUDGE_* above): only once the home screen
// is up, the marker is still absent, the broker's own injection has had
// its grace window, and the last nudge has drained. Re-submitting the
// idempotent readiness prompt lets a dropped spawn-time injection self-
// heal. This runs strictly before any workload and stops the instant the
// marker appears, so it cannot bleed into the workload it precedes.
if (
readinessNudge &&
OPENCODE_HOME_HINT.test(lastScreen) &&
Date.now() - startedAt >= READINESS_NUDGE_GRACE_MS &&
Date.now() - lastNudgeAt >= READINESS_NUDGE_INTERVAL_MS
) {
await submitPrompt(terminal, readinessNudge)
Comment on lines +146 to +150

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Restrict re-nudges to OpenCode's idle composer

When the initial readiness turn takes longer than the 6-second grace period, this condition can submit another prompt while that turn is still streaming: ctrl+p is a persistent OpenCode footer hint rather than an idle-only signal. Additional nudges can therefore be queued every eight seconds; once the first response prints the marker, startup returns and begins the terminal-fidelity workloads while a duplicate readiness turn remains queued or active, contaminating their PTY streams and results. Gate the nudge on a state that proves OpenCode is idle rather than on this footer text.

Useful? React with 👍 / 👎.

lastNudgeAt = Date.now()
}
} catch (error) {
if (error instanceof Error && error.message.includes('not authenticated')) throw error
}
Expand Down Expand Up @@ -152,6 +186,7 @@ export async function spawnRealAgent(harness: FidelityHarness): Promise<SpawnedA

const requestedName = `tf-${harness.cli}`
const marker = `TF_${harness.cli.toUpperCase()}_READY`
const readinessTask = `Reply with exactly one token made from the parts "TF", "${harness.cli.toUpperCase()}", "READY", joined with one underscore between adjacent parts. Do not use tools.`
const spawned = await harness.page.evaluate(async ({ projectId, root, cli, name, task, args }) => {
const api = (window as unknown as Window & {
pear: {
Expand Down Expand Up @@ -180,7 +215,7 @@ export async function spawnRealAgent(harness: FidelityHarness): Promise<SpawnedA
root: harness.projectRoot,
cli: harness.cli,
name: requestedName,
task: `Reply with exactly one token made from the parts "TF", "${harness.cli.toUpperCase()}", "READY", joined with one underscore between adjacent parts. Do not use tools.`,
task: readinessTask,
args: initialArgs(harness.cli)
})
const agentName = spawned.name || requestedName
Expand All @@ -199,7 +234,15 @@ export async function spawnRealAgent(harness: FidelityHarness): Promise<SpawnedA
{ message: `live xterm runtime should mount for ${agentName}`, timeout: 60_000 }
).toBeGreaterThan(0)

await acceptWorkspaceTrustIfShown(harness, terminal, agentName, marker)
await acceptWorkspaceTrustIfShown(
harness,
terminal,
agentName,
marker,
// OpenCode's slow OpenTUI boot can drop the broker's spawn-time readiness
// injection; re-nudge it. Other CLIs inject reliably, so leave them alone.
harness.cli === 'opencode' ? readinessTask : undefined
)
if (harness.cli === 'claude') {
// Normalize while the startup status is still visible. After workload 1,
// the bypass badge can scroll out even though the mode remains active.
Expand Down
Loading