From 495fc7cf153cb4f56559b79e892d7efb2ecc81bf Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Thu, 24 Sep 2026 00:50:31 -0700 Subject: [PATCH 1/8] Keep operations out of the relaunch gap in the host The rule that nothing may reach a browser between a relaunch's close and its reopen was enforced by the controller's drive gate, which saw only its own Surface's requests. The host serializes every launch and close, so it now refuses every operation but launch, attach and close on a browser a launch is replacing or a close is ending, whichever Surface asks. agent-browser starts a daemon at about:blank for any verb run with none up, so its provider now runs an operation only while the session's pid file names a live process. A daemon gone while its pane was hidden, or one in a socket directory the host does not share, is refused instead of restarted. With both in the host, the controller's gate no longer holds back an unparked pane's operations until its stream opens; only its catch-up waits, and a navigation driven directly supersedes one still pending. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NCV5Uq6FeZbBfzfpk3vrpe --- docs/specs/dor-browser.md | 27 ++++++---- docs/specs/dor-browser.rationale.md | 4 ++ .../agent-browser-surface-controller.test.ts | 14 +++-- .../wall/agent-browser-surface-controller.ts | 19 ++++--- lib/src/host/agent-browser-host.test.ts | 52 ++++++++++++++++--- lib/src/host/agent-browser-host.ts | 19 +++++-- lib/src/host/browser-host.test.ts | 24 +++++++++ lib/src/host/browser-host.ts | 30 +++++++++-- scripts/spec-word-budgets.json | 2 +- 9 files changed, 156 insertions(+), 35 deletions(-) diff --git a/docs/specs/dor-browser.md b/docs/specs/dor-browser.md index d9465c1f9..5c9f5b718 100644 --- a/docs/specs/dor-browser.md +++ b/docs/specs/dor-browser.md @@ -324,10 +324,12 @@ place of `live`; a `dor` handover moves any phase but `launching` and | `ended` | No browser; `error` says why | `relaunching`; a navigation rebinds, with its page | | `disposed` | Released | — | -- **Every daemon command must pass one gate (`driver`), open only in `live`, - and after an unpark only once its stream opens** — chrome and Display modal - actions, tabs, sync-to-pane, `get cdp-url`, edit chords, screenshots - (rationale). +- **Every browser operation must pass one gate (`driver`), open only in + `live`** — chrome and Display modal actions, tabs, sync-to-pane, `get + cdp-url`, edit chords, screenshots (rationale). **An unpark catches up — + sync, a pending intent — only once its stream opens.** The gate orders the + Surface's own intents; the host is what keeps an operation off a browser + mid-relaunch ([Browser Host](#browser-host)). - **A navigation asked for outside `live` is kept as the one latest intent**, run on the next `live`; **so is a pop-out or pop-in asked before the browser is bound** (`idle`, `launching`, `attaching`), run as a relaunch, and so is @@ -440,9 +442,9 @@ page to load** (rationale): its launch resolves once the *relaunched* daemon is up, asking `stream status` only after `open` returns. **A non-zero `open` exit with the daemon up is a page still loading, not a failed launch**; only a launch without a published port fails, including after a zero exit. **Never query the -daemon during the close/reopen gap** (rationale) — host-side, and in the -controller through its daemon gate — so **Dormouse supplies the active-tab URL -and the host trusts it**. +daemon during the close/reopen gap** (rationale) — the host refuses every +operation on a browser mid-launch ([Browser Host](#browser-host)) — so +**Dormouse supplies the active-tab URL and the host trusts it**. While popped out, Dormouse keeps a stream/CDP observer for same-tab URL/header updates and headed-window close auto-revert. @@ -492,6 +494,10 @@ list tabs, act, evaluate, screenshot, stream URL): must navigate it, never stop it** (a Tool re-announced); only one gone or in the other mode is relaunched (rationale). agent-browser cannot report its mode: one this host did not launch headed counts as headless. +- **Must refuse every operation but `launch`, `attach` and `close` on a + browser a launch is replacing or a close is ending**, until it is done, so + nothing reaches a browser in the close/reopen gap ([Pop-Out](#pop-out)), + whichever Surface asks. - **Must answer a launch inside `BROWSER_REQUEST_TIMEOUT_MS`**: startup, queueing included, gets 30 s from the request's arrival. A relaunch stops what runs the session first, then resolves once the provider reports the @@ -567,13 +573,16 @@ relaunch changes the mode. every other call runs in the host's. - **Every spawn passes the `binaryPath` gate in `runWithBinaryFallback`**, the host's `DORMOUSE_AGENT_BROWSER_BIN` being the exact-match override. +- **Only a launch's own steps may run a CLI verb with no daemon up**: an + operation runs only while `.pid` names a live process, since any + verb starts a daemon at `about:blank` to answer. - **`dor ab` must read the stream port itself after a command that may bind** (`stream status --json`, safe once the command made the daemon) and hand it over; the Surface streams from it without asking the host (rationale). **Never carry a socket directory to the host** — it kills the pid it reads there. **Host-side operations for a session in a socket directory the host - does not share are unsupported**: they run the CLI under the host's - environment. + does not share are refused**, as the host sees no daemon for it; a close runs + the CLI under the host's environment. - **VS Code must reach the stream through a loopback relay** — the agent-browser stream server rejects `vscode-webview://` origins. The relay grants one single-use, short-TTL token bound to one stream port and strips the Origin diff --git a/docs/specs/dor-browser.rationale.md b/docs/specs/dor-browser.rationale.md index b7162222c..e16dafc1c 100644 --- a/docs/specs/dor-browser.rationale.md +++ b/docs/specs/dor-browser.rationale.md @@ -108,6 +108,10 @@ A post-open blank-tab sweep can become such a query when a later relaunch, expli **Why a close runs after the launch in flight and supersedes one queued.** A failed swap's restore reopens the previous provider's session, whose `close` was issued at swap time; a fast failure (no Playwright installed) lands before that close does, so a reopen racing it was closed under the restored pane. A Tool re-run relaunching the `tool.` session its last run closed meets the same race. So does a Surface closed while its own launch, relaunch or relaunching attach into the session was in flight: that work closed the session when it landed, after the next launch into the name had opened it, and a close issued before the launch had bound the session registered nothing to wait on. The Tool serving loop that awaited each open never overlapped two launches; its replacement does (review of #775, 2026-09). The webview first also recorded each Surface's work in flight, to close the session again when it landed; the host, which already serialized a browser's launches and closes, now runs a close after the work already running, so a close is sent at once and never repeated. A launch still queued when a close of its browser arrives was sent by a Surface closed meanwhile (a Tool swapped away and back twice): run after that close it would reopen the session nobody shows. The host sees arrival order, which VS Code's message channel keeps but standalone's Tauri commands, run on a worker pool, can swap for two requests sent within the same instant (static reading, 2026-09). So the ordering takes both layers: a named launch waits in the webview for the answer to every close of its session it sent, which no transport can reorder, and the host orders a close after the work in flight and supersedes what is queued. The reverse swap — a Surface's own launch or relaunch reaching the host after its close — would find the close already done and bring a browser up for nobody, a headless one outliving even shutdown; so the close names those requests and the host refuses them when they arrive. Closing only after the work answered, as the webview once did, covers it too, but holds every such close up to 40 s and loses it to a webview reload. +**Why the host keeps operations out of the relaunch gap.** The controller's gate held back only its own Surface's requests, and only while it knew a relaunch was running: a second Surface on the session — a Workspace transfer's destination, the transient double binding — or a request its transport delivered late still reached the daemon mid-relaunch. The host serializes every launch and close, so it is the one place that sees each gap whole (review of the browser stack, 2026-09). + +**Why an agent-browser operation needs a live pid file.** A 0.31.1 daemon removes its `.pid`, `.sock` and `.stream` files when it exits cleanly and leaves `.config`: `~/.agent-browser` held 140 `.config` files beside the one live daemon's `.pid` (2026-09-24). So a missing pid file is a gone daemon — or one in a socket directory the host does not share — and any verb run for it starts a daemon at `about:blank`, as `set viewport` did when an unparked pane was resized over a daemon gone while hidden. + **Why one lifecycle for both providers.** The two hosts carried the same policies twice — headed tracking, relaunch generations, the blank-tab sweep, capture joins, the editing scripts — and the copies drifted: an empty copy clobbered the clipboard in one, the capture directory lacked its `chmod` in the other, and only Playwright serialized its closes with its relaunches, so the webview kept its own record of closes in flight for agent-browser (review of the browser stack, 2026-09). **Why the screenshot path is private.** The frame is a picture of the user's authenticated browser, written by an external process under the ambient umask, so a derivable name in the shared temp directory is readable by anything else on the machine for as long as it exists. Precedent: `standalone/sidecar/clipboard-ops.js` applies the same discipline, cleanup included, to clipboard images. diff --git a/lib/src/components/wall/agent-browser-surface-controller.test.ts b/lib/src/components/wall/agent-browser-surface-controller.test.ts index 80867f86c..784ac2d45 100644 --- a/lib/src/components/wall/agent-browser-surface-controller.test.ts +++ b/lib/src/components/wall/agent-browser-surface-controller.test.ts @@ -994,7 +994,7 @@ describe('attach', () => { ['answers', true, false], ['answers, with a page asked for meanwhile', true, true], ['has gone away', false, true], - ])('an unpark drives nothing until the port it parked at %s', async (_name, answers, askedMeanwhile) => { + ])('an unpark catches up only once the port it parked at %s', async (_name, answers, askedMeanwhile) => { vi.useFakeTimers(); try { const host = attachHost(async () => ({ ok: false, error: 'not running' })); @@ -1014,19 +1014,23 @@ describe('attach', () => { getAgentBrowserScreenController('id')!.chromeActions.navigate('https://next.example/'); host.browser.mockClear(); - // A daemon gone meanwhile would be started again by any CLI command. + // The resize and the page asked for while hidden wait for the stream, so + // a daemon gone meanwhile is found ended rather than driven. if (!answers) WebSocketMock.failPorts.add(1111); controller.setVisible(true); - // Asked for before the stream opens: kept, like any navigation the gate refuses. + // Asked for before the stream opens: sent at once — the host refuses it + // if the daemon is gone — superseding the page asked for while hidden. if (askedMeanwhile) getAgentBrowserScreenController('id')!.chromeActions.navigate('https://later.example/'); await vi.advanceTimersByTimeAsync(250); + const later = askedMeanwhile ? [onSess({ op: 'navigate', url: 'https://later.example/' })] : []; if (answers) { expect(sent()).toEqual([ + ...later, onSess({ op: 'viewport', width: 1000, height: 700, dpr: 1 }), - onSess({ op: 'navigate', url: askedMeanwhile ? 'https://later.example/' : 'https://next.example/' }), + ...(askedMeanwhile ? [] : [onSess({ op: 'navigate', url: 'https://next.example/' })]), ]); } else { - expect(sent()).toEqual([onSess({ op: 'attach', headed: false })]); + expect(sent()).toEqual([...later, onSess({ op: 'attach', headed: false })]); expect(controller.snapshot().phase).toBe('ended'); } } finally { diff --git a/lib/src/components/wall/agent-browser-surface-controller.ts b/lib/src/components/wall/agent-browser-surface-controller.ts index 7940ad63e..6b4b582fd 100644 --- a/lib/src/components/wall/agent-browser-surface-controller.ts +++ b/lib/src/components/wall/agent-browser-surface-controller.ts @@ -152,8 +152,8 @@ type Phase = /** Streaming from `port`. `seen`: the stream has reported its browser * connected, so a later drop means it went away — a headed window closed. * `resumed`: an unpark reconnecting to the port it parked at, not yet - * proven there — nothing drives the daemon until its stream opens, since a - * CLI command would start a new one if it went away while hidden. */ + * proven there — its catch-up waits until the stream opens, and a stream + * that fails asks the host where it moved. */ | { k: 'live'; port: number; seen: boolean; resumed: boolean } /** Hidden long enough to shed the stream; the daemon stays up at `port`. */ | { k: 'parked'; port: number } @@ -1515,13 +1515,14 @@ export class AgentBrowserSurfaceController { // --- the daemon gate --- - /** The browser to drive, only while `live` — and after an unpark, once its - * stream opens: every command, edit and capture takes it from here, since - * mid-launch, mid-attach or mid-relaunch, or for a daemon gone while hidden, - * a CLI command starts a competing daemon at about:blank - * (docs/specs/dor-browser.md → "Browser Connection"). */ + /** The browser to drive, only while `live`: every command, edit and capture + * takes it from here, and what is asked outside it waits as the pending + * intent. The host refuses whatever would reach a browser mid-launch or + * mid-close, or a daemon that is gone, so this gate orders the Surface's + * own intents rather than guarding the daemon (docs/specs/dor-browser.md → + * "Browser Connection"). */ private driver(): BrowserHandle | null { - if (this.phase.k !== 'live' || this.phase.resumed || !this.session) return null; + if (this.phase.k !== 'live' || !this.session) return null; return this.handle(); } @@ -1557,6 +1558,8 @@ export class AgentBrowserSurfaceController { private navigate(url: string): void { if (!url) return; if (this.driver()) { + // The latest navigation, superseding one an unpark has yet to catch up on. + delete this.pendingIntent.url; this.drive(`open ${url}`, (browser) => browser.navigate(url)); return; } diff --git a/lib/src/host/agent-browser-host.test.ts b/lib/src/host/agent-browser-host.test.ts index a1eaf0674..a98cdfae7 100644 --- a/lib/src/host/agent-browser-host.test.ts +++ b/lib/src/host/agent-browser-host.test.ts @@ -1,9 +1,9 @@ -import { spawn } from 'child_process'; +import { spawn, type ChildProcess } from 'child_process'; import { existsSync, mkdtempSync, promises as fsp, statSync, utimesSync, writeFileSync } from 'fs'; import { createServer, type Server } from 'net'; import { tmpdir } from 'os'; import { dirname, join } from 'path'; -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest'; import type { BrowserOp, BrowserRequestBinding } from '../lib/platform/browser-automation'; import { createAgentBrowserProvider } from './agent-browser-host'; import { createBrowserHost } from './browser-host'; @@ -83,6 +83,12 @@ function writeState(session: string, ext: 'pid' | 'stream', value: number): void writeFileSync(join(process.env.AGENT_BROWSER_SOCKET_DIR!, `${session}.${ext}`), `${value}\n`); } +/** Give each session a running daemon, as its pid file says: the host drives + * no other. This test process stands in for it. */ +function running(...sessions: string[]): void { + for (const session of sessions) writeState(session, 'pid', process.pid); +} + // The host spawns through dor-lib-common's spawnAndCapture; mock just that // boundary (not its internal cross-spawn — spawnAndCapture's own behavior is // covered by dor-lib-common's tests), keeping the package's other real exports. @@ -514,6 +520,29 @@ describe('agent-browser host attach', () => { expect(spawnMock).not.toHaveBeenCalled(); }); + // Any verb starts a daemon to answer when none runs, at about:blank: an + // operation on a daemon gone while its pane was hidden, or on one in a + // socket directory the host does not share, must start nothing. + it('runs no operation on a session whose daemon is not running', async () => { + const host = makeHost(); + const ops: BrowserOp[] = [ + { op: 'navigate', url: 'https://example.com/' }, + { op: 'history', dir: 'back' }, + { op: 'tab', action: 'select', tabId: 't1' }, + { op: 'viewport', width: 800, height: 600, dpr: 2 }, + { op: 'device', name: 'iPhone 16' }, + { op: 'edit', edit: 'copy' }, + { op: 'screenshot' }, + ]; + for (const pid of [undefined, DEAD_PID]) { + if (pid !== undefined) writeState(session, 'pid', pid); + for (const op of ops) { + expect(await ab(host, op, { session }), JSON.stringify(op)).toEqual({ ok: false, error: `agent-browser session '${session}' is not running` }); + } + } + expect(spawnMock).not.toHaveBeenCalled(); + }); + it('relaunches a gone daemon at the page named, headed for a pop-out, and tracks that window for shutdown', async () => { writeState(session, 'pid', DEAD_PID); writeState(session, 'stream', await closedPort()); @@ -588,9 +617,8 @@ describe('agent-browser host attach', () => { }); describe('agent-browser host screenshot transport', () => { - // Block body (not `() => spawnMock.mockReset()`): an arrow returning the mock - // makes vitest register it as a teardown hook and call it — a phantom spawn. - beforeEach(() => { spawnMock.mockReset(); }); + useTempSocketDir('dormouse-ab-shot-test-'); + beforeEach(() => { running('shotfile', 'dormouse.1.default', 'shotbytes', 'shutdown-sess', 'read-sess', 'queued', 'queued-bytes', 'retry-sess', 'sess', 'left', 'kept'); }); /** agent-browser's `screenshot `, writing `frames` in turn. */ function captureFrames(...frames: number[][]): string[][] { @@ -736,6 +764,15 @@ describe('agent-browser host screenshot transport', () => { } return spawnResult({ code: args.includes('open') ? 1 : 0 }); }); + // A relaunch kills the daemon its pid file names, so a child stands in. + const daemons: ChildProcess[] = []; + const daemon = () => { + const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1 << 30)'], { stdio: 'ignore' }); + daemons.push(child); + writeState('wedged', 'pid', child.pid!); + }; + onTestFinished(() => { for (const child of daemons) child.kill('SIGKILL'); }); + daemon(); const host = makeHost(); const capture = async () => { void abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'wedged' }); @@ -754,6 +791,7 @@ describe('agent-browser host screenshot transport', () => { // So does a relaunch, which reuses the session name. await ab(host, { op: 'launch', url: 'https://example.com/', headed: false }, { session: 'wedged' }); + daemon(); await capture(); expect(shots).toHaveLength(3); @@ -815,6 +853,7 @@ describe('agent-browser host screenshot transport', () => { describe('agent-browser host requests', () => { useTempSocketDir('dormouse-ab-argv-test-'); const session = { session: 'dormouse.1.gui-abc' }; + beforeEach(() => { running(session.session); }); it('renders each operation to exactly one fixed argv', async () => { const shapes: [BrowserOp, string[]][] = [ @@ -912,7 +951,8 @@ describe('agent-browser host requests', () => { }); describe('agent-browser host edit ops', () => { - beforeEach(() => { spawnMock.mockReset(); }); + useTempSocketDir('dormouse-ab-edit-test-'); + beforeEach(() => { running('sess'); }); // `op` is a TypeScript type, not a runtime check — it arrives from webview IPC // (`vscode-ext/src/message-router.ts`, `standalone/sidecar/main.js`) with no diff --git a/lib/src/host/agent-browser-host.ts b/lib/src/host/agent-browser-host.ts index 81f924643..6ba601437 100644 --- a/lib/src/host/agent-browser-host.ts +++ b/lib/src/host/agent-browser-host.ts @@ -217,6 +217,19 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): } } + /** + * One CLI call on a live browser. Any verb starts a daemon to answer when + * none runs, so it is refused unless the session's daemon runs as its pid + * file says: only a launch's own steps may start one (docs/specs/dor-browser.md + * → "Browser Host"). A session in a socket directory the host does not + * share has no pid file here, so it is refused too. + */ + async function drive(b: ProviderBinding, args: string[], options?: { timeoutMs?: number }): Promise { + const pid = await readStateNumber(b.session, 'pid'); + if (pid === undefined || !processAlive(pid)) throw new Error(`agent-browser session '${b.session}' is not running`); + return run(b, args, options); + } + /** The port `.stream` names, when something accepts on it. */ async function acceptingStreamPort(session: string): Promise { const port = await readStateNumber(session, 'stream'); @@ -340,7 +353,7 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): async act(b, act): Promise { const argv = actArgv(act); if (!argv) return { ok: false, error: 'invalid tab operation' }; - const result = await run(b, argv); + const result = await drive(b, argv); if (result.exitCode !== 0) return { ok: false, error: cliError(result) }; if (act.op !== 'cdpUrl') return { ok: true }; const url = parseCdpUrl(result.stdout); @@ -349,7 +362,7 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): // eval --json envelope: { success, data: { result }, error }. async evaluate(b, script) { - const result = await run(b, ['eval', script, '--json']); + const result = await drive(b, ['eval', script, '--json']); if (result.exitCode !== 0) throw new Error(result.stderr.trim() || `eval exited ${result.exitCode}`); let envelope: { success?: boolean; data?: { result?: unknown }; error?: unknown }; try { @@ -367,7 +380,7 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): const out = await file(); const args = ['screenshot', out, '--screenshot-format', format]; if (format === 'jpeg') args.push('--screenshot-quality', String(quality)); - const result = await run(b, args, { timeoutMs: CAPTURE_TIMEOUT_MS }); + const result = await drive(b, args, { timeoutMs: CAPTURE_TIMEOUT_MS }); if (result.exitCode !== 0) { log(`[agent-browser] screenshot failed (exit ${result.exitCode}): ${cliError(result)}`); throw new Error(result.stderr.trim() || `screenshot exited ${result.exitCode}`); diff --git a/lib/src/host/browser-host.test.ts b/lib/src/host/browser-host.test.ts index 03467efe9..218ebdbef 100644 --- a/lib/src/host/browser-host.test.ts +++ b/lib/src/host/browser-host.test.ts @@ -76,6 +76,30 @@ describe('createBrowserHost', () => { expect(await launch('r287')).toEqual({ ok: false, error: 'the browser was closed' }); }); + it('drives no browser a launch is replacing or a close is ending until it is done', async () => { + const fake = fakeProvider(); + const host = createBrowserHost({ writeClipboardText: vi.fn(), providers: { 'agent-browser': () => fake.provider } }); + const s1 = { provider: 'agent-browser', binding: { session: 's1' } } as const; + const drive = () => Promise.all([ + host.request({ ...s1, op: 'navigate', url: 'http://localhost:5173/next' }), + host.request({ ...s1, op: 'viewport', width: 800, height: 600, dpr: 2 }), + host.request({ ...s1, op: 'edit', edit: 'selectAll' }), + ]); + const refused = { ok: false, error: 'the browser is being relaunched or closed' }; + for (const [step, op] of [['stop s1', { op: 'launch', url: 'http://localhost:5173/', headed: true }], ['close s1', { op: 'close' }]] as const) { + fake.gate(step); + const settling = host.request({ ...s1, ...op }); + await flush(); + expect(await drive()).toEqual([refused, refused, refused]); + fake.release(step); + expect((await settling).ok).toBe(true); + } + // A pop-out's relaunch holds the browser from its stop until it is up. + expect(fake.calls).not.toContain('navigate s1'); + expect((await drive()).map((result) => result.ok)).toEqual([true, true, true]); + expect(fake.calls).toContain('navigate s1'); + }); + it('refuses a request id or cancel list it cannot bound', async () => { const host = createBrowserHost({ writeClipboardText: vi.fn(), providers: { 'agent-browser': () => fakeProvider().provider } }); const s1 = { provider: 'agent-browser', binding: { session: 's1' } } as const; diff --git a/lib/src/host/browser-host.ts b/lib/src/host/browser-host.ts index e52f2f747..339a4eace 100644 --- a/lib/src/host/browser-host.ts +++ b/lib/src/host/browser-host.ts @@ -346,6 +346,23 @@ export function createBrowserHost(deps: BrowserHostDeps) { return expires !== undefined && expires > Date.now(); } + // The browsers a launch is replacing or a close is ending right now. An + // operation reaching one meanwhile would drive the browser in the gap — + // agent-browser's CLI starts a competing daemon at about:blank to answer — + // so every operation but these three is refused until the launch or close + // is done. + const settling = new Map(); + async function settle(id: string, work: () => Promise): Promise { + settling.set(id, (settling.get(id) ?? 0) + 1); + try { + return await work(); + } finally { + const left = (settling.get(id) ?? 1) - 1; + if (left > 0) settling.set(id, left); + else settling.delete(id); + } + } + // Bumped by every launch and close: work begun for an earlier browser (a // post-launch sweep, a capture to join) must not reach the one that // replaced it. @@ -378,7 +395,11 @@ export function createBrowserHost(deps: BrowserHostDeps) { /** Launch `bound`'s browser at `url`, or blank without one: headed or not, * stopping what runs the session first unless it is `fresh`. Answers once * the browser is up, never waiting for the page. */ - async function launch(bound: Bound, url: string | undefined, isHeaded: boolean, fresh: boolean, requestDeadline: number): Promise { + function launch(bound: Bound, url: string | undefined, isHeaded: boolean, fresh: boolean, requestDeadline: number): Promise { + return settle(bound.id, () => bringUpBrowser(bound, url, isHeaded, fresh, requestDeadline)); + } + + async function bringUpBrowser(bound: Bound, url: string | undefined, isHeaded: boolean, fresh: boolean, requestDeadline: number): Promise { const { p, b, id } = bound; if (closed) throw new Error('the browser host is shutting down'); const deadline = requestDeadline - LAUNCH_CLOSE_RESERVE_MS; @@ -483,13 +504,13 @@ export function createBrowserHost(deps: BrowserHostDeps) { * superseded (`bringUp`), and one arriving later runs after. */ function closeSession(bound: Bound): Promise { closesArrived.set(bound.id, (closesArrived.get(bound.id) ?? 0) + 1); - return serialize(bound.id, async () => { + return serialize(bound.id, () => settle(bound.id, async () => { // The session is closed on purpose, so it is no longer shutdown's to // close. Invalidating released it. await invalidate(bound); headed.delete(bound.id); await bound.p.close(bound.b, CLOSE_TIMEOUT_MS); - }); + })); } // --- captures --- @@ -606,6 +627,9 @@ export function createBrowserHost(deps: BrowserHostDeps) { return { ok: true, ...p.describe(b), nativeIdentity: bound.id, wsPort: live.wsPort, ...(live.headed !== undefined ? { headed: live.headed } : {}) }; }; const requestDeadline = Date.now() + REQUEST_BUDGET_MS; + if (r.op !== 'launch' && r.op !== 'attach' && r.op !== 'close' && settling.has(bound.id)) { + return { ok: false, error: 'the browser is being relaunched or closed' }; + } switch (r.op) { case 'launch': { const fresh = r.binding.session === undefined; diff --git a/scripts/spec-word-budgets.json b/scripts/spec-word-budgets.json index 39eed55b4..b3ec7a1f6 100644 --- a/scripts/spec-word-budgets.json +++ b/scripts/spec-word-budgets.json @@ -5,7 +5,7 @@ "docs/specs/alert.md": 8150, "docs/specs/auto-update.md": 1150, "docs/specs/deploy.md": 1900, - "docs/specs/dor-browser.md": 7250, + "docs/specs/dor-browser.md": 7350, "docs/specs/dor-cli.md": 6300, "docs/specs/dor-tool.md": 4150, "docs/specs/glossary.md": 2950, From 1faca60ea5b55875168145cfc725d8258f178130 Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Thu, 24 Sep 2026 03:59:50 -0700 Subject: [PATCH 2/8] View both providers' browsers through one host-owned viewer socket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The webview reached a browser three ways — the agent-browser daemon's stream directly in standalone, through a VS Code-only TCP relay with its own grant table, and through Playwright's own guarded server — ran a screenshot loop whose every capture crossed each host's IPC twice, decoded base64 for every frame it painted, and held a browser-level CDP socket (`get cdp-url`) to follow a popped-out window. It also issued the daemon commands that raced relaunches. Now one loopback listener in the host serves a viewer socket per Surface, guarded by its own Host and a single-use grant the `view` operation issues: - The host subscribes to the provider: agent-browser's daemon stream, dialed on loopback only, its ~20 Hz re-broadcast dropped by raw compare and each changed frame decoded once; Playwright's CDP screencast, decoded once, its acks paced to ~20 fps. - Frames reach the webview as binary messages tagged provisional or crisp; status, tabs, url and a headed window's page as JSON; input comes back rebuilt field by field. A paste is text for both providers. - The two-stage paint runs host-side: which stream frames to send, and the crisp-capture loop with its pacing, input window and overdue rules. The webview decodes latest-only and keeps the canvas at crisp size. - A headed window's page is followed over CDP held in the host, and a browser that goes on its own is reported gone, so auto-revert no longer waits out three reconnects. - A launch or close ends every socket on its browser at once. Deletes the webview screenshot loop and CDP observer, the `screenshot`, `streamUrl` and `cdpUrl` operations, Tauri's `browser_screenshot`, the sidecar's file transport, the dev harness's capture re-encoding, and the VS Code relay with its loopback-lint exemption. The stream a launch or attach answers is renamed from `wsPort`, since a Playwright one is the host's number for its connection rather than a port. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NCV5Uq6FeZbBfzfpk3vrpe --- .../debug-standalone-agent-browser/SKILL.md | 26 +- docs/specs/dor-browser.md | 250 +++++---- docs/specs/dor-browser.rationale.md | 52 +- docs/specs/dor-cli.md | 2 +- docs/specs/dor-cli.rationale.md | 2 +- docs/specs/security-local.md | 2 +- docs/specs/security-local.rationale.md | 12 +- docs/specs/standalone.md | 3 +- docs/specs/vscode.md | 2 +- lib/src/components/Wall.test.tsx | 54 +- lib/src/components/WorkspaceWindow.test.tsx | 4 +- .../wall/AgentBrowserPanel.test.tsx | 164 +++--- .../wall/agent-browser-connection.test.ts | 319 +++-------- .../wall/agent-browser-connection.ts | 237 +++------ .../components/wall/agent-browser-input.ts | 17 - .../agent-browser-screenshot-loop.test.ts | 298 ----------- .../wall/agent-browser-screenshot-loop.ts | 216 -------- .../agent-browser-surface-controller.test.ts | 318 +++++------ .../wall/agent-browser-surface-controller.ts | 495 ++++++------------ lib/src/components/wall/browser-automation.ts | 25 +- lib/src/components/wall/tool-transfer.test.ts | 2 +- lib/src/components/wall/use-dor-control.ts | 17 +- lib/src/components/wall/wall-test-utils.ts | 5 +- lib/src/host/agent-browser-host.test.ts | 470 ++++++++++------- lib/src/host/agent-browser-host.ts | 226 +++++++- lib/src/host/browser-capture.ts | 70 +++ lib/src/host/browser-host-test-utils.ts | 71 ++- lib/src/host/browser-host.test.ts | 97 +++- lib/src/host/browser-host.ts | 198 +++---- lib/src/host/browser-stream-guard.test.ts | 22 +- lib/src/host/browser-stream-guard.ts | 17 +- lib/src/host/browser-viewer.test.ts | 305 +++++++++++ lib/src/host/browser-viewer.ts | 419 +++++++++++++++ .../host/playwright-host.lifecycle.test.ts | 167 +++--- lib/src/host/playwright-host.test.ts | 70 +-- lib/src/host/playwright-host.ts | 202 ++++--- lib/src/host/private-capture-dir.ts | 5 +- lib/src/lib/platform/browser-automation.ts | 130 ++++- lib/src/lib/platform/vscode-adapter.test.ts | 8 +- scripts/loopback-lint.mjs | 6 - scripts/spec-word-budgets.json | 4 +- standalone/scripts/dev-agent-browser.mjs | 19 +- standalone/sidecar/main.js | 10 +- standalone/src-tauri/src/lib.rs | 43 +- .../src/browser-sidecar-adapter.test.ts | 12 +- standalone/src/browser-sidecar-adapter.ts | 12 +- standalone/src/tauri-adapter.test.ts | 17 +- standalone/src/tauri-adapter.ts | 9 +- vscode-ext/src/agent-browser-host.ts | 104 +--- 49 files changed, 2648 insertions(+), 2587 deletions(-) delete mode 100644 lib/src/components/wall/agent-browser-screenshot-loop.test.ts delete mode 100644 lib/src/components/wall/agent-browser-screenshot-loop.ts create mode 100644 lib/src/host/browser-capture.ts create mode 100644 lib/src/host/browser-viewer.test.ts create mode 100644 lib/src/host/browser-viewer.ts diff --git a/.claude/skills/debug-standalone-agent-browser/SKILL.md b/.claude/skills/debug-standalone-agent-browser/SKILL.md index c674b0cff..2c0a953db 100644 --- a/.claude/skills/debug-standalone-agent-browser/SKILL.md +++ b/.claude/skills/debug-standalone-agent-browser/SKILL.md @@ -138,23 +138,23 @@ For click targeting, see **Clicking a link inside the screencast** above (1:1 ca ## What To Watch -The high-rate `[ab-panel]`/`[agent-browser]` stream and screenshot console -diagnostics are now **off by default** — they fire per frame (~20Hz). Enable them -before a run and reload (the flag is read once at module load): +The high-rate `[ab-panel]` viewer-socket console diagnostics are **off by +default** — they fire per stream event. Enable them before a run and reload (the +flag is read once, on the first log); the same flag has the host log each viewer +socket's rates every 5 s: ```js localStorage.setItem('dormouse.flags.abDebugLogs', 'true'); // then reload ``` -The connection's always-on `debugSnapshot()` ring is unaffected, and the -`stalled`/`failed`/`error` screenshot warnings stay unconditional. +The connection's always-on `debugSnapshot()` ring is unaffected. In the harness terminal, correlate (with the flag on): - `[sidecar] ...` for sidecar behavior -- `[browser log] [ab-panel] connecting stream ...` +- `[browser-viewer] {"perSecond":{"framesIn":…,"provisional":…,"crisp":…,"captures":…,"kbOut":…},"captureAvgMs":…,"eventLoopDelayMs":…}` — the host's per-socket rates and its event-loop delay (sidecar stderr) +- `[browser log] [ab-panel] connecting viewer ...` - `[browser log] [ab-panel] tabs msg ...` -- `[browser log] [agent-browser] screenshot start/done ...` - `[browser log] [measure] ...` For a clean `dor ab open dormouse.sh`, the first tab snapshot should look like one active tab: @@ -165,15 +165,15 @@ For a clean `dor ab open dormouse.sh`, the first tab snapshot should look like o If the first snapshot already contains GitHub or multiple Dormouse tabs, clear the nested session and rerun. -### Static-page screenshot churn (diagnosed + fixed) +### Static-page capture churn (diagnosed + fixed) -A *static* page should produce **zero** `screenshot start/done` and **zero** `tabs msg` once it settles. If you see them repeating (~20/sec) with unchanged tab snapshots, that is the known churn bug. +A *static* page should settle to **zero** `framesIn`, `crisp` and `captures` in the `[browser-viewer]` line, and **zero** `tabs msg`. If they stay near 20/sec with unchanged tab snapshots, that is the known churn bug. -Root cause: the external agent-browser **daemon re-broadcasts the current frame and tab list on a ~20Hz heartbeat even when nothing changes**. Each forwarded frame triggers a device-resolution screenshot — *a child-process spawn* (`agent-browser screenshot`) — which pokes the daemon into emitting again: a self-perpetuating feedback loop. Each redundant `tabs` message also forces a `setTabs` React re-render. +Root cause: the external agent-browser **daemon re-broadcasts the current frame and tab list on a ~20Hz heartbeat even when nothing changes**. Each forwarded frame triggers a device-resolution capture — *a child-process spawn* (`agent-browser screenshot`) — which pokes the daemon into emitting again: a self-perpetuating feedback loop. Each redundant `tabs` message also forces a `setTabs` React re-render. -Fix (in `lib/src/components/wall/agent-browser-connection.ts`): drop **byte-identical** frame and tab re-broadcasts (djb2 hash of the payload) before emitting `frame-pulse` / `tabs`, resetting the dedupe sentinels on reconnect. A genuine change (animation, navigation, new/closed/focused tab, title) alters the bytes and flows through. See `agent-browser-connection.test.ts` for the dedupe + reconnect-reprime tests. +Fix (host-side, `viewStream` in `lib/src/host/agent-browser-host.ts`): drop frame and tab re-broadcasts **byte-identical** to the last before they reach the viewer socket, per upstream connection, so a reconnect re-primes. A genuine change (animation, navigation, new/closed/focused tab, title) alters the bytes and flows through. See the `agent-browser host viewer` tests in `lib/src/host/agent-browser-host.test.ts`. -**Regression check:** open `dormouse.sh`, let it settle ~4s, then over 10s of idle confirm `grep -c "screenshot start"` and `grep -c "tabs msg"` are both **0**. To re-measure the daemon's raw vs. forwarded rate, temporarily add a 2s-window counter in the connection (count frames/tabs seen vs. dropped-as-duplicate) — on a static page it reads ~39 seen / 39 dropped per 2s before the dedupe takes effect. +**Regression check:** open `dormouse.sh`, let it settle ~4s, then over 10s of idle confirm the `[browser-viewer]` line reads `framesIn: 0` and `captures: 0`, and `grep -c "tabs msg"` is **0**. ## Validation @@ -185,7 +185,7 @@ pnpm --filter dormouse-standalone test pnpm --filter dormouse-standalone build ``` -After changing webview/lib code under `lib/src` (e.g. the agent-browser connection, panel, or screenshot loop), run from `lib/`: +After changing webview/lib code under `lib/src` (e.g. the agent-browser connection, panel, or controller), run from `lib/`: ```sh npx tsc --noEmit -p tsconfig.json diff --git a/docs/specs/dor-browser.md b/docs/specs/dor-browser.md index 5c9f5b718..3538cbf52 100644 --- a/docs/specs/dor-browser.md +++ b/docs/specs/dor-browser.md @@ -66,9 +66,9 @@ Invariants on the flat persisted `BrowserPanelParams`: `launchFallback`, `binaryPath`, `cwd`, `nativeIdentity`, `syncEngaged`, `key`), never nested but for a `launchFallback` restore's params. Pop-out is not a param — it derives from `renderMode` once, at controller construction. -- **Never carry a stream port in params**: the port `dor ab` reads, or the one - the Playwright host's `attach` answers for `dor pw`, goes straight to the - Surface's controller (rationale). +- **Never carry a stream in params**: the port `dor ab` reads, or the stream + the host's `attach` answers for `dor pw`, goes straight to the Surface's + controller (rationale). - **`contextPortKey` is declared and persisted like any other param.** Only a Surface the pane context menu opened for a port carries it, and reuse looks one up by it ([Pane Context Menu Connect](#pane-context-menu-connect)). @@ -82,7 +82,7 @@ Invariants on the flat persisted `BrowserPanelParams`: Source of truth: `lib/src/components/wall/BrowserPanel.tsx` (`BrowserPanelParams`), `lib/src/components/wall/browser-surface.ts`, `lib/src/components/Wall.tsx` (`replaceSurface`), `lib/src/components/wall/agent-browser-surface-controller.ts` -(`rememberRestorableUrl`, `handOverBrowserPort`), `lib/src/components/wall/IframePanel.tsx` (`applyFrameUrl`). +(`rememberRestorableUrl`, `handOverBrowserStream`), `lib/src/components/wall/IframePanel.tsx` (`applyFrameUrl`). ## Placement And Lifetime @@ -284,7 +284,7 @@ absolute path (`docs/specs/dor-cli.md` → Spawning External Binaries). native identity (agent-browser: the session; Playwright: installation, project scope and session, which a raw `--session` shares across one project's subdirectories). A command for a browser that has a Surface hands - its port over, refreshes `binaryPath` and reuses the pane — not an invariant, + its stream over, refreshes `binaryPath` and reuses the pane — not an invariant, though: a surface killed or render-swapped mid-command leaves the trailing request to mint a fresh pane (rationale). @@ -299,7 +299,7 @@ handle to its agent-browser session, and gates the rest` in ### Browser Connection A surface-id-keyed controller registry (mirroring `terminal-lifecycle.ts`) owns -one `AgentBrowserConnection` plus its screenshot loop. **The controller is +one `AgentBrowserConnection`, its end of a [Viewer Socket](#viewer-socket). **The controller is Surface-scoped, not panel-scoped** — it survives panel unmount. **Must keep the daemon/session alive while parked.** **A view must key its controller by provider as well as Surface id, and the registry must replace one driving the @@ -308,36 +308,37 @@ swap is restored in place, and a controller's provider is fixed for its life. **A view whose controller was released takes a new one on its next params change, never on the release itself** (a kill's, as its fade starts). -**Phase.** The controller holds one `Phase`; the stream connection and CDP -observer exist exactly in `live`. A hidden headless pane enters `parked` in -place of `live`; a `dor` handover moves any phase but `launching` and -`relaunching` to its port; a new `session` in params rebinds. +**Phase.** The controller holds one `Phase`; the viewer socket exists exactly +in `live`. A hidden headless pane enters `parked` in place of `live`; a `dor` +handover moves any phase but `launching` and `relaunching` to its stream; a new +`session` in params rebinds. A **stream** is what a launch or `attach` answers +for a live browser — agent-browser's daemon stream port, the host's number for +a Playwright connection — and what `view` takes back. | Phase | State | Next | | --- | --- | --- | -| `idle` | No view has started it | `launching` without a session; `live` at a handed-over port; else `attaching` with its page | -| `launching` | Opening `url`, in `launchSession` when set, then binding the session answered | `live` at the answered port, else `attaching`; `ended` | +| `idle` | No view has started it | `launching` without a session; `live` at a handed-over stream; else `attaching` with its page | +| `launching` | Opening `url`, in `launchSession` when set, then binding the session answered | `live` at the answered stream, else `attaching`; `ended` | | `attaching` | Asking the host (`attach`) where the session streams | `live`; `ended` | -| `live` | Streaming from its port | `parked` (hidden ≥1s, headless); `relaunching`; `ended` when a headless stream drops (rationale); `attaching` with no page when an unpark's port fails | -| `parked` | Stream released; daemon up at its port | `live` at that port on unpark (rationale); `relaunching` | -| `relaunching` | Headed↔headless relaunch | `live` at the host's port; else `attaching` with its page, headless | +| `live` | Viewing its stream | `parked` (hidden ≥1s, headless); `relaunching`; `ended` when a headless browser goes (rationale); `attaching` with no page when an unpark's stream fails | +| `parked` | Socket released; browser up at its stream | `live` at that stream on unpark (rationale); `relaunching` | +| `relaunching` | Headed↔headless relaunch | `live` at the host's stream; else `attaching` with its page, headless | | `ended` | No browser; `error` says why | `relaunching`; a navigation rebinds, with its page | | `disposed` | Released | — | - **Every browser operation must pass one gate (`driver`), open only in - `live`** — chrome and Display modal actions, tabs, sync-to-pane, `get - cdp-url`, edit chords, screenshots (rationale). **An unpark catches up — - sync, a pending intent — only once its stream opens.** The gate orders the - Surface's own intents; the host is what keeps an operation off a browser - mid-relaunch ([Browser Host](#browser-host)). + `live`** — chrome and Display modal actions, tabs, sync-to-pane, edit chords + (rationale). **An unpark catches up — sync, a pending intent — only once its + socket opens.** The gate orders the Surface's own intents; the host is what + keeps an operation off a browser mid-relaunch ([Browser Host](#browser-host)). - **A navigation asked for outside `live` is kept as the one latest intent**, run on the next `live`; **so is a pop-out or pop-in asked before the browser is bound** (`idle`, `launching`, `attaching`), run as a relaunch, and so is a new `url` in params while `launching` (rationale). **A launch or relaunch opens the pending page itself; one the host opened never loads again** (rationale). -- **The controller never asks a daemon-spawning CLI verb for a port**: ports - come from a launch or relaunch answer, a `dor` handover, or `attach`. +- **The controller never asks a daemon-spawning CLI verb for a stream**: + streams come from a launch or relaunch answer, a `dor` handover, or `attach`. - **A failed first launch is reported once to the Wall, which applies the Surface's `launchFallback`**: `close` the pane, `embed` (a Tool's iframe), or `{ restore }` the params a swap replaced. A param cleared on success, it @@ -352,48 +353,30 @@ place of `live`; a `dor` handover moves any phase but `launching` and `ended`). **A relaunch leaves `live` before headedness changes** (rationale). **Parking.** A Lath leaf is always mounted, so nothing else stops a hidden pane's -~20Hz stream and per-pulse screenshot loop (rationale). A pane that goes -off-screen — or whose view unmounts — parks after a ~1s debounce: connection and -screenshot loop disposed, daemon/session alive, daemon-side streaming stopping on -its own because clients trigger it. - -- **An unpark keeps the last good frame on screen**, re-priming from the stream's - re-broadcast frame/tabs; a fresh reattach mounts a blank canvas and shows the - placeholder until the first screenshot. +~20Hz stream and its crisp captures (rationale). A pane that goes off-screen — +or whose view unmounts — parks after a ~1s debounce: its viewer socket closed, +which ends the host's subscription, daemon/session alive, daemon-side streaming +stopping on its own because clients trigger it. + +- **An unpark keeps the last good frame on screen**, re-priming from the + stream's re-broadcast frame/tabs; a fresh reattach mounts a blank canvas and + asks the host to `repaint`, showing the placeholder until a frame lands. - **A resize made while not `live` is pushed on the next `live`.** -- **Never park a popped-out pane**: its stream/CDP observer must keep running for - window-close auto-revert, even while minimized. +- **Never park a popped-out pane**: its viewer socket brings the window's page + and its close, which auto-reverts, even while minimized. - **Never set `AGENT_BROWSER_IDLE_TIMEOUT_MS`** for Dormouse-managed sessions — daemon self-exit when idle would defeat "alive while parked". -The stream carries frames, status, tab snapshots, `url`, and native -`input_mouse` / `input_keyboard` input. **Control envelopes dispatch at any -size.** **`url` names the active tab at navigation commit; `tabs` -refreshes only when the driving command completes** -(for a slow page, after the load; rationale), so every commit clears the title -until `tabs` refreshes, even at the same URL. +**The webview paints what the socket sends, latest-only**: one frame decodes at +a time, and a newer arrival replaces the one waiting behind it. **A provisional +frame of the canvas's shape draws scaled into it**; only a crisp frame, or one +of another shape, sizes the canvas, so switching between the two reallocates +nothing (rationale). -**Two-stage paint.** A changed stream JPEG paints at once as a CSS-resolution -**provisional frame** — the first image, 250ms after any input (pointer, keys, -pasted text, editing chords; continuous input extends the window), and while a -capture is **overdue** — then a crisp device-resolution -host `screenshot` replaces it (rationale): - -- **Both paths are latest-only.** -- **No capture may start inside the provisional window** (rationale). -- **A capture is overdue past twice the average round trip, at least 400ms. It - is never re-issued while its host call is unresolved, and a paint made only - because it is overdue does not supersede it** (rationale). -- **Must leave the loop dirty when capture or bitmap decode becomes stale** - (rationale). Pinned by `agent-browser-screenshot-loop.test.ts`. -- **Any canvas writer but the crisp loop must bump the draw generation** in its - key, or the byte-identical-frame dedup drops its paint (rationale). -- **A host that cannot drive the provider paints every changed provisional - frame as its final image** rather than showing only the placeholder. - -High-rate `[ab-panel]`/`[agent-browser]` console diagnostics sit behind the +High-rate `[ab-panel]` console diagnostics sit behind the `dormouse.flags.abDebugLogs` localStorage flag, read lazily and memoized on the -first log (reload to apply); `debugSnapshot()`'s ring is always on. +first log (reload to apply), which also has the host log each viewer socket's +rates and its event-loop delay every 5 s; `debugSnapshot()`'s ring is always on. Input rules: @@ -404,8 +387,9 @@ Input rules: - **`windowsVirtualKeyCode` comes from a real key map, never `key.charCodeAt(0)`** (`.` is char 46 = VK_DELETE, so periods would otherwise become Delete presses). -- Local paste is replayed as per-character key input — as `input_text` on - [Playwright](#playwright). +- **Must send a local paste as `input_text` messages of at most 8192 + characters** (`viewerTextInputs`), never a key pair per character from the + webview; the host inserts each whole (rationale). - **Select-all/copy/cut go through the host `edit` operation on every platform**, since those chords do not survive CDP input. Undo/redo is not emulated. @@ -418,12 +402,79 @@ mid-relaunch` in `lib/src/components/wall/agent-browser-surface-controller.test. Source of truth: `lib/src/components/wall/AgentBrowserPanel.tsx` (`toDevice`, the tab strip, placeholders), `lib/src/components/wall/agent-browser-surface-controller.ts` -(`Phase`, `driver`, `launch`, `attach`, `whenBrowserLaunched`), `onBrowserLaunchFailed` in `lib/src/components/Wall.tsx`, -`lib/src/components/wall/agent-browser-connection.ts`, -`lib/src/components/wall/agent-browser-screenshot-loop.ts`, `lib/src/components/wall/agent-browser-input.ts`, +(`Phase`, `driver`, `launch`, `attach`, `paintFrame`, `paintBitmap`, `whenBrowserLaunched`), `onBrowserLaunchFailed` in `lib/src/components/Wall.tsx`, +`lib/src/components/wall/agent-browser-connection.ts`, `lib/src/components/wall/agent-browser-input.ts`, +`viewerTextInputs` in `lib/src/lib/platform/browser-automation.ts`, `lib/src/components/wall/use-surface-visibility.ts`, `lib/src/lib/agent-browser-tab.ts` (the tab record shared by the stream and `tab list --json`). +### Viewer Socket + +**The webview reaches a browser only through its host's viewer socket** — +never a daemon's stream, never CDP. One loopback listener in the host serves a +socket per Surface, onto the browser at the stream `view` names: + +| Message | Direction | Carries | +| --- | --- | --- | +| frame | host → webview, binary | A 12-byte header — kind (`provisional` / `crisp`), the viewport's CSS size — then the JPEG | +| `status`, `tabs` | host → webview | Whether the browser is up, and its viewport size; the tab list | +| `url` | host → webview | A navigation the active tab committed | +| `page` | host → webview | A popped-out window's page URL and title | +| `input_mouse`, `input_keyboard`, `input_text` | webview → host | Native input; a paste as text | +| `repaint` | webview → host | Resend the last frame to a canvas that mounted blank | + +- **Must authorize every upgrade with the listener's own loopback `Host` and a + single-use, 60-second grant for that one view**, capped at 1024; a plain + request is refused (`docs/specs/security-local.md` → Loopback Listeners). +- **Must rebuild every webview message field by field before a provider sees + it** (`parseViewerInput`); inbound messages are capped at 64 KiB. +- **Must send state only on change, and the current state to a socket that + connects — except `url`, a commit edge**: `tabs` refreshes only when the + driving command completes (rationale), so every commit clears the title + until `tabs` refreshes, even at the same URL. +- **A browser that goes on its own is reported `status { connected: false }` + before its socket closes**, ending a headless pane and auto-reverting a + headed one seen connected. **A launch or close of the browser ends every + socket on it**; a URL granted before one opens nothing. +- **A headed socket carries no frames**; a socket with more than 2 MB queued + skips a provisional one. + +**Two-stage paint, in the host.** A changed stream frame is sent at once as a +CSS-resolution **provisional frame** when it is the first, within 250 ms of +input (over the socket, or a host editing op), or while a capture is +**overdue**; otherwise it pulses the crisp loop, whose device-resolution capture +replaces it (rationale): + +- **One capture in flight**, the next no sooner than 1.5× the average capture + after the last began. +- **No capture may start inside the provisional window** (rationale). +- **A capture is overdue past twice the average round trip, at least 400ms; it + is never re-issued, and a paint made only because it is overdue does not + supersede it** (rationale). +- **A capture a provisional paint superseded while it ran is dropped and owed + again** (rationale). +- **A byte-identical capture is resent only after a provisional frame or a + `repaint`.** +- **An active-tab change owes a capture**; a browser the host cannot capture (a + daemon in a socket directory it does not share) has every changed frame sent + as provisional. + +Pinned by `lib/src/host/browser-viewer.test.ts`. + +**Upstreams.** agent-browser: the daemon's stream, dialed on `127.0.0.1` only; +**the host must drop its ~20 Hz re-broadcast by raw comparison against the last +frame and tab list**, decoding a changed frame once (rationale); **a tab list or +URL is state at any size**, never taken for a frame; a paste goes as key pairs. **A headed window's page is followed over its browser's +CDP, held host-side and dialed on loopback only** (rationale). Playwright: its +CDP screencast ([Playwright](#playwright)). + +Source of truth: `createViewerServer`, `BrowserView` and `parseViewerInput` in +`lib/src/host/browser-viewer.ts`; `BrowserStreamGrants` in +`lib/src/host/browser-stream-guard.ts`; `encodeViewerFrame` and `ViewerState` in +`lib/src/lib/platform/browser-automation.ts`; `viewStream` and `observePage` in +`lib/src/host/agent-browser-host.ts`. Pinned also by +`lib/src/host/agent-browser-host.test.ts` and `lib/src/host/browser-host.test.ts`. + ### Pop-Out A popout (`ab-popout`, `pw-popout`) relaunches the same session headed, because @@ -446,11 +497,14 @@ daemon during the close/reopen gap** (rationale) — the host refuses every operation on a browser mid-launch ([Browser Host](#browser-host)) — so **Dormouse supplies the active-tab URL and the host trusts it**. -While popped out, Dormouse keeps a stream/CDP observer for same-tab URL/header -updates and headed-window close auto-revert. +While popped out, the pane's viewer socket carries no frames: **the host +follows the window's page and reports the window going away** — agent-browser's +daemon stream and its browser's CDP, Playwright's connection +([Viewer Socket](#viewer-socket)) — and a window seen connected that goes +auto-reverts to the pane. -Source of truth: `lib/src/components/wall/agent-browser-surface-controller.ts` (pop-out state, CDP -observer, auto-revert), `lib/src/host/agent-browser-host.ts` (`killDaemon`), +Source of truth: `lib/src/components/wall/agent-browser-surface-controller.ts` (pop-out state, +auto-revert), `lib/src/host/agent-browser-host.ts` (`killDaemon`, `observePage`), VS Code/standalone shutdown wiring. ### Browser Host @@ -460,18 +514,17 @@ provider-tagged `BrowserRequest` — `{ provider, binding: { session?, cwd?, binaryPath? } }` plus one operation — answered by a `BrowserResult`. A host lists the providers it drives in `browserProviders`; one without them (the web demo) offers no automated renderer. VS Code runs the shared host in the extension -host; standalone runs the bundled copy in the sidecar behind one Rust command, -plus `browser_screenshot` for raw bytes. +host; standalone runs the bundled copy in the sidecar behind one Rust command. +**No frame rides a request's transport**: frames reach the webview over the +[Viewer Socket](#viewer-socket), never the sidecar stdio PTY traffic shares. | Operation | Contract | | --- | --- | | `launch` | Without a session, open an http(s) `url` in a new GUI session; with one, navigate it when it is up in the mode asked for, else relaunch it headed or headless at `url`, blank when that page is not http(s). Resolves when the browser is up, not when the page loads ([Pop-Out](#pop-out)). | -| `attach` | The live stream port, found without starting a browser ([agent-browser](#agent-browser), [Playwright](#playwright)), with headedness where the provider can tell. Only a gone browser, for a caller naming a page, is relaunched there (headed on request), answering `relaunched`; one it cannot view is left alone. | -| `screenshot` | One device-resolution JPEG/PNG frame. VS Code structured-clones the bytes; standalone passes Rust a temp-file **path** over the sidecar stdio, for Rust to read and delete (rationale). **One capture per session and format in flight**: a request made meanwhile joins it — never one from before the session's close or relaunch — and the capture's spawn is killed past 30s. | +| `attach` | The live browser's stream, found without starting a browser ([agent-browser](#agent-browser), [Playwright](#playwright)), with headedness where the provider can tell. Only a gone browser, for a caller naming a page, is relaunched there (headed on request), answering `relaunched`; one it cannot view is left alone. | +| `view` | A single-use viewer socket URL for the browser at a `stream`, `headed` for a popped-out pane ([Viewer Socket](#viewer-socket)). | | `edit` | select-all/copy/cut via fixed host-owned JS plus an OS clipboard write. | -| `streamUrl` | The URL the webview connects to for a stream port. | | `navigate`, `history`, `tab`, `viewport`, `device`, `close` | One fixed argv (agent-browser) or client call (Playwright) each. | -| `cdpUrl` | agent-browser only: the browser's CDP endpoint, for the popped-out URL observer. | **Every transport waits `BROWSER_REQUEST_TIMEOUT_MS` (40s) for any reply**, past agent-browser's 25s action timeout, so the webview never re-asks while the host @@ -479,7 +532,7 @@ still works. **The host runs one lifecycle for both providers**; a provider implements only the primitives that differ (`BrowserProvider`: find, stop, open, probe, close, -list tabs, act, evaluate, screenshot, stream URL): +list tabs, act, evaluate, screenshot, view): - **Must serialize a browser's launches, relaunching attaches and closes per native identity**, in arrival order, so two panes restoring one session @@ -495,9 +548,10 @@ list tabs, act, evaluate, screenshot, stream URL): the other mode is relaunched (rationale). agent-browser cannot report its mode: one this host did not launch headed counts as headless. - **Must refuse every operation but `launch`, `attach` and `close` on a - browser a launch is replacing or a close is ending**, until it is done, so - nothing reaches a browser in the close/reopen gap ([Pop-Out](#pop-out)), - whichever Surface asks. + browser a launch is replacing or a close is ending**, `view` included, until + it is done, so nothing reaches a browser in the close/reopen gap + ([Pop-Out](#pop-out)), whichever Surface asks; the launch or close has ended + every viewer socket on it first, so no capture does either. - **Must answer a launch inside `BROWSER_REQUEST_TIMEOUT_MS`**: startup, queueing included, gets 30 s from the request's arrival. A relaunch stops what runs the session first, then resolves once the provider reports the @@ -537,21 +591,23 @@ match. **A refused path is dropped, never fatal**, so the host's own candidates run. The webview applies the same predicate before sending one (`browserHandle`) or storing one. -**Screenshots are captured into a private per-process directory, and it is -removed** (rationale). **A tmpdir that cannot be created is answered -`{ ok: false }` and retried on the next capture, never memoized.** **Every -capture writes fresh files, one per caller** (rationale); a reader deletes its -own, and the browser's close or relaunch, or a capture past -`BROWSER_REQUEST_TIMEOUT_MS`, deletes one never read. +**Crisp captures are written into a private per-process directory, read back +and removed at once, and the directory is removed at shutdown** (rationale). +**One capture per browser is in flight**: a viewer socket asking meanwhile joins +it — never one from before the browser's close or relaunch — and an +agent-browser capture's spawn is killed past 30s. **A tmpdir that cannot be +created fails that capture and is retried on the next, never memoized.** The +file name is reused per browser, with a fresh one after its close or relaunch. Source of truth: `lib/src/host/browser-host.ts` (`parseBrowserRequest`, `createBrowserHost`, `BrowserProvider`), `BROWSER_PROVIDERS` (`isSessionName`, `isAllowedBinary`) in `dor-lib-common/src/browser-providers.ts`, `BrowserRequest` in `lib/src/lib/platform/browser-automation.ts`, `browserHandle` in `lib/src/components/wall/browser-automation.ts`, +`createBrowserCaptures` in `lib/src/host/browser-capture.ts`, `lib/src/host/private-capture-dir.ts`, `vscode-ext/src/agent-browser-host.ts`, `vscode-ext/src/webview-html.ts`, `standalone/src/tauri-adapter.ts`, -`standalone/src-tauri/src/lib.rs` (`browser_request`, `browser_screenshot`), +`standalone/src-tauri/src/lib.rs` (`browser_request`), `standalone/sidecar/main.js`. ### agent-browser @@ -578,36 +634,32 @@ relaunch changes the mode. verb starts a daemon at `about:blank` to answer. - **`dor ab` must read the stream port itself after a command that may bind** (`stream status --json`, safe once the command made the daemon) and hand it - over; the Surface streams from it without asking the host (rationale). - **Never carry a socket directory to the host** — it kills the pid it reads - there. **Host-side operations for a session in a socket directory the host - does not share are refused**, as the host sees no daemon for it; a close runs - the CLI under the host's environment. -- **VS Code must reach the stream through a loopback relay** — the agent-browser - stream server rejects `vscode-webview://` origins. The relay grants one - single-use, short-TTL token bound to one stream port and strips the Origin - header; standalone connects directly. + over; the host views it on loopback without reading the caller's socket + directory (rationale). **Never carry a socket directory to the host** — it + kills the pid it reads there. **Host-side operations and captures for a + session in a socket directory the host does not share are refused**, as the + host sees no daemon for it; its viewer socket still sends every changed + frame, and a close runs the CLI under the host's environment. Source of truth: `createAgentBrowserProvider` and `runWithBinaryFallback` in `lib/src/host/agent-browser-host.ts`, `isAllowedAgentBrowserBinary` and `streamStatusArgs` in `dor-lib-common/src/browser-providers.ts`, -`streamStatus` in `dor/src/commands/agent-browser.ts`, -`vscode-ext/src/agent-browser-host.ts`. Pinned by +`streamStatus` in `dor/src/commands/agent-browser.ts`. Pinned by `lib/src/host/agent-browser-host.test.ts`. ### Playwright **Must use the user's installed `@playwright/cli`, resolved from `DORMOUSE_PLAYWRIGHT_BIN` or `PATH`.** GUI launches use Chromium. Native commands retain Playwright semantics: `open` restarts, `goto` navigates; commands for unsupported engines still run, with a viewer warning. The viewer requires CLI 0.1.19’s local browser-binding endpoint; installation errors name this requirement. A Playwright session lives in its CLI project scope, so a binding's cwd and executable pin every later command, relative paths included; `--session` uses the caller's own scope. GUI Connect inherits the source terminal's cwd; a swap without one uses the host cwd. -**Must discover the native session in its CLI project scope and connect using that installation's matching Playwright client.** Accept only a unique registry entry matching session, workspace and library, with a local pipe endpoint and Chromium engine. Never load modules from the registry's library path. The host derives the client from the validated CLI installation. **`attach` relaunches at the page only when the registry lists no browser for the session**, never one it cannot view; `dor pw`'s binding attaches with no page, and reports the browser's headedness. Native CLI tabs and the pane share the selected tab; the host polls tab selection and metadata every 750ms while viewed, broadcasting only changes, and the current state to each connecting viewer. Screenshots reuse tab state for up to 750ms; explicit host controls refresh immediately. **The controller must apply a host-reported mode before the new viewer port**, so sync never sizes a headed window, except mid-relaunch or as an echo ([Browser Connection](#browser-connection)). +**Must discover the native session in its CLI project scope and connect using that installation's matching Playwright client.** Accept only a unique registry entry matching session, workspace and library, with a local pipe endpoint and Chromium engine. Never load modules from the registry's library path. The host derives the client from the validated CLI installation. **`attach` relaunches at the page only when the registry lists no browser for the session**, never one it cannot view; `dor pw`'s binding attaches with no page, and reports the browser's headedness. Native CLI tabs and the pane share the selected tab; the host polls tab selection and metadata every 750ms while viewed, broadcasting only changes, and the current state to each connecting viewer. Captures reuse tab state for up to 750ms; explicit host controls refresh immediately. **The controller must apply a host-reported mode before the new stream**, so sync never sizes a headed window, except mid-relaunch or as an echo ([Browser Connection](#browser-connection)). Arbitrary CLI arguments, JavaScript and CDP methods are unavailable through the webview channel; the trusted `dor pw` process retains native passthrough. Executable hints use the same filename/exact-host-override boundary as agent-browser, with `playwright-cli` as the accepted name; `dor pw` applies it to the executable a binding returns (`docs/specs/dor-cli.md` → Browser Surface Addressing). -A relaunch closes the previous CLI session, and a launch completes when the browser endpoint is ready. **Every CLI call a launch waits on is killed at its deadline; every other one at 10 s, except `open`**, which lasts as long as the page load and whose end could take its browser down; nothing waits on it past a launch's own bounds. Shutdown also disconnects viewers. Viewer disconnect alone leaves the CLI browser alive. Concurrent input/captures share CDP attachments; disposal releases late attachments. **A screencast that fails to start must forget its page**, so the next poll releases the attachment and retries. +A relaunch closes the previous CLI session, and a launch completes when the browser endpoint is ready; its stream is the host's number for that connection. **Every CLI call a launch waits on is killed at its deadline; every other one at 10 s, except `open`**, which lasts as long as the page load and whose end could take its browser down; nothing waits on it past a launch's own bounds. Shutdown also disconnects viewers. Viewer disconnect alone leaves the CLI browser alive; **a browser that disconnects on its own is reported gone to its viewers**. Concurrent input/captures share CDP attachments; disposal releases late attachments. **A screencast that fails to start must forget its page**, so the next poll releases the attachment and retries. -**Must authorize every stream upgrade with an own-loopback Host and a single-use, 60-second token bound to that viewer port.** Grants are capped at 1024; normal HTTP requests are refused. Input is limited to 64 KiB per message and 256 queued messages; frame backpressure drops frames above 2 MB queued. **Must send a paste as `input_text` messages of at most 8192 characters**, which the host inserts with CDP `Input.insertText`, never as a key pair per character (rationale). The same guarded stream serves all three hosts. +The viewer socket ([Viewer Socket](#viewer-socket)) takes its frames from the CDP screencast, **decoded once and acknowledged no faster than 20 a second** (rationale), and inserts a paste with CDP `Input.insertText`. **Input waiting on CDP past 256 messages closes the viewer socket.** -Source of truth: `followParamsHeadedness` in `lib/src/components/wall/agent-browser-surface-controller.ts`; `playwrightTextInputs` in `lib/src/lib/platform/browser-automation.ts`; `createPlaywrightProvider` in `lib/src/host/playwright-host.ts`; `resolvePlaywrightInstall` in `lib/src/host/playwright-install.ts`; `BrowserStreamGrants` in `lib/src/host/browser-stream-guard.ts`. Pinned by `lib/src/host/playwright-host.test.ts` (opt-in real CLI via `DORMOUSE_PLAYWRIGHT_TEST_BIN`), `lib/src/host/playwright-host.lifecycle.test.ts`, `lib/src/host/browser-stream-guard.test.ts`, and `AgentBrowserPanel Playwright params` in `lib/src/components/wall/AgentBrowserPanel.test.tsx`. +Source of truth: `followParamsHeadedness` in `lib/src/components/wall/agent-browser-surface-controller.ts`; `createPlaywrightProvider` in `lib/src/host/playwright-host.ts`; `resolvePlaywrightInstall` in `lib/src/host/playwright-install.ts`. Pinned by `lib/src/host/playwright-host.test.ts` (opt-in real CLI via `DORMOUSE_PLAYWRIGHT_TEST_BIN`), `lib/src/host/playwright-host.lifecycle.test.ts`, and `AgentBrowserPanel Playwright params` in `lib/src/components/wall/AgentBrowserPanel.test.tsx`. ## Iframe Renderer diff --git a/docs/specs/dor-browser.rationale.md b/docs/specs/dor-browser.rationale.md index e16dafc1c..05ffe9d52 100644 --- a/docs/specs/dor-browser.rationale.md +++ b/docs/specs/dor-browser.rationale.md @@ -48,23 +48,15 @@ views without weakening that first signal. ## Browser Connection -**What parking is worth.** Lath leaves stay mounted, so a background window would otherwise retain every pane's ~20Hz decode and screenshot round trips. The ~1s debounce rides through transient visibility flips and StrictMode remounts without rebuilding the connection. +**What parking is worth.** Lath leaves stay mounted, so a background window would otherwise retain every pane's ~20Hz stream and its crisp captures. The ~1s debounce rides through transient visibility flips and StrictMode remounts without rebuilding the connection. -**What the two-stage split buys.** Three things at once: input feedback that does not wait on a screenshot child-process round trip, a resting image sharp on HiDPI, and an idle animated page that does not pay to decode the stream continuously. Either path alone gives up one of the three. - -**Why keys open the window too.** A keystroke's echo is the most latency-sensitive paint there is; outside the window it waited a whole crisp capture — ~120ms, plus up to ~180ms of loop pacing in a burst — while a hover repainted from the stream in ~50ms. On HiDPI the typed text is CSS-resolution until 250ms after the last key, then sharpens, as hover already did. - -**Why no crisp capture starts inside the provisional window.** A host screenshot round trip is ~120ms against a ~20Hz stream, so *every* capture started while provisional frames are still landing is superseded before it resolves; the shots skipped would never have drawn anything. - -**Why an overdue capture paints the stream and is never re-issued.** `open` holds the daemon's queue until the page loads, up to 25s (see [Pop-Out](#pop-out)), and the URL bar, `dor ab open` and every relaunch run it. A capture issued meanwhile waits behind it, so the canvas stayed on the previous page for the whole load while the stream was already showing the new one. An 8s watchdog then freed the slot and spawned another `screenshot` into the same queue — about three blocked CLI processes by 25s — and on VS Code the first reply's unlink could delete the second capture's file. VS Code's adapter gave up on a reply after 10s and the webview re-asked the same way, posting the full JPEG to both requests once it came; every adapter now waits 30s, and the host still joins concurrent captures, since surfaces can share a session. When each overdue paint counted against the held capture, it was discarded on arrival and re-taken — an extra full capture on every slow navigation, the crisp frame a round trip late. Its wait's pulses still owe one follow-up: a capture queued behind `open` is taken at the end of its round trip, but one slow in itself may have been taken before the page's last change. The overdue round trip timed the page load, not a capture, so the loop clamps it before it enters the pacing average; otherwise the next slow load would wait ~16s to count as overdue. +**Why the canvas keeps its crisp size.** Provisional frames (CSS pixels) and crisp ones (device pixels) alternate on every hover, and sizing the canvas to each reallocated its backing store — about 14.7 MB at 2560×1440 — and changed the intrinsic size of a `max-w-full max-h-full` canvas, forcing layout, on every switch (static reading, 2026-09). Pointer mapping is unaffected: it scales by the device width over the canvas width, whatever the canvas's size. -**Why a stale-dropped capture must leave the loop dirty.** A provisional frame can supersede the host capture or its pending bitmap decode. Nothing is guaranteed to pulse the loop again: a single pointer move over a static page pulses exactly once, and that one pulse is consumed by the very capture the provisional paint supersedes — leaving the pane on the blurry provisional frame until the page happens to change on its own. +**Why a paste is text, not keys.** agent-browser's stream takes only key and mouse events, so a paste once went from the webview as a key down and up per character. Sent to the Playwright host, whose input queue closes the viewer (1008) at 256 queued messages, any paste over about 128 characters arriving as one burst truncated and dropped the pane into a 2 s reconnect (static reading, 2026-09). The host now expands a paste into key pairs only where the daemon takes them, and the 8192-character chunk keeps a message under the 64 KiB socket cap even when every character JSON-escapes to six bytes. -**Why every non-crisp painter must bump the draw generation.** The byte-dedup compares an incoming capture against the last crisp draw. A resting page whose crisp bytes match that draw dedups to a no-op and strands the pane on the blurry provisional frame; a freshly re-attached canvas mounts blank and has the same problem. +**Why the connection is dropped at relaunch start rather than left to fail.** The host closes the browser and kills the daemon, so the old socket's close is certain. Left connected, its three reconnect failures flagged the pane "ended" about three seconds into a pop-in that a slow page could hold open for 25s. When the popped-out URL observer still ran in the webview, its `get cdp-url` — issued the moment `poppedOut` flipped — landed in the close→reopen gap, where a daemon command spawns a competing headless daemon that the headed relaunch then reattaches to; flipping headedness before leaving `live` started it there too. Now it would open a headed viewer socket onto the browser being closed. -**Why the connection is dropped at relaunch start rather than left to fail.** The host closes the browser and kills the daemon, so the old socket's close is certain. Left connected, its three reconnect failures flagged the pane "ended" about three seconds into a pop-in that a slow page could hold open for 25s, and the popped-out CDP observer's `get cdp-url` — issued the moment `poppedOut` flipped — landed in the close→reopen gap, where a daemon command spawns a competing headless daemon that the headed relaunch then reattaches to. The same holds for flipping headedness before leaving `live`: the observer would start, gate open, in the gap. - -**Why one daemon gate, not a check per call site.** The rule against daemon commands in the relaunch gap was a `relaunching` flag each caller had to check, and most did not (static reading, 2026-09): the header's back/forward/reload/URL edit, the Display modal's device and custom viewport, tab clicks, sync-to-pane (reached by a resize once a pop-in had already flipped `poppedOut`), and Cmd-A/C/X all reached the daemon mid-relaunch. The lifecycle behind it was a dozen flags whose invariants lived in five predicates. +**Why one daemon gate, not a check per call site.** The rule against daemon commands in the relaunch gap was a `relaunching` flag each caller had to check, and most did not (static reading, 2026-09): the header's back/forward/reload/URL edit, the Display modal's device and custom viewport, tab clicks, sync-to-pane (reached by a resize once a pop-in had already flipped `poppedOut`), and Cmd-A/C/X all reached the daemon mid-relaunch. The lifecycle behind it was a dozen flags whose invariants lived in five predicates. Once captures and the popped-out URL observer moved into the host, and the host itself refused operations mid-relaunch (see [Browser Host](#browser-host)), the gate was left ordering the Surface's own intents. **Why an unpark streams from its parked port first, and attaches with no page.** Asking the host first cost every unpark an attach round trip (a pid and stream-file read plus a port probe) before the first frame, for a daemon that almost never moves while hidden (review, 2026-09). When it did move, or went away: a daemon gone while the pane was hidden is the same failure as one gone while visible, which the pane shows as ended; relaunching it on unpark would hide that failure and undo a `dor ab close` made meanwhile. A restore has no such history, so it reopens the page. @@ -76,12 +68,36 @@ views without weakening that first signal. **Why a new `url` mid-launch is a navigation.** Tool serving stopped awaiting the launch, so an announcement landing mid-launch wrote a new `url` with the session still unbound; nothing rebound, the launch went live on the old page, and serving had already recorded the announcement, so the pane was never re-framed (review of #775, 2026-09). -**Why an unpark drives nothing until its stream opens.** The unpark entered `live` at once, so sync-to-pane and a pending navigation ran before the parked port was proven. With the daemon gone while hidden (`dor ab close`, a crash) and the pane resized meanwhile, as restoring a Door does, `set viewport` started a fresh `about:blank` daemon through the CLI, and the failed stream's attach then streamed it instead of ending (review of #775, 2026-09). +**Why an unpark catches up only once its socket opens.** The unpark entered `live` at once, so sync-to-pane and a pending navigation ran before the parked port was proven. With the daemon gone while hidden (`dor ab close`, a crash) and the pane resized meanwhile, as restoring a Door does, `set viewport` started a fresh `about:blank` daemon through the CLI, and the failed stream's attach then streamed it instead of ending (review of #775, 2026-09). The host now refuses that command outright; waiting still keeps the resize and the page owed, rather than refused and lost, until the unpark finds its browser or ends. + +## Viewer Socket + +**Why the host owns the socket.** Three stream topologies grew beside each other (static reading, 2026-09): standalone dialed the daemon's stream directly, VS Code went through a TCP relay with its own grant table because the daemon rejects `vscode-webview://` origins, and Playwright had its own guarded server. Every crisp capture crossed the host boundary twice — webview → Rust `browser_screenshot` → sidecar stdio → a spawned `screenshot` into a file → Rust reads it → raw IPC back, with the VS Code message channel and the dev harness each carrying a copy — and the webview decoded base64 for every stream frame it painted. It also issued the daemon commands (captures, `stream status`, `get cdp-url`, tab selects) that raced relaunches. + +**What the host relay costs.** Measured on this machine with Node 24.18 (2026-09-24), the host viewing a synthetic daemon that sends a ~133 KB base64 frame and a tab list every 50 ms, with the webview's side of each socket in the same process (so an upper bound): a static page cost 1.3% of a core for one pane, 3.4% for four and 4.7% for eight, sending nothing; an animated one, every frame forwarded as binary (~1.9 MB/s per pane), 2.9%, 7.9% and 11.1%. Event-loop delay stayed at the 10 ms sampling floor (p99 12.5-12.7 ms, max ≤ 19.5 ms), so PTY traffic on the same loop is not held up and no worker thread is needed. Classifying, parsing and decoding one changed frame costs ~80 µs of it (15 µs to string, 44 µs `JSON.parse`, 10 µs base64), so decoding every changed frame, sent or not, is not worth deferring. + +**Why the heartbeat is compared raw, in the host.** agent-browser's daemon re-sent its current frame and tab list about 20 times a second (~39 of 39 dropped per 2 s on a static page, counted in the debug harness). The webview found each duplicate by running djb2 over the whole string on its main thread: 208 µs per ~133 KB frame in V8, against 2.7 µs for a raw string compare and 2.4 µs for `Buffer.equals` (Node 24.18, 2026-09-24) — about 4 ms of main-thread time per pane per second for nothing, plus a JSON parse and signature of every tab list. -**Why `url` is tracked separately from `tabs`.** Measured against agent-browser 0.31.1 (2026-09): on `open`, the stream sends `tabs` (about:blank), then `url` naming the target at navigation commit, and refreshes `tabs` only when the CLI command completes — after `load`. During a slow load the tab list still named the previous page, so a pop-out issued then relaunched the page before the one being loaded. +**What the two-stage split buys.** Three things at once: input feedback that does not wait on a screenshot child-process round trip, a resting image sharp on HiDPI, and an idle animated page that does not pay to decode the stream continuously. Either path alone gives up one of the three. + +**Why keys open the window too.** A keystroke's echo is the most latency-sensitive paint there is; outside the window it waited a whole crisp capture — ~120ms, plus up to ~180ms of loop pacing in a burst — while a hover repainted from the stream in ~50ms. On HiDPI the typed text is CSS-resolution until 250ms after the last key, then sharpens, as hover already did. Input reaches the host over the socket, and editing chords through its `edit` operation, so the host opens the window for both. + +**Why no crisp capture starts inside the provisional window.** A host screenshot round trip is ~120ms against a ~20Hz stream, so *every* capture started while provisional frames are still landing is superseded before it resolves; the shots skipped would never have drawn anything. + +**Why an overdue capture paints the stream and is never re-issued.** `open` holds the daemon's queue until the page loads, up to 25s (see [Pop-Out](#pop-out)), and the URL bar, `dor ab open` and every relaunch run it. A capture issued meanwhile waits behind it, so the canvas stayed on the previous page for the whole load while the stream was already showing the new one. An 8s watchdog then freed the slot and spawned another `screenshot` into the same queue — about three blocked CLI processes by 25s — and on VS Code the first reply's unlink could delete the second capture's file. VS Code's adapter gave up on a reply after 10s and the webview re-asked the same way, posting the full JPEG to both requests once it came. The host now bounds a capture at 30s and joins concurrent ones, since surfaces can share a session. When each overdue paint counted against the held capture, it was discarded on arrival and re-taken — an extra full capture on every slow navigation, the crisp frame a round trip late. Its wait's pulses still owe one follow-up: a capture queued behind `open` is taken at the end of its round trip, but one slow in itself may have been taken before the page's last change. The overdue round trip timed the page load, not a capture, so the loop clamps it before it enters the pacing average; otherwise the next slow load would wait ~16s to count as overdue. + +**Why a superseded capture is owed again.** Nothing is guaranteed to pulse the loop again: a single pointer move over a static page pulses exactly once, and that one pulse is consumed by the very capture the provisional paint supersedes — leaving the pane on the blurry provisional frame until the page happens to change on its own. With both paints on one ordered socket, and the webview decoding in arrival order, a capture can no longer be superseded during its decode, which the webview once had to guard against too. + +**Why a byte-identical capture is resent after a provisional frame or a repaint.** Dropping a capture identical to the last crisp one saves an encode, a send and a decode on a resting page — but only while that capture is what the canvas shows. After a provisional paint the canvas holds the blurry frame, and a freshly re-attached canvas is blank; skipping the identical capture then stranded the pane there. + +**Why `url` is a commit edge, apart from `tabs`.** Measured against agent-browser 0.31.1 (2026-09): on `open`, the stream sends `tabs` (about:blank), then `url` naming the target at navigation commit, and refreshes `tabs` only when the CLI command completes — after `load`. During a slow load the tab list still named the previous page, so a pop-out issued then relaunched the page before the one being loaded. **Whose limitation the CSS-resolution provisional frame is.** Chromium's `Page.startScreencast` captures in DIP and exposes no DPR knob, so the stream is CSS-resolution whatever the client asks for — upstream Chromium, not something agent-browser chose or could fix. +**Why a headed window's page is followed over CDP.** The daemon's stream lists the headed window's tabs, but refreshes them only when a CLI command completes, so a navigation made in the window itself never reached the header. The webview once followed it by holding a browser-level CDP socket from `get cdp-url`, which granted it `Runtime.evaluate` in any target and `file://` navigation (review of the browser stack, 2026-09); the same observer now runs in the host. + +**Why a browser that goes is reported, not left to reconnects.** A socket whose browser had gone reconnected three times — immediately, +2 s, +4 s — before a headless pane read "ended" or a closed headed window reverted to the pane, and a Playwright browser's viewers learned of a disconnect only that way (static reading, 2026-09). + ## Pop-Out **The symptom when the daemon is not killed first.** `agent-browser --headed open` against a live headless daemon reattaches to it and exits 0, so the host logs a successful headed open and the mode never changes. The user presses Pop out, gets the pane stub with no OS window anywhere, and nothing in the logs says why. @@ -96,7 +112,7 @@ A post-open blank-tab sweep can become such a query when a later relaunch, expli ## Browser Host -**Why standalone passes a screenshot path, not bytes.** The sidecar stdio is a JSON-lines pipe shared with PTY traffic; a base64 frame on it would bloat every capture and interleave with terminal output. +**Why no frame rides a request's transport.** The standalone sidecar's stdio is a JSON-lines pipe shared with PTY traffic, where a base64 frame would bloat every capture and hold up terminal output behind it; captures once detoured through a temp file Rust read back instead, and VS Code posted 100-700 KB typed arrays through the webview message channel. The viewer socket carries them in the host's own loop, off both. **Why the verb alone is no boundary.** agent-browser honors launch options after the verb: `agent-browser --session x open about:blank --executable-path /nonexistent` fails with `Failed to launch Chrome at "/nonexistent"` (checked against 0.31.1, 2026-09-23). A verb-only allowlist therefore let an allowed `open`, `back` or `tab` carry `--executable-path`, `--args`, `--extension`, `--init-script`, `--profile`, `--state` or `--proxy` past the `binaryPath` gate; it also passed `close --all` (every session), `tab new `, and `screenshot `, which writes an image over any file the user can write. A session name becomes `/.pid`, whose pid a relaunch SIGTERMs, so a `/` in it reaches outside that directory. The two hosts first parsed the same argv separately and drifted within a day: agent-browser took any URL scheme and any DPR, Playwright http(s) and DPR ≤ 10 — so one parser serves both. @@ -114,7 +130,7 @@ A post-open blank-tab sweep can become such a query when a later relaunch, expli **Why one lifecycle for both providers.** The two hosts carried the same policies twice — headed tracking, relaunch generations, the blank-tab sweep, capture joins, the editing scripts — and the copies drifted: an empty copy clobbered the clipboard in one, the capture directory lacked its `chmod` in the other, and only Playwright serialized its closes with its relaunches, so the webview kept its own record of closes in flight for agent-browser (review of the browser stack, 2026-09). -**Why the screenshot path is private.** The frame is a picture of the user's authenticated browser, written by an external process under the ambient umask, so a derivable name in the shared temp directory is readable by anything else on the machine for as long as it exists. Precedent: `standalone/sidecar/clipboard-ops.js` applies the same discipline, cleanup included, to clipboard images. +**Why the capture directory is private.** The frame is a picture of the user's authenticated browser, written by an external process under the ambient umask, so a derivable name in the shared temp directory is readable by anything else on the machine for as long as it exists. Precedent: `standalone/sidecar/clipboard-ops.js` applies the same discipline, cleanup included, to clipboard images. **Why a named launch into a live browser navigates.** A Tool re-announcing — its dev server moved — sends a named launch into the session it already has. Relaunching it stopped the daemon (`close`, then SIGTERM and SIGKILL), so an agent driving that Tool lost its tabs, page state and CDP clients on every move, and a `dor ab` command in flight failed or started a daemon mid-relaunch (review of #777, 2026-09). Only a change of mode needs a new browser. @@ -130,7 +146,7 @@ A post-open blank-tab sweep can become such a query when a later relaunch, expli ## Playwright -**Why a paste is text, not keys.** agent-browser's stream takes only key and mouse events, so its paste replays a key down and up per character. Sent to the Playwright host, whose input queue closes the viewer (1008) at 256 queued messages, any paste over about 128 characters arriving as one burst truncated and dropped the pane into a 2 s reconnect (static reading, 2026-09). The 8192-character chunk keeps a message under the 64 KiB socket cap even when every character JSON-escapes to six bytes. +**Why screencast acks are paced.** Chrome sends the next screencast frame only once the last is acknowledged, and the host acknowledged each on receipt, so the rate was bounded only by how fast Chrome could encode — each frame then JSON-parsed, re-stringified and base64-decoded again in the webview (static reading, 2026-09). Acknowledging no sooner than 50 ms after the last caps it at the ~20 Hz agent-browser's daemon streams at. ## Iframe Renderer diff --git a/docs/specs/dor-cli.md b/docs/specs/dor-cli.md index 10f3f0c21..83f1e5255 100644 --- a/docs/specs/dor-cli.md +++ b/docs/specs/dor-cli.md @@ -590,7 +590,7 @@ is each provider's descriptor: - **After a command that may bind succeeds, `surface.browser { provider, key?, session, cwd, binaryPath, wsPort? }` opens or reuses its Surface**: `dor ab` first reads the stream port itself (`docs/specs/dor-browser.md` → - agent-browser), and the host reports Playwright's. **The call must wait past + agent-browser), and the host reports Playwright's stream. **The call must wait past `BROWSER_REQUEST_TIMEOUT_MS`**, since the host's answer can queue behind a launch or close of the browser (rationale). A failure there adds a stderr warning without changing the command's success. diff --git a/docs/specs/dor-cli.rationale.md b/docs/specs/dor-cli.rationale.md index 3c45f4bf4..63c0f34d3 100644 --- a/docs/specs/dor-cli.rationale.md +++ b/docs/specs/dor-cli.rationale.md @@ -58,7 +58,7 @@ file `which` would" licenses the hijack — and that the X_OK probe was unpinned because `statSync().isFile()` already rejected the directory the test shadowed with. All three rounds were review findings on the fix, before it merged. -**What a missing `windowsHide` looks like.** cross-spawn routes `.cmd` shims through `cmd.exe`, which owns a real console window, and the browser panel's screenshot loop spawns one per stream-frame pulse — a live page flickers focus-stealing windows several times a second. +**What a missing `windowsHide` looks like.** cross-spawn routes `.cmd` shims through `cmd.exe`, which owns a real console window, and the browser host's crisp-capture loop spawns one per changed stream frame — a live page flickers focus-stealing windows several times a second. **Why none of the `exit`-vs-`close` trouble surfaced on macOS.** The `agent-browser` daemon double-forks and detaches from the inherited fds, so `close` fires normally; only on Windows, where the daemon holds the parent's stdout/stderr pipes for its whole life, does a `close`-only wait hang forever. diff --git a/docs/specs/security-local.md b/docs/specs/security-local.md index ae2bc3916..b6a2325a3 100644 --- a/docs/specs/security-local.md +++ b/docs/specs/security-local.md @@ -123,7 +123,7 @@ embedder, and the browser checks the whole chain (rationale). - **FAIL IF** the iframe proxy stops checking that `Host` names its own grant port, on either path. Its per-grant ephemeral port and one-fixed-upstream binding are real mitigations but neither is a secret, so the `Host` check is what makes DNS rebinding fail. - **FAIL IF** the iframe proxy drops upstream `X-Frame-Options` / CSP `frame-ancestors` without replacing them with exactly `frame-ancestors 'self' `, admits another source, or targets the shim anywhere but its own proxy origin and that chain's innermost origin. With no usable chain it must preserve the headers and inject nothing (rationale). - **FAIL IF** a request bearing a *foreign* `Origin` refreshes a grant's idle timer: a grant holds a live upstream binding, and a stranger polling it keeps a closed pane's binding open. An *absent* `Origin` must keep refreshing it — that is what a live frame's own navigations and sub-resources send. -- **FAIL IF** the stream relay's grant stops being single-use, TTL-bounded, and pinned to one target port, or if it begins rewriting `Origin` rather than dropping it. It needs no `Host` check while the token holds (rationale). +- **FAIL IF** the browser viewer listener upgrades without both its own loopback `Host` and a single-use, 60-second grant for that one view, or passes a provider a webview message it has not rebuilt; or the host dials a viewer upstream (an agent-browser stream, a headed window's CDP) off loopback: `createViewerServer` and `parseViewerInput` in `lib/src/host/browser-viewer.ts`, `viewStream` and `observePage` in `lib/src/host/agent-browser-host.ts`. The webview holds no CDP and reaches no daemon (rationale). Pinned by `lib/src/host/browser-viewer.test.ts`. - **FAIL IF** the browser-dev bridge drops any of its four gates — the per-run token, the loopback `Host` check, the `application/json` content-type required of every non-GET, and the exact-origin `access-control-allow-origin`. The first three live together in the gate that runs before routing, so a route that never reads a body is covered by all of them. It is dev-only and ships in nothing, but it dispatches `pty_spawn` with caller-supplied `shell`, `args`, `cwd` and `env` — arbitrary command execution on a maintainer or CI-agent machine (`docs/specs/security-ci.md` -> "Automated Maintainer (tend)"). The content-type rule is a security control, not tidiness (rationale). - **FAIL IF** the browser-dev Vite server permits cross-origin reads of token-bearing modules or disables its DNS-rebinding Host check. Pinned by `standalone/scripts/dev-agent-browser.test.mjs` (rationale). diff --git a/docs/specs/security-local.rationale.md b/docs/specs/security-local.rationale.md index 521cf3b01..cd2f83945 100644 --- a/docs/specs/security-local.rationale.md +++ b/docs/specs/security-local.rationale.md @@ -120,9 +120,15 @@ chain and fails the policy, and a different grant has a different origin. someone adds a listener — the same failure mode that once left `.vscode/` owned by nobody. -**Why the stream relay needs no `Host` check.** Rebinding exists to make -same-origin-looking requests to loopback, which buys nothing against a listener -demanding an unguessable one-shot secret. +**Why the webview gets one guarded viewer socket, never an upstream.** The +webview once held a browser-level CDP socket from `get cdp-url` for a popped-out +window's URL — `Runtime.evaluate` in any target, `file://` navigation — and in +VS Code dialed a relay that piped raw bytes to any loopback port it named, with +`Origin` dropped (review of the browser stack, 2026-09). The host now speaks +each upstream's protocol itself and relays only parsed state, frames and +rebuilt input, so a webview naming another loopback port gets nothing it could +not already reach with its own `ws://127.0.0.1:*` `connect-src`. The listener +checks `Host` as well as the token, like every listener built after the relay. **Why the browser-dev bridge's content-type gate is a security control.** Without it the endpoint is CORS-simple and needs no preflight to survive, and what it dispatches diff --git a/docs/specs/standalone.md b/docs/specs/standalone.md index 13511a414..ee60730e1 100644 --- a/docs/specs/standalone.md +++ b/docs/specs/standalone.md @@ -82,14 +82,13 @@ stdout. **stdout is the protocol** — sidecar diagnostics go to stderr, which R appends to the log file. Webview → Rust is Tauri invokes; the `#[tauri::command]` set and `TauriAdapter` -own the exact command list, most of them thin sidecar forwarders. Three carve-outs +own the exact command list, most of them thin sidecar forwarders. Two carve-outs are *not* forwarded: | Not forwarded | Handled | Why | |---|---|---| | `load_session` / `save_session` | Rust | the per-window session file is Rust's store (§Persistence) | | the `clipboard` readers (Windows only) | Rust (`clipboard_win.rs`) | native Win32 reads (`docs/specs/mouse-and-clipboard.md` §8.6) | -| `browser_screenshot` | Rust reads, then deletes, a sidecar-supplied temp-file *path* | images must never ride the JSON-lines pipe shared with PTY traffic (`docs/specs/dor-browser.md`) | Request/response commands block on the sidecar's reply under a timeout. `OPEN_PORT_TIMEOUT_MS` and `OPEN_PORT_TIMEOUT_PER_ID_MS` in `lib.rs` mirror the diff --git a/docs/specs/vscode.md b/docs/specs/vscode.md index d196415db..89ca7c37f 100644 --- a/docs/specs/vscode.md +++ b/docs/specs/vscode.md @@ -237,7 +237,7 @@ frame-src http://127.0.0.1:* http://localhost:* **`frame-src` is loopback-only** — `dor iframe` frames its target through the transparent proxy the extension host stands up, so the only origin ever embedded is loopback on an OS-assigned port; without the override `default-src 'none'` blocks the frame and leaves a blank white pane (`docs/specs/dor-browser.md`). -**The webview CSP carries no relay sources.** Its `connect-src` loopback `ws:` entries are for the agent-browser stream relay and guarded Playwright viewer — the Burrow holds its `/ws/burrow` socket from the *extension host*, which no CSP fences, so the origin allowlist is enforced there instead (see "Burrow: a service in the extension host"). +**The webview CSP carries no relay sources.** Its `connect-src` loopback `ws:` entries are for the host's guarded browser viewer sockets (`docs/specs/dor-browser.md` → Viewer Socket) — the Burrow holds its `/ws/burrow` socket from the *extension host*, which no CSP fences, so the origin allowlist is enforced there instead (see "Burrow: a service in the extension host"). **That allowlist is a build-time constant, never a runtime value**: `vscode-ext/scripts/esbuild.mjs` substitutes `__DORMOUSE_REMOTE_CONNECT_SRC__` into `dist/extension.js`. The default, the replace-not-add override rule, and the two build-time guards are `docs/specs/relay.md` → "Where a Burrow may reach a Relay". diff --git a/lib/src/components/Wall.test.tsx b/lib/src/components/Wall.test.tsx index 99d39e712..93e7f8b12 100644 --- a/lib/src/components/Wall.test.tsx +++ b/lib/src/components/Wall.test.tsx @@ -576,7 +576,7 @@ describe('Wall on the Lath engine', () => { const bound = await bind({ provider: 'agent-browser', session: 'mine', wsPort: 61218 }); expect(bound?.ok).toBe(true); expect(requests('attach')).toEqual([]); - expect(browser).toHaveBeenCalledWith(expect.objectContaining({ op: 'streamUrl', port: 61218 })); + expect(browser).toHaveBeenCalledWith(expect.objectContaining({ op: 'view', stream: 61218 })); expect((await bind({ provider: 'playwright', session: 'dormouse.pw.x', cwd: '/project', wsPort: 61219 }))?.ok).toBe(false); expect(requests('attach')).toEqual([{ provider: 'playwright', binding: { session: 'dormouse.pw.x', cwd: '/project' }, op: 'attach' }]); @@ -756,7 +756,7 @@ describe('Wall on the Lath engine', () => { }); it.each(['playwright', 'agent-browser'] as const)('never binds a %s key to a session a Surface in another Workspace still holds', async (provider) => { - const { requests } = hostBrowsers({ attach: async () => ({ ok: true, wsPort: 4555 }) }); + const { requests } = hostBrowsers({ attach: async () => ({ ok: true, stream: 4555 }) }); const untouchedSpy = vi.spyOn(terminalRegistry, 'isUntouched').mockReturnValue(false); const disposers: Array<() => void> = []; try { @@ -829,7 +829,7 @@ describe('Wall on the Lath engine', () => { it('reveals a restored context-port browser instead of opening a second', async () => { const untouchedSpy = vi.spyOn(terminalRegistry, 'isUntouched').mockReturnValue(false); const helperSpy = vi.spyOn(helpers, 'openHelper').mockResolvedValue({ id: 'context-helper', parentId: 'pane-a', command: '', status: 'preserved' }); - const { requests } = hostBrowsers({ launch: async () => ({ ok: true, session: 'second', wsPort: 1 }) }); + const { requests } = hostBrowsers({ launch: async () => ({ ok: true, session: 'second', stream: 1 }) }); try { // The key a pre-Playwright build persisted for this port's agent-browser pane. await act(async () => { @@ -866,8 +866,8 @@ describe('Wall on the Lath engine', () => { const helperSpy = vi.spyOn(helpers, 'openHelper').mockResolvedValue({ id: 'context-helper', parentId: 'pane-a', command: '', status: 'preserved' }); // A relaunch of the named session hangs; a mint answers at once. const { browser, requests } = hostBrowsers({ - launch: (request) => request.binding.session ? new Promise(() => {}) : Promise.resolve({ ok: true, session: 'second', wsPort: 1 }), - attach: async () => ({ ok: true, wsPort: 4321 }), + launch: (request) => request.binding.session ? new Promise(() => {}) : Promise.resolve({ ok: true, session: 'second', stream: 1 }), + attach: async () => ({ ok: true, stream: 4321 }), }); try { // The port's browser, navigated away from the port's page since. @@ -919,8 +919,8 @@ describe('Wall on the Lath engine', () => { const helperSpy = vi.spyOn(helpers, 'openHelper').mockResolvedValue({ id: 'context-helper', parentId: 'pane-a', command: '', status: 'preserved' }); // A relaunch of the named session hangs; a mint answers at once. const { requests } = hostBrowsers({ - launch: (request) => request.binding.session ? new Promise(() => {}) : Promise.resolve({ ok: true, session: 'second', wsPort: 1 }), - attach: async () => ({ ok: true, wsPort: 4321 }), + launch: (request) => request.binding.session ? new Promise(() => {}) : Promise.resolve({ ok: true, session: 'second', stream: 1 }), + attach: async () => ({ ok: true, stream: 4321 }), }); try { await act(async () => { @@ -959,7 +959,7 @@ describe('Wall on the Lath engine', () => { const helperSpy = vi.spyOn(helpers, 'openHelper').mockResolvedValue({ id: 'context-helper', parentId: 'pane-a', command: '', status: 'preserved' }); // A launch the host answers without a session or a reason of its own. const { browser } = hostBrowsers({ - launch: async (request) => request.provider === 'playwright' ? { ok: false } : { ok: true, session: 'ab', wsPort: 1 }, + launch: async (request) => request.provider === 'playwright' ? { ok: false } : { ok: true, session: 'ab', stream: 1 }, }); try { await act(async () => { @@ -992,13 +992,13 @@ describe('Wall on the Lath engine', () => { it('reuses and closes a parked browser that gains its session after minimization', async () => { const defaultSession = sessionForKey('default'); const untouchedSpy = vi.spyOn(terminalRegistry, 'isUntouched').mockReturnValue(false); - let resolveOpen!: (result: { ok: boolean; session: string; wsPort: number }) => void; - const openResult = new Promise<{ ok: boolean; session: string; wsPort: number }>((resolve) => { + let resolveOpen!: (result: { ok: boolean; session: string; stream: number }) => void; + const openResult = new Promise<{ ok: boolean; session: string; stream: number }>((resolve) => { resolveOpen = resolve; }); const { requests } = hostBrowsers({ launch: () => openResult, - attach: async () => ({ ok: true, wsPort: 4321 }), + attach: async () => ({ ok: true, stream: 4321 }), }); try { @@ -1052,7 +1052,7 @@ describe('Wall on the Lath engine', () => { // Boot completion writes `session` only to live parked metadata. The Door // record is intentionally still the session-less minimize-time snapshot. await act(async () => { - resolveOpen({ ok: true, session: defaultSession, wsPort: 4321 }); + resolveOpen({ ok: true, session: defaultSession, stream: 4321 }); await openResult; }); await flush(); @@ -1082,7 +1082,7 @@ describe('Wall on the Lath engine', () => { const untouchedSpy = vi.spyOn(terminalRegistry, 'isUntouched').mockReturnValue(true); const pwOpen = Promise.withResolvers<{ ok: boolean; error?: string }>(); const { requests } = hostBrowsers({ - launch: (request) => request.provider === 'playwright' ? pwOpen.promise : Promise.resolve({ ok: true, session: 'relaunched', wsPort: 4321 }), + launch: (request) => request.provider === 'playwright' ? pwOpen.promise : Promise.resolve({ ok: true, session: 'relaunched', stream: 4321 }), }); try { await act(async () => { @@ -1154,8 +1154,8 @@ describe('Wall on the Lath engine', () => { it('hands an eager render-swap session to a Surface minimized during launch', async () => { const untouchedSpy = vi.spyOn(terminalRegistry, 'isUntouched').mockReturnValue(true); - let resolveOpen!: (result: { ok: boolean; session?: string; wsPort?: number; binaryPath?: string }) => void; - const openResult = new Promise<{ ok: boolean; session?: string; wsPort?: number; binaryPath?: string }>((resolve) => { + let resolveOpen!: (result: { ok: boolean; session?: string; stream?: number; binaryPath?: string }) => void; + const openResult = new Promise<{ ok: boolean; session?: string; stream?: number; binaryPath?: string }>((resolve) => { resolveOpen = resolve; }); const { requests } = hostBrowsers({ launch: () => openResult }); @@ -1182,7 +1182,7 @@ describe('Wall on the Lath engine', () => { expect(container.querySelector(`[data-door-id="${eagerId}"]`)).not.toBeNull(); await act(async () => { - resolveOpen({ ok: true, session: 'dormouse.1.gui-minimized', wsPort: 4321, binaryPath: '/usr/bin/agent-browser' }); + resolveOpen({ ok: true, session: 'dormouse.1.gui-minimized', stream: 4321, binaryPath: '/usr/bin/agent-browser' }); await openResult; }); await flush(); @@ -1250,14 +1250,14 @@ describe('Wall on the Lath engine', () => { // URL and captures are not what this test watches. const { browser } = hostBrowsers(); browser.mockImplementation(async (request: BrowserRequest) => { - if (request.op === 'attach') return { ok: true, wsPort: 4321 }; + if (request.op === 'attach') return { ok: true, stream: 4321 }; if (request.provider === 'playwright') { return request.op === 'launch' ? new Promise((resolve) => { failPlaywright = resolve; }) : { ok: true }; } - if (request.op === 'streamUrl' || request.op === 'screenshot') return { ok: true }; + if (request.op === 'view') return { ok: true, url: `ws://127.0.0.1:${request.stream}` }; if (request.op === 'launch') { events.push(`launch ${request.binding.session} ${request.url}`); - return { ok: true, session: request.binding.session, wsPort: 5555 }; + return { ok: true, session: request.binding.session, stream: 5555 }; } events.push(`${request.op} ${request.binding.session}`); if (request.op === 'close') { @@ -1361,8 +1361,8 @@ describe('Wall on the Lath engine', () => { .mockReturnValue(['ab-screencast', 'ab-popout', 'pw-screencast', 'pw-popout', 'iframe']); const { browser, requests } = hostBrowsers({ launch: async (request) => request.provider === 'playwright' - ? { ok: true, session: 'gui-pw', wsPort: 4321 } - : { ok: true, session: 'gui-ab', wsPort: 4322 }, + ? { ok: true, session: 'gui-pw', stream: 4321 } + : { ok: true, session: 'gui-ab', stream: 4322 }, }); const playwrightRequests = () => browser.mock.calls.filter(([request]) => request.provider === 'playwright'); // Serving: a Tool whose command is not running retires its browser. @@ -1464,8 +1464,8 @@ describe('Wall on the Lath engine', () => { const untouchedSpy = vi.spyOn(terminalRegistry, 'isUntouched').mockReturnValue(true); const { requests } = hostBrowsers({ launch: async (request) => request.provider === 'playwright' - ? { ok: true, session: 'gui-pw', wsPort: 4322 } - : { ok: true, session: 'dormouse.1.gui-a1b2c3', wsPort: 4321 }, + ? { ok: true, session: 'gui-pw', stream: 4322 } + : { ok: true, session: 'dormouse.1.gui-a1b2c3', stream: 4321 }, }); const warn = vi.spyOn(console, 'warn').mockImplementation(() => {}); @@ -3661,7 +3661,7 @@ describe('Wall on the Lath engine', () => { }); await flush(); - const { requests } = hostBrowsers({ launch: async () => ({ ok: true, session: 'context-browser', wsPort: 4321 }) }); + const { requests } = hostBrowsers({ launch: async () => ({ ok: true, session: 'context-browser', stream: 4321 }) }); if (!fake.hasPty('pane-a')) fake.spawnPty('pane-a'); fake.setOpenPorts('pane-a', [{ protocol: 'tcp', @@ -4186,7 +4186,7 @@ describe('Wall on the Lath engine', () => { }); it('names the command that drives a browser run by the other provider', async () => { - hostBrowsers({ attach: async (request) => request.provider === 'playwright' ? { ok: true, wsPort: 4555 } : { ok: true } }); + hostBrowsers({ attach: async (request) => request.provider === 'playwright' ? { ok: true, stream: 4555 } : { ok: true } }); const untouchedSpy = vi.spyOn(terminalRegistry, 'isUntouched').mockReturnValue(false); try { await act(async () => root.render()); @@ -4254,7 +4254,7 @@ describe('Wall on the Lath engine', () => { }); it('opens a new https:// tab from an iframe as an agent-browser pane bound to its launch', async () => { - const launches: Array<(result: { ok: boolean; session?: string; wsPort?: number; error?: string }) => void> = []; + const launches: Array<(result: { ok: boolean; session?: string; stream?: number; error?: string }) => void> = []; const { requests } = hostBrowsers({ launch: () => new Promise((resolve) => { launches.push(resolve); }) }); (fake as PlatformAdapter).createIframeProxyUrl = vi.fn(async () => ({ ok: true as const, url: 'http://127.0.0.1:61234/' })); const untouchedSpy = vi.spyOn(terminalRegistry, 'isUntouched').mockReturnValue(false); @@ -4281,7 +4281,7 @@ describe('Wall on the Lath engine', () => { }]); // The pane is there at once; `dor ab --surface` has nothing to drive until the launch names it. expect(await dispatchResolveAgentBrowser(tab)).toMatchObject({ ok: false }); - await act(async () => { launches[0]({ ok: true, session: 'dormouse.1.gui-abc', wsPort: 4321 }); }); + await act(async () => { launches[0]({ ok: true, session: 'dormouse.1.gui-abc', stream: 4321 }); }); await flush(); expect(await dispatchResolveAgentBrowser(tab)).toMatchObject({ ok: true, result: { session: 'dormouse.1.gui-abc' } }); diff --git a/lib/src/components/WorkspaceWindow.test.tsx b/lib/src/components/WorkspaceWindow.test.tsx index 676b6190b..3af1bbb9e 100644 --- a/lib/src/components/WorkspaceWindow.test.tsx +++ b/lib/src/components/WorkspaceWindow.test.tsx @@ -559,7 +559,7 @@ describe('WorkspaceWindow', () => { it('refuses a Playwright surface.browser whose host answer lands after the close began', async () => { // The Playwright arm asks the host for the viewer before it creates // anything, so the guard above is not the last word. - const status = Promise.withResolvers<{ ok: boolean; wsPort: number; headed: boolean }>(); + const status = Promise.withResolvers<{ ok: boolean; stream: number; headed: boolean }>(); const browser = vi.fn(() => status.promise); Object.assign(fake, { browserProviders: ['agent-browser', 'playwright'], browser }); await render(); @@ -581,7 +581,7 @@ describe('WorkspaceWindow', () => { await flush(); expect(browser).toHaveBeenCalledWith(expect.objectContaining({ provider: 'playwright', op: 'attach', binding: expect.objectContaining({ session: 'late' }) })); await act(async () => { expect(await handle.closeAll('silent')).toBeNull(); }); - await act(async () => status.resolve({ ok: true, wsPort: 4321, headed: false })); + await act(async () => status.resolve({ ok: true, stream: 4321, headed: false })); await flush(); expect(respond).toHaveBeenCalledWith({ ok: false, error: 'this workspace is closing' }); diff --git a/lib/src/components/wall/AgentBrowserPanel.test.tsx b/lib/src/components/wall/AgentBrowserPanel.test.tsx index c19e3ead7..07af4309c 100644 --- a/lib/src/components/wall/AgentBrowserPanel.test.tsx +++ b/lib/src/components/wall/AgentBrowserPanel.test.tsx @@ -13,11 +13,12 @@ import { disposeAgentBrowserSurfaceController, disposeAllAgentBrowserSurfaceControllers, getAgentBrowserSurfaceController, - handOverBrowserPort, + handOverBrowserStream, } from './agent-browser-surface-controller'; import type { RenderMode } from './agent-browser-screen'; import { ModeContext, PaneWriteContext, SelectedIdContext, WallActionsContext, WorkspaceActiveContext, type PaneWriteActions } from './wall-context'; -import { installBrowserHost, stubWallActions as stubActions, type BrowserAnswers } from './wall-test-utils'; +import { installBrowserHost, stubWallActions as stubActions } from './wall-test-utils'; +import { encodeViewerFrame } from '../../lib/platform/browser-automation'; globalThis.IS_REACT_ACT_ENVIRONMENT = true; @@ -25,7 +26,7 @@ type TestPanelParams = { surfaceType: string; renderMode?: string; session: string; - wsPort?: number; + stream?: number; url?: string; poppedOut?: boolean; cwd?: string; @@ -33,10 +34,8 @@ type TestPanelParams = { const DEFAULT_PARAMS: TestPanelParams = { surfaceType: 'agent-browser', session: 'browser-session' }; -// The Playwright host serves its own viewer, so it names the stream URL. -const PLAYWRIGHT_STREAM: BrowserAnswers = { streamUrl: async ({ port }) => ({ ok: true, url: `ws://127.0.0.1:${port}` }) }; /** The operations that drive a live browser, rather than bind or view one. */ -const DRIVES = new Set(['navigate', 'history', 'tab', 'viewport', 'device', 'cdpUrl', 'close']); +const DRIVES = new Set(['navigate', 'history', 'tab', 'viewport', 'device', 'close']); class ResizeObserverMock { observe() {} @@ -47,7 +46,7 @@ class ResizeObserverMock { function paneProps(id: string, params: TestPanelParams = DEFAULT_PARAMS): PaneProps { const props = { id, title: 'Browser', params }; - handOverFixturePort(props); + handOverFixtureStream(props); return props; } @@ -82,7 +81,7 @@ class WebSocketMock { this.onclose?.(new CloseEvent('close')); } - emitMessage(data: string) { + emitMessage(data: string | ArrayBuffer) { this.onmessage?.({ data } as MessageEvent); } } @@ -94,6 +93,8 @@ beforeEach(() => { vi.stubGlobal('ResizeObserver', ResizeObserverMock); vi.stubGlobal('WebSocket', WebSocketMock); WebSocketMock.instances = []; + // A host granting every viewer socket, at the stream's number as its port. + installBrowserHost(); container = document.createElement('div'); document.body.appendChild(container); root = createRoot(container); @@ -115,11 +116,11 @@ afterEach(() => { // through params. These fixtures carry it in their params, so hand each new one // over the way the Wall does, with the params the command refreshed. const handedOver = new Map(); -function handOverFixturePort({ id, params }: PaneProps): void { +function handOverFixtureStream({ id, params }: PaneProps): void { const fixture = params as TestPanelParams | undefined; - if (fixture?.wsPort === undefined || handedOver.get(id) === fixture.wsPort) return; - handedOver.set(id, fixture.wsPort); - handOverBrowserPort(id, { ...fixture, renderMode: fixture.renderMode as RenderMode | undefined }, fixture.wsPort); + if (fixture?.stream === undefined || handedOver.get(id) === fixture.stream) return; + handedOver.set(id, fixture.stream); + handOverBrowserStream(id, { ...fixture, renderMode: fixture.renderMode as RenderMode | undefined }, fixture.stream); } async function renderPanel( @@ -167,8 +168,8 @@ describe('AgentBrowserPanel render mode controller', () => { it('relaunches screencast sessions as popout and publishes the mode immediately', async () => { const updateParameters = vi.fn(); const host = installBrowserHost({ - launch: async () => ({ ok: true, wsPort: 3456 }), - attach: async () => ({ ok: true, wsPort: 1234 }), + launch: async () => ({ ok: true, stream: 3456 }), + attach: async () => ({ ok: true, stream: 1234 }), }); await renderPanel(paneProps('ab-panel'), updateParameters); @@ -187,8 +188,8 @@ describe('AgentBrowserPanel render mode controller', () => { it('relaunches popped-out sessions back into screencast', async () => { const updateParameters = vi.fn(); const host = installBrowserHost({ - launch: async () => ({ ok: true, wsPort: 4567 }), - attach: async () => ({ ok: true, wsPort: 1234 }), + launch: async () => ({ ok: true, stream: 4567 }), + attach: async () => ({ ok: true, stream: 1234 }), }); await renderPanel( @@ -210,8 +211,8 @@ describe('AgentBrowserPanel render mode controller', () => { it('pop-in uses the latest observed headed-window tab URL over stale params', async () => { const updateParameters = vi.fn(); const host = installBrowserHost({ - launch: async () => ({ ok: true, wsPort: 4567 }), - attach: async () => ({ ok: true, wsPort: 1234 }), + launch: async () => ({ ok: true, stream: 4567 }), + attach: async () => ({ ok: true, stream: 1234 }), }); await renderPanel( @@ -219,7 +220,7 @@ describe('AgentBrowserPanel render mode controller', () => { surfaceType: 'browser', renderMode: 'ab-popout', session: 'browser-session', - wsPort: 1111, + stream: 1111, url: 'https://google.com/', }), updateParameters, @@ -245,14 +246,14 @@ describe('AgentBrowserPanel render mode controller', () => { it('mirrors popped-out stream tab URL updates when the stream reports id instead of tabId', async () => { const updateParameters = vi.fn(); - installBrowserHost({ attach: async () => ({ ok: true, wsPort: 1234 }) }); + installBrowserHost({ attach: async () => ({ ok: true, stream: 1234 }) }); await renderPanel( paneProps('ab-panel', { surfaceType: 'browser', renderMode: 'ab-popout', session: 'browser-session', - wsPort: 1111, + stream: 1111, url: 'https://google.com/', }), updateParameters, @@ -268,19 +269,16 @@ describe('AgentBrowserPanel render mode controller', () => { expect(updateParameters).toHaveBeenCalledWith({ url: 'https://example.com/' }); }); - it('mirrors popped-out manual navigation from CDP target events', async () => { + it('mirrors a popped-out window\'s page as the host observes it, holding no CDP itself', async () => { const updateParameters = vi.fn(); - const host = installBrowserHost({ - cdpUrl: async () => ({ ok: true, url: 'ws://127.0.0.1:9222/devtools/browser/test' }), - attach: async () => ({ ok: true, wsPort: 1234 }), - }); + const host = installBrowserHost({ attach: async () => ({ ok: true, stream: 1234 }) }); await renderPanel( paneProps('ab-panel', { surfaceType: 'browser', renderMode: 'ab-popout', session: 'browser-session', - wsPort: 1111, + stream: 1111, url: 'https://google.com/', }), updateParameters, @@ -291,24 +289,21 @@ describe('AgentBrowserPanel render mode controller', () => { await Promise.resolve(); }); - const cdpWs = WebSocketMock.instances.find((ws) => ws.url.includes('/devtools/browser/')); - expect(cdpWs).toBeTruthy(); + // Its one socket is the host's viewer, told it shows a headed window. + expect(host.requests('view')).toEqual([{ provider: 'agent-browser', binding: { session: 'browser-session' }, op: 'view', stream: 1111, headed: true }]); + expect(WebSocketMock.instances.map((ws) => ws.url)).toEqual(['ws://127.0.0.1:1111']); await act(async () => { - cdpWs?.emitMessage(JSON.stringify({ - method: 'Target.targetInfoChanged', - params: { targetInfo: { type: 'page', url: 'https://example.com/', title: 'Example Domain' } }, - })); + WebSocketMock.instances[0].emitMessage(JSON.stringify({ type: 'page', url: 'https://example.com/', title: 'Example Domain' })); }); - expect(host.requests('cdpUrl')).toEqual([{ provider: 'agent-browser', binding: { session: 'browser-session' }, op: 'cdpUrl' }]); expect(updateParameters).toHaveBeenCalledWith({ url: 'https://example.com/' }); }); it('actively selects a newly opened tab when the stream does not mark it active', async () => { const host = installBrowserHost(); - await renderPanel(paneProps('ab-panel', { surfaceType: 'browser', session: 'browser-session', wsPort: 1111 })); + await renderPanel(paneProps('ab-panel', { surfaceType: 'browser', session: 'browser-session', stream: 1111 })); await act(async () => { WebSocketMock.instances.at(-1)?.emitMessage(JSON.stringify({ @@ -333,7 +328,7 @@ describe('AgentBrowserPanel render mode controller', () => { it('does not force-select a provisional new tab that already reports active', async () => { const host = installBrowserHost(); - await renderPanel(paneProps('ab-panel', { surfaceType: 'browser', session: 'browser-session', wsPort: 1111 })); + await renderPanel(paneProps('ab-panel', { surfaceType: 'browser', session: 'browser-session', stream: 1111 })); await act(async () => { WebSocketMock.instances.at(-1)?.emitMessage(JSON.stringify({ @@ -358,7 +353,7 @@ describe('AgentBrowserPanel render mode controller', () => { it('selects a provisional new tab after it reaches its destination if it is not active', async () => { const host = installBrowserHost(); - await renderPanel(paneProps('ab-panel', { surfaceType: 'browser', session: 'browser-session', wsPort: 1111 })); + await renderPanel(paneProps('ab-panel', { surfaceType: 'browser', session: 'browser-session', stream: 1111 })); await act(async () => { WebSocketMock.instances.at(-1)?.emitMessage(JSON.stringify({ @@ -393,7 +388,7 @@ describe('AgentBrowserPanel render mode controller', () => { it('keeps the last known active tab when the stream emits a transient empty tab list', async () => { const updateParameters = vi.fn(); await renderPanel( - paneProps('ab-panel', { surfaceType: 'browser', session: 'browser-session', wsPort: 1111 }), + paneProps('ab-panel', { surfaceType: 'browser', session: 'browser-session', stream: 1111 }), updateParameters, ); @@ -443,13 +438,13 @@ describe('AgentBrowserPanel Playwright params', () => { vi.spyOn(HTMLElement.prototype, 'getBoundingClientRect').mockReturnValue( { width: 800, height: 600, left: 0, top: 0, right: 800, bottom: 600, x: 0, y: 0, toJSON() {} } as DOMRect, ); - const host = installBrowserHost(PLAYWRIGHT_STREAM); + const host = installBrowserHost(); // Every operation that drives the browser, in the directory it ran for. const commands = (cwd?: string) => host.browser.mock.calls .map(([request]) => request) .filter((request) => DRIVES.has(request.op) && (cwd === undefined || request.binding.cwd === cwd)) .map((request) => request.op === 'viewport' ? `set viewport ${request.width} ${request.height} ${request.dpr}` : request.op); - const params = { surfaceType: 'browser', renderMode: 'pw-screencast', session: 'app', cwd: '/first', wsPort: 4321 }; + const params = { surfaceType: 'browser', renderMode: 'pw-screencast', session: 'app', cwd: '/first', stream: 4321 }; await renderPanel(paneProps('pw-panel', params)); expect(commands()).toContain('set viewport 800 600 1'); @@ -457,7 +452,7 @@ describe('AgentBrowserPanel Playwright params', () => { await act(async () => { stream(4321).emitMessage(JSON.stringify({ type: 'status', connected: true, screencasting: true })); }); host.browser.mockClear(); - await renderPanel(paneProps('pw-panel', { ...params, renderMode: 'pw-popout', wsPort: 4322 })); + await renderPanel(paneProps('pw-panel', { ...params, renderMode: 'pw-popout', stream: 4322 })); expect(getAgentBrowserScreenController('pw-panel')?.snapshot().renderMode).toBe('pw-popout'); expect(container.textContent).toContain('This browser is running in a separate window.'); expect(commands()).toEqual([]); @@ -466,15 +461,15 @@ describe('AgentBrowserPanel Playwright params', () => { await act(async () => { stream(4322).emitMessage(JSON.stringify({ type: 'status', connected: false, screencasting: false })); }); expect(host.requests('launch')).toEqual([]); - await renderPanel(paneProps('pw-panel', { ...params, cwd: '/second', wsPort: 4323 })); + await renderPanel(paneProps('pw-panel', { ...params, cwd: '/second', stream: 4323 })); expect(getAgentBrowserScreenController('pw-panel')?.snapshot().renderMode).toBe('pw-screencast'); expect(commands('/second')).toContain('set viewport 800 600 1'); }); it('keeps its own mode write over params that predate it', async () => { - const host = installBrowserHost({ ...PLAYWRIGHT_STREAM, launch: async () => ({ ok: true, wsPort: 4330 }) }); + const host = installBrowserHost({ launch: async () => ({ ok: true, stream: 4330 }) }); const updateParameters = vi.fn(); - const popped = { surfaceType: 'browser', renderMode: 'pw-popout', session: 'app', cwd: '/p', wsPort: 4321 }; + const popped = { surfaceType: 'browser', renderMode: 'pw-popout', session: 'app', cwd: '/p', stream: 4321 }; await renderPanel(paneProps('pw-panel', popped), updateParameters); const stream = WebSocketMock.instances.findLast((ws) => ws.url === 'ws://127.0.0.1:4321')!; await act(async () => { stream.emitMessage(JSON.stringify({ type: 'status', connected: true, screencasting: false })); }); @@ -494,8 +489,8 @@ describe('AgentBrowserPanel Playwright params', () => { expect(getAgentBrowserScreenController('pw-panel')?.snapshot().renderMode).toBe('pw-screencast'); // Once params show it, a later host report is followed again. - await renderPanel(paneProps('pw-panel', { ...popped, renderMode: 'pw-screencast', wsPort: 4330 }), updateParameters); - await renderPanel(paneProps('pw-panel', { ...popped, wsPort: 4331 }), updateParameters); + await renderPanel(paneProps('pw-panel', { ...popped, renderMode: 'pw-screencast', stream: 4330 }), updateParameters); + await renderPanel(paneProps('pw-panel', { ...popped, stream: 4331 }), updateParameters); expect(getAgentBrowserScreenController('pw-panel')?.snapshot().renderMode).toBe('pw-popout'); }); }); @@ -504,10 +499,10 @@ describe('AgentBrowserPanel across a provider change', () => { // A minimized pane keeps this view mounted while its Wall restores a failed // cross-provider swap in place: same id, the other provider's params. function withBothProviders() { - installBrowserHost(PLAYWRIGHT_STREAM); + installBrowserHost(); } - const pw = { surfaceType: 'browser', renderMode: 'pw-screencast', session: 'failed-swap', wsPort: 4400 }; - const ab = { surfaceType: 'browser', renderMode: 'ab-screencast', session: 'relaunched', wsPort: 4401 }; + const pw = { surfaceType: 'browser', renderMode: 'pw-screencast', session: 'failed-swap', stream: 4400 }; + const ab = { surfaceType: 'browser', renderMode: 'ab-screencast', session: 'relaunched', stream: 4401 }; it('takes a fresh controller when the Wall disposes the old one and restores the other provider', async () => { withBothProviders(); @@ -530,7 +525,7 @@ describe('AgentBrowserPanel across a provider change', () => { await renderPanel(paneProps('swap-panel', pw)); const failed = getAgentBrowserSurfaceController('swap-panel')!; // No port handover, which would acquire the controller itself. - await renderPanel(paneProps('swap-panel', { ...ab, wsPort: undefined })); + await renderPanel(paneProps('swap-panel', { ...ab, stream: undefined })); await act(async () => { await Promise.resolve(); }); expect(getAgentBrowserSurfaceController('swap-panel')).not.toBe(failed); expect(getAgentBrowserScreenController('swap-panel')?.snapshot().renderMode).toBe('ab-screencast'); @@ -541,8 +536,8 @@ describe('AgentBrowserPanel across a provider change', () => { describe('AgentBrowserPanel after its controller is released', () => { it('takes a fresh controller for the params that follow, and none for the release alone', async () => { - const host = installBrowserHost({ attach: async () => ({ ok: true, wsPort: 4402 }) }); - const first = { surfaceType: 'browser', renderMode: 'ab-screencast', session: 'first-run', wsPort: 4400 }; + const host = installBrowserHost({ attach: async () => ({ ok: true, stream: 4402 }) }); + const first = { surfaceType: 'browser', renderMode: 'ab-screencast', session: 'first-run', stream: 4400 }; await renderPanel(paneProps('released-panel', first)); const released = getAgentBrowserSurfaceController('released-panel'); @@ -617,7 +612,7 @@ describe('AgentBrowserPanel visibility parking', () => { it('parks a hidden panel: closes the stream and opens no replacement', async () => { const { setVisible } = await renderVisibilityPanel({ - surfaceType: 'browser', session: 'browser-session', wsPort: 4321, + surfaceType: 'browser', session: 'browser-session', stream: 4321, }); const socket = liveStreamSocket(4321); @@ -634,7 +629,7 @@ describe('AgentBrowserPanel visibility parking', () => { it('parks a minimized (parked) panel even while the window stays in the foreground', async () => { const { setParked } = await renderVisibilityPanel({ - surfaceType: 'browser', session: 'browser-session', wsPort: 4321, + surfaceType: 'browser', session: 'browser-session', stream: 4321, }); const socket = liveStreamSocket(4321); @@ -657,7 +652,7 @@ describe('AgentBrowserPanel visibility parking', () => { it('idles a screencast whose Workspace is hidden, and resumes it on return', async () => { const { setWorkspaceActive } = await renderVisibilityPanel({ - surfaceType: 'browser', session: 'browser-session', wsPort: 4321, + surfaceType: 'browser', session: 'browser-session', stream: 4321, }); const socket = liveStreamSocket(4321); @@ -677,11 +672,10 @@ describe('AgentBrowserPanel visibility parking', () => { }); it('reconnects and repaints from the stream when it becomes visible again', async () => { - const screenshot = vi.fn(async () => ({ ok: false as const, error: 'test' })); - installBrowserHost({ screenshot }); + vi.stubGlobal('createImageBitmap', vi.fn(async () => ({ width: 4, height: 4, close: vi.fn() }))); const { setVisible } = await renderVisibilityPanel({ - surfaceType: 'browser', session: 'browser-session', wsPort: 4321, + surfaceType: 'browser', session: 'browser-session', stream: 4321, }); await act(async () => { setVisible(false); }); @@ -696,18 +690,17 @@ describe('AgentBrowserPanel visibility parking', () => { const reconnected = liveStreamSocket(4321); expect(reconnected?.readyState).toBe(1); - // A frame pulse over the reconnected stream drives a device screenshot. - screenshot.mockClear(); + // The host's first frame over the reconnected socket paints. await act(async () => { - reconnected?.emitMessage(JSON.stringify({ type: 'frame', data: 'x'.repeat(32) })); - await vi.advanceTimersByTimeAsync(300); + reconnected?.emitMessage(encodeViewerFrame({ kind: 'provisional', jpeg: new Uint8Array([0xff, 0xd8, 1]) }).buffer); + await vi.advanceTimersByTimeAsync(0); }); - expect(screenshot).toHaveBeenCalled(); + expect(createImageBitmap).toHaveBeenCalledOnce(); }); it('does not park a popped-out panel while it is hidden', async () => { const { setVisible } = await renderVisibilityPanel({ - surfaceType: 'browser', renderMode: 'ab-popout', session: 'browser-session', wsPort: 1111, + surfaceType: 'browser', renderMode: 'ab-popout', session: 'browser-session', stream: 1111, }); const socket = liveStreamSocket(1111); @@ -723,7 +716,7 @@ describe('AgentBrowserPanel visibility parking', () => { it('parks when the document is hidden (raw visibilitychange event)', async () => { await renderVisibilityPanel({ - surfaceType: 'browser', session: 'browser-session', wsPort: 4321, + surfaceType: 'browser', session: 'browser-session', stream: 4321, }); const socket = liveStreamSocket(4321); @@ -737,7 +730,7 @@ describe('AgentBrowserPanel visibility parking', () => { it('does not park when a hide is reversed within the delay', async () => { const { setVisible } = await renderVisibilityPanel({ - surfaceType: 'browser', session: 'browser-session', wsPort: 4321, + surfaceType: 'browser', session: 'browser-session', stream: 4321, }); const socket = liveStreamSocket(4321); @@ -762,7 +755,7 @@ describe('AgentBrowserPanel canvas input forwarding', () => { selectedId: string | null, workspaceActive = true, ): Promise { - const props = paneProps('ab-panel', { surfaceType: 'agent-browser', session: 'browser-session', wsPort: 4321 }); + const props = paneProps('ab-panel', { surfaceType: 'agent-browser', session: 'browser-session', stream: 4321 }); await act(async () => { root.render( @@ -851,7 +844,7 @@ describe('AgentBrowserPanel tab strip actions', () => { // just exercises the onClick → selectTab/closeTab wiring. async function renderWithTwoTabs(): Promise> { const host = installBrowserHost(); - const props = paneProps('ab-panel', { surfaceType: 'agent-browser', session: 'browser-session', wsPort: 4321 }); + const props = paneProps('ab-panel', { surfaceType: 'agent-browser', session: 'browser-session', stream: 4321 }); await renderPanel(props); const ws = WebSocketMock.instances[WebSocketMock.instances.length - 1]; await act(async () => { @@ -885,39 +878,6 @@ describe('AgentBrowserPanel tab strip actions', () => { expect(host.requests('tab')).toContainEqual({ provider: 'agent-browser', binding: { session: 'browser-session' }, op: 'tab', action: 'close', tabId: 't2' }); }); - it('captures a fresh frame when the active tab changes, but not on other tab edits', async () => { - // The daemon emits no screencast frame on a tab switch and the dedup'd stream - // is otherwise silent, so the panel forces one device screenshot so the canvas - // follows the newly-active tab. - const screenshot = vi.fn(async () => ({ ok: false as const, error: 'test' })); - installBrowserHost({ screenshot }); - const props = paneProps('ab-panel', { surfaceType: 'agent-browser', session: 'browser-session', wsPort: 4321 }); - await renderPanel(props); - const ws = WebSocketMock.instances[WebSocketMock.instances.length - 1]; - const tabs = (a: 't1' | 't2', extra = false) => JSON.stringify({ type: 'tabs', tabs: [ - { tabId: 't1', title: 'Dormouse', url: 'https://dormouse.sh/', active: a === 't1' }, - { tabId: 't2', title: 'GitHub', url: 'https://github.com/diffplug/dormouse', active: a === 't2' }, - ...(extra ? [{ tabId: 't3', title: 'GitHub', url: 'https://github.com/diffplug/dormouse', active: false }] : []), - ] }); - - await act(async () => { ws.emitMessage(tabs('t1')); }); - // Let the loop go fully idle before measuring tab-driven captures. Besides the - // priming capture, StrictMode remounts the panel, and the re-attach repaint - // pulse (a live connection survives the detach) schedules a throttled - // follow-up capture ~one shot-interval later — drain it before mockClear. - await new Promise((r) => setTimeout(r, 250)); - screenshot.mockClear(); - - // Adding a tab without changing which is active must NOT force a capture. - await act(async () => { ws.emitMessage(tabs('t1', true)); }); - await new Promise((r) => setTimeout(r, 120)); - expect(screenshot).not.toHaveBeenCalled(); - - // Switching the active tab does. - await act(async () => { ws.emitMessage(tabs('t2', true)); }); - await new Promise((r) => setTimeout(r, 120)); - expect(screenshot).toHaveBeenCalled(); - }); }); describe('the render modes a tool is offered (regression: PR #493 review)', () => { @@ -927,7 +887,7 @@ describe('the render modes a tool is offered (regression: PR #493 review)', () = // `agent-browser-surface-controller.ts`. The host can do everything, so a // refusal is the tool rule and not a missing capability. function withCapableHost() { - return installBrowserHost({ launch: async () => ({ ok: true, wsPort: 1 }) }); + return installBrowserHost({ launch: async () => ({ ok: true, stream: 1 }) }); } it('offers every mode on a plain browser surface', async () => { diff --git a/lib/src/components/wall/agent-browser-connection.test.ts b/lib/src/components/wall/agent-browser-connection.test.ts index ab11a5ac9..98c3dbd13 100644 --- a/lib/src/components/wall/agent-browser-connection.test.ts +++ b/lib/src/components/wall/agent-browser-connection.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { createAgentBrowserConnection } from './agent-browser-connection'; +import { encodeViewerFrame } from '../../lib/platform/browser-automation'; +import { createAgentBrowserConnection, type AgentBrowserConnectionDeps } from './agent-browser-connection'; class WebSocketMock { static instances: WebSocketMock[] = []; @@ -10,6 +11,7 @@ class WebSocketMock { onclose: ((event: CloseEvent) => void) | null = null; onerror: ((event: Event) => void) | null = null; readyState = 1; + binaryType = 'blob'; sent: string[] = []; closed = false; @@ -28,11 +30,22 @@ class WebSocketMock { this.onclose?.({ code: 1000, reason: '', wasClean: true } as CloseEvent); } - emitMessage(data: string) { + emitMessage(data: unknown) { this.onmessage?.({ data } as MessageEvent); } } +/** A connection to a host that grants `ws://viewer/` for each connect. */ +function connect(deps: Partial = {}) { + let granted = 0; + const viewUrl = vi.fn(async () => `ws://127.0.0.1:9/view/${++granted}`); + const connection = createAgentBrowserConnection({ session: 'dormouse.1.default', stream: 1234, viewUrl, ...deps }); + return { connection, viewUrl }; +} + +const socket = () => WebSocketMock.instances.at(-1)!; +const flush = async () => { for (let i = 0; i < 5; i++) await Promise.resolve(); }; + beforeEach(() => { vi.stubGlobal('WebSocket', WebSocketMock); WebSocketMock.instances = []; @@ -42,263 +55,91 @@ afterEach(() => { vi.restoreAllMocks(); }); -describe('agent-browser connection', () => { - it('forwards the stream\'s url message as a navigation event', async () => { - const connection = createAgentBrowserConnection({ session: 's', streamPort: 1234 }); - const events: unknown[] = []; - connection.subscribe((event) => { if (event.type === 'url') events.push(event); }); - await Promise.resolve(); - WebSocketMock.instances[0].emitMessage(JSON.stringify({ type: 'url', url: 'https://example.com/slow', timestamp: 1 })); - WebSocketMock.instances[0].emitMessage(JSON.stringify({ type: 'url' })); - expect(events).toEqual([{ type: 'url', url: 'https://example.com/slow' }]); - connection.dispose(); - }); - - it('closes the stream websocket when disposed', async () => { - const connection = createAgentBrowserConnection({ - session: 'dormouse.1.default', - streamPort: 1234, - }); - - await Promise.resolve(); - const ws = WebSocketMock.instances[0]; - expect(ws.url).toBe('ws://127.0.0.1:1234'); - +describe('viewer socket connection', () => { + it('dials the URL the host grants, taking frames as binary', async () => { + const { connection } = connect(); + await flush(); + expect(socket().url).toBe('ws://127.0.0.1:9/view/1'); + expect(socket().binaryType).toBe('arraybuffer'); connection.dispose(); - - expect(ws.closed).toBe(true); - }); - - it('ignores transient empty tabs after a real tab list', async () => { - const connection = createAgentBrowserConnection({ - session: 'dormouse.1.default', - streamPort: 1234, - }); - - await Promise.resolve(); - const ws = WebSocketMock.instances[0]; - ws.emitMessage(JSON.stringify({ - type: 'tabs', - tabs: [{ tabId: 't1', title: 'Dormouse', url: 'https://dormouse.sh/', active: true }], - })); - - expect(connection.snapshot().tabs).toHaveLength(1); - - ws.emitMessage(JSON.stringify({ type: 'tabs', tabs: [] })); - - expect(connection.snapshot().tabs).toEqual([ - { tabId: 't1', title: 'Dormouse', url: 'https://dormouse.sh/', active: true }, - ]); - }); - - it('drops byte-identical frame re-broadcasts (daemon heartbeat) but forwards real changes', async () => { - const connection = createAgentBrowserConnection({ - session: 'dormouse.1.default', - streamPort: 1234, - wantFrameData: () => true, - }); - let pulses = 0; - const provisionalFrames: Array = []; - connection.subscribe((event) => { - if (event.type === 'frame-pulse') { - pulses += 1; - provisionalFrames.push(event.data); - } - }); - - await Promise.resolve(); - const ws = WebSocketMock.instances[0]; - const frameA = JSON.stringify({ type: 'frame', data: 'AAAAAAAAAA' }); - const frameB = JSON.stringify({ type: 'frame', data: 'BBBBBBBBBB' }); - - ws.emitMessage(frameA); // first frame — primes - ws.emitMessage(frameA); // identical re-broadcast — dropped - ws.emitMessage(frameA); // identical re-broadcast — dropped - expect(pulses).toBe(1); - - ws.emitMessage(frameB); // real change — forwarded - ws.emitMessage(frameB); // identical — dropped - expect(pulses).toBe(2); - - ws.emitMessage(frameA); // changed again — forwarded - expect(pulses).toBe(3); - expect(provisionalFrames).toEqual(['AAAAAAAAAA', 'BBBBBBBBBB', 'AAAAAAAAAA']); - }); - - it('parses a large stream frame for provisional painting', async () => { - const connection = createAgentBrowserConnection({ - session: 'dormouse.1.default', - streamPort: 1234, - wantFrameData: () => true, - }); - const frames: string[] = []; - connection.subscribe((event) => { - if (event.type === 'frame-pulse' && event.data) frames.push(event.data); - }); - - await Promise.resolve(); - const data = 'A'.repeat(20_000); - WebSocketMock.instances[0].emitMessage(JSON.stringify({ - type: 'frame', - data, - metadata: { deviceWidth: 800, deviceHeight: 600 }, - })); - - expect(frames).toEqual([data]); + expect(socket().closed).toBe(true); }); - it('drops identical tab-snapshot re-broadcasts but forwards real changes', async () => { - const connection = createAgentBrowserConnection({ - session: 'dormouse.1.default', - streamPort: 1234, - }); - let tabsEvents = 0; - connection.subscribe((event) => { if (event.type === 'tabs') tabsEvents += 1; }); - - await Promise.resolve(); - const ws = WebSocketMock.instances[0]; - const snapshot = JSON.stringify({ - type: 'tabs', - tabs: [{ tabId: 't1', title: 'Dormouse', url: 'https://dormouse.sh/', active: true }], - }); - - ws.emitMessage(snapshot); // first — emits - ws.emitMessage(snapshot); // identical heartbeat — dropped - ws.emitMessage(snapshot); // identical heartbeat — dropped - expect(tabsEvents).toBe(1); - - // A genuine change (new tab) alters the signature and is forwarded. - ws.emitMessage(JSON.stringify({ - type: 'tabs', - tabs: [ - { tabId: 't1', title: 'Dormouse', url: 'https://dormouse.sh/', active: false }, - { tabId: 't2', title: 'GitHub', url: 'https://github.com/diffplug/dormouse', active: true }, - ], - })); - expect(tabsEvents).toBe(2); - }); - - it('routes an oversized tabs snapshot to the tab list instead of dropping it as a frame', async () => { - // Default deps → wantFrameData is absent → the idle hot path (false), which - // is the dominant real-world case on hosts with crisp screenshots. - const connection = createAgentBrowserConnection({ - session: 'dormouse.1.default', - streamPort: 1234, - }); - const tabsEventCounts: number[] = []; - connection.subscribe((event) => { if (event.type === 'tabs') tabsEventCounts.push(event.tabs.length); }); - - await Promise.resolve(); - const ws = WebSocketMock.instances[0]; - - // Many tabs with long URLs/titles push the control message past - // FRAME_PULSE_THRESHOLD (16384). Size alone must not misclassify it as a frame. - const tabs = Array.from({ length: 80 }, (_, i) => ({ - tabId: `t${i}`, - title: `Tab number ${i} — ${'x'.repeat(40)}`, - url: `https://example.com/very/long/path/segment/${i}?q=${'y'.repeat(120)}`, - active: i === 0, - })); - const payload = JSON.stringify({ type: 'tabs', tabs }); - expect(payload.length).toBeGreaterThan(16384); - - ws.emitMessage(payload); - - expect(connection.snapshot().tabs).toHaveLength(80); - expect(tabsEventCounts).toEqual([80]); - expect(connection.snapshot().tabs[0]).toMatchObject({ tabId: 't0', active: true }); - }); - - it('routes an oversized URL envelope as control data instead of a frame pulse', async () => { - const connection = createAgentBrowserConnection({ - session: 'dormouse.1.default', - streamPort: 1234, - }); + it('decodes each binary frame, and forwards url and a popped-out page as events', async () => { + const { connection } = connect(); const events: unknown[] = []; - connection.subscribe((event) => { - if (event.type === 'url' || event.type === 'frame-pulse') events.push(event); - }); - - await Promise.resolve(); - const url = `data:text/plain,${'x'.repeat(20_000)}`; - const payload = JSON.stringify({ type: 'url', url, timestamp: 1 }); - expect(payload.length).toBeGreaterThan(16384); - - WebSocketMock.instances[0].emitMessage(payload); - - expect(events).toEqual([{ type: 'url', url }]); + connection.subscribe((event) => { if (event.type !== 'debug') events.push(event); }); + await flush(); + const provisional = encodeViewerFrame({ kind: 'provisional', jpeg: new Uint8Array([0xff, 0xd8, 1]), size: { width: 800, height: 600 } }); + const crisp = encodeViewerFrame({ kind: 'crisp', jpeg: new Uint8Array([0xff, 0xd8, 2]) }); + socket().emitMessage(provisional.buffer); + socket().emitMessage(crisp.buffer); + // Not a frame: a short or unknown binary message is ignored. + socket().emitMessage(new Uint8Array([9, 0, 0]).buffer); + socket().emitMessage(JSON.stringify({ type: 'url', url: 'https://example.com/slow' })); + socket().emitMessage(JSON.stringify({ type: 'url' })); + socket().emitMessage(JSON.stringify({ type: 'page', url: 'https://example.com/headed', title: 'Headed' })); + expect(events.map((event) => { + const e = event as { type: string; kind?: string; jpeg?: Uint8Array }; + return e.type === 'frame' ? { ...e, jpeg: [...e.jpeg!] } : e; + })).toEqual([ + { type: 'connection-open', stream: 1234 }, + { type: 'frame', kind: 'provisional', jpeg: [0xff, 0xd8, 1], size: { width: 800, height: 600 } }, + { type: 'frame', kind: 'crisp', jpeg: [0xff, 0xd8, 2] }, + { type: 'url', url: 'https://example.com/slow' }, + { type: 'page', url: 'https://example.com/headed', title: 'Headed' }, + ]); connection.dispose(); }); - it('re-primes after a reconnect so the first identical frame/tabs still forwards', async () => { + it('asks the host for a fresh URL on every reconnect, counting a refusal as a failure', async () => { vi.useFakeTimers(); try { - const connection = createAgentBrowserConnection({ - session: 'dormouse.1.default', - streamPort: 1234, - }); - let pulses = 0; - let tabsEvents = 0; - connection.subscribe((event) => { - if (event.type === 'frame-pulse') pulses += 1; - if (event.type === 'tabs') tabsEvents += 1; - }); - - // Flush connect()'s async getStreamUrl microtask + the mock's queued onopen. + const { connection, viewUrl } = connect(); + const closes: number[] = []; + connection.subscribe((event) => { if (event.type === 'connection-close') closes.push(event.failures); }); await vi.advanceTimersByTimeAsync(0); - const ws = WebSocketMock.instances[0]; - const frame = JSON.stringify({ type: 'frame', data: 'AAAAAAAAAA' }); - const snapshot = JSON.stringify({ - type: 'tabs', - tabs: [{ tabId: 't1', title: 'Dormouse', url: 'https://dormouse.sh/', active: true }], - }); - ws.emitMessage(frame); - ws.emitMessage(snapshot); - expect(pulses).toBe(1); - expect(tabsEvents).toBe(1); - - // Socket drops; the connection resets dedupe state and schedules a reconnect - // (backoff ~2s for the first failure). Advance past it to open a new socket. - ws.onclose?.({ code: 1006, reason: '', wasClean: false } as CloseEvent); + socket().close(); await vi.advanceTimersByTimeAsync(2100); - const ws2 = WebSocketMock.instances[WebSocketMock.instances.length - 1]; - expect(ws2).not.toBe(ws); - - // The reconnected stream re-sends the same frame/tabs; they must re-prime, not - // be swallowed as duplicates of the pre-disconnect state. - ws2.emitMessage(frame); - ws2.emitMessage(snapshot); - expect(pulses).toBe(2); - expect(tabsEvents).toBe(2); - + // Grants are single-use: the reconnect asked again. + expect(viewUrl).toHaveBeenCalledTimes(2); + expect(socket().url).toBe('ws://127.0.0.1:9/view/2'); + // An open resets the count; a refused URL adds to it like a close. + viewUrl.mockRejectedValueOnce(new Error('the browser is being relaunched or closed')); + socket().close(); + await vi.advanceTimersByTimeAsync(2100); + expect(closes).toEqual([1, 1, 2]); + expect(viewUrl).toHaveBeenCalledTimes(3); connection.dispose(); } finally { vi.useRealTimers(); } }); - it('does not force-select an active provisional duplicate-url tab', async () => { - const runCommand = vi.fn(async () => ({ exitCode: 0, stdout: '', stderr: '' })); - const connection = createAgentBrowserConnection({ - session: 'dormouse.1.default', - streamPort: 1234, - runCommand, - }); - - await Promise.resolve(); - const ws = WebSocketMock.instances[0]; - ws.emitMessage(JSON.stringify({ + it('ignores transient empty tabs after a real tab list', async () => { + const { connection } = connect(); + await flush(); + socket().emitMessage(JSON.stringify({ type: 'tabs', tabs: [{ tabId: 't1', title: 'Dormouse', url: 'https://dormouse.sh/', active: true }], })); - ws.emitMessage(JSON.stringify({ - type: 'tabs', - tabs: [ - { tabId: 't1', title: 'Dormouse', url: 'https://dormouse.sh/', active: false }, - { tabId: 't2', title: 'Dormouse', url: 'https://dormouse.sh/', active: true }, - ], - })); + expect(connection.snapshot().tabs).toHaveLength(1); + socket().emitMessage(JSON.stringify({ type: 'tabs', tabs: [] })); + expect(connection.snapshot().tabs).toEqual([ + { tabId: 't1', title: 'Dormouse', url: 'https://dormouse.sh/', active: true }, + ]); + }); - expect(runCommand).not.toHaveBeenCalled(); + it('selects a newly opened tab that is not active, and not one already active on a duplicate URL', async () => { + const selectTab = vi.fn(async () => ({ ok: true })); + const { connection } = connect({ selectTab }); + await flush(); + const tab = (tabId: string, active: boolean, url = 'https://dormouse.sh/') => ({ tabId, title: 'Dormouse', url, active }); + socket().emitMessage(JSON.stringify({ type: 'tabs', tabs: [tab('t1', true)] })); + socket().emitMessage(JSON.stringify({ type: 'tabs', tabs: [tab('t1', false), tab('t2', true)] })); + expect(selectTab).not.toHaveBeenCalled(); + socket().emitMessage(JSON.stringify({ type: 'tabs', tabs: [tab('t1', false), tab('t2', true), tab('t3', false, 'https://other.example/')] })); + expect(selectTab).toHaveBeenCalledExactlyOnceWith('t3'); + connection.dispose(); }); }); diff --git a/lib/src/components/wall/agent-browser-connection.ts b/lib/src/components/wall/agent-browser-connection.ts index cafafc541..826800249 100644 --- a/lib/src/components/wall/agent-browser-connection.ts +++ b/lib/src/components/wall/agent-browser-connection.ts @@ -1,24 +1,12 @@ -import type { BrowserResult } from '../../lib/platform/browser-automation'; +import { decodeViewerFrame, type BrowserResult, type ViewerFrame, type ViewerState } from '../../lib/platform/browser-automation'; import { type AgentBrowserTab, parseAgentBrowserTabs } from '../../lib/agent-browser-tab'; // Re-exported so existing importers keep resolving the tab type/parser from here. export type { AgentBrowserTab }; export { parseAgentBrowserTabs }; -// Stream messages above this size are frames (a base64 JPEG); status/tabs are -// small JSON control messages. Large frames parse only while the consumer asks -// for provisional low-latency hover feedback; the idle hot path stays hash+pulse. -const FRAME_PULSE_THRESHOLD = 16384; const DEBUG_RING_LIMIT = 300; -// Fast non-cryptographic string hash (djb2) for cheap byte-identity checks on -// stream payloads. Used to detect redundant frames/tabs the daemon re-broadcasts. -function djb2(s: string): number { - let h = 5381; - for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) | 0; - return h; -} - export type AgentBrowserConnectionState = 'connecting' | 'open' | 'closed' | 'failed'; export interface AgentBrowserStreamStatus { @@ -28,71 +16,49 @@ export interface AgentBrowserStreamStatus { viewportHeight?: number; } -/** A frame's device dims. Both are required: a partial or zero-sized report can - * neither size the surface nor key a resize, so it degrades to absent metadata - * rather than to a half-filled record (see `frameMetadata`). */ -export interface AgentBrowserFramePulse { - deviceWidth: number; - deviceHeight: number; -} - -/** Read device dims off a stream frame's envelope, in the one place both the - * large (>16KB) and small frame paths share. */ -function frameMetadata(raw: { deviceWidth?: unknown; deviceHeight?: unknown } | undefined): AgentBrowserFramePulse | undefined { - const w = raw?.deviceWidth; - const h = raw?.deviceHeight; - return typeof w === 'number' && w > 0 && typeof h === 'number' && h > 0 - ? { deviceWidth: w, deviceHeight: h } - : undefined; -} - export interface AgentBrowserSnapshot { connection: AgentBrowserConnectionState; session: string; - streamPort: number; + stream: number; tabs: AgentBrowserTab[]; status: AgentBrowserStreamStatus | null; lastError?: string; } export type AgentBrowserConnectionEvent = - | { type: 'connection-open'; port: number } - | { type: 'connection-close'; port: number; failures: number; code: number; reason: string; wasClean: boolean } - | { type: 'connection-error'; port: number } + | { type: 'connection-open'; stream: number } + | { type: 'connection-close'; stream: number; failures: number; code: number; reason: string; wasClean: boolean } + | { type: 'connection-error'; stream: number } | { type: 'status'; status: AgentBrowserStreamStatus } | { type: 'tabs'; tabs: AgentBrowserTab[]; previousTabs: AgentBrowserTab[] } /** The active tab committed a navigation. Fires at commit; the `tabs` * snapshot refreshes only when the driving command completes, which for a - * slow page is the whole load (docs/specs/dor-browser.md → "Agent-Browser - * Connection"). */ + * slow page is the whole load (docs/specs/dor-browser.md → "Viewer + * Socket"). */ | { type: 'url'; url: string } - | { - type: 'frame-pulse'; - metadata?: AgentBrowserFramePulse; - /** CSS-resolution stream JPEG, base64 encoded. Present only when the - * consumer requested a provisional paint. */ - data?: string; - } + /** A popped-out window's page, as its browser reports it. */ + | { type: 'page'; url: string; title: string | null } + /** A frame to paint: provisional (CSS resolution) or crisp. */ + | ({ type: 'frame' } & ViewerFrame) | { type: 'debug'; event: AgentBrowserDebugEvent }; export interface AgentBrowserDebugEvent { ts: number; session: string; - port: number; + stream: number; event: string; data?: unknown; } export interface AgentBrowserConnectionDeps { session: string; - streamPort: number; - getStreamUrl?: (port: number) => Promise; + /** The live browser's stream, as the host's launch or attach named it. */ + stream: number; + /** A fresh single-use URL for the host's viewer socket on it. */ + viewUrl: () => Promise; /** Make `tabId` the active tab. */ selectTab?: (tabId: string) => Promise; canSelectTabs?: () => boolean; - /** Whether the current stream frame's JPEG bytes are useful to the consumer. - * False keeps the idle hot path at hash+pulse without parsing the large JSON. */ - wantFrameData?: () => boolean; log?: (message: string) => void; } @@ -100,6 +66,12 @@ export function createAgentBrowserConnection(deps: AgentBrowserConnectionDeps): return new AgentBrowserConnection(deps); } +/** + * The webview's end of one viewer socket (docs/specs/dor-browser.md → "Viewer + * Socket"): frames arrive binary, state as JSON, both already deduplicated by + * the host; input goes back as JSON. A socket that closes is dialed again with + * a fresh URL, backing off, and the consumer counts the failures. + */ export class AgentBrowserConnection { private readonly listeners = new Set<(event: AgentBrowserConnectionEvent) => void>(); private readonly debugEvents: AgentBrowserDebugEvent[] = []; @@ -111,26 +83,15 @@ export class AgentBrowserConnection { private pendingNewTab: { tabId: string; initialUrl: string; seenAtMs: number } | null = null; private snap: AgentBrowserSnapshot; - // The agent-browser daemon re-broadcasts the current frame and tab list on a - // ~20Hz heartbeat even when nothing changes, so a *static* page would otherwise - // drive ~20 device-resolution screenshots/sec (each a child-process spawn) plus - // ~20 `setTabs` re-renders/sec. We drop byte-identical re-broadcasts here so an - // unchanged page costs nothing downstream (the screenshot loop's own contract: - // "a static page produces no pulses, so no shots and no cost"). `0`/`''` are - // pre-first-message sentinels, and reset on reconnect so a fresh stream always - // re-primes the canvas/tabs. - private lastFrameKey = 0; - private lastTabsSig = ''; - constructor(private readonly deps: AgentBrowserConnectionDeps) { this.snap = { connection: 'connecting', session: deps.session, - streamPort: deps.streamPort, + stream: deps.stream, tabs: [], status: null, }; - this.connect(); + void this.connect(); } subscribe(listener: (event: AgentBrowserConnectionEvent) => void): () => void { @@ -168,7 +129,7 @@ export class AgentBrowserConnection { const item: AgentBrowserDebugEvent = { ts: Date.now(), session: this.deps.session, - port: this.deps.streamPort, + stream: this.deps.stream, event, ...(data !== undefined ? { data } : {}), }; @@ -186,125 +147,68 @@ export class AgentBrowserConnection { } private async connect(): Promise { - let url: string | undefined; + let url: string; try { - url = await this.deps.getStreamUrl?.(this.deps.streamPort); + url = await this.deps.viewUrl(); } catch (err) { - this.debug('stream-url-error', { error: err instanceof Error ? err.message : String(err) }); + if (this.disposed) return; + const reason = err instanceof Error ? err.message : String(err); + this.debug('view-url-error', { error: reason }); + this.closed({ code: 0, reason, wasClean: false }); + return; } if (this.disposed) return; - const wsUrl = url ?? `ws://127.0.0.1:${this.deps.streamPort}`; - this.log(`[ab-panel] connecting stream ${JSON.stringify({ wsPort: this.deps.streamPort, url: wsUrl })}`); - this.debug('connect', { url: wsUrl }); - this.socket = new WebSocket(wsUrl); - this.socket.onopen = () => { + this.log(`[ab-panel] connecting viewer ${JSON.stringify({ stream: this.deps.stream })}`); + this.debug('connect'); + const socket = this.socket = new WebSocket(url); + socket.binaryType = 'arraybuffer'; + socket.onopen = () => { this.failures = 0; this.patch({ connection: 'open' }); - this.log(`[ab-panel] stream open ${JSON.stringify({ wsPort: this.deps.streamPort })}`); + this.log(`[ab-panel] viewer open ${JSON.stringify({ stream: this.deps.stream })}`); this.debug('open'); - this.emit({ type: 'connection-open', port: this.deps.streamPort }); + this.emit({ type: 'connection-open', stream: this.deps.stream }); }; - this.socket.onmessage = (ev) => this.handleMessage(ev.data); - this.socket.onerror = () => { - this.patch({ lastError: 'stream socket error' }); - this.log(`[ab-panel] stream error ${JSON.stringify({ wsPort: this.deps.streamPort })}`); + socket.onmessage = (ev) => this.handleMessage(ev.data); + socket.onerror = () => { + this.patch({ lastError: 'viewer socket error' }); + this.log(`[ab-panel] viewer error ${JSON.stringify({ stream: this.deps.stream })}`); this.debug('error'); - this.emit({ type: 'connection-error', port: this.deps.streamPort }); + this.emit({ type: 'connection-error', stream: this.deps.stream }); }; - this.socket.onclose = (ev) => { + socket.onclose = (ev) => { this.socket = null; - // A reconnected stream re-sends the current frame/tabs; clear the dedupe - // sentinels so that first post-reconnect snapshot always re-primes the - // canvas and tab list rather than being dropped as a "duplicate". - this.lastFrameKey = 0; - this.lastTabsSig = ''; if (this.disposed) return; - this.failures += 1; - if (this.failures >= 3) this.patch({ connection: 'failed' }); - else this.patch({ connection: 'closed' }); - const data = { wsPort: this.deps.streamPort, failures: this.failures, code: ev.code, reason: ev.reason, wasClean: ev.wasClean }; - this.log(`[ab-panel] stream close ${JSON.stringify(data)}`); - this.debug('close', data); - this.emit({ type: 'connection-close', port: this.deps.streamPort, failures: this.failures, code: ev.code, reason: ev.reason, wasClean: ev.wasClean }); - this.retryTimer = setTimeout(() => this.connect(), Math.min(1000 * 2 ** this.failures, 10000)); + this.closed({ code: ev.code, reason: ev.reason, wasClean: ev.wasClean }); }; } - // Drop a frame whose pixels (and device dims) match the previous one — the - // daemon's heartbeat re-broadcasts an unchanged page, and redrawing it is pure - // cost. Returns true when the frame is a duplicate the caller should ignore. - // The dims are mixed into the hash rather than into the payload string: a frame - // is ~100KB of base64 at ~20Hz, so a `${data}@WxH` key would copy all of it just - // to hash it and throw it away. - private isDuplicateFrame(payload: string, metadata?: AgentBrowserFramePulse): boolean { - let key = djb2(payload) ^ (payload.length | 0); - if (metadata) key = (key ^ Math.imul(metadata.deviceWidth, 31) ^ metadata.deviceHeight) | 0; - if (key === this.lastFrameKey) return true; - this.lastFrameKey = key; - return false; - } - - /** The single frame-emission point. `withData` carries the base64 body to the - * consumer for a provisional paint; without it the event is a bare pulse that - * only paces the crisp screenshot loop. */ - private emitFrame(data: string, metadata: AgentBrowserFramePulse | undefined, withData: boolean): void { - if (this.isDuplicateFrame(data, metadata)) return; - this.emit({ type: 'frame-pulse', metadata, ...(withData ? { data } : {}) }); + /** The socket closed, or could not be asked for: count it, and dial again. */ + private closed({ code, reason, wasClean }: { code: number; reason: string; wasClean: boolean }): void { + this.failures += 1; + this.patch({ connection: this.failures >= 3 ? 'failed' : 'closed' }); + const data = { stream: this.deps.stream, failures: this.failures, code, reason, wasClean }; + this.log(`[ab-panel] viewer close ${JSON.stringify(data)}`); + this.debug('close', data); + this.emit({ type: 'connection-close', ...data }); + if (this.disposed) return; + this.retryTimer = setTimeout(() => void this.connect(), Math.min(1000 * 2 ** this.failures, 10000)); } private handleMessage(raw: unknown): void { - if (typeof raw !== 'string') return; - if (raw.length > FRAME_PULSE_THRESHOLD) { - // Size alone can't discriminate a frame from a control message: a `tabs` - // snapshot or URL with enough text crosses the threshold too, and routing - // it as a frame would silently drop the update. A frame's bulk is a base64 - // JPEG body, whose alphabet contains no `"` or `:`, so these compact type - // substrings cannot occur inside a real frame — they are zero-false-positive - // markers for oversized control messages. Only those pay a parse; frames - // keep the hash+pulse fast path untouched. - if ( - raw.includes('"type":"tabs"') - || raw.includes('"type":"status"') - || raw.includes('"type":"url"') - ) { - this.dispatchControl(raw); - return; - } - // `wantFrameData` gates only the parse itself: on the large path it is the - // difference between JSON.parsing ~100KB and hashing it, which is the whole - // point of the threshold. Emission policy lives in emitFrame. Asked here (and - // below) rather than once up top so `status`/`tabs` never pay for it. - if (this.deps.wantFrameData?.()) { - try { - const msg = JSON.parse(raw) as { type?: unknown; data?: unknown; metadata?: { deviceWidth?: unknown; deviceHeight?: unknown } }; - if (msg.type === 'frame' && typeof msg.data === 'string') { - this.emitFrame(msg.data, frameMetadata(msg.metadata), true); - return; - } - } catch { - // Older/raw daemons may send the JPEG body without a JSON envelope. It - // still drives the crisp capture, just without a provisional paint. - } - } - if (this.isDuplicateFrame(raw)) return; - this.emit({ type: 'frame-pulse' }); + if (raw instanceof ArrayBuffer) { + const frame = decodeViewerFrame(raw); + if (frame) this.emit({ type: 'frame', ...frame }); return; } - this.dispatchControl(raw); - } - - // Parse a JSON envelope and route it to the frame/status/tabs handlers. Shared - // by the small-message path and the oversized-control-message path above. - private dispatchControl(raw: string): void { - let msg: any; + if (typeof raw !== 'string') return; + let msg: ViewerState; try { - msg = JSON.parse(raw); + msg = JSON.parse(raw) as ViewerState; } catch { return; } - if (msg.type === 'frame' && typeof msg.data === 'string') { - this.emitFrame(msg.data, frameMetadata(msg.metadata), this.deps.wantFrameData?.() ?? false); - } else if (msg.type === 'status') { + if (msg.type === 'status') { const status: AgentBrowserStreamStatus = { connected: msg.connected === true, screencasting: msg.screencasting === true, @@ -318,28 +222,23 @@ export class AgentBrowserConnection { } else if (msg.type === 'url' && typeof msg.url === 'string') { this.debug('url', { url: msg.url }); this.emit({ type: 'url', url: msg.url }); + } else if (msg.type === 'page' && typeof msg.url === 'string') { + this.debug('page', { url: msg.url }); + this.emit({ type: 'page', url: msg.url, title: typeof msg.title === 'string' ? msg.title : null }); } } private handleTabs(next: AgentBrowserTab[]): void { const previousTabs = this.snap.tabs; if (next.length === 0 && previousTabs.length > 0) { - this.log(`[ab-panel] empty tabs snapshot ignored ${JSON.stringify({ w: this.deps.streamPort, previous: previousTabs.length })}`); + this.log(`[ab-panel] empty tabs snapshot ignored ${JSON.stringify({ stream: this.deps.stream, previous: previousTabs.length })}`); this.debug('tabs-empty-ignored', { previous: previousTabs.length }); return; } - // Drop an identical tab-snapshot re-broadcast (same ids, active flags, urls, - // titles): it would otherwise re-run tab-selection and force a `setTabs` - // re-render every heartbeat. A real change (new/closed tab, navigation, - // focus, title) alters the signature and falls through. - const fullSig = JSON.stringify(next.map((t) => `${t.tabId}:${t.active ? 'A' : '-'}:${t.url}:${t.title ?? ''}`)); - if (fullSig === this.lastTabsSig) return; - this.lastTabsSig = fullSig; - this.maybeSelectNewTab(next, previousTabs); this.knownTabIds = new Set(next.map((t) => t.tabId)); - const sig = JSON.stringify({ w: this.deps.streamPort, t: next.map((t) => `${t.tabId}:${t.active ? 'A' : '-'}:${t.url}`) }); + const sig = JSON.stringify({ stream: this.deps.stream, t: next.map((t) => `${t.tabId}:${t.active ? 'A' : '-'}:${t.url}`) }); this.log(`[ab-panel] tabs msg ${sig}`); this.debug('tabs', { tabs: next }); this.patch({ tabs: next }); diff --git a/lib/src/components/wall/agent-browser-input.ts b/lib/src/components/wall/agent-browser-input.ts index 2bc6231f9..9273b2900 100644 --- a/lib/src/components/wall/agent-browser-input.ts +++ b/lib/src/components/wall/agent-browser-input.ts @@ -72,20 +72,3 @@ export const MOUSE_BUTTON_MASKS: Record = { 0: 1, 1: 4, 2: 2 }; export function modifiers(e: { altKey: boolean; ctrlKey: boolean; metaKey: boolean; shiftKey: boolean }): number { return (e.altKey ? 1 : 0) | (e.ctrlKey ? 2 : 0) | (e.metaKey ? 4 : 0) | (e.shiftKey ? 8 : 0); } - -/** A paste as agent-browser stream messages: its stream takes only key and - * mouse events, so a key down and up per character, a newline as Enter. */ -export function keyPairTextInputs(text: string): Record[] { - const messages: Record[] = []; - for (const ch of text) { - if (ch === '\r') continue; - if (ch === '\n') { - messages.push({ type: 'input_keyboard', eventType: 'keyDown', key: 'Enter', code: 'Enter', text: '\r', windowsVirtualKeyCode: 13, modifiers: 0 }); - messages.push({ type: 'input_keyboard', eventType: 'keyUp', key: 'Enter', code: 'Enter', text: '', windowsVirtualKeyCode: 13, modifiers: 0 }); - } else { - messages.push({ type: 'input_keyboard', eventType: 'keyDown', key: ch, code: '', text: ch, windowsVirtualKeyCode: 0, modifiers: 0 }); - messages.push({ type: 'input_keyboard', eventType: 'keyUp', key: ch, code: '', text: '', windowsVirtualKeyCode: 0, modifiers: 0 }); - } - } - return messages; -} diff --git a/lib/src/components/wall/agent-browser-screenshot-loop.test.ts b/lib/src/components/wall/agent-browser-screenshot-loop.test.ts deleted file mode 100644 index 3fc5d2b37..000000000 --- a/lib/src/components/wall/agent-browser-screenshot-loop.test.ts +++ /dev/null @@ -1,298 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import type { BrowserResult } from '../../lib/platform/browser-automation'; -import { createScreenshotLoop } from './agent-browser-screenshot-loop'; - -// Drive the loop directly (no controller/React): it owns backpressure, byte -// dedup, and the draw-generation key. The host capture is a fake the loop's -// `capture` calls; createImageBitmap is stubbed (jsdom has none). - -type HostScreenshot = (opts: { format: 'jpeg'; quality: number }) => Promise; -let hostScreenshot: HostScreenshot | undefined; -function setScreenshot(fn: HostScreenshot): void { - hostScreenshot = fn; -} -const capture = (opts: { format: 'jpeg'; quality: number }) => hostScreenshot?.(opts) ?? null; - -beforeEach(() => { - vi.useFakeTimers(); - vi.stubGlobal('createImageBitmap', vi.fn(async () => ({ width: 4, height: 4, close: vi.fn() } as unknown as ImageBitmap))); -}); - -afterEach(() => { - vi.useRealTimers(); - vi.restoreAllMocks(); - hostScreenshot = undefined; -}); - -describe('screenshot loop byte dedup', () => { - it('draws once for byte-identical captures (still captures, but skips the redundant decode+draw)', async () => { - const bytes = new Uint8Array([1, 2, 3, 4, 5]); - const screenshot = vi.fn(async () => ({ ok: true as const, bytes, mime: 'image/jpeg' })); - setScreenshot(screenshot); - const draw = vi.fn(); - const loop = createScreenshotLoop({ - capture, - isCapable: () => true, - draw, - }); - - loop.pulse(); - await vi.advanceTimersByTimeAsync(300); - expect(screenshot).toHaveBeenCalledTimes(1); - expect(draw).toHaveBeenCalledTimes(1); - - // A second pulse still captures, but the bytes match what's on the canvas, so - // the decode+draw is skipped. - loop.pulse(); - await vi.advanceTimersByTimeAsync(300); - expect(screenshot).toHaveBeenCalledTimes(2); - expect(draw).toHaveBeenCalledTimes(1); - - loop.dispose(); - }); - - it('repaints byte-identical captures after the draw generation bumps', async () => { - const bytes = new Uint8Array([9, 8, 7]); - const screenshot = vi.fn(async () => ({ ok: true as const, bytes, mime: 'image/jpeg' })); - setScreenshot(screenshot); - const draw = vi.fn(); - let generation = 0; - const loop = createScreenshotLoop({ - capture, - isCapable: () => true, - draw, - getDrawGeneration: () => generation, - }); - - loop.pulse(); - await vi.advanceTimersByTimeAsync(300); - expect(draw).toHaveBeenCalledTimes(1); - - // Same bytes, but a fresh canvas (generation bumped on re-attach) must repaint. - generation = 1; - loop.pulse(); - await vi.advanceTimersByTimeAsync(300); - expect(draw).toHaveBeenCalledTimes(2); - - loop.dispose(); - }); -}); - -describe('screenshot loop backpressure', () => { - it('retries after a provisional supersedes the only capture in flight', async () => { - // A single pointer move over an otherwise static page: one stream frame paints - // provisionally, its pulse is consumed by the capture that frame started, and - // the stream then goes quiet. Dropping the stale result without leaving work - // pending would strand the pane on the blurry frame until the page changes. - let release: ((res: BrowserResult) => void) | undefined; - const screenshot = vi.fn(() => new Promise((r) => { release = r; })); - setScreenshot(screenshot as unknown as HostScreenshot); - const draw = vi.fn(); - let provisionalGeneration = 0; - const loop = createScreenshotLoop({ - capture, - isCapable: () => true, - draw, - getProvisionalGeneration: () => provisionalGeneration, - }); - - loop.pulse(); - await vi.advanceTimersByTimeAsync(300); - expect(screenshot).toHaveBeenCalledTimes(1); - - // The provisional lands mid-capture; no further pulses ever arrive. - provisionalGeneration += 1; - release?.({ ok: true, bytes: new Uint8Array([5, 5, 5]), mime: 'image/jpeg' }); - await vi.advanceTimersByTimeAsync(0); - expect(draw).not.toHaveBeenCalled(); - - // A retry has to come from the loop itself: nothing is left to pulse it. - await vi.advanceTimersByTimeAsync(1000); - expect(screenshot).toHaveBeenCalledTimes(2); - - // That retry is not superseded, so the crisp frame reaches the canvas. - release?.({ ok: true, bytes: new Uint8Array([5, 5, 5]), mime: 'image/jpeg' }); - await vi.advanceTimersByTimeAsync(300); - expect(draw).toHaveBeenCalledTimes(1); - - loop.dispose(); - }); - - it('spends no captures while provisional paints are still landing', async () => { - const screenshot = vi.fn(async () => ({ ok: true as const, bytes: new Uint8Array([1, 2]), mime: 'image/jpeg' })); - setScreenshot(screenshot); - const draw = vi.fn(); - let provisionalGeneration = 0; - let provisionalDeadline = 0; - const loop = createScreenshotLoop({ - capture, - isCapable: () => true, - draw, - getProvisionalGeneration: () => provisionalGeneration, - getProvisionalDeadline: () => provisionalDeadline, - }); - - // Sustained hover: a stream frame every 50ms paints provisionally, pushes the - // 250ms window out, and pulses the loop. Every capture started in here would - // resolve after a newer provisional had already painted. - for (let i = 0; i < 12; i++) { - provisionalDeadline = performance.now() + 250; - provisionalGeneration += 1; - loop.pulse(); - await vi.advanceTimersByTimeAsync(50); - } - expect(screenshot).not.toHaveBeenCalled(); - - // Pointer stops: the window lapses and exactly one settled capture sharpens the - // resting frame. - await vi.advanceTimersByTimeAsync(600); - expect(screenshot).toHaveBeenCalledTimes(1); - expect(draw).toHaveBeenCalledTimes(1); - - loop.dispose(); - }); - - it('retries a crisp decode superseded by a provisional paint', async () => { - let resolveBitmap: ((bitmap: ImageBitmap) => void) | undefined; - vi.stubGlobal('createImageBitmap', vi.fn(() => new Promise((resolve) => { resolveBitmap = resolve; }))); - setScreenshot(async () => ({ ok: true, bytes: new Uint8Array([7, 8, 9]), mime: 'image/jpeg' })); - const draw = vi.fn(); - let provisionalGeneration = 0; - const loop = createScreenshotLoop({ - capture, - isCapable: () => true, - draw, - getProvisionalGeneration: () => provisionalGeneration, - }); - - loop.pulse(); - await vi.advanceTimersByTimeAsync(300); - expect(createImageBitmap).toHaveBeenCalledTimes(1); - - // The host bytes returned first, but their bitmap decode is still pending - // when a newer low-latency frame becomes visible. - provisionalGeneration += 1; - resolveBitmap?.({ width: 4, height: 4, close: vi.fn() } as unknown as ImageBitmap); - await vi.advanceTimersByTimeAsync(0); - expect(draw).not.toHaveBeenCalled(); - - // No new pulse is needed to sharpen the now-static provisional frame. - await vi.advanceTimersByTimeAsync(300); - expect(createImageBitmap).toHaveBeenCalledTimes(2); - resolveBitmap?.({ width: 4, height: 4, close: vi.fn() } as unknown as ImageBitmap); - await vi.advanceTimersByTimeAsync(0); - expect(draw).toHaveBeenCalledTimes(1); - - loop.dispose(); - }); - - it('coalesces pulses during an in-flight capture into a single follow-up', async () => { - // A capture that stays in flight until we resolve it, so we can pulse during it. - const releases: Array<(res: BrowserResult) => void> = []; - const screenshot = vi.fn(() => new Promise((resolve) => { releases.push(resolve); })); - setScreenshot(screenshot as unknown as HostScreenshot); - const draw = vi.fn(); - let provisionalGeneration = 0; - const loop = createScreenshotLoop({ - capture, - isCapable: () => true, - draw, - getProvisionalGeneration: () => provisionalGeneration, - }); - - loop.pulse(); - await vi.advanceTimersByTimeAsync(300); - expect(screenshot).toHaveBeenCalledTimes(1); - - // Several pulses arrive while capture #1 is still in flight — they must - // collapse to ONE follow-up, not one capture each. - loop.pulse(); - loop.pulse(); - loop.pulse(); - expect(screenshot).toHaveBeenCalledTimes(1); - - // A newer provisional frame paints while capture #1 is in flight. The crisp - // result is stale and must NOT overwrite that responsive frame. - provisionalGeneration += 1; - releases[0]?.({ ok: true, bytes: new Uint8Array([42]), mime: 'image/jpeg' }); - await vi.advanceTimersByTimeAsync(300); - expect(draw).not.toHaveBeenCalled(); - // Exactly one follow-up capture ran for the three coalesced pulses. - expect(screenshot).toHaveBeenCalledTimes(2); - - // The coalesced follow-up is current and becomes the crisp resting frame. - releases[1]?.({ ok: true, bytes: new Uint8Array([43]), mime: 'image/jpeg' }); - await vi.advanceTimersByTimeAsync(300); - expect(draw).toHaveBeenCalledTimes(1); - - loop.dispose(); - }); -}); - -describe('screenshot loop behind a blocking command', () => { - it('reports the capture overdue, never re-issues it, and draws it when it lands', async () => { - const releases: Array<(res: BrowserResult) => void> = []; - const screenshot = vi.fn(() => new Promise((resolve) => { releases.push(resolve); })); - setScreenshot(screenshot as unknown as HostScreenshot); - const draw = vi.fn(); - const loop = createScreenshotLoop({ - capture, - isCapable: () => true, - draw, - }); - - loop.pulse(); - await vi.advanceTimersByTimeAsync(300); - expect(screenshot).toHaveBeenCalledTimes(1); - expect(loop.captureOverdue()).toBe(false); - await vi.advanceTimersByTimeAsync(500); - expect(loop.captureOverdue()).toBe(true); - - // The page keeps changing through the rest of the load. - for (let i = 0; i < 20; i++) { - loop.pulse(); - await vi.advanceTimersByTimeAsync(1000); - } - expect(screenshot).toHaveBeenCalledTimes(1); - - // `open` returns: the capture is drawn, and since when in the wait it was - // taken is unknown, the wait's pulses are owed one more. - releases[0]({ ok: true, bytes: new Uint8Array([1]), mime: 'image/jpeg' }); - await vi.advanceTimersByTimeAsync(10); - expect(loop.captureOverdue()).toBe(false); - expect(draw).toHaveBeenCalledTimes(1); - expect(screenshot).toHaveBeenCalledTimes(2); - - // The wait timed the page load, not a capture, so the next slow load is - // overdue just as soon. - await vi.advanceTimersByTimeAsync(500); - expect(loop.captureOverdue()).toBe(true); - - loop.dispose(); - }); - - it('still owes a shot for the wait when the overdue capture fails', async () => { - const releases: Array<(res: BrowserResult) => void> = []; - const screenshot = vi.fn(() => new Promise((resolve) => { releases.push(resolve); })); - setScreenshot(screenshot as unknown as HostScreenshot); - vi.spyOn(console, 'warn').mockImplementation(() => {}); - const loop = createScreenshotLoop({ capture, isCapable: () => true, draw: vi.fn() }); - - loop.pulse(); - await vi.advanceTimersByTimeAsync(300); - for (let i = 0; i < 5; i++) { - loop.pulse(); - await vi.advanceTimersByTimeAsync(1000); - } - // An adapter gives up on the reply (VS Code answers its timeout `{ ok: false }`): - // nothing was drawn, so the pane is still on the stream's frames. - releases[0]({ ok: false, error: 'agent-browser screenshot timed out' }); - await vi.advanceTimersByTimeAsync(10); - expect(screenshot).toHaveBeenCalledTimes(2); - - loop.dispose(); - }); -}); diff --git a/lib/src/components/wall/agent-browser-screenshot-loop.ts b/lib/src/components/wall/agent-browser-screenshot-loop.ts deleted file mode 100644 index c1fae3220..000000000 --- a/lib/src/components/wall/agent-browser-screenshot-loop.ts +++ /dev/null @@ -1,216 +0,0 @@ -import type { BrowserResult } from '../../lib/platform/browser-automation'; - -// Fast non-cryptographic hash (djb2, xor length) over the raw screenshot bytes — -// the byte analogue of the connection's frame dedup. A static page the daemon -// keeps re-pulsing produces byte-identical captures, so this lets us skip the -// decode+draw entirely. -function djb2Bytes(bytes: Uint8Array): number { - let h = 5381; - for (let i = 0; i < bytes.length; i++) h = ((h << 5) + h + bytes[i]) | 0; - return (h ^ (bytes.length | 0)) | 0; -} - -export interface ScreenshotLoopDeps { - /** Start one host capture, or null when none can be taken now — the - * controller's daemon gate decides. */ - capture: (opts: { format: 'jpeg'; quality: number }) => Promise | null; - isCapable: () => boolean; - draw: (bitmap: ImageBitmap) => void; - /** A monotonic draw-target generation. Included in the byte-dedup key so a - * fresh canvas (bumped on re-attach) still repaints even when the bytes match - * the last displayed frame. Absent ⇒ generation 0 (dedup on bytes alone). */ - getDrawGeneration?: () => number; - /** Monotonic count of provisional stream paints. A crisp capture whose start - * predates a newer provisional paint is stale and must not overwrite it. */ - getProvisionalGeneration?: () => number; - /** `performance.now()` timestamp through which provisional stream paints are - * still expected (continued input pushes it out). While it is in the - * future, a capture is certain to be superseded mid-flight and dropped, so the - * loop waits it out and takes one settled shot at the end instead of burning a - * host round trip per pulse. Absent ⇒ no window; capture on the pacing rule - * alone. */ - getProvisionalDeadline?: () => number; - /** Optional gated logger for the high-rate per-shot start/done diagnostics; - * absent ⇒ silent. Warnings (stall/failure/error) stay unconditional. */ - log?: (message: string) => void; -} - -export interface ScreenshotLoop { - /** A "page changed" signal — schedule a fresh shot (coalesced + throttled). */ - pulse(): void; - /** Whether the capture in flight is overdue (see `createScreenshotLoop`). */ - captureOverdue(): boolean; - dispose(): void; -} - -const OVERDUE_FLOOR_MS = 400; -// A capture this slow is flagged `stalled` in its done/error log. -const STALL_WARNING_MS = 8000; - -/** - * Display crisp HiDPI screenshots, paced by stream-frame "pulses". The - * screencast is CSS-resolution only (Chromium's `Page.startScreencast` ignores - * deviceScaleFactor), so the panel paints it provisionally for responsiveness - * and uses this loop to replace it with device-resolution host screenshots. - * - * Backpressure (we only ever want the latest, and must slow down if capture - * can't keep up): at most one screenshot in flight; a pulse during a shot sets - * `dirty` (no queue — bursts collapse to one follow-up, latest wins); the next - * shot won't start until at least one shot-duration (adaptive EWMA) has passed - * since the last one began, so a slow capture self-throttles. A static page - * produces no pulses, so no shots and no cost. - * - * While the panel is painting provisional stream frames, a shot can't win the race - * anyway (`getProvisionalDeadline`), so the loop waits the window out and takes one - * settled shot at its end rather than one per pulse. Whatever the loop drops — a - * deferred shot or one superseded mid-flight — it stays `dirty`, because the canvas - * is left on a CSS-resolution frame and only a later crisp shot can sharpen it. - * - * A shot out past twice the usual round trip (at least 400ms) is overdue: queued - * behind a blocking daemon command, such as an `open` waiting on its page load, - * or slow itself, while the panel paints the stream instead. It is never - * re-issued — a second one would only queue behind it, and every host adapter - * bounds the wait. When in the round trip its image was taken is unknown, so a - * pulse during it leaves one shot owed like any other; its round trip is - * clamped before it enters the pacing average. - */ -export function createScreenshotLoop(deps: ScreenshotLoopDeps): ScreenshotLoop { - let inFlight = false; - let dirty = false; - let seq = 0; - // The newest shot drawn: an older one never paints over it. - let drawnSeq = 0; - let lastStart = 0; - let avgMs = 120; - let timer: ReturnType | undefined; - let disposed = false; - // The `bytes:generation` of the frame currently on the canvas. Skips decoding a - // capture we've already displayed onto this same draw target. - let lastDrawnKey: string | null = null; - - const overdueAfterMs = () => Math.max(2 * avgMs, OVERDUE_FLOOR_MS); - - const display = (bytes: Uint8Array, mime: string, mySeq: number, provisionalAtStart: number) => { - // Identical bytes drawn onto the same canvas generation → nothing to repaint; - // skip the decode. A fresh canvas bumps the generation, so the same bytes - // still draw after a re-attach. - const gen = deps.getDrawGeneration?.() ?? 0; - const key = `${djb2Bytes(bytes)}:${gen}`; - if (key === lastDrawnKey) return; - // The bytes are ArrayBuffer-backed (readFile → structured clone); narrow off - // the ArrayBufferLike default so they satisfy BlobPart. - const part = bytes as Uint8Array; - createImageBitmap(new Blob([part], { type: mime })).then((bitmap) => { - // A newer shot drew first (or we're gone) — drop this one. A newer shot - // merely started is no reason to withhold this image, which is still - // newer than what is on the canvas. - if (disposed || mySeq < drawnSeq) { - bitmap.close(); - return; - } - if ((deps.getProvisionalGeneration?.() ?? 0) !== provisionalAtStart) { - bitmap.close(); - // A provisional can land during decode as well as during the host - // capture. The stream may now be quiet; keep the sharpening shot owed. - dirty = true; - schedule(); - return; - } - // Record only once actually drawn, so a shot dropped by the seq guard never - // suppresses a later identical capture that must still paint. - lastDrawnKey = key; - drawnSeq = mySeq; - deps.draw(bitmap); - }).catch((err) => console.warn('[agent-browser] screenshot decode failed:', err)); - }; - - const take = () => { - const capture = deps.capture({ format: 'jpeg', quality: 85 }); - if (!capture) return; - inFlight = true; - dirty = false; - const mySeq = ++seq; - const provisionalAtStart = deps.getProvisionalGeneration?.() ?? 0; - lastStart = performance.now(); - deps.log?.(`[agent-browser] screenshot start ${JSON.stringify({ seq: mySeq })}`); - const stalled = () => performance.now() - lastStart > STALL_WARNING_MS; - capture.then((res) => { - const elapsedMs = performance.now() - lastStart; - deps.log?.(`[agent-browser] screenshot done ${JSON.stringify({ seq: mySeq, ok: res.ok, bytes: res.bytes?.byteLength ?? 0, elapsedMs: Math.round(elapsedMs), stalled: stalled(), dirty })}`); - avgMs = avgMs * 0.6 + Math.min(elapsedMs, overdueAfterMs()) * 0.4; - inFlight = false; - // A provisional stream frame painted during this capture is visibly newer. - // Do not let the stale crisp result overwrite it. Mere `dirty` pulses do not - // suppress drawing — an idle animated page still needs periodic crisp frames - // even without input. - const stale = (deps.getProvisionalGeneration?.() ?? 0) !== provisionalAtStart; - if (res.ok && res.bytes) { - if (stale) { - // Dropping the result leaves CSS-resolution pixels on the canvas, and the - // stream owes us nothing more: a single pointer move over a static page - // pulses once, and that pulse was consumed by this very capture. Keep the - // work pending so the resting frame still sharpens. - dirty = true; - } else { - display(res.bytes, res.mime || 'image/jpeg', mySeq, provisionalAtStart); - } - } else { - console.warn('[agent-browser] screenshot failed:', res.error ?? '(no data)'); - } - if (dirty) schedule(); - }).catch((err) => { - console.warn(`[agent-browser] screenshot error ${JSON.stringify({ seq: mySeq, stalled: stalled() })}:`, err); - inFlight = false; - if (dirty) schedule(); - }); - }; - - const schedule = () => { - if (disposed) return; - if (inFlight) { - dirty = true; - return; - } - const now = performance.now(); - // Two independent reasons to wait; the longer wins. - // - // Pacing: space shots to ~1.5× the measured capture time since the last START. - // The slower capture gets, the more we back off (≈⅔ duty cycle), and the 50ms - // floor stops a fast/cached/error return from spinning a tight loop. - const paceWait = lastStart + Math.max(50, avgMs * 1.5) - now; - // Provisional window: while stream paints are still landing (~20Hz) any capture - // (~120ms) is superseded before it resolves, so every one of them is a host - // round trip whose bytes we throw away. The dropped shots were never drawing - // anything, so skipping them costs no pixels — the frame that actually lands is - // the first one started after the last provisional paint either way. - const provisionalWait = (deps.getProvisionalDeadline?.() ?? 0) - now; - const wait = Math.max(paceWait, provisionalWait); - if (wait > 0) { - dirty = true; - if (timer === undefined) { - timer = setTimeout(() => { - timer = undefined; - // Re-enter `schedule`, not `take`: continued input pushes the - // provisional window past this timer, and re-checking re-arms for the new - // end instead of spending a shot that window would supersede. - if (dirty && !inFlight) schedule(); - }, wait); - } - return; - } - take(); - }; - - return { - pulse: () => { - if (disposed || !deps.isCapable()) return; - dirty = true; - schedule(); - }, - captureOverdue: () => inFlight && performance.now() - lastStart > overdueAfterMs(), - dispose: () => { - disposed = true; - if (timer !== undefined) clearTimeout(timer); - }, - }; -} diff --git a/lib/src/components/wall/agent-browser-surface-controller.test.ts b/lib/src/components/wall/agent-browser-surface-controller.test.ts index 784ac2d45..214a4e7c4 100644 --- a/lib/src/components/wall/agent-browser-surface-controller.test.ts +++ b/lib/src/components/wall/agent-browser-surface-controller.test.ts @@ -4,16 +4,15 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { FakePtyAdapter, setPlatform } from '../../lib/platform'; import type { PlatformAdapter } from '../../lib/platform/types'; -import { PLAYWRIGHT_TEXT_INPUT_MAX, type BrowserRequest, type BrowserResult } from '../../lib/platform/browser-automation'; +import { VIEWER_TEXT_INPUT_MAX, encodeViewerFrame, type BrowserRequest, type BrowserResult } from '../../lib/platform/browser-automation'; import { getAgentBrowserScreenController } from './agent-browser-screen'; import { forgetLaunchBinaryPaths, launchBinaryPath, rememberLaunchBinaryPath } from './browser-automation'; import { HIDDEN_PARK_DELAY_MS, - PROVISIONAL_INPUT_WINDOW_MS, acquireAgentBrowserSurfaceController, closeBrowserSurface, disposeAgentBrowserSurfaceController, - handOverBrowserPort, + handOverBrowserStream, whenBrowserLaunched, type AgentBrowserSurfaceController, type AgentBrowserSurfaceParams, @@ -50,6 +49,7 @@ class WebSocketMock { queueMicrotask(() => this.onopen?.(new Event('open'))); } + binaryType = 'blob'; send(data: string) { this.sent.push(data); } close() { @@ -57,7 +57,7 @@ class WebSocketMock { this.onclose?.(new CloseEvent('close')); } - emitMessage(data: string) { + emitMessage(data: string | ArrayBuffer) { this.onmessage?.({ data } as MessageEvent); } } @@ -84,6 +84,11 @@ function makeSink(): AgentBrowserViewSink & { }; } +/** A frame the host sends over a viewer socket. */ +function emitFrame(socket: WebSocketMock | undefined, kind: 'provisional' | 'crisp' = 'provisional', n = 1, size?: { width: number; height: number }) { + socket?.emitMessage(encodeViewerFrame({ kind, jpeg: new Uint8Array([0xff, 0xd8, n]), ...(size ? { size } : {}) }).buffer); +} + /** A controller whose first start streams from `port`, as `dor ab` hands * one over. */ function withPort(id: string, params: AgentBrowserSurfaceParams, port: number): AgentBrowserSurfaceController { @@ -119,7 +124,8 @@ beforeEach(() => { vi.stubGlobal('ResizeObserver', ResizeObserverMock); WebSocketMock.instances = []; WebSocketMock.failPorts = new Set(); - setPlatform(new FakePtyAdapter()); + // A host that grants every viewer socket, at the stream's number as its port. + installBrowserHost(); }); afterEach(() => { @@ -189,138 +195,91 @@ describe('view attachment', () => { }); }); -describe('provisional stream paint', () => { - type Shot = { ok: true; bytes: Uint8Array; mime: string }; - /** An attached pane on `sess:4321` with the clock, the host capture and the - * canvas under the test's control. Captures never answer unless `screenshot` says. */ - async function paintFixture( - screenshot: () => Promise = () => new Promise(() => {}), - clipboardText?: string, - ) { - const clock = { now: 1000 }; - vi.spyOn(performance, 'now').mockImplementation(() => clock.now); - const host = installBrowserHost({ screenshot }); +describe('painting', () => { + /** An attached pane viewing `sess:4321`, its decodes answered by the test, + * in order, and its canvas recording what is drawn. */ + async function paintFixture(clipboardText?: string) { + const host = installBrowserHost(); const platform: PlatformAdapter = host.platform; if (clipboardText !== undefined) platform.readClipboardText = vi.fn(async () => clipboardText); - const bitmap = { width: 40, height: 30, close: vi.fn() } as unknown as ImageBitmap; - vi.stubGlobal('createImageBitmap', vi.fn(async () => bitmap)); + const decodes: { bytes: number; resolve: (bitmap: ImageBitmap) => void }[] = []; + vi.stubGlobal('createImageBitmap', vi.fn(async (blob: Blob) => { + const bytes = new Uint8Array(await blob.arrayBuffer()); + return new Promise((resolve) => decodes.push({ bytes: bytes[2], resolve })); + })); const sink = makeSink(); const drawImage = vi.fn(); sink.canvas.getContext = vi.fn(() => ({ drawImage })) as unknown as typeof sink.canvas.getContext; const controller = withPort('id', { session: 'sess' }, 4321); controller.attachView(sink); await flushMicrotasks(); - const frame = async (label: string) => { - streamSocket(4321)?.emitMessage(JSON.stringify({ type: 'frame', data: btoa(label), metadata: { deviceWidth: 40, deviceHeight: 30 } })); + const bitmap = (width: number, height: number) => ({ width, height, close: vi.fn() }) as unknown as ImageBitmap; + /** Decode the next waiting frame as a `width`×`height` image. */ + const decode = async (width: number, height: number) => { + await vi.waitFor(() => expect(decodes.length).toBeGreaterThan(0)); + decodes.shift()!.resolve(bitmap(width, height)); await flushMicrotasks(); }; - const decodes = () => vi.mocked(createImageBitmap).mock.calls.length; - return { clock, host, platform, sink, bitmap, drawImage, controller, frame, decodes }; + return { host, sink, drawImage, controller, decodes, decode }; } - it('draws the native stream frame before the crisp screenshot resolves', async () => { - const { clock, host, sink, bitmap, drawImage, controller, frame, decodes } = await paintFixture(); + it('draws each frame over the whole canvas, sized by the crisp one so a provisional one reallocates nothing', async () => { + const { sink, drawImage, controller, decode } = await paintFixture(); + const sizes: string[] = []; + const record = () => sizes.push(`${sink.canvas.width}x${sink.canvas.height}`); - controller.send({ type: 'input_mouse', eventType: 'mouseMoved', x: 1, y: 1 }); - await frame('low-latency-frame'); - expect(host.requests('screenshot').length).toBeGreaterThan(0); - expect(drawImage).toHaveBeenCalledWith(bitmap, 0, 0); - expect(sink.canvas.width).toBe(40); - expect(sink.canvas.height).toBe(30); + emitFrame(streamSocket(4321), 'provisional', 1, { width: 40, height: 30 }); + await decode(40, 30); + record(); expect(controller.snapshot().hasFrame).toBe(true); - - // Once pointer activity is old, an animated page must not keep decoding its - // CSS-resolution stream at frame rate; the throttled crisp path remains. - clock.now += PROVISIONAL_INPUT_WINDOW_MS + 1; - await frame('idle-animation-frame'); - expect(decodes()).toBe(1); - }); - - it('paints the stream frame after keys, pasted text and editing chords, not only after pointer input', async () => { - const { clock, host, platform, controller, frame, decodes } = await paintFixture(undefined, 'pasted'); - await frame('first'); - expect(decodes()).toBe(1); - - // At rest, a changed frame only pulses the crisp loop. - clock.now += PROVISIONAL_INPUT_WINDOW_MS + 1; - await frame('idle'); - expect(decodes()).toBe(1); - + // The device-resolution capture that replaces it sizes the canvas. + emitFrame(streamSocket(4321), 'crisp', 2); + await decode(80, 60); + record(); + // Hover: the CSS-resolution stream paints into that same canvas, scaled. + emitFrame(streamSocket(4321), 'provisional', 3); + await decode(40, 30); + record(); + emitFrame(streamSocket(4321), 'crisp', 4); + await decode(80, 60); + record(); + expect(sizes).toEqual(['40x30', '80x60', '80x60', '80x60']); + expect(drawImage.mock.calls.map((call) => call.slice(1))).toEqual([[0, 0, 40, 30], [0, 0, 80, 60], [0, 0, 80, 60], [0, 0, 80, 60]]); + // The frame's viewport size drives the screen indicator and sync. + expect(controller.getDeviceSize()).toEqual({ width: 40, height: 30 }); + + // A resized viewport's frame is another shape: the canvas follows it. + emitFrame(streamSocket(4321), 'provisional', 5, { width: 50, height: 30 }); + await decode(50, 30); + expect(`${sink.canvas.width}x${sink.canvas.height}`).toBe('50x30'); + }); + + it('decodes the newest frame only: one at a time, a later arrival replacing the one waiting', async () => { + const { drawImage, decodes, decode } = await paintFixture(); + for (let n = 1; n <= 4; n++) emitFrame(streamSocket(4321), 'provisional', n); + await flushMicrotasks(); + expect(decodes.map((d) => d.bytes)).toEqual([1]); + await decode(40, 30); + // Frames 2 and 3 were replaced while 1 decoded; 4 paints next. + expect(decodes.map((d) => d.bytes)).toEqual([4]); + await decode(40, 30); + expect(drawImage).toHaveBeenCalledTimes(2); + }); + + it('sends input, pasted text and repaint requests over the viewer socket, and editing chords to the host', async () => { + const { host, controller } = await paintFixture('pasted\r\ntext'); + const sent = () => streamSocket(4321)!.sent.map((raw) => JSON.parse(raw)); controller.handleKeyDownLike({ key: 'a', code: 'KeyA', ctrlKey: false, metaKey: false, altKey: false, shiftKey: false }); - await frame('typed'); - expect(decodes()).toBe(2); - - // A paste is replayed as key input once the clipboard read resolves. - clock.now += PROVISIONAL_INPUT_WINDOW_MS + 1; + expect(sent().at(-1)).toMatchObject({ type: 'input_keyboard', eventType: 'keyDown', key: 'a', text: 'a' }); + // A paste goes as text, whichever provider: the host inserts it. controller.handleKeyDownLike({ key: 'v', code: 'KeyV', ctrlKey: true, metaKey: false, altKey: false, shiftKey: false }); await flushMicrotasks(); - expect(platform.readClipboardText).toHaveBeenCalled(); - await frame('pasted'); - expect(decodes()).toBe(3); - - // A select-all runs through the host rather than the stream. - clock.now += PROVISIONAL_INPUT_WINDOW_MS + 1; + expect(sent().at(-1)).toEqual({ type: 'input_text', text: 'pasted\ntext' }); controller.handleKeyDownLike({ key: 'a', code: 'KeyA', ctrlKey: true, metaKey: false, altKey: false, shiftKey: false }); expect(host.requests('edit')).toEqual([onSess({ op: 'edit', edit: 'selectAll' })]); - await frame('selected'); - expect(decodes()).toBe(4); - }); - - it('paints the stream while a crisp capture waits behind a blocking command, then draws that capture', async () => { - const releases: Array<(shot: Shot) => void> = []; - const { clock, host, drawImage, frame, decodes } = await paintFixture(() => new Promise((resolve) => { releases.push(resolve); })); - // The first image paints from the stream, superseding the capture it pulsed; - // its replacement is the one a page-loading `open` then holds. - await frame('previous page'); - clock.now += 300; - releases[0]({ ok: true, bytes: new Uint8Array([1]), mime: 'image/jpeg' }); - await flushMicrotasks(); - expect(host.requests('screenshot')).toHaveLength(2); - const decoded = decodes(); - - clock.now += PROVISIONAL_INPUT_WINDOW_MS + 1; - await frame('loading'); - expect(decodes()).toBe(decoded); - // Overdue now: the loading page paints from the stream. - clock.now += 400; - await frame('still loading'); - expect(decodes()).toBe(decoded + 1); - - // `open` returns: the held capture is drawn on arrival rather than dropped - // as older than the overdue paints (the follow-up its wait's pulses owe - // comes after). - const drawn = drawImage.mock.calls.length; - clock.now += 1000; - releases[1]({ ok: true, bytes: new Uint8Array([2]), mime: 'image/jpeg' }); - await flushMicrotasks(); - await flushMicrotasks(); - expect(drawImage.mock.calls.length).toBe(drawn + 1); - }); - - it('repaints a byte-identical crisp capture over a provisional paint', async () => { - // A provisional paint changes the canvas behind the loop's byte-dedup, so a - // resting page's byte-identical capture must still repaint over the blur. - const { clock, host, drawImage, controller, frame } = await paintFixture( - async () => ({ ok: true, bytes: new Uint8Array([9, 9, 9]), mime: 'image/jpeg' }), - ); - await frame('first'); - expect(controller.snapshot().hasFrame).toBe(true); - - // Past the input window a frame is a bare pulse, so this capture lands as - // the crisp resting frame the loop records. - clock.now += PROVISIONAL_INPUT_WINDOW_MS + 1; - await frame('rest'); - const afterCrisp = drawImage.mock.calls.length; - expect(host.requests('screenshot').length).toBeGreaterThan(0); - - controller.send({ type: 'input_mouse', eventType: 'mouseMoved', x: 1, y: 1 }); - await frame('hover'); - const afterProvisional = drawImage.mock.calls.length; - expect(afterProvisional).toBeGreaterThan(afterCrisp); - - clock.now += PROVISIONAL_INPUT_WINDOW_MS + 1; - await frame('settled'); - expect(drawImage.mock.calls.length).toBeGreaterThan(afterProvisional); + // A view remounted over the open socket asks the host for the last frame. + controller.attachView(makeSink()); + expect(sent().at(-1)).toEqual({ type: 'repaint' }); }); }); @@ -387,7 +346,7 @@ describe('sync-to-pane while parked', () => { it('pushes a resize made behind a hidden pane once it is live again', async () => { vi.useFakeTimers(); try { - const host = installBrowserHost({ attach: async () => ({ ok: true, wsPort: 4321 }) }); + const host = installBrowserHost({ attach: async () => ({ ok: true, stream: 4321 }) }); const observers: ResizeObserverCallback[] = []; vi.stubGlobal('ResizeObserver', class { constructor(callback: ResizeObserverCallback) { observers.push(callback); } @@ -426,8 +385,6 @@ describe('parking', () => { afterEach(() => vi.useRealTimers()); it('detach parks after the debounce and resets hasFrame', async () => { - const screenshot = vi.fn(async () => ({ ok: true as const, bytes: new Uint8Array([1, 2, 3]), mime: 'image/jpeg' })); - installBrowserHost({ screenshot }); // Give the draw path a bitmap so hasFrame can flip true without a real canvas. vi.stubGlobal('createImageBitmap', vi.fn(async () => ({ width: 4, height: 4, close: vi.fn() }))); @@ -438,10 +395,9 @@ describe('parking', () => { const socket = streamSocket(4321); expect(socket?.readyState).toBe(1); - // Drive one frame → screenshot → draw → hasFrame true. - socket?.emitMessage(JSON.stringify({ type: 'frame', data: 'x'.repeat(32) })); - await vi.advanceTimersByTimeAsync(300); - expect(screenshot).toHaveBeenCalled(); + // One frame → draw → hasFrame true. + emitFrame(socket, 'crisp'); + await vi.advanceTimersByTimeAsync(0); expect(controller.snapshot().hasFrame).toBe(true); // Detach immediately drops hasFrame (the canvas DOM died with the unmount). @@ -457,36 +413,6 @@ describe('parking', () => { }); }); -describe('re-attach repaint', () => { - it('schedules a repaint capture when re-attaching to a live connection', async () => { - vi.useFakeTimers(); - try { - const screenshot = vi.fn(async () => ({ ok: true as const, bytes: new Uint8Array([1, 2, 3]), mime: 'image/jpeg' })); - installBrowserHost({ screenshot }); - vi.stubGlobal('createImageBitmap', vi.fn(async () => ({ width: 4, height: 4, close: vi.fn() }))); - - const controller = withPort('id', { session: 'sess' }, 4321); - const first = makeSink(); - const h1 = controller.attachView(first); - await vi.advanceTimersByTimeAsync(0); - expect(streamSocket(4321)?.readyState).toBe(1); - - // Detach but stay within the park debounce, so the connection (and its - // screenshot loop) survive. - h1.detach(); - screenshot.mockClear(); - - // Re-attach to that live, unparked connection → one repaint capture, so a - // view remounted within the debounce doesn't sit blank. - controller.attachView(makeSink()); - await vi.advanceTimersByTimeAsync(300); - expect(screenshot).toHaveBeenCalled(); - } finally { - vi.useRealTimers(); - } - }); -}); - describe('param-write buffering', () => { it('buffers writes while detached and flushes them on the next attach', async () => { // Answering no CDP endpoint keeps the popped-out CDP observer from opening @@ -555,11 +481,11 @@ describe('updateParams', () => { describe('launch', () => { function launchHost(launch: NonNullable) { - return installBrowserHost({ launch, attach: async () => ({ ok: true, wsPort: 9999 }) }); + return installBrowserHost({ launch, attach: async () => ({ ok: true, stream: 9999 }) }); } it('a session-less pane opens its page, binds the session the host answers with, and streams', async () => { - const host = launchHost(async () => ({ ok: true, session: 'dormouse.1.gui-abc', wsPort: 4321, binaryPath: '/usr/bin/agent-browser' })); + const host = launchHost(async () => ({ ok: true, session: 'dormouse.1.gui-abc', stream: 4321, binaryPath: '/usr/bin/agent-browser' })); const launched = whenBrowserLaunched('id'); // A restored pane whose launch never landed is the same pane. const controller = acquireAgentBrowserSurfaceController('id', { @@ -593,7 +519,7 @@ describe('launch', () => { }); it('opens in the session params name, headed for a pop-out, and binds it', async () => { - const host = launchHost(async () => ({ ok: true, session: 'dormouse.1.tool.t', wsPort: 4321 })); + const host = launchHost(async () => ({ ok: true, session: 'dormouse.1.tool.t', stream: 4321 })); const controller = acquireAgentBrowserSurfaceController('id', { renderMode: 'ab-popout', url: 'http://localhost:6006/', launchSession: 'dormouse.1.tool.t', }); @@ -629,14 +555,14 @@ describe('launch', () => { expect(await launched).toBeNull(); expect(host.requests('close')).toEqual([]); - launch.resolve({ ok: true, session: 'dormouse.1.gui-late', wsPort: 4321 }); + launch.resolve({ ok: true, session: 'dormouse.1.gui-late', stream: 4321 }); await flushMicrotasks(); expect(host.requests('close')).toEqual([{ provider: 'agent-browser', binding: { session: 'dormouse.1.gui-late' }, op: 'close' }]); expect(WebSocketMock.instances).toHaveLength(0); }); it('a navigation out of a failed launch launches at its page, and loads it once', async () => { - const answers: BrowserResult[] = [{ ok: false, error: 'boom' }, { ok: true, session: 'dormouse.1.gui-n', wsPort: 4321 }]; + const answers: BrowserResult[] = [{ ok: false, error: 'boom' }, { ok: true, session: 'dormouse.1.gui-n', stream: 4321 }]; const host = launchHost(async () => answers.shift()!); acquireAgentBrowserSurfaceController('id', { renderMode: 'ab-screencast', url: 'https://page.example/' }).attachView(makeSink()); await flushMicrotasks(); @@ -659,14 +585,14 @@ describe('launch', () => { // A new announcement: the same session, another page. controller.updateParams({ renderMode: 'ab-screencast', url: 'http://localhost:6007/docs', launchSession: 'dormouse.1.tool.t' }); - launch.resolve({ ok: true, session: 'dormouse.1.tool.t', wsPort: 4321 }); + launch.resolve({ ok: true, session: 'dormouse.1.tool.t', stream: 4321 }); await flushMicrotasks(); expect(streamSocket(4321)?.readyState).toBe(1); expect(opens(host)).toEqual(['http://localhost:6007/docs']); }); it('ignores params that predate the session its launch bound', async () => { - const host = launchHost(async () => ({ ok: true, session: 'dormouse.1.gui-abc', wsPort: 4321 })); + const host = launchHost(async () => ({ ok: true, session: 'dormouse.1.gui-abc', stream: 4321 })); const controller = acquireAgentBrowserSurfaceController('id', { renderMode: 'ab-screencast', url: 'https://page.example/' }); controller.attachView(makeSink()); await flushMicrotasks(); @@ -689,14 +615,14 @@ describe('launch', () => { controller.updateParams({ renderMode: 'ab-screencast', url: 'http://localhost:6006/', session: 'dormouse.1.tool.t' }); controller.handOver(4321); - launch.resolve({ ok: true, session: 'dormouse.1.tool.t', wsPort: 4321 }); + launch.resolve({ ok: true, session: 'dormouse.1.tool.t', stream: 4321 }); await flushMicrotasks(); expect(host.requests('close')).toEqual([]); expect(streamSocket(4321)?.readyState).toBe(1); }); it('launches with the binary `dor ab` last resolved, and remembers the one it ran', async () => { - const host = launchHost(async () => ({ ok: true, session: 'dormouse.1.gui-b', wsPort: 4321, binaryPath: '/opt/ab/agent-browser' })); + const host = launchHost(async () => ({ ok: true, session: 'dormouse.1.gui-b', stream: 4321, binaryPath: '/opt/ab/agent-browser' })); rememberLaunchBinaryPath('agent-browser', '/usr/local/bin/agent-browser'); acquireAgentBrowserSurfaceController('id', { renderMode: 'ab-screencast', url: 'https://page.example/' }).attachView(makeSink()); await flushMicrotasks(); @@ -731,8 +657,8 @@ describe('launch', () => { // A Workspace transfer: the destination opens the same named session. disposeAgentBrowserSurfaceController('named'); disposeAgentBrowserSurfaceController('minted'); - answers[0]({ ok: true, session: 'dormouse.1.tool.t', wsPort: 4321 }); - answers[1]({ ok: true, session: 'dormouse.1.gui-x', wsPort: 4322 }); + answers[0]({ ok: true, session: 'dormouse.1.tool.t', stream: 4321 }); + answers[1]({ ok: true, session: 'dormouse.1.gui-x', stream: 4322 }); await flushMicrotasks(); expect(host.requests('close')).toEqual([{ provider: 'agent-browser', binding: { session: 'dormouse.1.gui-x' }, op: 'close' }]); }); @@ -910,7 +836,7 @@ describe('attach', () => { } it('a restored pane attaches at the page and presentation it had', async () => { - const host = attachHost(async () => ({ ok: true, wsPort: 2222 })); + const host = attachHost(async () => ({ ok: true, stream: 2222 })); const controller = acquireAgentBrowserSurfaceController('id', { session: 'sess', renderMode: 'ab-popout', url: 'https://restored.example/', }); @@ -929,7 +855,7 @@ describe('attach', () => { await flushMicrotasks(); expect(controller.snapshot().phase).toBe('ended'); - handOverBrowserPort('id', { session: 'sess', url: 'https://page.example/' }, 4321); + handOverBrowserStream('id', { session: 'sess', url: 'https://page.example/' }, 4321); await flushMicrotasks(); expect(controller.snapshot().phase).toBe('live'); expect(streamSocket(4321)?.readyState).toBe(1); @@ -961,7 +887,7 @@ describe('attach', () => { it('never attaches while parked, and an unpark whose port still answers asks the host nothing', async () => { vi.useFakeTimers(); try { - const { host, controller } = await parkedAt1111(async () => ({ ok: true, wsPort: 2222 })); + const { host, controller } = await parkedAt1111(async () => ({ ok: true, stream: 2222 })); // Hidden and shown again, a headless pane never left `live` for its view. expect(controller.snapshot().phase).toBe('live'); controller.setVisible(true); @@ -977,7 +903,7 @@ describe('attach', () => { it('an unpark whose port fails asks the host, without a page, where the stream moved', async () => { vi.useFakeTimers(); try { - const { host, controller } = await parkedAt1111(async () => ({ ok: true, wsPort: 2222 })); + const { host, controller } = await parkedAt1111(async () => ({ ok: true, stream: 2222 })); WebSocketMock.failPorts.add(1111); controller.setVisible(true); await vi.advanceTimersByTimeAsync(0); @@ -999,7 +925,7 @@ describe('attach', () => { try { const host = attachHost(async () => ({ ok: false, error: 'not running' })); // What reaches the browser: a stream URL is only the host's to build. - const sent = () => host.browser.mock.calls.map(([request]) => request).filter((request) => request.op !== 'streamUrl'); + const sent = () => host.browser.mock.calls.map(([request]) => request).filter((request) => request.op !== 'view'); const sink = makeSink(); let size = { width: 800, height: 600 }; sink.viewport.getBoundingClientRect = () => ({ ...size }) as DOMRect; @@ -1052,7 +978,7 @@ describe('attach', () => { }); it('a headless browser that drops ends, reached again only through attach or a handed-over port', async () => { - const host = attachHost(async () => ({ ok: true, wsPort: 3333 })); + const host = attachHost(async () => ({ ok: true, stream: 3333 })); const controller = withPort('id', { session: 'sess', url: 'https://page.example/' }, 1111); controller.attachView(makeSink()); await flushMicrotasks(); @@ -1067,7 +993,7 @@ describe('attach', () => { expect(host.requests('attach')).toEqual([]); // `dor ab open` brings it back on the port it had. - handOverBrowserPort('id', { session: 'sess', url: 'https://page.example/' }, 1111); + handOverBrowserStream('id', { session: 'sess', url: 'https://page.example/' }, 1111); await flushMicrotasks(); expect(controller.snapshot().phase).toBe('live'); expect(streamSockets(1111)).toHaveLength(2); @@ -1089,7 +1015,7 @@ describe('attach', () => { const host = attachHost(async () => ({ ok: false, error: 'not running' })); acquireAgentBrowserSurfaceController('id', { session: 'sess', url: 'https://page.example/' }).attachView(makeSink()); await flushMicrotasks(); - host.answers.attach = async () => ({ ok: true, wsPort: 3333, relaunched: true }); + host.answers.attach = async () => ({ ok: true, stream: 3333, relaunched: true }); getAgentBrowserScreenController('id')!.chromeActions.navigate('https://next.example/'); await flushMicrotasks(); @@ -1099,7 +1025,7 @@ describe('attach', () => { }); it('does not query the daemon while a relaunch is in flight', async () => { - const host = attachHost(async () => ({ ok: true, wsPort: 9999 })); + const host = attachHost(async () => ({ ok: true, stream: 9999 })); host.answers.launch = () => new Promise(() => {}); const controller = withPort('id', { session: 'sess' }, 1111); @@ -1158,7 +1084,7 @@ describe('closeBrowserSurface', () => { // The host runs that close after the relaunch, so closing again when it // lands would close whoever launched the session next. - resolvePopOut({ ok: true, wsPort: 3456 }); + resolvePopOut({ ok: true, stream: 3456 }); await flushMicrotasks(); expect(closes()).toHaveLength(1); expect(streamSockets(3456)).toHaveLength(0); @@ -1175,14 +1101,14 @@ describe('closeBrowserSurface', () => { // Through the binding the launch used, so the host orders the two. expect(host.requests('close')).toEqual([{ provider: 'agent-browser', binding: host.requests('launch')[0].binding, op: 'close' }]); expect(host.requests('close')[0].binding).toMatchObject({ session: 'dormouse.1.tool.t', binaryPath: '/opt/agent-browser' }); - launch.resolve({ ok: true, session: 'dormouse.1.tool.t', wsPort: 4321 }); + launch.resolve({ ok: true, session: 'dormouse.1.tool.t', stream: 4321 }); await flushMicrotasks(); expect(host.requests('close')).toHaveLength(1); }); it('cancels only its own requests the host has not answered', async () => { const popIn = pending(); - const answers = [Promise.resolve({ ok: true, wsPort: 3456 }), popIn.promise]; + const answers = [Promise.resolve({ ok: true, stream: 3456 }), popIn.promise]; const host = installBrowserHost({ launch: () => answers.shift()! }); const controller = withPort('id', { session: 'sess' }, 1111); controller.attachView(makeSink()); @@ -1208,7 +1134,7 @@ describe('closeBrowserSurface', () => { getAgentBrowserScreenController('id')?.actions.setRenderMode?.('ab-popout'); disposeAgentBrowserSurfaceController('id'); - resolvePopOut({ ok: true, wsPort: 3456 }); + resolvePopOut({ ok: true, stream: 3456 }); await flushMicrotasks(); expect(closes()).toHaveLength(0); }); @@ -1227,8 +1153,8 @@ describe('relaunch (pop-out / pop-in)', () => { function relaunchHost() { const popOut = pending(); const host = installBrowserHost({ - attach: async () => ({ ok: true, wsPort: 9999 }), - launch: (request) => request.headed ? popOut.promise : Promise.resolve({ ok: true, wsPort: 5555 }), + attach: async () => ({ ok: true, stream: 9999 }), + launch: (request) => request.headed ? popOut.promise : Promise.resolve({ ok: true, stream: 5555 }), }); /** The relaunches of a bound session, headed (pop-outs) or not (pop-ins). */ const relaunches = (headed: boolean) => host.requests('launch').filter((request) => request.binding.session !== undefined && request.headed === headed); @@ -1251,16 +1177,16 @@ describe('relaunch (pop-out / pop-in)', () => { expect(old?.readyState).toBe(3); expect(controller.snapshot().phase).toBe('relaunching'); expect(controller.snapshot().poppedOut).toBe(true); - // No daemon command while the relaunch is in flight: not even the popped-out - // CDP observer's `get cdp-url`. - expect(host.requests('cdpUrl')).toEqual([]); + // No viewer socket while the relaunch is in flight, headed or not. + expect(host.requests('view')).toEqual([onSess({ op: 'view', stream: 1111 })]); - host.resolvePopOut({ ok: true, wsPort: 3456 }); + host.resolvePopOut({ ok: true, stream: 3456 }); await flushMicrotasks(); expect(controller.snapshot().phase).toBe('live'); expect(streamSockets(3456).length).toBe(1); expect(streamSockets(1111).length).toBe(1); - expect(host.requests('cdpUrl')).toEqual([onSess({ op: 'cdpUrl' })]); + // The popped-out window's viewer: its page and its close, no frames. + expect(host.requests('view').at(-1)).toEqual(onSess({ op: 'view', stream: 3456, headed: true })); expect(host.requests('attach')).toEqual([]); }); @@ -1279,7 +1205,7 @@ describe('relaunch (pop-out / pop-in)', () => { expect(host.relaunches(false)).toHaveLength(0); expect(controller.snapshot().poppedOut).toBe(true); - host.resolvePopOut({ ok: true, wsPort: 3456 }); + host.resolvePopOut({ ok: true, stream: 3456 }); await flushMicrotasks(); controller.popIn(); expect(host.relaunches(false)).toHaveLength(1); @@ -1414,7 +1340,7 @@ describe('relaunch (pop-out / pop-in)', () => { expect(host.browser).not.toHaveBeenCalled(); // The relaunch lands: only the latest navigation runs, once. - host.resolvePopOut({ ok: true, wsPort: 3456 }); + host.resolvePopOut({ ok: true, stream: 3456 }); await vi.advanceTimersByTimeAsync(0); expect(host.requests('navigate')).toEqual([onSess({ op: 'navigate', url: 'https://latest.example/' })]); } finally { @@ -1439,7 +1365,7 @@ describe('relaunch (pop-out / pop-in)', () => { getAgentBrowserScreenController('id')!.actions.engageSync(); expect(host.requests('viewport')).toEqual([]); - popIn.resolve({ ok: true, wsPort: 5555 }); + popIn.resolve({ ok: true, stream: 5555 }); await flushMicrotasks(); expect(host.requests('viewport')).toEqual([onSess({ op: 'viewport', width: 800, height: 600, dpr: 1 })]); }); @@ -1453,7 +1379,7 @@ describe('relaunch (pop-out / pop-in)', () => { // The pane context menu's reuse of an existing port target. getAgentBrowserScreenController('id')?.actions.setRenderMode?.('ab-popout', { url: 'http://localhost:5173/' }); expect(host.relaunches(true)).toEqual([onSess({ op: 'launch', url: 'http://localhost:5173/', headed: true })]); - host.resolvePopOut({ ok: true, wsPort: 3456 }); + host.resolvePopOut({ ok: true, stream: 3456 }); await flushMicrotasks(); expect(host.requests('navigate')).toEqual([]); }); @@ -1466,7 +1392,7 @@ describe('relaunch (pop-out / pop-in)', () => { getAgentBrowserScreenController('id')?.actions.setRenderMode?.('ab-popout'); getAgentBrowserScreenController('id')!.chromeActions.navigate('https://page.example/'); - host.resolvePopOut({ ok: true, wsPort: 3456 }); + host.resolvePopOut({ ok: true, stream: 3456 }); await flushMicrotasks(); expect(host.relaunches(true)).toEqual([expect.objectContaining({ url: 'https://page.example/' })]); expect(opens(host)).toEqual([]); @@ -1489,7 +1415,7 @@ describe('relaunch (pop-out / pop-in)', () => { getAgentBrowserScreenController('id')!.chromeActions.navigate('https://next.example/'); getAgentBrowserScreenController('id')?.actions.setRenderMode?.('ab-popout', asked ? { url: asked } : undefined); expect(host.relaunches(true)).toEqual([expect.objectContaining({ url: opened })]); - host.resolvePopOut({ ok: true, wsPort: 3456 }); + host.resolvePopOut({ ok: true, stream: 3456 }); await vi.advanceTimersByTimeAsync(0); expect(opens(host)).toEqual([]); } finally { @@ -1509,7 +1435,7 @@ describe('relaunch (pop-out / pop-in)', () => { expect(host.requests('attach')).toHaveLength(1); expect(host.relaunches(true)).toEqual([]); - attach.resolve({ ok: true, wsPort: 1111 }); + attach.resolve({ ok: true, stream: 1111 }); await flushMicrotasks(); expect(host.relaunches(true)).toEqual([onSess({ op: 'launch', url: 'http://localhost:5173/', headed: true })]); }); @@ -1547,8 +1473,8 @@ describe('relaunch (pop-out / pop-in)', () => { describe('Playwright provider', () => { it('pastes as whole-text messages the host inserts, not a key pair per character', async () => { - const host = installBrowserHost({ streamUrl: async (request) => ({ ok: true, url: `ws://127.0.0.1:${request.port}` }) }); - const pasted = `${'x'.repeat(PLAYWRIGHT_TEXT_INPUT_MAX + 10)}\r\nend`; + const host = installBrowserHost(); + const pasted = `${'x'.repeat(VIEWER_TEXT_INPUT_MAX + 10)}\r\nend`; (host.platform as PlatformAdapter).readClipboardText = vi.fn(async () => pasted); const controller = withPort('pw', { renderMode: 'pw-screencast', session: 's' }, 4321); controller.attachView(makeSink()); @@ -1576,7 +1502,7 @@ describe('Playwright provider', () => { it('uses the shared controller with provider-scoped host calls and cwd', async () => { // The swap back to agent-browser is offered only where the host can launch one. - const host = installBrowserHost({ streamUrl: async (request) => ({ ok: true, url: `ws://127.0.0.1:${request.port}` }) }); + const host = installBrowserHost(); const controller = withPort('pw', { renderMode: 'pw-screencast', session: 'shared-name', cwd: '/first-project' }, 4321); const sink = makeSink(); controller.attachView(sink); diff --git a/lib/src/components/wall/agent-browser-surface-controller.ts b/lib/src/components/wall/agent-browser-surface-controller.ts index 6b4b582fd..554bcb3e5 100644 --- a/lib/src/components/wall/agent-browser-surface-controller.ts +++ b/lib/src/components/wall/agent-browser-surface-controller.ts @@ -4,7 +4,15 @@ * released by Wall on kill/render swap: `closeBrowserSurface` closes the * session too, `disposeAgentBrowserSurfaceController` only the client side. */ -import { BROWSER_CLOSE_MAX_CANCELS, isBlankUrl, isBrowsableUrl, type BrowserAutomationProvider, type BrowserResult } from '../../lib/platform/browser-automation'; +import { + BROWSER_CLOSE_MAX_CANCELS, + isBlankUrl, + isBrowsableUrl, + viewerTextInputs, + type BrowserAutomationProvider, + type BrowserResult, + type ViewerFrame, +} from '../../lib/platform/browser-automation'; import { isAllowedBinaryFor } from '../../lib/agent-browser-binary'; import { readTextFromClipboard } from '../../lib/clipboard'; import { isAbDebugLogsEnabled } from '../../lib/feature-flags'; @@ -41,7 +49,6 @@ import { modifiers, virtualKeyCode, } from './agent-browser-input'; -import { createScreenshotLoop, type ScreenshotLoop } from './agent-browser-screenshot-loop'; import { createAgentBrowserConnection, type AgentBrowserConnection, @@ -53,20 +60,19 @@ import { // immediately, so quick visibility flips — or a StrictMode unmount→remount — // don't tear down and rebuild the stream connection. export const HIDDEN_PARK_DELAY_MS = 1000; -/** Keep low-latency stream painting active briefly after input — pointer, keys, - * pasted text, editing chords. Continuous input extends the window; idle - * animated pages stay on the cheaper crisp path. */ -export const PROVISIONAL_INPUT_WINDOW_MS = 250; - -// The high-rate `[ab-panel]` stream/screenshot diagnostics fire per frame -// (~20Hz), so the flag is read ONCE — lazily, on the first log — and memoized: -// toggling needs a reload, which is the right trade for a hot loop. The -// connection's always-on debug ring is unaffected. + +// The high-rate `[ab-panel]` diagnostics fire per stream event, so the flag is +// read ONCE — lazily, on the first log — and memoized: toggling needs a +// reload, which is the right trade for a hot loop. The same flag has the host +// log each viewer socket's rates. The connection's always-on debug ring is +// unaffected. // `localStorage.setItem('dormouse.flags.abDebugLogs', 'true')` + reload to enable. let abDebugLogsEnabled: boolean | undefined; +function abDebugLogsOn(): boolean { + return abDebugLogsEnabled ??= isAbDebugLogsEnabled(); +} function abDebugLog(message: string): void { - if (abDebugLogsEnabled === undefined) abDebugLogsEnabled = isAbDebugLogsEnabled(); - if (abDebugLogsEnabled) console.log(message); + if (abDebugLogsOn()) console.log(message); } // SYNCED is "browser viewport CSS size == pane CSS size". The screencast is @@ -106,7 +112,7 @@ export type KeyLike = { /** Canonical params for a browser surface, as the view reads them. Pop-out is * deliberately absent: it is derived from `renderMode`, never stored; nor is - * the stream port, which `dor` hands straight to the controller + * the stream, which `dor` hands straight to the controller * (docs/specs/dor-browser.md → "Canonical Params"). */ export interface AgentBrowserSurfaceParams { surfaceType?: string; @@ -147,18 +153,18 @@ type Phase = * success — `named` when the launch names one, which a close of the * Surface meanwhile closes too, through the launch's own binding. */ | { k: 'launching'; named?: { session: string; browser: BrowserHandle } } - /** The session's stream port is being asked of the host (`attach`). */ + /** The session's stream is being asked of the host (`attach`). */ | { k: 'attaching' } - /** Streaming from `port`. `seen`: the stream has reported its browser - * connected, so a later drop means it went away — a headed window closed. - * `resumed`: an unpark reconnecting to the port it parked at, not yet - * proven there — its catch-up waits until the stream opens, and a stream - * that fails asks the host where it moved. */ - | { k: 'live'; port: number; seen: boolean; resumed: boolean } - /** Hidden long enough to shed the stream; the daemon stays up at `port`. */ - | { k: 'parked'; port: number } + /** Viewing `stream` over the host's viewer socket. `seen`: the stream has + * reported its browser connected, so a later drop means it went away — a + * headed window closed. `resumed`: an unpark reconnecting to the stream it + * parked at, not yet proven there — its catch-up waits until the socket + * opens, and one that fails asks the host where it moved. */ + | { k: 'live'; stream: number; seen: boolean; resumed: boolean } + /** Hidden long enough to shed the socket; the browser stays up at `stream`. */ + | { k: 'parked'; stream: number } /** A headed↔headless relaunch is in flight: the host closes the browser and - * kills its daemon before reopening on a new port. */ + * kills its daemon before reopening on a new stream. */ | { k: 'relaunching' } /** Nothing to show: the browser went away, or `error` kept it from opening. */ | { k: 'ended'; error?: string } @@ -170,7 +176,7 @@ export type BrowserSurfacePhase = Phase['k']; /** The live DOM bindings a mounted view lends the controller. `attachView` * wires these; `detach()` returns them. */ export interface AgentBrowserViewSink { - /** Draw target for device-resolution screenshots. */ + /** Draw target for the viewer socket's frames. */ canvas: HTMLCanvasElement; /** The content area — observed for resize and read for pane rect / pop-out * positioning. */ @@ -236,24 +242,23 @@ export class AgentBrowserSurfaceController { private session: string | undefined; private launchSession: string | undefined; private binaryPath: string | undefined; - /** A port handed over before the first start, which then streams from it. */ - private initialPort: number | undefined; + /** A stream handed over before the first start, which then views it. */ + private initialStream: number | undefined; private paramsUrl: string | undefined; private paramsKey: string | null; private paramsSyncEngaged: boolean | undefined; - // --- stream connection (exists only while live) --- + // --- viewer socket (exists only while live) --- private connection: AgentBrowserConnection | null = null; - private screenshotLoop: ScreenshotLoop | null = null; private connectionUnsub: (() => void) | null = null; private connectionKey: string | null = null; - // The `session:port` the connection last (re)connected to. An unpark + // The `session:stream` the connection last (re)connected to. An unpark // reconnects to the same identity, so the last good frame is still valid and // must not be blanked to the placeholder; only a real identity change resets // hasFrame. private lastConnectedIdentity: string | null = null; - /** The port of the last live/parked phase, whose viewport sync still holds. */ - private boundPort: number | undefined; + /** The stream of the last live/parked phase, whose viewport sync still holds. */ + private boundStream: number | undefined; // --- view state (the snapshot) --- private status: StreamStatus | null = null; @@ -323,30 +328,13 @@ export class AgentBrowserSurfaceController { private readonly screenActions: ScreenActions; private readonly chromeActions: ChromeActions; - // --- CDP observer (while popped out) --- - private cdpKey: string | null = null; - private cdpTeardown: (() => void) | null = null; - // --- view binding --- private sink: AgentBrowserViewSink | null = null; private attachToken: object | null = null; - // "The canvas changed without the crisp loop drawing it." Bumped on attachView - // (a fresh view mounts blank) and on every provisional paint (CSS-resolution - // pixels land behind the loop's back). The screenshot loop folds this into its - // byte-dedup key so an identical capture still repaints — otherwise it would - // skip the redundant bytes and leave the canvas blank, or blurry, until the - // page happens to change. - private drawGeneration = 0; - // Latest-wins generation shared by provisional stream decodes and crisp host - // screenshots. A late low-resolution decode must never overwrite a newer crisp - // frame (or a provisional frame from a later pointer move). - private frameDrawSeq = 0; - private provisionalUntil = 0; - // Counts the provisional paints that supersede a crisp capture in flight — - // not those made only while one is overdue, which it is newer than - // (`createScreenshotLoop`). - private provisionalPaintGeneration = 0; - private paintingForOverdue = false; + // Latest-only decoding: one frame decoding, and at most one waiting behind + // it, which a newer arrival replaces. Frames paint in the order they came. + private decoding = false; + private pendingFrame: ViewerFrame | null = null; // Param writes buffered while detached (a minimized popped-out pane can still // observe URL changes); flushed on the next attach. private pendingParams = new Map(); @@ -506,8 +494,8 @@ export class AgentBrowserSurfaceController { }); this.lastPublishedScreen = null; this.publishScreen(); - this.bind(this.initialPort); - this.initialPort = undefined; + this.bind(this.initialStream); + this.initialStream = undefined; } // A display-scale (DPR) change resizes nothing, so the pane's ResizeObserver @@ -574,10 +562,6 @@ export class AgentBrowserSurfaceController { const token = {}; this.attachToken = token; this.sink = sink; - // A fresh canvas mounts blank; bump the draw generation so the screenshot - // loop repaints it even if the next capture's bytes match the last frame. - this.drawGeneration += 1; - this.frameDrawSeq += 1; // Seed + observe the pane size cache before ensureStarted so the first // computeScreenSnapshot reads a real size. this.setupPaneSizeObserver(); @@ -601,20 +585,18 @@ export class AgentBrowserSurfaceController { if (this.syncEngaged) this.issueSyncToPane(); this.updateParkState(); this.publishScreen(); - // After a (re)attach, if a live in-pane connection exists, force one capture - // so a view remounted within the park debounce repaints instead of sitting - // blank — the connection's own frame dedup swallows the heartbeat rebroadcast, - // and the bumped generation defeats the screenshot loop's byte dedup. - if (this.phase.k === 'live' && !this.headed) this.screenshotLoop?.pulse(); + // A view remounted within the park debounce mounts a blank canvas over a + // socket still open, whose host sends only changes: ask it for the last frame. + if (this.phase.k === 'live' && !this.headed) this.connection?.send({ type: 'repaint' }); return { // Guard by identity: a stale handle's detach must no-op if a newer view // has already attached (StrictMode attach A → detach A → attach B can // interleave), and dispose already released everything. detach: () => { if (this.phase.k === 'disposed' || this.attachToken !== token) return; - // A provisional decode aimed at the old canvas needs no cancelling here: - // drawProvisionalFrame captures its sink and drops the bitmap when - // `this.sink` has moved on, which clearing it below guarantees. + // A decode aimed at the old canvas needs no cancelling here: + // `decodeFrame` captures its sink and drops the bitmap when `this.sink` + // has moved on, which clearing it below guarantees. this.sink = null; this.attachToken = null; // The observed viewport died with the unmount; drop the cache (and the @@ -623,7 +605,7 @@ export class AgentBrowserSurfaceController { this.paneSize = null; // The canvas DOM died with the unmount; on reattach a fresh canvas // mounts blank, so drop hasFrame to match the minimize/reattach - // placeholder → first-screenshot sequence. + // placeholder → first-frame sequence. this.setHasFrame(false); this.updateParkState(); }, @@ -637,8 +619,8 @@ export class AgentBrowserSurfaceController { // Mirror every field first, then rebind once: binding per field would bind // a new session with the cwd or binary of the old. if (params.cwd !== undefined) this.cwd = params.cwd; - // First, so neither a stream this rebinds nor a port handed over next - // (`handOverBrowserPort`) inherits a `set viewport` meant for the old mode. + // First, so neither a stream this rebinds nor one handed over next + // (`handOverBrowserStream`) inherits a `set viewport` meant for the old mode. if (params.renderMode && this.echoed('renderMode', params.renderMode)) this.followParamsHeadedness(params.renderMode); const sessionChanged = this.echoed('session', params.session) && params.session !== this.session; if (sessionChanged) this.session = params.session; @@ -663,14 +645,13 @@ export class AgentBrowserSurfaceController { } /** - * A port a `dor ab` / `dor pw` command just learned for this Surface's - * session — the same one again included — to stream from, leaving `ended` - * too. + * A stream a `dor ab` / `dor pw` command just learned for this Surface's + * session — the same one again included — to view, leaving `ended` too. */ - handOver(port: number): void { + handOver(stream: number): void { if (this.phase.k === 'disposed') return; - if (this.phase.k === 'idle') this.initialPort = port; - else this.adopt(port); + if (this.phase.k === 'idle') this.initialStream = stream; + else this.adopt(stream); } /** Whether params may set `field` to `value`: not while they have yet to @@ -717,7 +698,7 @@ export class AgentBrowserSurfaceController { // --- phase --- - /** Enter `next`, with the stream connection and CDP observer following it. */ + /** Enter `next`, with the viewer socket following it. */ private setPhase(next: Phase): void { this.phase = next; if (next.k === 'live' || next.k === 'parked') { @@ -725,15 +706,14 @@ export class AgentBrowserSurfaceController { // engaged, reclaim the pane size. Clearing lastIssued is essential — it // otherwise still holds the previous browser's pane size and // issueSyncToPane would no-op, leaving the fresh browser unsynced. - if (next.port !== this.boundPort) { - this.boundPort = next.port; + if (next.stream !== this.boundStream) { + this.boundStream = next.stream; this.lastIssued = null; } } this.reconcileConnection(); - this.reconcileCdp(); this.emitView(); - // An unpark's port is proven only once its stream opens. + // An unpark's stream is proven only once its socket opens. if (next.k === 'live' && !next.resumed) this.drivable(); } @@ -749,30 +729,30 @@ export class AgentBrowserSurfaceController { else if (url) this.drive(`open ${url}`, (browser) => browser.navigate(url)); } - /** (Re)bind the current params: no session launches one, a session without a - * port learns it from the host, one whose port was just handed over streams - * from it at once. */ + /** (Re)bind the current params: no session launches one, a session without + * a stream learns it from the host, one whose stream was just handed over + * is viewed at once. */ private bind(handover?: number): void { if (!this.session) this.launch(); else if (handover) this.goLive(handover); else this.attach(true); } - /** A port handed over for the bound session: stream from it. A launch or + /** A stream handed over for the bound session: view it. A launch or * relaunch in flight ignores it — the host's answer is the authoritative - * port. */ - private adopt(port: number): void { + * stream. */ + private adopt(stream: number): void { const phase = this.phase; if (phase.k === 'relaunching' || phase.k === 'launching') return; - if (phase.k === 'live' && phase.port === port) return; - if (phase.k === 'parked') this.setPhase({ k: 'parked', port }); - else this.goLive(port); + if (phase.k === 'live' && phase.stream === stream) return; + if (phase.k === 'parked') this.setPhase({ k: 'parked', stream }); + else this.goLive(stream); } - private goLive(port: number, resumed = false): void { + private goLive(stream: number, resumed = false): void { this.setPhase(this.parkRequested && !this.headed - ? { k: 'parked', port } - : { k: 'live', port, seen: false, resumed }); + ? { k: 'parked', stream } + : { k: 'live', stream, seen: false, resumed }); } /** @@ -838,7 +818,7 @@ export class AgentBrowserSurfaceController { }); this.launchSession = undefined; this.openedByHost(url); - if (res.wsPort) this.goLive(res.wsPort); + if (res.stream) this.goLive(res.stream); else this.attach(false); settleLaunch(this.id, null); }); @@ -869,7 +849,7 @@ export class AgentBrowserSurfaceController { * Ask the host where the session's stream is (`attach`, which never starts a * daemon to answer). With `relaunch`, a session whose daemon is gone is * reopened at the page this Surface had, so a restore after a reboot comes - * back where it was. Without, as on an unpark whose port failed, a gone + * back where it was. Without, as on an unpark whose stream failed, a gone * daemon is `ended`. */ private attach(relaunch: boolean): void { @@ -891,7 +871,7 @@ export class AgentBrowserSurfaceController { // Only a gone daemon is relaunched at the page; a live one was only // found, so a navigation pending to it still has to run. if (res.relaunched) this.openedByHost(url); - if (res.ok && res.wsPort) this.goLive(res.wsPort); + if (res.ok && res.stream) this.goLive(res.stream); else this.setPhase({ k: 'ended', error: relaunch ? res.error : undefined }); }); } @@ -912,8 +892,9 @@ export class AgentBrowserSurfaceController { private updateParkState(): void { if (this.parkTimer) { clearTimeout(this.parkTimer); this.parkTimer = undefined; } - // Detached ⇒ hidden. Popped out is exempt: its stream/CDP observer detects a - // headed window close and drives auto-revert, so parking it would break that. + // Detached ⇒ hidden. Popped out is exempt: its viewer socket brings the + // headed window's close, which drives auto-revert, and its page as it + // navigates, so parking it would break both. const shouldPark = !this.headed && (!this.visible || !this.sink); if (!shouldPark) { this.setParkRequested(false); return; } this.parkTimer = setTimeout(() => { @@ -932,19 +913,20 @@ export class AgentBrowserSurfaceController { if (this.parkRequested === parkRequested) return; this.parkRequested = parkRequested; const phase = this.phase; - // A parked pane holds no stream/screenshot loop; the daemon/session stays - // alive and re-broadcasts on reconnect. An unpark streams from the port it - // parked at at once, and asks the host only if that fails: the daemon may - // have moved while no client was alive. - if (parkRequested && phase.k === 'live') this.setPhase({ k: 'parked', port: phase.port }); - else if (!parkRequested && phase.k === 'parked') this.goLive(phase.port, true); + // A parked pane holds no viewer socket; the daemon/session stays alive and + // re-broadcasts on reconnect. An unpark views the stream it parked at at + // once, and asks the host only if that fails: the daemon may have moved + // while no client was alive. + if (parkRequested && phase.k === 'live') this.setPhase({ k: 'parked', stream: phase.stream }); + else if (!parkRequested && phase.k === 'parked') this.goLive(phase.stream, true); } - // --- stream connection (keyed; exists exactly while live) --- + // --- viewer socket (keyed; exists exactly while live) --- private reconcileConnection(): void { const phase = this.phase; - const key = phase.k === 'live' ? `${this.session}:${phase.port}` : null; + // Headedness too: a headed viewer is sent no frames. + const key = phase.k === 'live' ? `${this.session}:${phase.stream}:${this.headed}` : null; if (key === this.connectionKey) return; if (this.connection) { @@ -952,58 +934,38 @@ export class AgentBrowserSurfaceController { this.connectionUnsub = null; this.connection.dispose(); this.connection = null; - this.screenshotLoop?.dispose(); - this.screenshotLoop = null; } this.connectionKey = key; if (phase.k !== 'live') return; const session = this.session!; - const streamPort = phase.port; - - // Per-connection pairing: the screenshot loop and the connection are created - // and disposed together. The loop lives here (not in a separate effect) so a - // reconnect always re-creates it — a disposed loop would silently drop every - // frame pulse. - const screenshotLoop = createScreenshotLoop({ - capture: (opts) => this.driver()?.screenshot(opts) ?? null, - // Checked per stream frame: the gate, without building a handle. - isCapable: () => this.phase.k === 'live' && !!this.session && this.hosted, - draw: this.drawBitmap, - // A re-attach bumps drawGeneration so a fresh (blank) canvas repaints even - // when the capture bytes are identical to the last displayed frame. - getDrawGeneration: () => this.drawGeneration, - getProvisionalGeneration: () => this.provisionalPaintGeneration, - getProvisionalDeadline: () => this.provisionalUntil, - log: abDebugLog, - }); + const stream = phase.stream; + const headed = this.headed; const connection = createAgentBrowserConnection({ session, - streamPort, - getStreamUrl: async (port) => { - const answer = await this.handle()?.streamUrl(port); - if (answer && !answer.ok) throw new Error(answer.error ?? `${this.label} stream unavailable`); - return answer?.url; + stream, + viewUrl: async () => { + const answer = await this.handle()?.view(stream, { headed, debug: abDebugLogsOn() }); + if (!answer?.ok || !answer.url) throw new Error(answer?.error ?? `${this.label} viewer unavailable`); + return answer.url; }, selectTab: (tabId) => this.driver()?.tab('select', tabId) ?? Promise.resolve({ ok: false, error: `${this.label} commands unavailable` }), canSelectTabs: () => !this.headed, - wantFrameData: () => this.wantsProvisionalFrame(), log: abDebugLog, }); this.connection = connection; - this.screenshotLoop = screenshotLoop; this.connectionUnsub = connection.subscribe((event) => { const phase = this.phase; if (phase.k !== 'live') return; if (event.type === 'connection-open') { - // The port it parked at still answers: the daemon it drives is there. + // The stream it parked at still answers: the daemon it drives is there. if (phase.resumed) { phase.resumed = false; this.drivable(); } } else if (event.type === 'connection-close') { - // An unpark's daemon may have moved while nothing streamed. + // An unpark's daemon may have moved while nothing viewed it. if (phase.resumed) this.attach(false); else if (event.failures >= 3) this.streamLost(); } else if (event.type === 'status') { @@ -1023,187 +985,74 @@ export class AgentBrowserSurfaceController { // the header follows, and a relaunch mid-load carries the page being // loaded rather than the one before it. this.applyStreamUrl(event.url); + } else if (event.type === 'page') { + this.applyObservedNavigation(event.url, event.title); } else if (event.type === 'tabs') { - const prevActiveId = event.previousTabs.find((t) => t.active)?.tabId; - const nextActiveId = event.tabs.find((t) => t.active)?.tabId; this.setTabs(event.tabs); - // Switching the active tab doesn't make the daemon emit a screencast - // frame, and the dedup'd stream is otherwise silent on a static page, so - // force one capture so the surface follows the tab the user just selected. - if (nextActiveId && nextActiveId !== prevActiveId && !this.headed) { - screenshotLoop.pulse(); - } - } else if (event.type === 'frame-pulse') { - if (event.metadata) { - this.device = { width: event.metadata.deviceWidth, height: event.metadata.deviceHeight }; + } else if (event.type === 'frame') { + if (event.size) { + this.device = { width: event.size.width, height: event.size.height }; + this.maybeDisengageSync(); + this.publishScreen(); } - // The native stream frame is CSS-resolution but arrives immediately after - // hover/animation changes. Paint it as a provisional response, then let the - // host screenshot loop replace it with the crisp device-resolution frame. - // The body rides along only when we asked for it (via `wantFrameData`), so - // its presence is the request — re-testing `wantsProvisionalFrame` here would - // only race its own `provisionalUntil` deadline and drop a frame we wanted. - if (event.data) this.drawProvisionalFrame(event.data, this.paintingForOverdue); - this.maybeDisengageSync(); - this.publishScreen(); - if (!this.headed) screenshotLoop.pulse(); + this.paintFrame(event); } }); // The new stream reports its own status; the last one came from whatever // this Surface streamed before. this.status = null; - // Unparking reconnects to the same session/port; the last good frame is still - // valid, so only blank to the placeholder when the identity actually changed. - const identity = `${session}:${streamPort}`; + // Unparking reconnects to the same session/stream; the last good frame is + // still valid, so only blank to the placeholder when the identity changed. + const identity = `${session}:${stream}`; if (this.lastConnectedIdentity !== identity) { this.lastConnectedIdentity = identity; this.setHasFrame(false); } } - private paintBitmap(bitmap: ImageBitmap): void { - const canvas = this.sink?.canvas; - if (!canvas) { - bitmap.close(); - return; - } - if (canvas.width !== bitmap.width) canvas.width = bitmap.width; - if (canvas.height !== bitmap.height) canvas.height = bitmap.height; - canvas.getContext('2d')?.drawImage(bitmap, 0, 0); - bitmap.close(); - this.setHasFrame(true); - } + // --- painting --- - private drawBitmap = (bitmap: ImageBitmap): void => { - // A crisp host screenshot supersedes every provisional decode already in - // flight, even when that decode resolves later. - this.frameDrawSeq += 1; - this.paintBitmap(bitmap); - }; + /** Paint a frame from the viewer socket, latest-only: one decodes at a + * time, and a newer arrival replaces the one waiting behind it. */ + private paintFrame(frame: ViewerFrame): void { + if (this.decoding) this.pendingFrame = frame; + else this.decodeFrame(frame); + } - private drawProvisionalFrame(data: string, forOverdueCapture: boolean): void { + private decodeFrame(frame: ViewerFrame): void { const sink = this.sink; if (!sink || typeof createImageBitmap !== 'function') return; - let bytes: Uint8Array; - try { - const binary = atob(data); - bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i += 1) bytes[i] = binary.charCodeAt(i); - } catch { - return; - } - // Latest-only, like the crisp loop: a newer pulse arriving while this decode - // is in flight bumps the sequence, and the stale bitmap is dropped rather - // than painted over the newer frame. - const mySeq = ++this.frameDrawSeq; - createImageBitmap(new Blob([bytes], { type: 'image/jpeg' })).then((bitmap) => { - if (this.phase.k === 'disposed' || mySeq !== this.frameDrawSeq || this.sink !== sink) { - bitmap.close(); - return; - } - if (!forOverdueCapture) this.provisionalPaintGeneration += 1; - // This paint puts CSS-resolution pixels on the canvas behind the crisp - // loop's back, so its byte-dedup (`lastDrawnKey`) no longer describes what - // is on screen: a resting page whose crisp bytes match the last crisp draw - // would dedup to a no-op and strand the pane on the blur. Bump the draw - // generation for the same reason a re-attach does — the canvas changed - // underneath the loop, so the next crisp capture must repaint regardless - // of its bytes. - this.drawGeneration += 1; - this.paintBitmap(bitmap); - }).catch(() => { - // The crisp screenshot path remains authoritative; a malformed/unsupported - // provisional frame is only a missed latency optimization. + this.decoding = true; + // The frame's JPEG is a view over the socket's ArrayBuffer. + createImageBitmap(new Blob([frame.jpeg as Uint8Array], { type: 'image/jpeg' })).then((bitmap) => { + if (this.phase.k === 'disposed' || this.sink !== sink) bitmap.close(); + else this.paintBitmap(sink.canvas, bitmap, frame.kind); + }, () => { + // A malformed frame is only a missed paint; the next one replaces it. + }).finally(() => { + this.decoding = false; + const next = this.pendingFrame; + this.pendingFrame = null; + if (next) this.decodeFrame(next); }); } - private wantsProvisionalFrame(): boolean { - const forInput = !this.hasFrame || !this.hosted || performance.now() <= this.provisionalUntil; - this.paintingForOverdue = !forInput; - return forInput || !!this.screenshotLoop?.captureOverdue(); - } - - // agent-browser's stream publishes the initial headed tab list but not every - // same-tab manual navigation. While popped out, subscribe directly to Chrome - // DevTools Protocol target/page events so the Dormouse URL/header tracks the - // headed window without polling. - private reconcileCdp(): void { - if (this.provider === 'playwright') return; // The host stream also observes headed navigation. - const phase = this.phase; - // `get cdp-url` is a daemon command, so it waits for `live` like every other. - const desired = phase.k === 'live' && this.headed && this.hosted; - const key = desired ? `${this.session}:${phase.port}` : null; - if (key === this.cdpKey) return; - this.cdpTeardown?.(); - this.cdpTeardown = null; - this.cdpKey = key; - if (!desired) return; - this.cdpTeardown = this.startCdpObserver(); - } - - private startCdpObserver(): () => void { - let disposed = false; - let ws: WebSocket | null = null; - let nextId = 1; - - const send = (method: string, params?: Record) => { - if (ws?.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ id: nextId++, method, ...(params ? { params } : {}) })); - }; - const handleTargetInfo = (targetInfo: unknown) => { - if (!targetInfo || typeof targetInfo !== 'object') return; - const info = targetInfo as { type?: unknown; url?: unknown; title?: unknown }; - if (info.type !== 'page') return; - this.applyObservedNavigation( - typeof info.url === 'string' ? info.url : null, - typeof info.title === 'string' ? info.title : null, - ); - }; - const handleCdpMessage = (raw: unknown) => { - if (typeof raw !== 'string') return; - let msg: any; - try { msg = JSON.parse(raw); } catch { return; } - if (msg.method === 'Target.targetCreated' || msg.method === 'Target.targetInfoChanged') { - handleTargetInfo(msg.params?.targetInfo); - } else if (msg.method === 'Target.targetDestroyed') { - abDebugLog(`[ab-panel] cdp target destroyed ${JSON.stringify({ targetId: msg.params?.targetId })}`); - } else if (msg.method === 'Page.frameNavigated') { - const frame = msg.params?.frame; - if (!frame?.parentId) { - this.applyObservedNavigation( - typeof frame?.url === 'string' ? frame.url : null, - typeof frame?.name === 'string' ? frame.name : null, - ); - } - } else if (Array.isArray(msg.result?.targetInfos)) { - for (const targetInfo of msg.result.targetInfos) handleTargetInfo(targetInfo); - } - }; - - const connect = async () => { - const result = await this.driver()?.cdpUrl(); - const cdpUrl = result?.ok ? result.url ?? null : null; - if (result && !cdpUrl) abDebugLog(`[ab-panel] cdp-url failed ${JSON.stringify({ error: result.error })}`); - if (disposed || !cdpUrl) return; - abDebugLog(`[ab-panel] connecting cdp ${JSON.stringify({ cdpUrl })}`); - ws = new WebSocket(cdpUrl); - ws.onopen = () => { - abDebugLog('[ab-panel] cdp open'); - send('Target.setDiscoverTargets', { discover: true }); - send('Target.getTargets'); - // If get cdp-url ever returns a page websocket instead of the browser - // websocket, these page-level events are the navigation source. - send('Page.enable'); - }; - ws.onmessage = (ev) => handleCdpMessage(ev.data); - ws.onclose = () => { if (!disposed) abDebugLog('[ab-panel] cdp close'); }; - ws.onerror = () => abDebugLog('[ab-panel] cdp error'); - }; - - void connect(); - return () => { - disposed = true; - ws?.close(); - }; + /** + * Draw `bitmap` over the whole canvas. A crisp frame sizes the canvas; a + * provisional one, at CSS resolution, is drawn scaled into a canvas of the + * same shape rather than resizing it, so switching between the two never + * reallocates the backing store or relayouts the pane. + */ + private paintBitmap(canvas: HTMLCanvasElement, bitmap: ImageBitmap, kind: ViewerFrame['kind']): void { + const sameShape = canvas.width > 0 && canvas.height > 0 + && Math.abs(canvas.width / canvas.height - bitmap.width / bitmap.height) < 0.01; + if (kind === 'crisp' || !sameShape) { + if (canvas.width !== bitmap.width) canvas.width = bitmap.width; + if (canvas.height !== bitmap.height) canvas.height = bitmap.height; + } + canvas.getContext('2d')?.drawImage(bitmap, 0, 0, canvas.width, canvas.height); + bitmap.close(); + this.setHasFrame(true); } // --- view-snapshot field setters (notify on real change) --- @@ -1233,7 +1082,7 @@ export class AgentBrowserSurfaceController { this.emitView(); // Push the render-mode flip (screencast ↔ popout) to the header/modal. this.publishScreen(); - this.reconcileCdp(); + this.reconcileConnection(); this.updateParkState(); } @@ -1394,7 +1243,7 @@ export class AgentBrowserSurfaceController { if (!this.driver()) return; // A popped-out surface is a real headed OS window the user drives directly; // never force its viewport to the (now-stub) pane size. Sync resumes when it - // pops back in — the new port's reclaim re-issues against the fresh session. + // pops back in — the new stream's reclaim re-issues against the fresh session. if (this.headed) return; // A host that cannot drive the provider (e.g. the web demo) can't size // the viewport; stay silent rather than warn on every resize — the surface @@ -1448,10 +1297,10 @@ export class AgentBrowserSurfaceController { * Pop-Out / pop-in: relaunch this session's browser headed as a native OS * window, or back headless in the pane, at `url` or the page it is on. The * host closes the browser and kills its daemon before reopening on a new - * port, so the stream is dropped up front — its close would read as the - * window closing, and a daemon command in the gap spawns a competing daemon — - * and reconnects to the port the host hands back. One relaunch at a time, of - * a bound browser: anything else keeps only the navigation asked for. + * stream, so the viewer socket is dropped up front — its close would read as + * the window closing — and reconnects to the stream the host hands back. One + * relaunch at a time, of a bound browser: anything else keeps only the + * navigation asked for. */ private relaunch(headed: boolean, url?: string): void { const session = this.session; @@ -1471,8 +1320,8 @@ export class AgentBrowserSurfaceController { this.pendingIntent.url = url; } const target = this.launchUrl(); - // The phase first: flipping headedness while still live would start the - // CDP observer, whose `get cdp-url` would land in the close/reopen gap. + // The phase first: flipping headedness while still live would reopen the + // viewer socket on the browser the relaunch is closing. const phase: Phase = { k: 'relaunching' }; this.setPhase(phase); this.setHeaded(headed); @@ -1482,9 +1331,9 @@ export class AgentBrowserSurfaceController { abDebugLog(`[ab-panel] relaunch result ${JSON.stringify(res)}`); // Closed meanwhile, the host closes what this brought up after it. if (this.phase !== phase) return; - if (res.ok && res.wsPort) { + if (res.ok && res.stream) { this.openedByHost(target); - this.goLive(res.wsPort); + this.goLive(res.stream); return; } // Failed: back in the pane at the page it was on, relaunching headless @@ -1499,8 +1348,8 @@ export class AgentBrowserSurfaceController { /** * The stream says its browser is gone, or the stream itself is. A headless - * one has `ended`: the gate shuts, so nothing reaches a daemon whose port this - * controller would never learn. A headed one seen connected auto-reverts — + * one has `ended`: the gate shuts, so nothing reaches a daemon whose stream + * this controller would never learn. A headed one seen connected auto-reverts — * its window closed, so relaunch headless in the pane; one not yet seen is * still opening. A Dormouse teardown (pane kill, a render-swap away) releases * the controller before it closes the session, so no stream is left to see @@ -1572,17 +1421,12 @@ export class AgentBrowserSurfaceController { // --- input bridging --- + /** One input message over the viewer socket, whose host also paints the + * stream for a while after it. */ send(payload: Record): void { - // Typing is the most latency-sensitive input there is: its echo must not - // wait a crisp capture round trip any more than a hover does. - if (typeof payload.type === 'string' && payload.type.startsWith('input_')) this.openProvisionalWindow(); this.connection?.send(payload); } - private openProvisionalWindow(): void { - this.provisionalUntil = performance.now() + PROVISIONAL_INPUT_WINDOW_MS; - } - selectTab(tab: StreamTab): void { if (!tab.active) this.drive(`tab ${tab.tabId}`, (browser) => browser.tab('select', tab.tabId)); } @@ -1616,9 +1460,9 @@ export class AgentBrowserSurfaceController { // cmd/ctrl-V types the LOCAL clipboard into the page. Plain key forwarding // would trigger paste of the embedded Chromium's own (empty) clipboard, so - // bridge by replaying the text as the provider's viewer takes it. + // bridge by sending the text, which the host inserts. private insertText(text: string): void { - for (const message of BROWSER_PROVIDER_GUI[this.provider].pasteMessages(text)) this.send(message); + for (const message of viewerTextInputs(text)) this.send(message); } handleKeyDownLike(e: KeyLike): void { @@ -1638,8 +1482,6 @@ export class AgentBrowserSurfaceController { if (mod && !e.altKey && !e.shiftKey) { const op = EDIT_OPS[e.key.toLowerCase() as keyof typeof EDIT_OPS]; if (op && this.hosted && this.session) { - if (!this.driver()) return; - this.openProvisionalWindow(); this.drive(op, (browser) => browser.edit(op)); return; } @@ -1691,8 +1533,7 @@ export class AgentBrowserSurfaceController { if (this.parkTimer) { clearTimeout(this.parkTimer); this.parkTimer = undefined; } this.teardownPaneSizeObserver(); this.paneSize = null; - // Leaving `live` drops the connection, its screenshot loop and the CDP - // observer. + // Leaving `live` drops the viewer socket. this.setPhase({ k: 'disposed' }); this.dprQuery?.removeEventListener('change', this.onDprChange); this.dprQuery = null; @@ -1735,8 +1576,8 @@ export function getAgentBrowserSurfaceController(id: string): AgentBrowserSurfac return registry.get(id) ?? null; } -/** Release all CLIENT-side resources for a surface (connection, screenshot loop, - * CDP observer, timers, screen registration), leaving its session running — for +/** Release all CLIENT-side resources for a surface (viewer socket, timers, + * screen registration), leaving its session running — for * a Surface whose browser lives on elsewhere. A kill or a swap away uses * `closeBrowserSurface`. A safe no-op for a surface with no controller * (iframe/terminal). */ @@ -1782,15 +1623,15 @@ function closeInFlight(provider: BrowserAutomationProvider, session: string): Pr return closesInFlight.get(closeKey(provider, session)); } -/** Hand `id`'s controller a port a `dor` command just learned, with the +/** Hand `id`'s controller a stream a `dor` command just learned, with the * params that command just refreshed — acquired from them if no view has - * mounted it yet, so its first start streams at once. The params go first, so - * a host-reported presentation or cwd applies before the new stream: sync + * mounted it yet, so its first start views it at once. The params go first, + * so a host-reported presentation or cwd applies before the new stream: sync * never sizes a headed window. */ -export function handOverBrowserPort(id: string, params: AgentBrowserSurfaceParams, port: number): void { +export function handOverBrowserStream(id: string, params: AgentBrowserSurfaceParams, stream: number): void { const controller = acquireAgentBrowserSurfaceController(id, params); controller.updateParams(params); - controller.handOver(port); + controller.handOver(stream); } /** Ask `id`'s browser for a render mode and a page — acquired from `params` diff --git a/lib/src/components/wall/browser-automation.ts b/lib/src/components/wall/browser-automation.ts index c81f679ca..7cecc922c 100644 --- a/lib/src/components/wall/browser-automation.ts +++ b/lib/src/components/wall/browser-automation.ts @@ -13,7 +13,6 @@ import { } from 'dor-lib-common/browser-providers'; import { getPlatform } from '../../lib/platform'; import { - playwrightTextInputs, type BrowserEditOp, type BrowserOp, type BrowserRequest, @@ -23,7 +22,6 @@ import { import { isAllowedBinaryFor } from '../../lib/agent-browser-binary'; import { messageOf } from '../../lib/errors'; import { isToolRender } from '../../lib/platform/tool-types'; -import { keyPairTextInputs } from './agent-browser-input'; import type { RenderMode } from './agent-browser-screen'; interface BrowserProviderGui { @@ -36,13 +34,11 @@ interface BrowserProviderGui { devices: readonly string[]; /** What to run from a terminal to size the viewport on a host that cannot. */ viewportHint: string; - /** A paste as the stream messages the provider's viewer takes. */ - pasteMessages(text: string): Record[]; } -function gui(provider: BrowserAutomationProvider, fields: { devices: readonly string[]; viewportVerb: string; pasteMessages(text: string): Record[] }): BrowserProviderGui { +function gui(provider: BrowserAutomationProvider, fields: { devices: readonly string[]; viewportVerb: string }): BrowserProviderGui { const cli = `dor ${BROWSER_PROVIDERS[provider].alias}`; - return { label: BROWSER_PROVIDERS[provider].label, cli, devices: fields.devices, viewportHint: `${cli} ${fields.viewportVerb} …`, pasteMessages: fields.pasteMessages }; + return { label: BROWSER_PROVIDERS[provider].label, cli, devices: fields.devices, viewportHint: `${cli} ${fields.viewportVerb} …` }; } export const BROWSER_PROVIDER_GUI: Record = { @@ -51,13 +47,10 @@ export const BROWSER_PROVIDER_GUI: Record; attach(opts?: { url?: string; headed?: boolean; requestId?: string }): Promise; - streamUrl(port: number): Promise; - screenshot(opts: { format?: 'jpeg' | 'png'; quality?: number }): Promise; + /** A viewer socket URL on the browser at `stream`. */ + view(stream: number, opts: { headed: boolean; debug: boolean }): Promise; edit(edit: BrowserEditOp): Promise; navigate(url: string): Promise; history(dir: 'back' | 'forward' | 'reload'): Promise; tab(action: 'select' | 'close', tabId: string): Promise; viewport(width: number, height: number, dpr: number): Promise; device(name: string): Promise; - cdpUrl(): Promise; /** `cancels`: the closing Surface's own requests still unanswered. */ close(cancels?: readonly string[]): Promise; } @@ -167,20 +159,13 @@ export function browserHandle(provider: BrowserAutomationProvider, binding: Omit provider, launch: (url, headed, requestId) => send({ op: 'launch', ...(url !== undefined ? { url } : {}), headed, ...(requestId !== undefined ? { requestId } : {}) }), attach: (opts = {}) => send({ op: 'attach', ...opts }), - streamUrl: (port) => send({ op: 'streamUrl', port }), - screenshot: async (opts) => { - const result = await send({ op: 'screenshot', ...opts }); - // JSON transports materialize a Uint8Array as an ordinary array. - if (result.bytes && !(result.bytes instanceof Uint8Array)) result.bytes = new Uint8Array(result.bytes); - return result; - }, + view: (stream, { headed, debug }) => send({ op: 'view', stream, ...(headed ? { headed } : {}), ...(debug ? { debug } : {}) }), edit: (edit) => send({ op: 'edit', edit }), navigate: (url) => send({ op: 'navigate', url }), history: (dir) => send({ op: 'history', dir }), tab: (action, tabId) => send({ op: 'tab', action, tabId }), viewport: (width, height, dpr) => send({ op: 'viewport', width, height, dpr }), device: (name) => send({ op: 'device', name }), - cdpUrl: () => send({ op: 'cdpUrl' }), close: (cancels = []) => send({ op: 'close', ...(cancels.length ? { cancels: [...cancels] } : {}) }), }; } diff --git a/lib/src/components/wall/tool-transfer.test.ts b/lib/src/components/wall/tool-transfer.test.ts index 79a700178..d5c765d2a 100644 --- a/lib/src/components/wall/tool-transfer.test.ts +++ b/lib/src/components/wall/tool-transfer.test.ts @@ -6,7 +6,7 @@ import type { RestoredSession } from '../../lib/session-restore'; const params = { surfaceType: 'tool', command: 'pnpm storybook', toolRender: 'ab-screencast', url: 'http://localhost:6006/edited', renderMode: 'ab-screencast', - session: 'dormouse.1.tool-one', wsPort: 9222, toolAnnouncedPort: 6006, toolAnnouncedPath: '/token/view', + session: 'dormouse.1.tool-one', stream: 9222, toolAnnouncedPort: 6006, toolAnnouncedPath: '/token/view', }; function engine(initial = params) { diff --git a/lib/src/components/wall/use-dor-control.ts b/lib/src/components/wall/use-dor-control.ts index 6cd2f90b7..7da63bf7b 100644 --- a/lib/src/components/wall/use-dor-control.ts +++ b/lib/src/components/wall/use-dor-control.ts @@ -34,7 +34,7 @@ import { isSurfaceClosing } from '../../lib/notepad/notepad-store'; import { clearToolAnnounce } from '../../lib/tool-announce-store'; import { isWorkspaceTransferPending } from '../../lib/window-session-aggregator'; import { stringParam } from './dor-control-shared'; -import { handOverBrowserPort } from './agent-browser-surface-controller'; +import { handOverBrowserStream } from './agent-browser-surface-controller'; import { callerStillPlaceable, callerStillRunnable, @@ -138,7 +138,8 @@ type EnsureBrowserSurface = (args: { cwd?: string; key?: string; session: string; - wsPort?: number; + /** The browser's stream, for the Surface's controller to view at once. */ + stream?: number; binaryPath?: string; /** Resolved lazily, only when a fresh surface must be created: the reuse path * must succeed without a visible reference (e.g. `dor ab` from a minimized @@ -777,7 +778,7 @@ export function useDorControl({ nativeIdentity, cwd, session, - wsPort, + stream, binaryPath, reference, minimized = false, @@ -791,10 +792,10 @@ export function useDorControl({ ...(headed !== undefined ? { renderMode: headedRenderMode(provider, headed) } : {}), ...(binaryPath !== undefined ? { binaryPath } : {}), }; - // The stream port is no param: the command just learned it, so the - // Surface's controller streams from it at once, even when it is unchanged. + // The stream is no param: the command just learned it, so the Surface's + // controller views it at once, even when it is unchanged. const handOver = (id: string) => { - if (wsPort !== undefined) handOverBrowserPort(id, lath.getMeta(id)?.params ?? {}, wsPort); + if (stream !== undefined) handOverBrowserStream(id, lath.getMeta(id)?.params ?? {}, stream); }; const existing = findBrowserSurface(provider, key !== undefined ? { key } : { nativeIdentity }); @@ -1684,7 +1685,7 @@ export function useDorControl({ // streamed from as it is; the host's state files may not describe that // daemon at all (docs/specs/dor-browser.md → "agent-browser"). const callerPort = provider === 'agent-browser' && isTcpPort(params.wsPort) ? params.wsPort : undefined; - const status: BrowserResult = callerPort === undefined ? await browser.attach() : { ok: true, wsPort: callerPort }; + const status: BrowserResult = callerPort === undefined ? await browser.attach() : { ok: true, stream: callerPort }; if (!status.ok) { detail.respond({ ok: false, error: status.error ?? `${BROWSER_PROVIDER_GUI[provider].label} connection failed` }); return; @@ -1700,7 +1701,7 @@ export function useDorControl({ session, cwd, binaryPath, - wsPort: status.wsPort, + stream: status.stream, headed: status.headed, // agent-browser's is its session; every host answer names one. nativeIdentity: status.nativeIdentity ?? session, diff --git a/lib/src/components/wall/wall-test-utils.ts b/lib/src/components/wall/wall-test-utils.ts index 5a5fa2a6c..4b636236c 100644 --- a/lib/src/components/wall/wall-test-utils.ts +++ b/lib/src/components/wall/wall-test-utils.ts @@ -210,7 +210,9 @@ export type BrowserAnswers = { /** * Install a platform whose host drives `providers`, answering each typed * browser request from `answers` by operation — `{ ok: true }` where none is - * given. `answers` stays live, so a test may swap one mid-flight; `requests` + * given, and for `view` a viewer socket URL naming the stream as its port, so + * a test finds a Surface's socket by the stream it views. `answers` stays + * live, so a test may swap one mid-flight; `requests` * reads back every request of one kind, in order, without the ids a Surface * mints for its launches and attaches and a close's `cancels` of them — fresh * UUIDs no test can predict (`browser`'s calls keep them). @@ -226,6 +228,7 @@ export function installBrowserHost( ) { const browser = vi.fn(async (request: BrowserRequest): Promise => { const answer = answers[request.op] as ((r: BrowserRequest) => BrowserResult | Promise) | undefined; + if (!answer && request.op === 'view') return { ok: true, url: `ws://127.0.0.1:${request.stream}` }; return (await answer?.(request)) ?? { ok: true }; }); const platform = Object.assign(new FakePtyAdapter(), { browserProviders: providers, browser }); diff --git a/lib/src/host/agent-browser-host.test.ts b/lib/src/host/agent-browser-host.test.ts index a98cdfae7..7492c6e1d 100644 --- a/lib/src/host/agent-browser-host.test.ts +++ b/lib/src/host/agent-browser-host.test.ts @@ -5,8 +5,11 @@ import { tmpdir } from 'os'; import { dirname, join } from 'path'; import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest'; import type { BrowserOp, BrowserRequestBinding } from '../lib/platform/browser-automation'; +import { WebSocketServer, type WebSocket } from 'ws'; import { createAgentBrowserProvider } from './agent-browser-host'; +import { createBrowserCaptures, type BrowserCaptures } from './browser-capture'; import { createBrowserHost } from './browser-host'; +import { openViewer } from './browser-host-test-utils'; type SpawnResult = { stdout?: string; stderr?: string; code?: number }; @@ -109,11 +112,6 @@ function ab(host: Host, op: BrowserOp, binding: BrowserRequestBinding = {}) { return host.request({ provider: 'agent-browser', binding, ...op }); } -/** The same, answering a screenshot with its file — the sidecar's transport. */ -function abFile(host: Host, op: BrowserOp, binding: BrowserRequestBinding = {}) { - return host.requestFile({ provider: 'agent-browser', binding, ...op }); -} - function enqueueSpawnResults(results: SpawnResult[]) { const queue = [...results]; spawnMock.mockImplementation((binary: string, args: string[]) => { @@ -153,7 +151,7 @@ describe('agent-browser host relaunch', () => { const result = await ab(host, { op: 'launch', url: 'https://example.com/', headed: true }, { session: 'dormouse.1.default', binaryPath: '/usr/local/bin/agent-browser' }); expect(result).toEqual({ - ok: true, wsPort: 61218, headed: true, session: 'dormouse.1.default', nativeIdentity: 'dormouse.1.default', binaryPath: '/usr/local/bin/agent-browser', + ok: true, stream: 61218, headed: true, session: 'dormouse.1.default', nativeIdentity: 'dormouse.1.default', binaryPath: '/usr/local/bin/agent-browser', }); await vi.waitFor(() => { expect(spawnMock).toHaveBeenCalledWith( @@ -189,7 +187,7 @@ describe('agent-browser host relaunch', () => { try { writeState(session, 'pid', DEAD_PID + 1); writeState(session, 'stream', port); - expect(await popOut).toEqual({ ok: true, wsPort: port, headed: true, session, nativeIdentity: session }); + expect(await popOut).toEqual({ ok: true, stream: port, headed: true, session, nativeIdentity: session }); // `open` has not returned, so no daemon command (the blank-tab sweep) has // been queued behind it. expect(calls.some((args) => args.includes('tab'))).toBe(false); @@ -234,7 +232,7 @@ describe('agent-browser host relaunch', () => { try { writeState(session, 'pid', DEAD_PID + 1); writeState(session, 'stream', port); - expect(await popOut).toEqual({ ok: true, wsPort: port, headed: true, session, nativeIdentity: session }); + expect(await popOut).toEqual({ ok: true, stream: port, headed: true, session, nativeIdentity: session }); // The second relaunch invalidates the first one's post-open tail before // its close queues behind that still-pending `open` command. @@ -276,7 +274,7 @@ describe('agent-browser host relaunch', () => { try { writeState(session, 'pid', DEAD_PID + 1); writeState(session, 'stream', port); - expect(await popOut).toEqual({ ok: true, wsPort: port, headed: true, session, nativeIdentity: session }); + expect(await popOut).toEqual({ ok: true, stream: port, headed: true, session, nativeIdentity: session }); // Pane kill/render-swap enters command('close') and invalidates the // relaunch tail synchronously, before the close queues behind open. @@ -323,7 +321,7 @@ describe('agent-browser host relaunch', () => { try { writeState(session, 'pid', DEAD_PID + 1); writeState(session, 'stream', port); - expect(await popOut).toEqual({ ok: true, wsPort: port, headed: true, session, nativeIdentity: session }); + expect(await popOut).toEqual({ ok: true, stream: port, headed: true, session, nativeIdentity: session }); await host.close(); await new Promise((resolve) => setTimeout(resolve, 0)); @@ -348,7 +346,7 @@ describe('agent-browser host relaunch', () => { stream: () => ({ stdout: JSON.stringify({ port: 61219 }) }), }); expect(await ab(host, { op: 'launch', url: 'https://slow.example/', headed: false })).toEqual({ - ok: true, session: expect.stringMatching(/^dormouse\.1\.gui-/), nativeIdentity: session, wsPort: 61219, headed: false, + ok: true, session: expect.stringMatching(/^dormouse\.1\.gui-/), nativeIdentity: session, stream: 61219, headed: false, }); // Failed with no daemon at all: fail, and close so nothing half-launched @@ -400,7 +398,7 @@ describe('agent-browser host daemon stop', () => { const kill = vi.spyOn(process, 'kill').mockImplementation((pid, signal) => (signal === 0 ? realKill(pid, 0) : true)); try { // A Tool re-announced: the same session, headless, another page. - expect(await ab(makeHost(), { op: 'launch', url: page, headed: false }, { session })).toMatchObject({ ok: true, wsPort: port, headed: false }); + expect(await ab(makeHost(), { op: 'launch', url: page, headed: false }, { session })).toMatchObject({ ok: true, stream: port, headed: false }); await vi.waitFor(() => expect(calls).toEqual([['--session', session, 'open', page]])); expect(kill.mock.calls.filter(([, signal]) => signal !== 0)).toEqual([]); } finally { @@ -502,7 +500,7 @@ describe('agent-browser host attach', () => { writeState(session, 'pid', process.pid); writeState(session, 'stream', port); const host = makeHost(); - expect(await ab(host, { op: 'attach', url: 'https://example.com/' }, { session })).toEqual({ ok: true, wsPort: port, session, nativeIdentity: session }); + expect(await ab(host, { op: 'attach', url: 'https://example.com/' }, { session })).toEqual({ ok: true, stream: port, session, nativeIdentity: session }); expect(spawnMock).not.toHaveBeenCalled(); } finally { await closeServer(server); @@ -532,13 +530,15 @@ describe('agent-browser host attach', () => { { op: 'viewport', width: 800, height: 600, dpr: 2 }, { op: 'device', name: 'iPhone 16' }, { op: 'edit', edit: 'copy' }, - { op: 'screenshot' }, ]; + const provider = createAgentBrowserProvider(); for (const pid of [undefined, DEAD_PID]) { if (pid !== undefined) writeState(session, 'pid', pid); for (const op of ops) { expect(await ab(host, op, { session }), JSON.stringify(op)).toEqual({ ok: false, error: `agent-browser session '${session}' is not running` }); } + // Nor a capture for a viewer socket. + await expect(provider.screenshot({ session }, async () => '/nonexistent/shot.jpg')).rejects.toThrow('is not running'); } expect(spawnMock).not.toHaveBeenCalled(); }); @@ -568,8 +568,8 @@ describe('agent-browser host attach', () => { // Opened at the page, so the caller has no navigation left to run. // Opened at the page, so the caller has no navigation left to run; the // second found the browser the first brought up, at the first's page. - expect(first).toEqual({ ok: true, wsPort: port, relaunched: true, session, nativeIdentity: session }); - expect(second).toEqual({ ok: true, wsPort: port, session, nativeIdentity: session }); + expect(first).toEqual({ ok: true, stream: port, relaunched: true, session, nativeIdentity: session }); + expect(second).toEqual({ ok: true, stream: port, session, nativeIdentity: session }); expect(calls).toEqual([ ['--session', session, 'close'], ['--session', session, '--headed', 'open', 'https://example.com/'], @@ -602,7 +602,7 @@ describe('agent-browser host attach', () => { try { const binding = { session: 'dormouse.1.tool.a' }; expect(await ab(host, { op: 'launch', url: 'http://localhost:5173/', headed: false }, binding)).toEqual({ - ok: true, session: 'dormouse.1.tool.a', nativeIdentity: 'dormouse.1.tool.a', wsPort: port, headed: false, + ok: true, session: 'dormouse.1.tool.a', nativeIdentity: 'dormouse.1.tool.a', stream: port, headed: false, }); // Whatever held the session is closed first, so the launch lands headless. expect(calls[0]).toEqual(['--session', 'dormouse.1.tool.a', 'close']); @@ -616,213 +616,288 @@ describe('agent-browser host attach', () => { }); }); -describe('agent-browser host screenshot transport', () => { - useTempSocketDir('dormouse-ab-shot-test-'); - beforeEach(() => { running('shotfile', 'dormouse.1.default', 'shotbytes', 'shutdown-sess', 'read-sess', 'queued', 'queued-bytes', 'retry-sess', 'sess', 'left', 'kept'); }); +describe('agent-browser host viewer', () => { + const session = 'dormouse.1.default'; + useTempSocketDir('dormouse-ab-view-test-'); + const hosts: Host[] = []; + const servers: WebSocketServer[] = []; + afterEach(async () => { + await Promise.all(hosts.splice(0).map((host) => host.close())); + for (const server of servers.splice(0)) { + for (const client of server.clients) client.terminate(); + server.close(); + } + }); - /** agent-browser's `screenshot `, writing `frames` in turn. */ - function captureFrames(...frames: number[][]): string[][] { - const queue = [...frames]; - const calls: string[][] = []; - spawnMock.mockImplementation(async (_binary: string, args: string[]) => { - calls.push(args); - if (args.includes('screenshot')) writeFileSync(args[args.indexOf('screenshot') + 1], Uint8Array.from(queue.shift() ?? [0])); - return spawnResult({}); + /** A loopback WebSocket server standing in for the daemon's stream, or for + * its browser's CDP endpoint. */ + async function fakeServer(onMessage: (message: Record, client: WebSocket) => void = () => {}) { + const server = new WebSocketServer({ port: 0, host: '127.0.0.1' }); + servers.push(server); + await new Promise((resolve) => server.once('listening', resolve)); + const received: Record[] = []; + let client: WebSocket | undefined; + server.on('connection', (ws) => { + client = ws; + ws.on('message', (data) => { + const message = JSON.parse(data.toString()); + received.push(message); + onMessage(message, ws); + }); }); - return calls; + return { + port: (server.address() as { port: number }).port, + received, + connected: () => vi.waitFor(() => expect(client).toBeDefined()).then(() => client!), + send: (message: unknown) => client!.send(typeof message === 'string' ? message : JSON.stringify(message)), + }; } - const read = async (file: string) => Array.from(await fsp.readFile(file)); - const filesIn = async (dir: string) => (await fsp.readdir(dir).catch(() => [] as string[])).sort(); - it('hands the file transport a fresh file per capture, which a later capture never rewrites', async () => { - const calls = captureFrames([1, 1], [2, 2]); + /** The host's viewer socket onto a daemon streaming on `port`. */ + async function view(port: number, headed = false) { const host = makeHost(); - const binding = { session: 'shotfile', binaryPath: '/usr/local/bin/agent-browser' }; - const first = await abFile(host, { op: 'screenshot', format: 'jpeg', quality: 85 }, binding); - expect(first).toEqual({ ok: true, path: expect.any(String), mime: 'image/jpeg' }); - expect(calls[0]).toEqual(['--session', 'shotfile', 'screenshot', expect.any(String), '--screenshot-format', 'jpeg', '--screenshot-quality', '85']); + hosts.push(host); + const { url } = await ab(host, { op: 'view', stream: port, ...(headed ? { headed } : {}) }, { session }); + return openViewer(url!); + } - // The next frame is taken while the reader has yet to read the first. - const second = await abFile(host, { op: 'screenshot', format: 'jpeg' }, binding); - expect(second.path).not.toBe(first.path); - expect(await read(first.path!)).toEqual([1, 1]); - expect(await read(second.path!)).toEqual([2, 2]); - // Only the files handed out remain: each capture's own file went once read. - expect(await filesIn(dirname(first.path!))).toEqual([first.path!, second.path!].map((file) => file.slice(dirname(file).length + 1)).sort()); - await host.close(); + // A frame's base64 body, large enough to be told from a control message by size. + const frame = (fill: number, deviceWidth = 800) => ({ type: 'frame', data: Buffer.alloc(13_000, fill).toString('base64'), metadata: { deviceWidth, deviceHeight: 600 } }); + + it('relays the daemon stream, dropping its unchanged re-broadcasts and decoding each changed frame once', async () => { + running(session); + const daemon = await fakeServer(); + writeState(session, 'stream', daemon.port); + spawnMock.mockImplementation(async (_binary: string, args: string[]) => { + if (args.includes('screenshot')) writeFileSync(args[3], Uint8Array.from([0xff, 0xd8, 0x99])); + return spawnResult({}); + }); + const viewer = await view(daemon.port); + await daemon.connected(); + const tabs = { type: 'tabs', tabs: [{ tabId: 't1', url: 'https://example.com/', title: 'Example', active: true }] }; + daemon.send({ type: 'status', connected: true, screencasting: true, viewportWidth: 800, viewportHeight: 600 }); + daemon.send(tabs); + daemon.send(tabs); + // Input opens the window in which every changed frame paints at once. + viewer.send({ type: 'input_mouse', eventType: 'mouseMoved', x: 1, y: 1 }); + await vi.waitFor(() => expect(daemon.received).toHaveLength(1)); + daemon.send(frame(1)); + daemon.send(frame(1)); + daemon.send(frame(2, 900)); + // A commit edge is never deduplicated: a reload commits the same URL. + daemon.send({ type: 'url', url: 'https://example.com/' }); + daemon.send({ type: 'url', url: 'https://example.com/' }); + await vi.waitFor(() => expect(viewer.states).toHaveLength(4)); + expect(viewer.states).toEqual([ + { type: 'status', connected: true, screencasting: true, viewportWidth: 800, viewportHeight: 600 }, + { ...tabs, tabs: [{ ...tabs.tabs[0] }] }, + { type: 'url', url: 'https://example.com/' }, + { type: 'url', url: 'https://example.com/' }, + ]); + await vi.waitFor(() => expect(viewer.frames.filter((f) => f.kind === 'crisp').length).toBeGreaterThan(0)); + const provisional = viewer.frames.filter((f) => f.kind === 'provisional'); + expect(provisional.map((f) => [f.jpeg[0], f.jpeg.byteLength, f.size?.width])).toEqual([[1, 13_000, 800], [2, 13_000, 900]]); + expect([...viewer.frames.find((f) => f.kind === 'crisp')!.jpeg]).toEqual([0xff, 0xd8, 0x99]); }); - // The frame is a picture of the user's authenticated browser, written by an - // external process under the ambient umask. A derivable path straight in - // os.tmpdir() let any other local account read every frame, or pre-create the - // name as a symlink and have agent-browser clobber the target. - it('captures into a private, unguessable directory rather than a derivable tmp path', async () => { - const calls = captureFrames([1]); - const host = makeHost(); - const shot = await abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'dormouse.1.default', binaryPath: '/usr/local/bin/agent-browser' }); - if (!shot.ok) throw new Error('expected a path'); - for (const file of [shot.path!, calls[0][3]]) { - // Nothing about the path is derivable from the session name. - expect(file).not.toContain('dormouse.1.default'); - expect(dirname(file)).not.toBe(tmpdir()); - expect(statSync(dirname(file)).mode & 0o777).toBe(0o700); + it('routes a tab list or URL too large to tell from a frame by size as state', async () => { + running(session); + const daemon = await fakeServer(); + writeState(session, 'stream', daemon.port); + const viewer = await view(daemon.port); + await daemon.connected(); + const tabs = Array.from({ length: 80 }, (_, i) => ({ + tabId: `t${i}`, + title: `Tab number ${i} — ${'x'.repeat(40)}`, + url: `https://example.com/very/long/path/segment/${i}?q=${'y'.repeat(120)}`, + active: i === 0, + })); + const url = `https://example.com/?q=${'x'.repeat(20_000)}`; + for (const message of [{ type: 'tabs', tabs }, { type: 'url', url, timestamp: 1 }]) { + expect(JSON.stringify(message).length).toBeGreaterThan(16_384); + daemon.send(message); } - await host.close(); + await vi.waitFor(() => expect(viewer.states).toHaveLength(2)); + expect(viewer.states).toEqual([{ type: 'tabs', tabs }, { type: 'url', url }]); + expect(viewer.frames).toEqual([]); }); - it('answers the bytes transport with the frame, leaving no file', async () => { - captureFrames([0xff, 0xd8, 0xff, 0x01]); - const host = makeHost(); - const result = await ab(host, { op: 'screenshot', format: 'jpeg', quality: 85 }, { session: 'shotbytes', binaryPath: '/usr/local/bin/agent-browser' }); - expect(result.mime).toBe('image/jpeg'); - expect(Array.from(result.bytes ?? [])).toEqual([0xff, 0xd8, 0xff, 0x01]); - const probe = await abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'shotbytes' }); - expect(await filesIn(dirname(probe.path!))).toEqual([probe.path!.slice(dirname(probe.path!).length + 1)]); - await host.close(); + it('sends the daemon only validated input, a paste as a key pair per character', async () => { + running(session); + const daemon = await fakeServer(); + writeState(session, 'stream', daemon.port); + const viewer = await view(daemon.port); + await daemon.connected(); + viewer.send({ type: 'input_keyboard', eventType: 'keyDown', key: 'a', code: 'KeyA', text: 'a', windowsVirtualKeyCode: 65, modifiers: 0, commands: ['selectAll'] }); + viewer.send({ type: 'input_text', text: 'x\n' }); + viewer.send({ type: 'navigate', url: 'file:///etc/passwd' }); + await vi.waitFor(() => expect(daemon.received).toHaveLength(5)); + expect(daemon.received).toEqual([ + { type: 'input_keyboard', eventType: 'keyDown', key: 'a', code: 'KeyA', text: 'a', windowsVirtualKeyCode: 65, modifiers: 0 }, + { type: 'input_keyboard', eventType: 'keyDown', key: 'x', code: '', text: 'x', windowsVirtualKeyCode: 0, modifiers: 0 }, + { type: 'input_keyboard', eventType: 'keyUp', key: 'x', code: '', text: '', windowsVirtualKeyCode: 0, modifiers: 0 }, + { type: 'input_keyboard', eventType: 'keyDown', key: 'Enter', code: 'Enter', text: '\r', windowsVirtualKeyCode: 13, modifiers: 0 }, + { type: 'input_keyboard', eventType: 'keyUp', key: 'Enter', code: 'Enter', text: '', windowsVirtualKeyCode: 13, modifiers: 0 }, + ]); }); - it('deletes frames never read when their browser closes or relaunches, and any older than every reader\'s wait', async () => { - captureFrames([1], [2], [3], [4]); - const host = makeHost(); - const unread = await abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'left' }); - await ab(host, { op: 'close' }, { session: 'left' }); - await vi.waitFor(() => expect(existsSync(unread.path!)).toBe(false)); - - // A reader that gave up never comes for its frame. - const stale = await abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'kept' }); - const now = Date.now(); - const clock = vi.spyOn(Date, 'now').mockReturnValue(now + 40_001); - try { - const next = await abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'kept' }); - await vi.waitFor(() => expect(existsSync(stale.path!)).toBe(false)); - expect(existsSync(next.path!)).toBe(true); - } finally { - clock.mockRestore(); - } - await host.close(); + it('tells the viewer when the daemon goes away, and ends a viewer of a port nothing streams on', async () => { + running(session); + const daemon = await fakeServer(); + writeState(session, 'stream', daemon.port); + const viewer = await view(daemon.port); + (await daemon.connected()).terminate(); + expect(await viewer.closed).toBe(1000); + expect(viewer.states).toEqual([{ type: 'status', connected: false, screencasting: false }]); + + const nowhere = await view(await closedPort()); + expect(await nowhere.closed).toBe(1011); + expect(nowhere.states).toEqual([]); + // Nor anything but a TCP port. + const offRange = await view(70_000); + expect(await offRange.closed).toBe(1011); }); - it('drops the capture directory on shutdown', async () => { - captureFrames([1, 2, 3]); - const host = makeHost(); - const shot = await abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'shutdown-sess', binaryPath: '/usr/local/bin/agent-browser' }); - if (!shot.ok) throw new Error('expected a path'); - const dir = dirname(shot.path!); - - await host.close(); - - // A frame of the user's authenticated browser must not outlive the process - // that took it, waiting on whenever the OS gets round to reaping tmp. - expect(existsSync(dir)).toBe(false); + // `dor ab` under the caller's own AGENT_BROWSER_SOCKET_DIR hands over a port + // the host's socket directory knows nothing of. + it('watches a daemon in a socket directory it does not share: every changed frame, and no capture', async () => { + const daemon = await fakeServer(); + const viewer = await view(daemon.port); + await daemon.connected(); + for (let fill = 1; fill <= 3; fill++) daemon.send(frame(fill)); + await vi.waitFor(() => expect(viewer.frames).toHaveLength(3)); + expect(viewer.frames.map((f) => `${f.kind} ${f.jpeg[0]}`)).toEqual(['provisional 1', 'provisional 2', 'provisional 3']); + await new Promise((resolve) => setTimeout(resolve, 300)); + expect(spawnMock).not.toHaveBeenCalled(); }); - it('joins a capture already in flight for the session instead of spawning another', async () => { - const release = deferred(); - spawnMock.mockImplementation(async (_binary: string, args: string[]) => { - const result = await release.promise; - writeFileSync(args[3], Uint8Array.from([0xff, 0xd8, 0x01])); - return spawnResult(result); + it('follows a headed window\'s page over its browser\'s CDP, and sends it no frames', async () => { + running(session); + const cdp = await fakeServer((message, client) => { + if (message.method === 'Target.getTargets') { + client.send(JSON.stringify({ id: message.id, result: { targetInfos: [{ type: 'page', url: 'https://one.example/', title: 'One' }, { type: 'service_worker', url: 'https://one.example/sw.js' }] } })); + } }); - const host = makeHost(); - - const paths = [ - abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'queued' }), - abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'queued' }), - ]; - const bytes = [ - ab(host, { op: 'screenshot', format: 'jpeg' }, { session: 'queued-bytes' }), - ab(host, { op: 'screenshot', format: 'jpeg' }, { session: 'queued-bytes' }), - ]; - await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledTimes(2)); // one per session - await new Promise((resolve) => setTimeout(resolve, 20)); - expect(spawnMock).toHaveBeenCalledTimes(2); - - release.resolve({}); - // Each file caller gets its own copy, which its reader deletes. - const [first, second] = await Promise.all(paths); - expect(second.path).not.toBe(first.path); - for (const result of [first, second]) expect(await read(result.path!)).toEqual([0xff, 0xd8, 0x01]); - for (const result of await Promise.all(bytes)) { - expect(Array.from(result.bytes ?? [])).toEqual([0xff, 0xd8, 0x01]); - } + const daemon = await fakeServer(); + writeState(session, 'stream', daemon.port); + enqueueSpawnResults([{ stdout: `ws://127.0.0.1:${cdp.port}/devtools/browser/abc\n` }]); + const viewer = await view(daemon.port, true); + await daemon.connected(); + await cdp.connected(); + await vi.waitFor(() => expect(cdp.received.map((m) => m.method)).toEqual(['Target.setDiscoverTargets', 'Target.getTargets', 'Page.enable'])); + cdp.send({ method: 'Target.targetInfoChanged', params: { targetInfo: { type: 'page', url: 'https://two.example/', title: 'Two' } } }); + cdp.send({ method: 'Page.frameNavigated', params: { frame: { parentId: 'p', url: 'https://ad.example/' } } }); + daemon.send(frame(1)); + await vi.waitFor(() => expect(viewer.states).toHaveLength(2)); + expect(viewer.states).toEqual([ + { type: 'page', url: 'https://one.example/', title: 'One' }, + { type: 'page', url: 'https://two.example/', title: 'Two' }, + ]); + expect(viewer.frames).toEqual([]); + expect(spawnMock.mock.calls.map((call) => (call[1] as string[]).slice(2))).toEqual([['get', 'cdp-url']]); + }); - // Once it has answered, the next request captures afresh. - spawnMock.mockReset(); - captureFrames([1]); - await abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'queued' }); - expect(spawnMock).toHaveBeenCalledTimes(1); - await host.close(); + it('dials no CDP endpoint off loopback', async () => { + running(session); + const daemon = await fakeServer(); + writeState(session, 'stream', daemon.port); + enqueueSpawnResults([{ stdout: 'ws://10.0.0.1:9222/devtools/browser/abc\n' }]); + const log = vi.fn(); + const host = createBrowserHost({ writeClipboardText: vi.fn(), providers: { 'agent-browser': () => createAgentBrowserProvider({ log }) } }); + hosts.push(host); + const { url } = await ab(host, { op: 'view', stream: daemon.port, headed: true }, { session }); + await openViewer(url!); + await vi.waitFor(() => expect(log).toHaveBeenCalledWith('[agent-browser] refused a CDP endpoint off loopback: ws://10.0.0.1:9222/devtools/browser/abc')); }); +}); - it('joins no capture from before a close or relaunch, and bounds every capture', async () => { - // Every screenshot hangs; closes and relaunch steps answer at once. - const shots: string[] = []; +describe('agent-browser host captures', () => { + useTempSocketDir('dormouse-ab-shot-test-'); + const b = { session: 'dormouse.1.default', binaryPath: '/usr/local/bin/agent-browser' }; + beforeEach(() => { running(b.session); }); + // Stand in for agent-browser writing the frame where it is told, once + // `release` lets it: each capture writes the next byte. + function writesFrames() { + let shots = 0; + let gate: Promise | null = null; + let open = () => {}; spawnMock.mockImplementation(async (_binary: string, args: string[]) => { - if (args.includes('screenshot')) { - shots.push(args[3]); - return new Promise(() => {}); - } - return spawnResult({ code: args.includes('open') ? 1 : 0 }); + await gate; + writeFileSync(args[3], Uint8Array.from([0xff, 0xd8, ++shots])); + return spawnResult({}); }); - // A relaunch kills the daemon its pid file names, so a child stands in. - const daemons: ChildProcess[] = []; - const daemon = () => { - const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1 << 30)'], { stdio: 'ignore' }); - daemons.push(child); - writeState('wedged', 'pid', child.pid!); - }; - onTestFinished(() => { for (const child of daemons) child.kill('SIGKILL'); }); - daemon(); - const host = makeHost(); - const capture = async () => { - void abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'wedged' }); - await vi.waitFor(() => expect(shots.length).toBeGreaterThan(0)); - await new Promise((resolve) => setTimeout(resolve, 20)); + return { + hold: () => { gate = new Promise((resolve) => { open = resolve; }); }, + release: () => { gate = null; open(); }, }; + } + const take = (captures: BrowserCaptures, provider = createAgentBrowserProvider()) => captures.take(b.session, (file) => provider.screenshot(b, file)); - await capture(); - await capture(); - expect(shots).toHaveLength(1); - - // A close ends the session those captures were for. - await ab(host, { op: 'close' }, { session: 'wedged' }); - await capture(); - expect(shots).toHaveLength(2); - - // So does a relaunch, which reuses the session name. - await ab(host, { op: 'launch', url: 'https://example.com/', headed: false }, { session: 'wedged' }); - daemon(); - await capture(); - expect(shots).toHaveLength(3); - - // Each replacement writes a file the capture it replaced cannot overwrite. - expect(new Set(shots).size).toBe(3); - // And no capture can pin the slot: the spawn itself is bounded past the - // CLI's 25s action timeout. - for (const call of spawnMock.mock.calls.filter((c) => (c[1] as string[]).includes('screenshot'))) { - expect(call[2]).toEqual({ timeoutMs: 30_000 }); - } - await host.close(); + // The frame is a picture of the user's authenticated browser, written by an + // external process under the ambient umask. A derivable path straight in + // os.tmpdir() let any other local account read every frame, or pre-create the + // name as a symlink and have agent-browser clobber the target. + it('captures into a private, unguessable file, reads it back and removes it, bounded', async () => { + writesFrames(); + const captures = createBrowserCaptures(); + expect([...await take(captures)]).toEqual([0xff, 0xd8, 1]); + const [binary, args, options] = spawnMock.mock.calls[0] as [string, string[], unknown]; + const file = args[3]; + expect(binary).toBe(b.binaryPath); + expect(args).toEqual(['--session', b.session, 'screenshot', file, '--screenshot-format', 'jpeg', '--screenshot-quality', '85']); + // Past the CLI's 25s action timeout, so a wedged capture cannot pin its slot. + expect(options).toEqual({ timeoutMs: 30_000 }); + expect(file).not.toContain(b.session); + expect(existsSync(file)).toBe(false); + const dir = dirname(file); + expect(dir).not.toBe(tmpdir()); + expect(statSync(dir).mode & 0o777).toBe(0o700); + // Reused per browser, so frames do not accumulate. + expect([...await take(captures)]).toEqual([0xff, 0xd8, 2]); + expect((spawnMock.mock.calls[1][1] as string[])[3]).toBe(file); + // No frame of the user's browser outlives the host that took it. + await captures.remove(); + expect(existsSync(dir)).toBe(false); }); - it('answers a capture-directory failure as a result, and retries the next time', async () => { + it('joins a capture of the browser already running, and none it was told to forget', async () => { + const frames = writesFrames(); + const captures = createBrowserCaptures(); + frames.hold(); + const joined = [take(captures), take(captures)]; + await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledTimes(1)); + // Its browser closed or relaunched: the next capture is a fresh one, in a + // file the running one cannot overwrite. + captures.forget(b.session); + const fresh = take(captures); + await vi.waitFor(() => expect(spawnMock).toHaveBeenCalledTimes(2)); + frames.release(); + const [first, second] = await Promise.all(joined); + expect(second).toBe(first); + expect(await fresh).not.toBe(first); + expect((spawnMock.mock.calls[1][1] as string[])[3]).not.toBe((spawnMock.mock.calls[0][1] as string[])[3]); + await captures.remove(); + }); + + it('fails a capture whose directory it could not create, and retries it the next time', async () => { // `??=` on the mkdtemp promise would memoize a rejection, so one transient // EACCES/ENOSPC on tmpdir would disable screenshots for the whole process. const mkdtemp = vi.spyOn(fsp, 'mkdtemp').mockRejectedValueOnce(new Error('ENOSPC: no space left on device')); - const host = makeHost(); - - const failed = await abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'retry-sess', binaryPath: '/usr/local/bin/agent-browser' }); - expect(failed.ok).toBe(false); - expect(failed.ok === false && failed.error).toContain('ENOSPC'); + writesFrames(); + const captures = createBrowserCaptures(); + await expect(take(captures)).rejects.toThrow('ENOSPC'); expect(spawnMock).not.toHaveBeenCalled(); // never spawned without a path - mkdtemp.mockRestore(); - captureFrames([1]); - const recovered = await abFile(host, { op: 'screenshot', format: 'jpeg' }, { session: 'retry-sess', binaryPath: '/usr/local/bin/agent-browser' }); - expect(recovered.ok).toBe(true); - await host.close(); + expect([...await take(captures)]).toEqual([0xff, 0xd8, 1]); + await captures.remove(); }); +}); + +describe('agent-browser host binary', () => { + useTempSocketDir('dormouse-ab-binary-test-'); + beforeEach(() => { running('sess'); }); + // `binaryPath` crosses from the webview realm and off the persisted session // blob, so an unchecked one is arbitrary local execution in the extension host // or the Tauri sidecar. The gate is at the spawn, so it covers attach / @@ -877,11 +952,6 @@ describe('agent-browser host requests', () => { expect(await ab(host, op, session)).toEqual({ ok: true }); expect(spawnMock.mock.calls[0][1]).toEqual(['--session', 'dormouse.1.gui-abc', ...argv]); } - // The CDP endpoint is read from what the CLI prints. - spawnMock.mockReset(); - enqueueSpawnResults([{ stdout: 'ws://127.0.0.1:9222/devtools/browser/abc\n' }]); - expect(await ab(host, { op: 'cdpUrl' }, session)).toEqual({ ok: true, url: 'ws://127.0.0.1:9222/devtools/browser/abc' }); - expect(spawnMock.mock.calls[0][1]).toEqual(['--session', 'dormouse.1.gui-abc', 'get', 'cdp-url']); }); it('refuses any other request, including a field carrying options', async () => { @@ -901,6 +971,13 @@ describe('agent-browser host requests', () => { { op: 'device', name: '--state=/tmp/s.json' }, { op: 'screenshotTo', path: '/Users/someone/.zshrc' }, { op: 'eval', script: 'document.cookie' }, + // The webview holds no CDP, captures nothing itself, and names no port to dial but a stream's. + { op: 'cdpUrl' }, + { op: 'screenshot' }, + { op: 'streamUrl', port: 9222 }, + { op: 'view', stream: -1 }, + { op: 'view', stream: 1.5 }, + { op: 'view' }, { op: 'constructor' }, {}, ]; @@ -920,7 +997,7 @@ describe('agent-browser host requests', () => { const ops: BrowserOp[] = [ { op: 'close' }, { op: 'edit', edit: 'copy' }, - { op: 'screenshot' }, + { op: 'view', stream: 4321 }, { op: 'attach', url: 'https://example.com/' }, { op: 'launch', url: 'https://example.com/', headed: true }, { op: 'launch', url: 'https://example.com/', headed: false }, @@ -928,7 +1005,6 @@ describe('agent-browser host requests', () => { for (const session of ['--executable-path', '-x', '../../tmp/evil', 'a/b', 'a\\b', 'a\nb', '']) { for (const op of ops) { expect(await ab(host, op, { session })).toEqual({ ok: false, error: 'a valid session name is required' }); - expect((await abFile(host, op, { session })).ok).toBe(false); } } expect(spawnMock).not.toHaveBeenCalled(); diff --git a/lib/src/host/agent-browser-host.ts b/lib/src/host/agent-browser-host.ts index 6ba601437..c402426c5 100644 --- a/lib/src/host/agent-browser-host.ts +++ b/lib/src/host/agent-browser-host.ts @@ -4,14 +4,11 @@ * imported by the VS Code extension host, bundled for the * standalone sidecar. What is genuinely agent-browser's lives here — its * per-session daemon and the state files it leaves beside its socket, the pid - * kill a headed/headless relaunch needs, and each operation's one fixed argv. - * The host owns everything the two providers share. + * kill a headed/headless relaunch needs, each operation's one fixed argv, and + * the daemon's stream the host relays to its viewer socket. The host owns + * everything the two providers share. * - * Plain Node (child_process / fs), so the same code runs on both hosts. The - * VS Code stream relay is NOT here: it works around the `vscode-webview://` - * origin the agent-browser stream server rejects, which is a VS-Code-only - * concern (the standalone webview's `tauri://localhost` origin is accepted, so - * it connects directly). It stays in the VS Code host, injected as `streamUrl`. + * Plain Node (child_process / fs / ws), so the same code runs on both hosts. */ import * as net from 'net'; import * as os from 'os'; @@ -31,10 +28,12 @@ import { AGENT_BROWSER_SOCKET_DIR_ENV, DEFAULT_AGENT_BROWSER_BIN, } from 'dor-lib-common'; +import { WebSocket } from 'ws'; import { isAllowedAgentBrowserBinary } from '../lib/agent-browser-binary'; import { parseAgentBrowserTabs } from '../lib/agent-browser-tab'; -import type { BrowserResult } from '../lib/platform/browser-automation'; +import { CAPTURE_JPEG_QUALITY, type BrowserResult, type ViewerInput } from '../lib/platform/browser-automation'; import type { BrowserAct, BrowserProvider, LiveBrowser, ProviderBinding } from './browser-host'; +import type { Upstream, ViewerSink } from './browser-viewer'; const SESSION_ARGS = BROWSER_PROVIDERS['agent-browser'].sessionArgs; @@ -53,7 +52,6 @@ function actArgv(act: BrowserAct): string[] | null { return act.action === 'select' ? ['tab', act.tabId] : ['tab', 'close', act.tabId]; case 'viewport': return ['set', 'viewport', String(act.width), String(act.height), String(act.dpr)]; case 'device': return ['set', 'device', act.name]; - case 'cdpUrl': return ['get', 'cdp-url']; } } @@ -85,15 +83,47 @@ const CAPTURE_TIMEOUT_MS = 30_000; const STREAM_PORT_READ_ATTEMPTS = 4; const STREAM_PORT_READ_DELAY_MS = 150; const PORT_PROBE_TIMEOUT_MS = 500; +const STREAM_CONNECT_TIMEOUT_MS = 5000; +// A stream message above this size is a frame (a base64 JPEG); status, tabs +// and url are small — unless a long tab list or URL crosses it too. +const FRAME_THRESHOLD_BYTES = 16384; const delay = (ms: number): Promise => new Promise((resolve) => setTimeout(resolve, ms)); export interface AgentBrowserProviderDeps { /** Optional diagnostic logger. */ log?: (message: string) => void; - /** The stream URL for a port; absent, the webview dials it directly. */ - streamUrl?: (port: number) => Promise; } +/** + * A paste for the daemon's stream, which takes only key and mouse events: a + * key down and up per character, a newline as Enter. + */ +export function keyPairTextInputs(text: string): Extract[] { + const messages: Extract[] = []; + for (const ch of text) { + if (ch === '\r') continue; + if (ch === '\n') { + messages.push({ type: 'input_keyboard', eventType: 'keyDown', key: 'Enter', code: 'Enter', text: '\r', windowsVirtualKeyCode: 13, modifiers: 0 }); + messages.push({ type: 'input_keyboard', eventType: 'keyUp', key: 'Enter', code: 'Enter', text: '', windowsVirtualKeyCode: 13, modifiers: 0 }); + } else { + messages.push({ type: 'input_keyboard', eventType: 'keyDown', key: ch, code: '', text: ch, windowsVirtualKeyCode: 0, modifiers: 0 }); + messages.push({ type: 'input_keyboard', eventType: 'keyUp', key: ch, code: '', text: '', windowsVirtualKeyCode: 0, modifiers: 0 }); + } + } + return messages; +} + +/** The viewport's CSS size a stream frame's metadata carries, when whole. */ +function frameSize(metadata: { deviceWidth?: unknown; deviceHeight?: unknown } | undefined): { width: number; height: number } | undefined { + const width = metadata?.deviceWidth; + const height = metadata?.deviceHeight; + return typeof width === 'number' && width > 0 && typeof height === 'number' && height > 0 ? { width, height } : undefined; +} + +// A frame's bulk is base64, whose alphabet has no `"` or `:`: these mark a +// control message large enough to pass for a frame. +const CONTROL_MARKERS = ['"type":"tabs"', '"type":"status"', '"type":"url"']; + export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): BrowserProvider { const log = deps.log ?? (() => {}); @@ -275,7 +305,7 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): const pid = await readStateNumber(b.session, 'pid'); if (pid !== undefined && processAlive(pid)) { const port = await acceptingStreamPort(b.session); - if (port !== undefined) return { wsPort: port }; + if (port !== undefined) return { stream: port }; throw new Error(`agent-browser session '${b.session}' is not streaming`); } return { gone: `agent-browser session '${b.session}' is not running`, named: pid !== undefined }; @@ -321,11 +351,11 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): if (opened) { if (opened.exitCode !== 0 && !daemonUp) return { failed: opened.stderr.trim() || `agent-browser open exited ${opened.exitCode}` }; const port = await readStreamPort(b, deadline); - return port !== undefined ? { wsPort: port } : { failed: 'agent-browser published no stream port' }; + return port !== undefined ? { stream: port } : { failed: 'agent-browser published no stream port' }; } if (!daemonUp) return undefined; const port = await acceptingStreamPort(b.session); - return port !== undefined ? { wsPort: port } : undefined; + return port !== undefined ? { stream: port } : undefined; }, async close(b, timeoutMs) { @@ -354,10 +384,7 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): const argv = actArgv(act); if (!argv) return { ok: false, error: 'invalid tab operation' }; const result = await drive(b, argv); - if (result.exitCode !== 0) return { ok: false, error: cliError(result) }; - if (act.op !== 'cdpUrl') return { ok: true }; - const url = parseCdpUrl(result.stdout); - return url ? { ok: true, url } : { ok: false, error: 'agent-browser printed no CDP endpoint' }; + return result.exitCode === 0 ? { ok: true } : { ok: false, error: cliError(result) }; }, // eval --json envelope: { success, data: { result }, error }. @@ -376,10 +403,9 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): // agent-browser's `screenshot` honors the session's viewport/DPR, unlike // the CSS-resolution screencast, and writes the frame where it is told. - async screenshot(b, { format, quality }, file) { + async screenshot(b, file) { const out = await file(); - const args = ['screenshot', out, '--screenshot-format', format]; - if (format === 'jpeg') args.push('--screenshot-quality', String(quality)); + const args = ['screenshot', out, '--screenshot-format', 'jpeg', '--screenshot-quality', String(CAPTURE_JPEG_QUALITY)]; const result = await drive(b, args, { timeoutMs: CAPTURE_TIMEOUT_MS }); if (result.exitCode !== 0) { log(`[agent-browser] screenshot failed (exit ${result.exitCode}): ${cliError(result)}`); @@ -388,6 +414,162 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): return { path: out }; }, - streamUrl: async (port) => (deps.streamUrl ? deps.streamUrl(port) : `ws://127.0.0.1:${port}`), + view: (b, port, { headed }, sink) => viewStream(b, port, headed, sink), }; + + /** + * The daemon's stream at `port`, relayed to one viewer socket. The daemon + * re-sends its current frame and tab list ~20 times a second whether or not + * they changed, so each is compared raw against the last and dropped when + * equal: only a changed frame is parsed, decoded once and passed on. A + * headed viewer gets no frames, and its page is followed over the browser's + * CDP (`observePage`), which the stream does not report for navigations + * made in the window itself. + * + * `port` may be one `dor ab` read under a socket directory the host does + * not share: it is only ever dialed on loopback, only the stream's own + * messages come back from it, and only validated input goes to it. + */ + async function viewStream(b: ProviderBinding, port: number, headed: boolean, sink: ViewerSink): Promise { + if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('an agent-browser stream is a TCP port'); + // Only a daemon in the host's own socket directory is one it can capture. + const capturable = (await readStateNumber(b.session, 'stream')) === port; + const socket = new WebSocket(`ws://127.0.0.1:${port}`, { handshakeTimeout: STREAM_CONNECT_TIMEOUT_MS, perMessageDeflate: false }); + let opened = false; + let closing = false; + let observer: { close(): void } | null = null; + let lastFrame: Buffer | undefined; + const lastState = new Map(); + socket.on('error', (error) => log(`[agent-browser] stream error: ${error.message}`)); + const open = new Promise((resolve, reject) => { + socket.once('open', () => { + opened = true; + resolve(); + }); + socket.on('close', () => { + observer?.close(); + if (!opened) reject(new Error(`no agent-browser stream answers on port ${port}`)); + else if (!closing) sink.gone(); + }); + }); + socket.on('message', (data: Buffer, isBinary) => { + if (isBinary) return; + const large = data.length > FRAME_THRESHOLD_BYTES && !CONTROL_MARKERS.some((marker) => data.includes(marker)); + if (large) { + if (headed || lastFrame?.equals(data)) return; + lastFrame = data; + } + const text = data.toString(); + let message: { type?: unknown; data?: unknown; metadata?: { deviceWidth?: unknown; deviceHeight?: unknown }; connected?: unknown; screencasting?: unknown; viewportWidth?: unknown; viewportHeight?: unknown; tabs?: unknown; url?: unknown }; + try { + message = JSON.parse(text); + } catch { + return; + } + if (message.type === 'frame' && typeof message.data === 'string') { + if (headed) return; + if (!large) { + if (lastFrame?.equals(data)) return; + lastFrame = data; + } + sink.frame(Buffer.from(message.data, 'base64'), frameSize(message.metadata)); + return; + } + if (message.type === 'url') { + // A commit edge, never deduplicated: a reload commits the same URL. + if (typeof message.url === 'string') sink.state({ type: 'url', url: message.url }); + return; + } + if (message.type !== 'status' && message.type !== 'tabs') return; + if (lastState.get(message.type) === text) return; + lastState.set(message.type, text); + if (message.type === 'tabs') { + if (Array.isArray(message.tabs)) sink.state({ type: 'tabs', tabs: parseAgentBrowserTabs(message.tabs) }); + return; + } + sink.state({ + type: 'status', + connected: message.connected === true, + screencasting: message.screencasting === true, + ...(typeof message.viewportWidth === 'number' ? { viewportWidth: message.viewportWidth } : {}), + ...(typeof message.viewportHeight === 'number' ? { viewportHeight: message.viewportHeight } : {}), + }); + }); + await open; + if (headed) observer = observePage(b, sink); + const forward = (message: object) => { + if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify(message)); + }; + return { + capturable, + input(message) { + if (message.type === 'input_text') for (const key of keyPairTextInputs(message.text)) forward(key); + else forward(message); + return true; + }, + close() { + closing = true; + observer?.close(); + socket.close(); + }, + }; + } + + /** + * Follow a headed window's page over its browser's CDP: the URL and title + * of each page target as it is created or changes, and each main-frame + * navigation. The endpoint is asked of the daemon, which is up while its + * stream is; the CDP socket stays in the host. + */ + function observePage(b: ProviderBinding, sink: ViewerSink): { close(): void } { + let closed = false; + let cdp: WebSocket | null = null; + const page = (url: unknown, title: unknown) => { + if (typeof url === 'string') sink.state({ type: 'page', url, title: typeof title === 'string' ? title : null }); + }; + void drive(b, ['get', 'cdp-url']).then((result) => { + const url = result.exitCode === 0 ? parseCdpUrl(result.stdout) : null; + if (closed || !url) { + if (!url) log(`[agent-browser] no CDP endpoint for ${b.session}: ${cliError(result)}`); + return; + } + // The browser's own endpoint, and only ever on loopback. + if (!/^ws:\/\/(127\.0\.0\.1|localhost):\d+\//.test(url)) { + log(`[agent-browser] refused a CDP endpoint off loopback: ${url}`); + return; + } + const socket = cdp = new WebSocket(url, { perMessageDeflate: false }); + let nextId = 1; + const send = (method: string, params?: Record) => socket.send(JSON.stringify({ id: nextId++, method, ...(params ? { params } : {}) })); + socket.on('open', () => { + send('Target.setDiscoverTargets', { discover: true }); + send('Target.getTargets'); + // Were the endpoint a page's rather than the browser's, its own events + // would be the navigation source. + send('Page.enable'); + }); + socket.on('error', (error) => log(`[agent-browser] CDP observer error: ${error.message}`)); + socket.on('message', (data: Buffer) => { + let message: { method?: string; params?: { targetInfo?: unknown; frame?: { parentId?: unknown; url?: unknown; name?: unknown } }; result?: { targetInfos?: unknown } }; + try { + message = JSON.parse(data.toString()); + } catch { + return; + } + const target = (info: unknown) => { + const t = info as { type?: unknown; url?: unknown; title?: unknown } | null; + if (t && typeof t === 'object' && t.type === 'page') page(t.url, t.title); + }; + if (message.method === 'Target.targetCreated' || message.method === 'Target.targetInfoChanged') target(message.params?.targetInfo); + else if (message.method === 'Page.frameNavigated' && !message.params?.frame?.parentId) page(message.params?.frame?.url, message.params?.frame?.name); + else if (Array.isArray(message.result?.targetInfos)) for (const info of message.result.targetInfos) target(info); + }); + }).catch((error: unknown) => log(`[agent-browser] CDP observer: ${error instanceof Error ? error.message : String(error)}`)); + return { + close() { + closed = true; + cdp?.close(); + }, + }; + } } diff --git a/lib/src/host/browser-capture.ts b/lib/src/host/browser-capture.ts new file mode 100644 index 000000000..69233833f --- /dev/null +++ b/lib/src/host/browser-capture.ts @@ -0,0 +1,70 @@ +/** + * The crisp captures behind the viewer sockets (docs/specs/dor-browser.md → + * "Viewer Socket"): one device-resolution JPEG of a browser per ask, which a + * CLI writes into the host's private capture directory and the host reads + * back and removes at once, so no frame waits on disk. + */ +import { randomBytes } from 'crypto'; +import * as path from 'path'; +import { promises as fs } from 'fs'; +import { privateCaptureDir } from './private-capture-dir'; + +/** One capture by a provider: written to `file()` by a CLI, or its bytes. */ +export type Shoot = (file: () => Promise) => Promise<{ path: string } | { bytes: Uint8Array }>; + +export interface BrowserCaptures { + /** A JPEG of browser `id`, joining one of it already running. */ + take(id: string, shoot: Shoot): Promise; + /** Join none of `id`'s running captures, and give its next a fresh file, + * so one still running cannot overwrite it: its browser was closed or + * replaced. */ + forget(id: string): void; + /** Drop the directory and every frame in it. */ + remove(): Promise; +} + +export function createBrowserCaptures(): BrowserCaptures { + // Screenshots of the user's authenticated browser land here, written by an + // external process under the ambient umask — which is why the private + // directory, not the file mode, is the control. + const dir = privateCaptureDir('dormouse-browser-'); + // One file per browser, so frames don't litter; one capture of it at a time + // (below), so reusing the name is safe. The random name keeps it unguessable + // from the session alone. + const names = new Map(); + // Surfaces can share a session, so a capture another viewer asks for + // meanwhile joins rather than repeats. + const inFlight = new Map>(); + + async function file(id: string): Promise { + let name = names.get(id); + if (name === undefined) names.set(id, name = randomBytes(12).toString('hex')); + return path.join(await dir.get(), `shot-${name}.jpg`); + } + + return { + take(id, shoot) { + const pending = inFlight.get(id); + if (pending) return pending; + // Joined whole, read and unlink included: a caller joining only the + // capture would read a file the first caller has already removed. + const taking: Promise = (async () => { + const shot = await shoot(() => file(id)); + if ('bytes' in shot) return shot.bytes; + const buffer = await fs.readFile(shot.path); + await fs.unlink(shot.path).catch(() => {}); + return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + })().finally(() => { if (inFlight.get(id) === taking) inFlight.delete(id); }); + inFlight.set(id, taking); + return taking; + }, + forget(id) { + inFlight.delete(id); + names.delete(id); + }, + async remove() { + await dir.remove(); + names.clear(); + }, + }; +} diff --git a/lib/src/host/browser-host-test-utils.ts b/lib/src/host/browser-host-test-utils.ts index be9b40766..2ef29bdaa 100644 --- a/lib/src/host/browser-host-test-utils.ts +++ b/lib/src/host/browser-host-test-utils.ts @@ -2,10 +2,23 @@ * A fake `BrowserProvider` for tests that run the real browser host * (`createBrowserHost`) — its own tests, and the webview's against it. */ +import { WebSocket } from 'ws'; import type { BrowserProvider, ProviderBinding } from './browser-host'; +import type { Upstream, ViewerSink } from './browser-viewer'; +import { decodeViewerFrame, type ViewerFrame, type ViewerInput, type ViewerState } from '../lib/platform/browser-automation'; + +/** One subscription the host made through the fake provider's `view`. */ +export interface FakeView { + session: string; + stream: number; + headed: boolean; + sink: ViewerSink; + inputs: Exclude[]; + closed: boolean; +} /** A provider that records every primitive the host calls, in order; `stop`, - * `close` and `open` can be held open by a test. */ + * `close`, `open` and `screenshot` can be held open by a test. */ export function fakeProvider() { const calls: string[] = []; const held = new Map void>(); @@ -16,6 +29,8 @@ export function fakeProvider() { if (gates.has(name)) await hold(name); }; let tabs: { tabId: string; url: string }[] = []; + const views: FakeView[] = []; + let shot = 0; const provider: BrowserProvider = { pollMs: 1, bind: (binding) => binding, @@ -27,7 +42,7 @@ export function fakeProvider() { await step(`open ${b.session} ${url ?? 'blank'}${headed ? ' headed' : ''}`); return { exitCode: 0, stderr: '' }; }, - probe: async () => ({ wsPort: 4321 }), + probe: async () => ({ stream: 4321 }), close: async (b) => { await step(`close ${b.session}`); }, listTabs: async () => tabs, closeTab: async (b, tabId) => { calls.push(`tab ${b.session} close ${tabId}`); }, @@ -36,15 +51,63 @@ export function fakeProvider() { return { ok: true }; }, evaluate: async () => '', - screenshot: async () => ({ bytes: new Uint8Array([1]) }), - streamUrl: async (port) => `ws://127.0.0.1:${port}`, + // Each capture a distinct JPEG-ish frame, so none dedups against the last. + screenshot: async (b) => { + await step(`screenshot ${b.session}`); + shot += 1; + return { bytes: new Uint8Array([0xff, 0xd8, shot]) }; + }, + view: async (b, stream, { headed }, sink): Promise => { + calls.push(`view ${b.session} ${stream}${headed ? ' headed' : ''}`); + const view: FakeView = { session: b.session, stream, headed, sink, inputs: [], closed: false }; + views.push(view); + return { + capturable: true, + input: (message) => { view.inputs.push(message); return true; }, + close: () => { view.closed = true; }, + }; + }, }; return { provider, calls, + views, /** Hold the named primitive call until `release(name)`. */ gate: (name: string) => gates.add(name), release: (name: string) => { gates.delete(name); held.get(name)?.(); }, setTabs: (next: typeof tabs) => { tabs = next; }, }; } + +/** The webview's end of a viewer socket, as a test holds it: what arrived, + * frames decoded, and a way to send input. */ +export interface TestViewer { + socket: WebSocket; + frames: ViewerFrame[]; + states: ViewerState[]; + send(message: object): void; + /** Settles with the close code once the host ends the socket. */ + closed: Promise; +} + +/** Connect to a viewer socket URL as the webview does, once it is open. */ +export async function openViewer(url: string): Promise { + const socket = new WebSocket(url); + const frames: ViewerFrame[] = []; + const states: ViewerState[] = []; + socket.on('error', () => {}); + socket.on('message', (data: Buffer, isBinary) => { + if (!isBinary) { + states.push(JSON.parse(data.toString()) as ViewerState); + return; + } + const frame = decodeViewerFrame(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer); + if (frame) frames.push(frame); + }); + const closed = new Promise((resolve) => socket.once('close', (code) => resolve(code))); + await new Promise((resolve, reject) => { + socket.once('open', () => resolve()); + socket.once('unexpected-response', (_req, res) => reject(new Error(`viewer refused: ${res.statusCode}`))); + }); + return { socket, frames, states, send: (message) => socket.send(JSON.stringify(message)), closed }; +} diff --git a/lib/src/host/browser-host.test.ts b/lib/src/host/browser-host.test.ts index 218ebdbef..b69de1902 100644 --- a/lib/src/host/browser-host.test.ts +++ b/lib/src/host/browser-host.test.ts @@ -1,7 +1,7 @@ // @vitest-environment node import { describe, expect, it, vi } from 'vitest'; import { createBrowserHost } from './browser-host'; -import { fakeProvider } from './browser-host-test-utils'; +import { fakeProvider, openViewer } from './browser-host-test-utils'; const flush = async () => { for (let i = 0; i < 10; i++) await new Promise((resolve) => setImmediate(resolve)); }; @@ -18,7 +18,7 @@ describe('createBrowserHost', () => { expect(fake.calls).toEqual(['close dormouse.1.default']); fake.release('close dormouse.1.default'); expect(await closing).toEqual({ ok: true }); - expect(await launching).toMatchObject({ ok: true, session: 'dormouse.1.default', wsPort: 4321 }); + expect(await launching).toMatchObject({ ok: true, session: 'dormouse.1.default', stream: 4321 }); expect(fake.calls).toEqual(['close dormouse.1.default', 'stop dormouse.1.default', 'open dormouse.1.default http://localhost:5173/']); }); @@ -38,10 +38,10 @@ describe('createBrowserHost', () => { closes.push(host.request({ ...tool, op: 'close' })); const third = host.request(launch); fake.release('stop dormouse.1.tool.t'); - expect(await first).toMatchObject({ ok: true, wsPort: 4321 }); + expect(await first).toMatchObject({ ok: true, stream: 4321 }); expect(await superseded).toEqual({ ok: false, error: 'the browser was closed' }); expect(await Promise.all(closes)).toEqual([{ ok: true }, { ok: true }]); - expect(await third).toMatchObject({ ok: true, wsPort: 4321 }); + expect(await third).toMatchObject({ ok: true, stream: 4321 }); expect(fake.calls).toEqual([ 'stop dormouse.1.tool.t', 'open dormouse.1.tool.t http://localhost:6006/', 'close dormouse.1.tool.t', 'close dormouse.1.tool.t', @@ -84,20 +84,103 @@ describe('createBrowserHost', () => { host.request({ ...s1, op: 'navigate', url: 'http://localhost:5173/next' }), host.request({ ...s1, op: 'viewport', width: 800, height: 600, dpr: 2 }), host.request({ ...s1, op: 'edit', edit: 'selectAll' }), + host.request({ ...s1, op: 'view', stream: 4321 }), ]); const refused = { ok: false, error: 'the browser is being relaunched or closed' }; for (const [step, op] of [['stop s1', { op: 'launch', url: 'http://localhost:5173/', headed: true }], ['close s1', { op: 'close' }]] as const) { fake.gate(step); const settling = host.request({ ...s1, ...op }); await flush(); - expect(await drive()).toEqual([refused, refused, refused]); + expect(await drive()).toEqual([refused, refused, refused, refused]); fake.release(step); expect((await settling).ok).toBe(true); } // A pop-out's relaunch holds the browser from its stop until it is up. expect(fake.calls).not.toContain('navigate s1'); - expect((await drive()).map((result) => result.ok)).toEqual([true, true, true]); + expect((await drive()).map((result) => result.ok)).toEqual([true, true, true, true]); expect(fake.calls).toContain('navigate s1'); + await host.close(); + }); + + it('relays a browser over one viewer socket: its state, a provisional frame, the capture that sharpens it, and input back', async () => { + const fake = fakeProvider(); + const host = createBrowserHost({ writeClipboardText: vi.fn(), providers: { 'agent-browser': () => fake.provider } }); + const s1 = { provider: 'agent-browser', binding: { session: 's1' } } as const; + try { + const { url } = await host.request({ ...s1, op: 'view', stream: 4321 }); + const viewer = await openViewer(url!); + await vi.waitFor(() => expect(fake.views).toHaveLength(1)); + const [view] = fake.views; + expect(view).toMatchObject({ session: 's1', stream: 4321, headed: false }); + view.sink.state({ type: 'url', url: 'http://localhost:5173/' }); + view.sink.frame(new Uint8Array([0xff, 0xd8, 0xaa]), { width: 800, height: 600 }); + await vi.waitFor(() => expect(viewer.frames.map((frame) => frame.kind)).toEqual(['provisional', 'crisp'])); + expect(viewer.states).toEqual([{ type: 'url', url: 'http://localhost:5173/' }]); + expect([...viewer.frames[1].jpeg]).toEqual([0xff, 0xd8, 1]); + viewer.send({ type: 'input_text', text: 'hi' }); + await vi.waitFor(() => expect(view.inputs).toEqual([{ type: 'input_text', text: 'hi' }])); + // The URL was good for this one socket. + await expect(openViewer(url!)).rejects.toThrow('403'); + viewer.socket.close(); + await vi.waitFor(() => expect(view.closed).toBe(true)); + } finally { + await host.close(); + } + }); + + it('joins one capture for every viewer of a browser, and ends them all when a launch or close replaces it', async () => { + const fake = fakeProvider(); + const host = createBrowserHost({ writeClipboardText: vi.fn(), providers: { 'agent-browser': () => fake.provider } }); + const s1 = { provider: 'agent-browser', binding: { session: 's1' } } as const; + const view = async () => openViewer((await host.request({ ...s1, op: 'view', stream: 4321 })).url!); + try { + // Two Surfaces on one session: a Workspace transfer's two ends. + const viewers = [await view(), await view()]; + await vi.waitFor(() => expect(fake.views).toHaveLength(2)); + fake.gate('screenshot s1'); + for (const { sink } of fake.views) sink.frame(new Uint8Array([0xff, 0xd8, 0xaa])); + await vi.waitFor(() => expect(fake.calls).toContain('screenshot s1')); + fake.release('screenshot s1'); + await vi.waitFor(() => { + for (const viewer of viewers) expect(viewer.frames.map((frame) => frame.kind)).toEqual(['provisional', 'crisp']); + }); + expect(fake.calls.filter((call) => call === 'screenshot s1')).toHaveLength(1); + + // A URL granted before a relaunch opens nothing on the browser after it. + const { url: granted } = await host.request({ ...s1, op: 'view', stream: 4321 }); + await host.request({ ...s1, op: 'launch', url: 'http://localhost:5173/', headed: true }); + expect(await Promise.all(viewers.map((viewer) => viewer.closed))).toEqual([1001, 1001]); + await vi.waitFor(() => expect(fake.views.map((v) => v.closed)).toEqual([true, true])); + const late = await openViewer(granted!); + expect(await late.closed).toBe(1001); + expect(fake.views).toHaveLength(2); + } finally { + await host.close(); + } + }); + + it('joins no capture of the browser a relaunch replaced', async () => { + const fake = fakeProvider(); + const host = createBrowserHost({ writeClipboardText: vi.fn(), providers: { 'agent-browser': () => fake.provider } }); + const s1 = { provider: 'agent-browser', binding: { session: 's1' } } as const; + const view = async () => openViewer((await host.request({ ...s1, op: 'view', stream: 4321 })).url!); + try { + await view(); + await vi.waitFor(() => expect(fake.views).toHaveLength(1)); + // The old browser's capture is still running when the relaunch lands. + fake.gate('screenshot s1'); + fake.views[0].sink.frame(new Uint8Array([0xff, 0xd8, 0xaa])); + await vi.waitFor(() => expect(fake.calls).toContain('screenshot s1')); + await host.request({ ...s1, op: 'launch', url: 'http://localhost:5173/', headed: false }); + const viewer = await view(); + await vi.waitFor(() => expect(fake.views).toHaveLength(2)); + fake.views[1].sink.frame(new Uint8Array([0xff, 0xd8, 0xbb])); + await vi.waitFor(() => expect(fake.calls.filter((call) => call === 'screenshot s1')).toHaveLength(2)); + fake.release('screenshot s1'); + await vi.waitFor(() => expect(viewer.frames.map((frame) => frame.kind)).toEqual(['provisional', 'crisp'])); + } finally { + await host.close(); + } }); it('refuses a request id or cancel list it cannot bound', async () => { @@ -149,7 +232,7 @@ describe('createBrowserHost', () => { flush().then(() => null), ]); // Up before the page loads: nothing reaches the browser while `open` holds it. - expect(launched).toMatchObject({ ok: true, wsPort: 4321 }); + expect(launched).toMatchObject({ ok: true, stream: 4321 }); expect(fake.calls.filter((call) => call.startsWith('tab'))).toEqual([]); fake.release('open s1 http://localhost:5173/'); await flush(); diff --git a/lib/src/host/browser-host.ts b/lib/src/host/browser-host.ts index 339a4eace..c1645f237 100644 --- a/lib/src/host/browser-host.ts +++ b/lib/src/host/browser-host.ts @@ -5,14 +5,13 @@ * as one provider-tagged `BrowserRequest`, is validated here once — the * security boundary for both providers — and runs under one lifecycle: launches * and closes serialized per native identity, the post-launch blank-tab sweep, - * capture joins and their private directory, the editing scripts, headed + * the viewer sockets and their crisp captures, the editing scripts, headed * tracking and shutdown. A provider implements only the primitives that - * genuinely differ (`BrowserProvider`): agent-browser its daemon state files - * and argv, Playwright its install discovery, registry and CDP viewer. + * genuinely differ (`BrowserProvider`): agent-browser its daemon state files, + * stream and argv, Playwright its install discovery, registry and CDP. */ import { randomBytes } from 'crypto'; import * as path from 'path'; -import { promises as fs } from 'fs'; import { BROWSER_PROVIDERS, isBrowserProvider, @@ -33,20 +32,22 @@ import { type BrowserRequestBinding, type BrowserResult, } from '../lib/platform/browser-automation'; -import { privateCaptureDir } from './private-capture-dir'; +import { createBrowserCaptures } from './browser-capture'; +import type { WebSocket } from 'ws'; +import { BrowserView, createViewerServer, type Upstream, type ViewerSink } from './browser-viewer'; /** An operation on a live browser that each provider maps to its own call: * a fixed agent-browser argv, or a Playwright client call. */ -export type BrowserAct = Extract; +export type BrowserAct = Extract; /** The binding a provider runs one request with: the session named, or minted * for a new launch. */ export type ProviderBinding = BrowserBinding; -/** A browser that is up: where it streams, and whether it runs headed when the - * provider can tell. */ +/** A browser that is up: its stream — what `view` subscribes to — and + * whether it runs headed when the provider can tell. */ export interface LiveBrowser { - wsPort: number; + stream: number; headed?: boolean; } @@ -95,10 +96,13 @@ export interface BrowserProvider { act(b: B, act: BrowserAct): Promise; /** Run one of the host's fixed editing scripts in the page. */ evaluate(b: B, script: string): Promise; - /** One device-resolution frame: written to `file()` by a CLI, or its bytes. */ - screenshot(b: B, opts: { format: 'jpeg' | 'png'; quality: number }, file: () => Promise): Promise<{ path: string } | { bytes: Uint8Array }>; - /** The URL the webview connects to for a stream port. */ - streamUrl(port: number): Promise; + /** One device-resolution JPEG at `CAPTURE_JPEG_QUALITY`: written to + * `file()` by a CLI, or its bytes. */ + screenshot(b: B, file: () => Promise): Promise<{ path: string } | { bytes: Uint8Array }>; + /** Subscribe `sink` to the browser at `stream` (a `find` or `probe` + * answer): its changed frames — none for a `headed` viewer — and state. + * Rejects when that browser is not live. */ + view(b: B, stream: number, opts: { headed: boolean }, sink: ViewerSink): Promise; /** Shutdown: release every client-side resource. */ dispose?(): Promise; } @@ -158,12 +162,6 @@ function dimension(value: unknown, max: number): number | null { return typeof value === 'number' && Number.isFinite(value) && value > 0 && value <= max ? value : null; } -/** A capture's JPEG quality: an integer in 1..100, defaulting to 85. */ -function jpegQuality(quality: unknown): number { - if (typeof quality !== 'number' || !Number.isFinite(quality)) return 85; - return Math.min(100, Math.max(1, Math.round(quality))); -} - /** * `raw`, as a request the providers may run, or why it may not. The request * arrives from webview IPC unvalidated, so the result is rebuilt field by @@ -192,10 +190,9 @@ function parseBrowserRequest(raw: unknown): BrowserRequest | string { // A new session opens where it was asked; a relaunch only carries its page // along, reopening blank on one it may not navigate to. if (binding.session === undefined && !isBrowsableUrl(op.url)) return 'Browser navigation requires an http(s) URL'; - } else if (op.op !== 'streamUrl' && binding.session === undefined) { + } else if (binding.session === undefined) { return 'a valid session name is required'; } - if (op.op === 'cdpUrl' && provider !== 'agent-browser') return `${provider} has no cdpUrl operation`; return { provider, binding, ...op }; } @@ -210,13 +207,11 @@ function parseOp(r: Record): BrowserOp | string { ? { op: 'launch', ...url, headed: r.headed === true, ...id } : { op: 'attach', ...url, ...(r.headed === true ? { headed: true } : {}), ...id }; } - case 'streamUrl': { - const port = r.port; - return typeof port === 'number' && Number.isInteger(port) && port > 0 && port <= 65535 ? { op: 'streamUrl', port } : 'a stream port is required'; + case 'view': { + const stream = r.stream; + if (typeof stream !== 'number' || !Number.isSafeInteger(stream) || stream <= 0) return 'a stream is required'; + return { op: 'view', stream, ...(r.headed === true ? { headed: true } : {}), ...(r.debug === true ? { debug: true } : {}) }; } - case 'screenshot': - // Normalized where it is taken (`screenshot`). - return { op: 'screenshot', ...(r.format === 'png' ? { format: 'png' } : {}), ...(typeof r.quality === 'number' ? { quality: r.quality } : {}) }; case 'edit': return editScript(r.edit) !== undefined ? { op: 'edit', edit: r.edit as BrowserEditOp } : `unknown edit op '${String(r.edit)}'`; case 'navigate': @@ -234,8 +229,6 @@ function parseOp(r: Record): BrowserOp | string { } case 'device': return typeof r.name === 'string' && DEVICE_NAME.test(r.name) ? { op: 'device', name: r.name } : 'invalid device name'; - case 'cdpUrl': - return { op: 'cdpUrl' }; case 'close': { const cancels = r.cancels; if (cancels === undefined) return { op: 'close' }; @@ -263,10 +256,6 @@ const OPEN_SETTLE_MS = 4_000; const LAUNCH_CLOSE_RESERVE_MS = OPEN_SETTLE_MS + 4_000; const wait = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); -/** Screenshots answer with their bytes, or — to the sidecar, whose Rust - * caller reads the file itself — with the private file holding them. */ -type Transport = 'bytes' | 'file'; - /** One request, resolved: its provider, the provider's binding, and the * native identity everything per-browser keys on. */ type Bound = { p: BrowserProvider; b: unknown; id: string }; @@ -370,7 +359,9 @@ export function createBrowserHost(deps: BrowserHostDeps) { async function invalidate({ p, b, id }: Bound): Promise { const generation = (generations.get(id) ?? 0) + 1; generations.set(id, generation); - forgetInFlight(id); + captures.forget(id); + // Their browser is going: whoever still views it asks again. + for (const view of views.get(id) ?? []) view.close(1001, 'the browser was relaunched or closed'); await p.release?.(b); return generation; } @@ -424,7 +415,7 @@ export function createBrowserHost(deps: BrowserHostDeps) { } catch (error) { why = messageOf(error); } - if (probe && 'wsPort' in probe) { + if (probe && 'stream' in probe) { sweepAfter(opening, bound, generation); return probe; } @@ -477,7 +468,7 @@ export function createBrowserHost(deps: BrowserHostDeps) { async function reuse(bound: Bound, url: string | undefined, isHeaded: boolean): Promise { const { p, b, id } = bound; const found = await p.find(b).catch(() => undefined); - if (!found || !('wsPort' in found) || (found.headed ?? headed.has(id)) !== isHeaded) return undefined; + if (!found || !('stream' in found) || (found.headed ?? headed.has(id)) !== isHeaded) return undefined; if (isBrowsableUrl(url)) { void p.act(b, { op: 'navigate', url }).then((result) => { if (!result.ok) log(`navigating ${id} to ${url} failed: ${result.error ?? 'no reason given'}`); @@ -493,7 +484,7 @@ export function createBrowserHost(deps: BrowserHostDeps) { function attach(bound: Bound, url: string | undefined, isHeaded: boolean, requestDeadline: number): Promise { return bringUp(bound.id, async () => { const found = await bound.p.find(bound.b); - if ('wsPort' in found) return found; + if ('stream' in found) return found; if (!isBrowsableUrl(url)) throw new Error(found.gone); return { ...await launch(bound, url, isHeaded, !found.named, requestDeadline), relaunched: true }; }); @@ -513,87 +504,57 @@ export function createBrowserHost(deps: BrowserHostDeps) { })); } - // --- captures --- + // --- viewer sockets and their captures --- - // Screenshots of the user's authenticated browser land here, written by an - // external process under the ambient umask — which is why the private - // directory, not the file mode, is the control. Every file is a fresh - // random name — unguessable, and never one a reader may still be reading. - const captures = privateCaptureDir('dormouse-browser-'); - async function freshCapturePath(format: 'jpeg' | 'png'): Promise { - return path.join(await captures.get(), `shot-${randomBytes(12).toString('hex')}.${format === 'png' ? 'png' : 'jpg'}`); - } + const viewers = createViewerServer(); + // The viewer sockets open on each browser: a launch or close ends them. + const views = new Map>(); - // Files handed to the file transport's reader, which deletes each once read - // (the Tauri `browser_screenshot` command). Kept per browser until then: its - // close or relaunch removes those still there, and each new one removes any - // older than every reader's wait, which no reader will come for. - const handedOut = new Map>(); - async function handOut(id: string, bytes: Uint8Array, format: 'jpeg' | 'png'): Promise { - const file = await freshCapturePath(format); - await fs.writeFile(file, bytes, { mode: 0o600 }); - let files = handedOut.get(id); - if (!files) handedOut.set(id, files = new Map()); - const now = Date.now(); - for (const [old, at] of files) { - if (at > now - BROWSER_REQUEST_TIMEOUT_MS) break; - files.delete(old); - void fs.unlink(old).catch(() => {}); + /** Open one viewer socket on the browser at `stream`, unless a launch or + * close of it began since its URL was granted (`generation`). */ + function openView(socket: WebSocket, bound: Bound, stream: number, isHeaded: boolean, debug: boolean, generation: number): void { + if (closed || (generations.get(bound.id) ?? 0) !== generation) { + socket.close(1001, 'the browser was relaunched or closed'); + return; } - files.set(file, now); - return file; - } - - // A capture a caller asking meanwhile joins rather than repeats, one per - // browser and format: surfaces can share a session, and a caller re-asks - // after its adapter's timeout. Never one from before the browser's close or - // relaunch (`forgetInFlight`). - const inFlight = new Map }>(); - function joinInFlight(id: string, format: string, work: () => Promise): Promise { - const key = `${format}\0${id}`; - const pending = inFlight.get(key); - if (pending) return pending.promise; - const entry = { id, promise: work().finally(() => { if (inFlight.get(key) === entry) inFlight.delete(key); }) }; - inFlight.set(key, entry); - return entry.promise; + const view = new BrowserView(socket, { + headed: isHeaded, + capture: () => crisp(bound), + onClose: () => { + const open = views.get(bound.id); + open?.delete(view); + if (open?.size === 0) views.delete(bound.id); + }, + ...(debug && deps.log ? { log: deps.log } : {}), + }); + let open = views.get(bound.id); + if (!open) views.set(bound.id, open = new Set()); + open.add(view); + bound.p.view(bound.b, stream, { headed: isHeaded }, view).then( + (upstream) => view.attach(upstream), + (error: unknown) => view.close(1011, messageOf(error)), + ); } - /** Join none of `id`'s pending captures, and delete the frames of its page - * still handed out, rather than leave them on disk until shutdown. */ - function forgetInFlight(id: string): void { - for (const [key, entry] of inFlight) if (entry.id === id) inFlight.delete(key); - for (const file of handedOut.get(id)?.keys() ?? []) void fs.unlink(file).catch(() => {}); - handedOut.delete(id); - } + const captures = createBrowserCaptures(); - async function screenshot({ p, b, id }: Bound, asked: { format?: 'jpeg' | 'png'; quality?: number }, transport: Transport): Promise { - const opts = { format: asked.format === 'png' ? 'png' as const : 'jpeg' as const, quality: jpegQuality(asked.quality) }; - const mime = opts.format === 'png' ? 'image/png' : 'image/jpeg'; - let bytes: Uint8Array; + /** One device-resolution JPEG of `bound`'s browser, for its viewer sockets' + * crisp paint; undefined when none can be taken. A launch or close ends + * every viewer socket of the browser first, so none asks mid-relaunch. */ + async function crisp({ p, b, id }: Bound): Promise { try { - // The frame is held in memory and its capture file gone before anyone - // joined reads it, so each caller gets its own copy. - bytes = await joinInFlight(id, opts.format, async () => { - const shot = await p.screenshot(b, opts, () => freshCapturePath(opts.format)); - if ('bytes' in shot) return shot.bytes; - try { - const buffer = await fs.readFile(shot.path); - return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); - } finally { - await fs.unlink(shot.path).catch(() => {}); - } - }); + return await captures.take(id, (file) => p.screenshot(b, file)); } catch (error) { - return { ok: false, error: messageOf(error) }; + log(error); + return undefined; } - if (transport === 'bytes') return { ok: true, bytes, mime }; - if (closed) return { ok: false, error: 'the browser host is shutting down' }; - return { ok: true, path: await handOut(id, bytes, opts.format), mime }; } // --- editing --- - async function edit({ p, b }: Bound, op: BrowserEditOp): Promise { + async function edit({ p, b, id }: Bound, op: BrowserEditOp): Promise { + // The page answers an editing chord the way it answers a keystroke. + for (const view of views.get(id) ?? []) view.openProvisionalWindow(); const result = await p.evaluate(b, EDIT_SCRIPTS[op]); if (op === 'selectAll') return { ok: true }; const text = typeof result === 'string' ? result : ''; @@ -610,7 +571,7 @@ export function createBrowserHost(deps: BrowserHostDeps) { // --- dispatch --- - async function run(raw: unknown, transport: Transport): Promise { + async function run(raw: unknown): Promise { const r = parseBrowserRequest(raw); if (typeof r === 'string') return { ok: false, error: r }; try { @@ -619,12 +580,11 @@ export function createBrowserHost(deps: BrowserHostDeps) { // nothing for the Surface that close was for. if ((r.op === 'launch' || r.op === 'attach') && wasCancelled(r.requestId)) throw new Error('the browser was closed'); const p = providerFor(r.provider); - if (r.op === 'streamUrl') return { ok: true, url: await p.streamUrl(r.port) }; const b = p.bind({ ...r.binding, session: r.binding.session ?? generateGuiSession() }); const bound: Bound = { p, b, id: p.identity(b) }; const answer = (live: LiveBrowser): BrowserResult => { trackHeaded(bound, live.headed); - return { ok: true, ...p.describe(b), nativeIdentity: bound.id, wsPort: live.wsPort, ...(live.headed !== undefined ? { headed: live.headed } : {}) }; + return { ok: true, ...p.describe(b), nativeIdentity: bound.id, stream: live.stream, ...(live.headed !== undefined ? { headed: live.headed } : {}) }; }; const requestDeadline = Date.now() + REQUEST_BUDGET_MS; if (r.op !== 'launch' && r.op !== 'attach' && r.op !== 'close' && settling.has(bound.id)) { @@ -645,8 +605,11 @@ export function createBrowserHost(deps: BrowserHostDeps) { cancelRequests(r.cancels); await closeSession(bound); return { ok: true }; - case 'screenshot': - return await screenshot(bound, r, transport); + case 'view': { + const { stream, headed: viewHeaded = false, debug = false } = r; + const generation = generations.get(bound.id) ?? 0; + return { ok: true, url: await viewers.grant((socket) => openView(socket, bound, stream, viewHeaded, debug, generation)) }; + } case 'edit': return await edit(bound, r.edit); default: @@ -658,21 +621,20 @@ export function createBrowserHost(deps: BrowserHostDeps) { } return { - /** One request from the webview; a screenshot answers with its bytes. */ - request: (raw: unknown) => run(raw, 'bytes'), - /** The same, but a screenshot answers with a private file's path. */ - requestFile: (raw: unknown) => run(raw, 'file'), - /** Shutdown: close every headed window — so quitting orphans none — and - * drop the capture directory, so no frame of the user's browser outlives - * the process that took it. */ + /** One request from the webview. */ + request: run, + /** Shutdown: end every viewer socket, close every headed window — so + * quitting orphans none — and drop the capture directory, so no frame of + * the user's browser outlives the process that took it. */ close: async () => { // Every launch and sweep still pending now finds itself superseded. closed = true; const windows = [...headed.values()]; headed.clear(); await Promise.all([ + viewers.close(), ...windows.map((bound) => shut(bound).catch(log)), - captures.remove().then(() => handedOut.clear()), + captures.remove(), ]); await settleAllWithin([...lifecycle.values()], CLOSE_TIMEOUT_MS, undefined); await Promise.all([...providers.values()].map((provider) => provider.dispose?.())); diff --git a/lib/src/host/browser-stream-guard.test.ts b/lib/src/host/browser-stream-guard.test.ts index 7095d2a82..58547b9c1 100644 --- a/lib/src/host/browser-stream-guard.test.ts +++ b/lib/src/host/browser-stream-guard.test.ts @@ -1,19 +1,19 @@ // @vitest-environment node import { expect, test, vi } from 'vitest'; import { BrowserStreamGrants } from './browser-stream-guard'; -test('stream grants are port-bound, single-use, expiring and bounded', () => { +test('stream grants are single-use, expiring and bounded, and answer only what they were issued for', () => { vi.useFakeTimers(); try { - const grants = new BrowserStreamGrants(); - const token = grants.issue(1234); - expect(grants.consume(token, 1234)).toBe(true); - expect(grants.consume(token, 1234)).toBe(false); - expect(grants.consume(grants.issue(1234), 5678)).toBe(false); - const expired = grants.issue(1234); + const grants = new BrowserStreamGrants(); + const token = grants.issue('view-a'); + expect(grants.consume(token)).toBe('view-a'); + expect(grants.consume(token)).toBeUndefined(); + expect(grants.consume('0'.repeat(64))).toBeUndefined(); + const expired = grants.issue('view-a'); vi.advanceTimersByTime(60000); - expect(grants.consume(expired, 1234)).toBe(false); - const evicted = grants.issue(1234); - for (let i = 0; i < 1024; i++) grants.issue(1234); - expect(grants.consume(evicted, 1234)).toBe(false); + expect(grants.consume(expired)).toBeUndefined(); + const evicted = grants.issue('view-a'); + for (let i = 0; i < 1024; i++) grants.issue('view-b'); + expect(grants.consume(evicted)).toBeUndefined(); } finally { vi.useRealTimers(); } }); diff --git a/lib/src/host/browser-stream-guard.ts b/lib/src/host/browser-stream-guard.ts index e37505986..ac4d126e3 100644 --- a/lib/src/host/browser-stream-guard.ts +++ b/lib/src/host/browser-stream-guard.ts @@ -1,19 +1,22 @@ -/** Single-use, short-lived grants authorize one browser stream, never an upstream address. */ +/** Single-use, short-lived grants that each authorize one viewer socket onto + * the view they were issued for, never an address the caller names. */ import { randomBytes } from 'node:crypto'; -export class BrowserStreamGrants { - private grants = new Map(); - issue(port: number): string { + +export class BrowserStreamGrants { + private grants = new Map(); + issue(value: T): string { const now = Date.now(); for (const [token, grant] of this.grants) if (grant.expires <= now) this.grants.delete(token); // A host should never accumulate unbounded grants from a malfunctioning view. if (this.grants.size >= 1024) this.grants.delete(this.grants.keys().next().value!); const token = randomBytes(32).toString('hex'); - this.grants.set(token, { port, expires: now + 60_000 }); + this.grants.set(token, { value, expires: now + 60_000 }); return token; } - consume(token: string, port: number): boolean { + /** What `token` was issued for, once: undefined when unknown, spent or expired. */ + consume(token: string): T | undefined { const grant = this.grants.get(token); this.grants.delete(token); - return !!grant && grant.port === port && grant.expires > Date.now(); + return grant && grant.expires > Date.now() ? grant.value : undefined; } } diff --git a/lib/src/host/browser-viewer.test.ts b/lib/src/host/browser-viewer.test.ts new file mode 100644 index 000000000..76cd761b5 --- /dev/null +++ b/lib/src/host/browser-viewer.test.ts @@ -0,0 +1,305 @@ +// @vitest-environment node +import { EventEmitter } from 'node:events'; +import { request as httpRequest } from 'node:http'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { WebSocket } from 'ws'; +import { decodeViewerFrame, type ViewerFrame, type ViewerState } from '../lib/platform/browser-automation'; +import { BrowserView, PROVISIONAL_INPUT_WINDOW_MS, createViewerServer, parseViewerInput, type Upstream } from './browser-viewer'; +import { openViewer } from './browser-host-test-utils'; + +/** The webview's socket as a view sees it: what was sent, and input to send. */ +class FakeSocket extends EventEmitter { + readyState: number = WebSocket.OPEN; + bufferedAmount = 0; + sent: (string | Uint8Array)[] = []; + closedWith: number | undefined; + send(data: string | Uint8Array) { this.sent.push(data); } + close(code: number) { + this.readyState = WebSocket.CLOSED; + this.closedWith = code; + this.emit('close', code); + } + frames(): ViewerFrame[] { + return this.sent.filter((data): data is Uint8Array => typeof data !== 'string') + .map((data) => decodeViewerFrame(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer)!); + } + kinds() { return this.frames().map((frame) => `${frame.kind} ${frame.jpeg[0]}`); } + states(): ViewerState[] { return this.sent.filter((data): data is string => typeof data === 'string').map((data) => JSON.parse(data)); } + input(message: object) { this.emit('message', Buffer.from(JSON.stringify(message)), false); } +} + +const jpeg = (n: number) => new Uint8Array([n]); + +/** A view on a fake socket whose captures the test answers, in order. */ +function makeView(opts: { headed?: boolean; capturable?: boolean } = {}) { + const socket = new FakeSocket(); + const captures: ((jpeg: Uint8Array | undefined) => void)[] = []; + const capture = vi.fn(() => new Promise((resolve) => { captures.push(resolve); })); + const inputs: unknown[] = []; + const upstream: Upstream = { + capturable: opts.capturable ?? true, + input: (message) => { inputs.push(message); return true; }, + close: vi.fn(), + }; + const view = new BrowserView(socket as unknown as WebSocket, { headed: opts.headed ?? false, capture, onClose: vi.fn() }); + view.attach(upstream); + return { socket, view, capture, captures, inputs, upstream }; +} + +describe('a viewer socket', () => { + beforeEach(() => { vi.useFakeTimers(); }); + afterEach(() => { vi.useRealTimers(); }); + + it('sends the first stream frame at once, then the capture that sharpens it, and no other stream frame', async () => { + const { socket, view, capture, captures } = makeView(); + view.frame(jpeg(1), { width: 800, height: 600 }); + expect(socket.kinds()).toEqual(['provisional 1']); + expect(socket.frames()[0].size).toEqual({ width: 800, height: 600 }); + await vi.advanceTimersByTimeAsync(100); + expect(capture).toHaveBeenCalledOnce(); + captures[0](jpeg(9)); + await vi.advanceTimersByTimeAsync(0); + expect(socket.kinds()).toEqual(['provisional 1', 'crisp 9']); + // An animated page: its frames only pulse the loop, which keeps capturing. + view.frame(jpeg(2)); + await vi.advanceTimersByTimeAsync(300); + expect(capture).toHaveBeenCalledTimes(2); + expect(socket.kinds()).toEqual(['provisional 1', 'crisp 9']); + // A static page pulses nothing, so it costs nothing. + captures[1](jpeg(10)); + await vi.advanceTimersByTimeAsync(5000); + expect(capture).toHaveBeenCalledTimes(2); + // The crisp frame carries the viewport's size too. + expect(socket.frames().at(-1)).toMatchObject({ kind: 'crisp', size: { width: 800, height: 600 } }); + }); + + it('sends a byte-identical capture only over a provisional paint, or to a canvas that asks again', async () => { + const { socket, view, captures } = makeView(); + view.frame(jpeg(1)); + await vi.advanceTimersByTimeAsync(100); + captures[0](jpeg(9)); + await vi.advanceTimersByTimeAsync(0); + view.frame(jpeg(2)); + await vi.advanceTimersByTimeAsync(300); + captures[1](jpeg(9)); + await vi.advanceTimersByTimeAsync(0); + // Still what the canvas shows: not sent again. + expect(socket.kinds()).toEqual(['provisional 1', 'crisp 9']); + + // Input paints the stream over it, so the same capture sharpens it again. + socket.input({ type: 'input_mouse', eventType: 'mouseMoved', x: 1, y: 1 }); + view.frame(jpeg(3)); + await vi.advanceTimersByTimeAsync(PROVISIONAL_INPUT_WINDOW_MS + 100); + captures[2](jpeg(9)); + await vi.advanceTimersByTimeAsync(0); + expect(socket.kinds()).toEqual(['provisional 1', 'crisp 9', 'provisional 3', 'crisp 9']); + + // A canvas that mounted blank gets the last frame back. + socket.input({ type: 'repaint' }); + expect(socket.kinds().at(-1)).toBe('crisp 9'); + }); + + it('keeps a capture a provisional paint superseded owed, with nothing left to pulse it', async () => { + const { socket, view, capture, captures } = makeView(); + view.frame(jpeg(1)); + await vi.advanceTimersByTimeAsync(100); + expect(capture).toHaveBeenCalledOnce(); + // A single pointer move mid-capture: one provisional paint, then quiet. + socket.input({ type: 'input_mouse', eventType: 'mouseMoved', x: 1, y: 1 }); + view.frame(jpeg(2)); + captures[0](jpeg(9)); + await vi.advanceTimersByTimeAsync(0); + expect(socket.kinds()).toEqual(['provisional 1', 'provisional 2']); + await vi.advanceTimersByTimeAsync(1000); + expect(capture).toHaveBeenCalledTimes(2); + captures[1](jpeg(9)); + await vi.advanceTimersByTimeAsync(0); + expect(socket.kinds()).toEqual(['provisional 1', 'provisional 2', 'crisp 9']); + }); + + it('spends no captures while input keeps the stream painting, then one settled capture', async () => { + const { socket, view, capture, captures } = makeView(); + view.frame(jpeg(1)); + await vi.advanceTimersByTimeAsync(100); + captures[0](jpeg(9)); + await vi.advanceTimersByTimeAsync(0); + // Sustained typing: a key and a changed frame every 50 ms. + for (let i = 0; i < 12; i++) { + socket.input({ type: 'input_keyboard', eventType: 'keyDown', key: 'a', code: 'KeyA', text: 'a' }); + view.frame(jpeg(20 + i)); + await vi.advanceTimersByTimeAsync(50); + } + expect(capture).toHaveBeenCalledOnce(); + expect(socket.kinds().filter((kind) => kind.startsWith('provisional'))).toHaveLength(13); + await vi.advanceTimersByTimeAsync(600); + expect(capture).toHaveBeenCalledTimes(2); + }); + + it('coalesces the frames that arrive during a capture into one follow-up', async () => { + const { view, capture, captures } = makeView(); + view.frame(jpeg(1)); + await vi.advanceTimersByTimeAsync(100); + for (let i = 0; i < 3; i++) view.frame(jpeg(2 + i)); + expect(capture).toHaveBeenCalledOnce(); + captures[0](jpeg(9)); + await vi.advanceTimersByTimeAsync(300); + expect(capture).toHaveBeenCalledTimes(2); + captures[1](jpeg(10)); + await vi.advanceTimersByTimeAsync(1000); + expect(capture).toHaveBeenCalledTimes(2); + }); + + it('paints the stream while a capture is overdue, never re-issues it, and sends it when it lands', async () => { + const { socket, view, capture, captures } = makeView(); + view.frame(jpeg(1)); + await vi.advanceTimersByTimeAsync(100); + // Held behind a page-loading `open`: past twice the average, at least 400 ms. + await vi.advanceTimersByTimeAsync(500); + for (let i = 0; i < 20; i++) { + view.frame(jpeg(10 + i)); + await vi.advanceTimersByTimeAsync(1000); + } + expect(capture).toHaveBeenCalledOnce(); + expect(socket.kinds()).toHaveLength(21); + // Newer than every overdue paint, so it is sent; the wait's frames owe one more. + captures[0](jpeg(9)); + await vi.advanceTimersByTimeAsync(10); + expect(socket.kinds().at(-1)).toBe('crisp 9'); + expect(capture).toHaveBeenCalledTimes(2); + // The wait timed the page load, not a capture: the next is overdue as soon. + await vi.advanceTimersByTimeAsync(500); + view.frame(jpeg(50)); + expect(socket.kinds().at(-1)).toBe('provisional 50'); + + // One that fails still leaves the wait's frames a capture owed. + captures[1](undefined); + await vi.advanceTimersByTimeAsync(10); + expect(capture).toHaveBeenCalledTimes(3); + }); + + it('paints every changed frame of a browser it cannot capture, and none for a headed viewer', async () => { + const watched = makeView({ capturable: false }); + for (let i = 0; i < 3; i++) watched.view.frame(jpeg(i)); + await vi.advanceTimersByTimeAsync(1000); + expect(watched.socket.kinds()).toEqual(['provisional 0', 'provisional 1', 'provisional 2']); + expect(watched.capture).not.toHaveBeenCalled(); + + const headed = makeView({ headed: true }); + headed.view.frame(jpeg(1)); + headed.view.state({ type: 'url', url: 'https://example.com/' }); + await vi.advanceTimersByTimeAsync(1000); + expect(headed.socket.frames()).toEqual([]); + expect(headed.capture).not.toHaveBeenCalled(); + expect(headed.socket.states()).toEqual([{ type: 'url', url: 'https://example.com/' }]); + }); + + it('captures the tab an active-tab change shows, and nothing for other tab edits', async () => { + const { view, capture, captures } = makeView(); + view.frame(jpeg(1)); + await vi.advanceTimersByTimeAsync(100); + captures[0](jpeg(9)); + await vi.advanceTimersByTimeAsync(1000); + const tabs = (active: string, title: string) => ({ + type: 'tabs' as const, + tabs: ['t1', 't2'].map((tabId) => ({ tabId, url: `https://${tabId}.example/`, title, active: tabId === active })), + }); + view.state(tabs('t1', 'One')); + view.state(tabs('t1', 'Renamed')); + await vi.advanceTimersByTimeAsync(1000); + expect(capture).toHaveBeenCalledOnce(); + view.state(tabs('t2', 'Renamed')); + await vi.advanceTimersByTimeAsync(1000); + expect(capture).toHaveBeenCalledTimes(2); + }); + + it('forwards only validated input, and closes a socket whose input backs up', async () => { + const { socket, inputs, view, upstream } = makeView(); + socket.input({ type: 'input_mouse', eventType: 'mousePressed', x: 3, y: 4, button: 'left', buttons: 1, clickCount: 1, modifiers: 2, extra: 'dropped' }); + socket.input({ type: 'input_mouse', eventType: 'mouseMoved', x: Infinity, y: 0 }); + socket.input({ type: 'eval', script: 'document.cookie' }); + socket.emit('message', Buffer.from('not json'), false); + socket.emit('message', Buffer.from([1, 2, 3]), true); + expect(inputs).toEqual([{ type: 'input_mouse', eventType: 'mousePressed', x: 3, y: 4, button: 'left', buttons: 1, clickCount: 1, modifiers: 2 }]); + + upstream.input = () => false; + socket.input({ type: 'input_text', text: 'x' }); + expect(socket.closedWith).toBe(1008); + // Closing ends the provider's subscription. + expect(upstream.close).toHaveBeenCalledOnce(); + view.frame(jpeg(1)); + expect(socket.frames()).toEqual([]); + }); + + it('ends at once when the host closes it, capturing nothing while the webview has yet to answer', async () => { + const { socket, view, capture, captures, upstream } = makeView(); + view.frame(jpeg(1)); + await vi.advanceTimersByTimeAsync(100); + view.frame(jpeg(2)); + // The browser is relaunching: a slow webview has not answered the close. + socket.close = vi.fn() as unknown as FakeSocket['close']; + view.close(1001, 'the browser was relaunched or closed'); + expect(upstream.close).toHaveBeenCalledOnce(); + captures[0](jpeg(9)); + await vi.advanceTimersByTimeAsync(1000); + expect(capture).toHaveBeenCalledOnce(); + expect(socket.kinds()).toEqual(['provisional 1']); + }); + + it('tells the webview a browser that went away on its own is gone', () => { + const { socket, view } = makeView(); + view.gone(); + expect(socket.states()).toEqual([{ type: 'status', connected: false, screencasting: false }]); + expect(socket.closedWith).toBe(1000); + }); +}); + +describe('parseViewerInput', () => { + it('rebuilds each shape field by field, bounded, and refuses the rest', () => { + expect(parseViewerInput(JSON.stringify({ type: 'input_mouse', eventType: 'mouseWheel', x: 1, y: 2, button: 'evil', buttons: 255, clickCount: 9, modifiers: 255, deltaX: 'x', deltaY: 5 }))) + .toEqual({ type: 'input_mouse', eventType: 'mouseWheel', x: 1, y: 2, button: 'none', buttons: 31, clickCount: 3, modifiers: 15, deltaX: 0, deltaY: 5 }); + expect(parseViewerInput(JSON.stringify({ type: 'input_keyboard', eventType: 'keyDown', key: 'a', code: 'c'.repeat(200), text: 't'.repeat(2000), windowsVirtualKeyCode: 65.5, modifiers: 16 }))) + .toEqual({ type: 'input_keyboard', eventType: 'keyDown', key: 'a', code: 'c'.repeat(100), text: 't'.repeat(1000), windowsVirtualKeyCode: 0, modifiers: 0 }); + for (const refused of [ + { type: 'input_mouse', eventType: 'click', x: 1, y: 1 }, + { type: 'input_mouse', eventType: 'mouseMoved', x: 1e7, y: 1 }, + { type: 'input_keyboard', eventType: 'keyPress', key: 'a' }, + { type: 'input_keyboard', eventType: 'keyDown', key: 'k'.repeat(101) }, + { type: 'input_text', text: 'x'.repeat(8193) }, + { type: 'input_touch' }, + null, + 7, + ]) { + expect(parseViewerInput(JSON.stringify(refused)), JSON.stringify(refused)).toBeNull(); + } + expect(parseViewerInput('{')).toBeNull(); + }); +}); + +describe('the viewer listener', () => { + it('upgrades a granted socket once, addressed by its own loopback name, and refuses everything else', async () => { + const server = createViewerServer(); + const opened: WebSocket[] = []; + try { + const url = await server.grant((socket) => opened.push(socket)); + const { port, pathname } = new URL(url); + expect(url).toMatch(/^ws:\/\/127\.0\.0\.1:\d+\/view\/[a-f0-9]{64}$/); + + // A rebinding page arrives under its own name. + const rebound = new WebSocket(url, { headers: { host: `attacker.example:${port}` } }); + const refused = await new Promise((resolve) => rebound.once('unexpected-response', (_req, res) => resolve(res.statusCode!))); + expect(refused).toBe(403); + // A guessed token, and a plain request, get nothing. + await expect(openViewer(`ws://127.0.0.1:${port}/view/${'0'.repeat(64)}`)).rejects.toThrow('403'); + const status = await new Promise((resolve) => httpRequest({ host: '127.0.0.1', port, path: pathname }, (res) => resolve(res.statusCode!)).end()); + expect(status).toBe(403); + + const viewer = await openViewer(url); + await vi.waitFor(() => expect(opened).toHaveLength(1)); + // Single-use: the same URL opens nothing again. + await expect(openViewer(url)).rejects.toThrow('403'); + viewer.socket.close(); + } finally { + await server.close(); + } + }); +}); diff --git a/lib/src/host/browser-viewer.ts b/lib/src/host/browser-viewer.ts new file mode 100644 index 000000000..c6ee9b0e2 --- /dev/null +++ b/lib/src/host/browser-viewer.ts @@ -0,0 +1,419 @@ +/** + * The viewer socket (docs/specs/dor-browser.md → "Viewer Socket"): the one + * WebSocket per Surface over which the host relays a provider's browser to the + * webview — frames as binary messages, state as JSON text — and the webview's + * input back. The webview never reaches a daemon or CDP itself. + * + * One loopback listener serves every socket, guarded like every loopback + * listener (`./loopback-guard.ts`: its own `Host`) and by a single-use grant + * per socket. Each socket's `BrowserView` runs the two-stage paint's host + * half: which stream frames to send as provisional, and the crisp-capture + * loop that replaces them. + */ +import { createServer, type Server } from 'node:http'; +import { monitorEventLoopDelay, type IntervalHistogram } from 'node:perf_hooks'; +import { WebSocketServer, WebSocket } from 'ws'; +import { + VIEWER_TEXT_INPUT_MAX, + encodeViewerFrame, + type ViewerFrameKind, + type ViewerInput, + type ViewerState, +} from '../lib/platform/browser-automation'; +import { BrowserStreamGrants } from './browser-stream-guard'; +import { isLoopbackHost } from './loopback-guard'; + +/** What a provider pushes to one viewer of a live browser. */ +export interface ViewerSink { + /** A screencast frame that differs from the last one: its JPEG, and the + * viewport's CSS size when the provider knows it. */ + frame(jpeg: Uint8Array, size?: { width: number; height: number }): void; + /** A state change. */ + state(message: ViewerState): void; + /** The browser went away on its own — its window closed, its daemon or + * connection died — never because the host closed it. */ + gone(): void; +} + +/** A provider's subscription behind one viewer socket. */ +export interface Upstream { + /** Whether the host can capture this browser: false for one it can only + * watch, an agent-browser daemon in a socket directory it does not share. */ + readonly capturable: boolean; + /** Forward one validated input message; false when the provider's input + * backlog is full. */ + input(message: Exclude): boolean; + close(): void; +} + +// --- the listener --- + +export interface ViewerServer { + /** A single-use URL, good for 60 s, whose socket `open` takes. */ + grant(open: (socket: WebSocket) => void): Promise; + /** Shutdown: end every socket and stop listening. */ + close(): Promise; +} + +const VIEW_PATH = /^\/view\/([a-f0-9]{64})$/; +// Only input travels webview → host; its largest message is an `input_text` +// of VIEWER_TEXT_INPUT_MAX characters, six bytes each when JSON-escaped. +const MAX_INBOUND_BYTES = 65536; + +export function createViewerServer(): ViewerServer { + const grants = new BrowserStreamGrants<(socket: WebSocket) => void>(); + const wss = new WebSocketServer({ noServer: true, maxPayload: MAX_INBOUND_BYTES }); + let listening: Promise<{ server: Server; port: number }> | null = null; + let closed = false; + + function listen(): Promise<{ server: Server; port: number }> { + listening ??= new Promise<{ server: Server; port: number }>((resolve, reject) => { + const server = createServer((_req, res) => { res.writeHead(403); res.end(); }); + let port = 0; + server.on('upgrade', (req, socket, head) => { + const token = VIEW_PATH.exec(req.url ?? '')?.[1]; + const open = token && isLoopbackHost(req.headers.host, port) ? grants.consume(token) : undefined; + if (!open) { + socket.end('HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n'); + return; + } + wss.handleUpgrade(req, socket, head, open); + }); + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.off('error', reject); + server.unref(); + port = (server.address() as { port: number }).port; + resolve({ server, port }); + }); + }).catch((error: unknown) => { + // Retried by the next grant, never memoized. + listening = null; + throw error; + }); + return listening; + } + + return { + async grant(open) { + if (closed) throw new Error('the browser host is shutting down'); + const { port } = await listen(); + return `ws://127.0.0.1:${port}/view/${grants.issue(open)}`; + }, + async close() { + closed = true; + for (const socket of wss.clients) socket.terminate(); + const bound = await listening?.catch(() => null); + await new Promise((resolve) => (bound ? bound.server.close(() => resolve()) : resolve())); + }, + }; +} + +// --- one socket --- + +/** Continued input keeps the stream painting this long after the last. */ +export const PROVISIONAL_INPUT_WINDOW_MS = 250; +const OVERDUE_FLOOR_MS = 400; +// A capture this slow is flagged `stalled` in its debug log. +const STALL_WARNING_MS = 8000; +// A provisional frame is superseded by the next, so a socket backed up past +// this skips it. +const FRAME_BACKLOG_BYTES = 2_000_000; +const STATS_INTERVAL_MS = 5000; + +export interface BrowserViewDeps { + /** The Surface shows its browser as its own window: no frame is sent. */ + headed: boolean; + /** One device-resolution JPEG of the browser, or undefined when none can + * be taken now. */ + capture(): Promise; + /** Called once the socket has closed, however it closed. */ + onClose(): void; + /** Set to log this socket's rates every few seconds. */ + log?(message: string): void; +} + +/** + * One viewer socket: the provider's sink on one side, the webview's socket on + * the other. A stream frame is sent as a provisional paint when it is the + * first, inside the input window, or while a capture is overdue; any changed + * frame pulses the crisp loop, whose capture replaces it. At most one capture + * is in flight; one starts no sooner than 1.5× the average capture after the + * last began, never inside the input window; one out past twice the average + * (at least 400 ms) is overdue and never re-issued; one a provisional paint + * superseded while it ran is dropped and owed again. + */ +export class BrowserView implements ViewerSink { + private upstream: Upstream | null = null; + private capturable = false; + private closed = false; + // What the socket last carried, which is what the webview's canvas shows. + private last: { kind: ViewerFrameKind; jpeg: Uint8Array; message: Uint8Array } | null = null; + private size: { width: number; height: number } | undefined; + private provisionalUntil = 0; + private activeTab: string | undefined; + // Counts the provisional paints that supersede a capture in flight — not + // those made only because one is overdue, which it is newer than. + private provisionalGeneration = 0; + // --- the crisp loop --- + private inFlight = false; + private dirty = false; + private lastStart = -Infinity; + private avgMs = 120; + private timer: ReturnType | undefined; + private readonly stats = { framesIn: 0, provisional: 0, crisp: 0, captures: 0, captureMs: 0, bytesOut: 0 }; + private statsTimer: ReturnType | undefined; + + constructor(private readonly socket: WebSocket, private readonly deps: BrowserViewDeps) { + socket.on('message', (raw, isBinary) => this.receive(raw, isBinary)); + socket.on('close', () => this.dispose()); + socket.on('error', () => {}); + if (deps.log) { + startLagMonitor(); + this.statsTimer = setInterval(() => this.logStats(), STATS_INTERVAL_MS); + this.statsTimer.unref(); + } + } + + /** The provider's subscription, once it is live. */ + attach(upstream: Upstream): void { + if (this.closed) { + upstream.close(); + return; + } + this.upstream = upstream; + this.capturable = upstream.capturable; + // A frame that beat the subscription was painted as it came; sharpen it. + if (this.last) this.pulse(); + } + + /** End the socket: the host relaunched or closed its browser, or the + * provider could not subscribe. The view ends at once — no capture starts + * while the webview has yet to answer the close. */ + close(code: number, reason: string): void { + this.dispose(); + this.socket.close(code, reason.slice(0, 120)); + } + + /** Paint the stream for a while: input reached the page another way (a + * host editing op). */ + openProvisionalWindow(): void { + this.provisionalUntil = performance.now() + PROVISIONAL_INPUT_WINDOW_MS; + } + + // --- the provider's sink --- + + frame(jpeg: Uint8Array, size?: { width: number; height: number }): void { + if (this.closed || this.deps.headed) return; + this.stats.framesIn += 1; + if (size) this.size = size; + const now = performance.now(); + const forInput = !this.last || !this.capturable || now <= this.provisionalUntil; + if (forInput || this.captureOverdue(now)) { + this.send('provisional', jpeg); + if (forInput) this.provisionalGeneration += 1; + } + if (this.capturable) this.pulse(); + } + + state(message: ViewerState): void { + if (this.closed || this.socket.readyState !== WebSocket.OPEN) return; + this.socket.send(JSON.stringify(message)); + if (message.type !== 'tabs') return; + // Selecting another tab sends no screencast frame, and the stream is + // otherwise quiet on a static page: capture the tab now shown. + const active = message.tabs.find((tab) => tab.active)?.tabId; + if (active !== undefined && this.activeTab !== undefined && active !== this.activeTab && this.last) this.pulse(); + if (active !== undefined) this.activeTab = active; + } + + gone(): void { + this.state({ type: 'status', connected: false, screencasting: false }); + this.close(1000, 'the browser went away'); + } + + // --- the webview's socket --- + + private receive(raw: { toString(): string }, isBinary: boolean): void { + if (this.closed || isBinary) return; + const message = parseViewerInput(raw.toString()); + if (!message) return; + if (message.type === 'repaint') { + // A canvas that mounted blank: what it would have shown. + if (this.last) this.transmit(this.last.message); + return; + } + this.openProvisionalWindow(); + if (this.upstream && !this.upstream.input(message)) this.close(1008, 'Input backlog exceeded'); + } + + private send(kind: ViewerFrameKind, jpeg: Uint8Array): void { + if (this.socket.readyState !== WebSocket.OPEN) return; + if (kind === 'provisional' && this.socket.bufferedAmount > FRAME_BACKLOG_BYTES) return; + const message = encodeViewerFrame({ kind, jpeg, ...(this.size ? { size: this.size } : {}) }); + this.last = { kind, jpeg, message }; + this.stats[kind] += 1; + this.transmit(message); + } + + private transmit(message: Uint8Array): void { + this.stats.bytesOut += message.byteLength; + this.socket.send(message); + } + + // --- the crisp loop --- + + private overdueAfterMs(): number { + return Math.max(2 * this.avgMs, OVERDUE_FLOOR_MS); + } + + private captureOverdue(now: number): boolean { + return this.inFlight && now - this.lastStart > this.overdueAfterMs(); + } + + /** A "page changed" signal: a fresh capture is owed, coalesced and paced. */ + private pulse(): void { + if (this.closed || !this.capturable) return; + this.dirty = true; + this.schedule(); + } + + private schedule(): void { + if (this.closed || this.inFlight) return; + const now = performance.now(); + // Pacing: ~1.5× the measured capture since the last start, so a slow + // capture self-throttles; the 50 ms floor stops a fast failure spinning. + const paceWait = this.lastStart + Math.max(50, this.avgMs * 1.5) - now; + // Inside the input window every capture is superseded before it lands. + const provisionalWait = this.provisionalUntil - now; + const wait = Math.max(paceWait, provisionalWait); + if (wait > 0) { + this.timer ??= setTimeout(() => { + this.timer = undefined; + // Re-enter `schedule`: continued input moves the window's end. + if (this.dirty) this.schedule(); + }, wait); + return; + } + this.take(); + } + + private take(): void { + this.inFlight = true; + this.dirty = false; + const generation = this.provisionalGeneration; + const started = this.lastStart = performance.now(); + this.stats.captures += 1; + void this.deps.capture().catch(() => undefined).then((jpeg) => { + const elapsedMs = performance.now() - started; + this.stats.captureMs += elapsedMs; + if (elapsedMs > STALL_WARNING_MS) this.deps.log?.(`[browser-viewer] capture stalled ${Math.round(elapsedMs)}ms`); + // An overdue capture timed a page load, not a capture: clamp it. + this.avgMs = this.avgMs * 0.6 + Math.min(elapsedMs, this.overdueAfterMs()) * 0.4; + this.inFlight = false; + if (this.closed) return; + if (jpeg) { + // A provisional paint landed meanwhile and is newer: keep the + // sharpening capture owed rather than paint over it. + if (this.provisionalGeneration !== generation) this.dirty = true; + else if (!(this.last?.kind === 'crisp' && sameBytes(this.last.jpeg, jpeg))) this.send('crisp', jpeg); + } + if (this.dirty) this.schedule(); + }); + } + + private logStats(): void { + const { stats } = this; + const lag = lagMonitor; + this.deps.log?.(`[browser-viewer] ${JSON.stringify({ + perSecond: { + framesIn: stats.framesIn / (STATS_INTERVAL_MS / 1000), + provisional: stats.provisional / (STATS_INTERVAL_MS / 1000), + crisp: stats.crisp / (STATS_INTERVAL_MS / 1000), + captures: stats.captures / (STATS_INTERVAL_MS / 1000), + kbOut: Math.round(stats.bytesOut / 1024 / (STATS_INTERVAL_MS / 1000)), + }, + captureAvgMs: stats.captures ? Math.round(stats.captureMs / stats.captures) : null, + eventLoopDelayMs: lag ? { p99: Math.round(lag.percentile(99) / 1e6), max: Math.round(lag.max / 1e6) } : null, + })}`); + for (const key of Object.keys(stats) as (keyof typeof stats)[]) stats[key] = 0; + lag?.reset(); + } + + private dispose(): void { + if (this.closed) return; + this.closed = true; + if (this.timer) clearTimeout(this.timer); + if (this.statsTimer) clearInterval(this.statsTimer); + this.upstream?.close(); + this.deps.onClose(); + } +} + +let lagMonitor: IntervalHistogram | null = null; +/** The host's event-loop delay, sampled once any viewer logs its rates. */ +function startLagMonitor(): void { + if (lagMonitor) return; + lagMonitor = monitorEventLoopDelay({ resolution: 10 }); + lagMonitor.enable(); +} + +function sameBytes(a: Uint8Array, b: Uint8Array): boolean { + return a.byteLength === b.byteLength && Buffer.compare(a, b) === 0; +} + +const MOUSE_EVENTS = new Set(['mouseMoved', 'mousePressed', 'mouseReleased', 'mouseWheel']); +const MOUSE_BUTTONS = new Set(['left', 'right', 'middle', 'none']); +const finiteCoordinate = (n: unknown): n is number => typeof n === 'number' && Number.isFinite(n) && Math.abs(n) <= 1e6; +const integer = (n: unknown): number => (Number.isInteger(n) ? n as number : 0); + +/** + * One message from the webview, rebuilt field by field from what it sent: + * each provider forwards only these shapes, bounded, to its browser. Null for + * anything else. + */ +export function parseViewerInput(raw: string): ViewerInput | null { + let data: Record; + try { + const parsed: unknown = JSON.parse(raw); + if (!parsed || typeof parsed !== 'object') return null; + data = parsed as Record; + } catch { + return null; + } + switch (data.type) { + case 'input_mouse': { + if (typeof data.eventType !== 'string' || !MOUSE_EVENTS.has(data.eventType)) return null; + if (!finiteCoordinate(data.x) || !finiteCoordinate(data.y)) return null; + const eventType = data.eventType as Extract['eventType']; + return { + type: 'input_mouse', + eventType, + x: data.x, + y: data.y, + button: typeof data.button === 'string' && MOUSE_BUTTONS.has(data.button) ? data.button as 'left' : 'none', + buttons: integer(data.buttons) & 31, + clickCount: Math.min(3, Math.max(0, integer(data.clickCount))), + modifiers: integer(data.modifiers) & 15, + ...(eventType === 'mouseWheel' ? { deltaX: Number(data.deltaX) || 0, deltaY: Number(data.deltaY) || 0 } : {}), + }; + } + case 'input_keyboard': + if ((data.eventType !== 'keyDown' && data.eventType !== 'keyUp') || typeof data.key !== 'string' || data.key.length > 100) return null; + return { + type: 'input_keyboard', + eventType: data.eventType, + key: data.key, + code: typeof data.code === 'string' ? data.code.slice(0, 100) : '', + text: typeof data.text === 'string' ? data.text.slice(0, 1000) : '', + windowsVirtualKeyCode: integer(data.windowsVirtualKeyCode), + modifiers: integer(data.modifiers) & 15, + }; + case 'input_text': + return typeof data.text === 'string' && data.text.length <= VIEWER_TEXT_INPUT_MAX ? { type: 'input_text', text: data.text } : null; + case 'repaint': + return { type: 'repaint' }; + default: + return null; + } +} diff --git a/lib/src/host/playwright-host.lifecycle.test.ts b/lib/src/host/playwright-host.lifecycle.test.ts index da9f783ac..9443fa18f 100644 --- a/lib/src/host/playwright-host.lifecycle.test.ts +++ b/lib/src/host/playwright-host.lifecycle.test.ts @@ -1,12 +1,11 @@ // @vitest-environment node import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest'; import { EventEmitter } from 'node:events'; -import { readFileSync } from 'node:fs'; -import { Server } from 'node:http'; -import { WebSocket } from 'ws'; -import { createBrowserHost } from './browser-host'; +import { createBrowserCaptures } from './browser-capture'; +import { createBrowserHost, type BrowserProvider } from './browser-host'; +import { openViewer } from './browser-host-test-utils'; import { createPlaywrightProvider } from './playwright-host'; -import { BROWSER_REQUEST_TIMEOUT_MS, PLAYWRIGHT_TEXT_INPUT_MAX, playwrightTextInputs, type BrowserOp, type BrowserRequestBinding } from '../lib/platform/browser-automation'; +import { BROWSER_REQUEST_TIMEOUT_MS, VIEWER_TEXT_INPUT_MAX, viewerTextInputs, type BrowserOp, type BrowserRequestBinding } from '../lib/platform/browser-automation'; const mocks = vi.hoisted(() => ({ cli: vi.fn(), connect: vi.fn(), clipboard: vi.fn() })); vi.mock('dor-lib-common', async importOriginal => ({ ...await importOriginal(), spawnAndCapture: mocks.cli })); @@ -16,6 +15,7 @@ vi.mock('./playwright-install', () => ({ })); let host: ReturnType; +let provider: BrowserProvider; let page: EventEmitter & Record; let browser: EventEmitter & Record; let cdp: { send: ReturnType; detach: ReturnType; on: ReturnType }; @@ -23,6 +23,18 @@ let attach: ReturnType; const binding = { cwd: process.cwd(), session: 'test' }; /** One Playwright request through the shared host, bound to `b`. */ const pw = (op: BrowserOp, b: BrowserRequestBinding = binding) => host.request({ provider: 'playwright', binding: b, ...op }); +let captures: ReturnType; +/** One crisp capture, as the host takes it for a viewer socket: joined per + * browser. */ +const capture = () => { + const b = provider.bind({ ...binding }); + return captures.take(provider.identity(b), (file) => provider.screenshot(b, file)).then(() => true, () => false); +}; +/** A viewer socket onto the session's live browser. */ +const viewSession = async () => { + const { stream } = await pw({ op: 'attach' }); + return openViewer((await pw({ op: 'view', stream: stream! })).url!); +}; beforeEach(() => { vi.clearAllMocks(); @@ -45,19 +57,20 @@ beforeEach(() => { endpoint: '/tmp/test-playwright.pipe', browser: { browserName: 'chromium' }, }] } : { result: '- 0: (current) Test' }), })); - host = createBrowserHost({ writeClipboardText: mocks.clipboard, providers: { playwright: () => createPlaywrightProvider() } }); + provider = createPlaywrightProvider(); + captures = createBrowserCaptures(); + host = createBrowserHost({ writeClipboardText: mocks.clipboard, providers: { playwright: () => provider } }); }); -afterEach(async () => { await host.close(); vi.restoreAllMocks(); }); +afterEach(async () => { await host.close(); await captures.remove(); vi.restoreAllMocks(); }); test('concurrent captures join one, a concurrent control shares its CDP attachment, and close detaches it once', async () => { page.setViewportSize = vi.fn(async () => {}); const [first, second, sized] = await Promise.all([ - pw({ op: 'screenshot' }), - pw({ op: 'screenshot' }), + capture(), + capture(), pw({ op: 'viewport', width: 640, height: 480, dpr: 1 }), ]); - expect(first.error).toBeUndefined(); - expect(second).toEqual(first); + expect([first, second]).toEqual([true, true]); expect(sized.ok).toBe(true); expect(attach).toHaveBeenCalledTimes(1); expect(cdp.send.mock.calls.filter(([method]) => method === 'Page.captureScreenshot')).toHaveLength(1); @@ -69,13 +82,14 @@ test('concurrent captures join one, a concurrent control shares its CDP attachme test('closing during a CDP attachment releases it without capturing', async () => { let complete!: (value: typeof cdp) => void; attach.mockImplementation(() => new Promise(resolve => { complete = resolve; })); - const { wsPort } = await pw({ op: 'attach' }); - const capture = pw({ op: 'screenshot' }); + await pw({ op: 'attach' }); + const captured = capture(); await vi.waitFor(() => expect(attach).toHaveBeenCalledTimes(1)); const closing = pw({ op: 'close' }); - await vi.waitFor(async () => expect((await pw({ op: 'streamUrl', port: wsPort! }, {})).ok).toBe(false)); + // The close releases the connection, whose disposal waits on the attachment. + await new Promise((resolve) => setTimeout(resolve, 20)); complete(cdp); - expect((await capture).ok).toBe(false); + expect(await captured).toBe(false); expect((await closing).ok).toBe(true); expect(cdp.send).not.toHaveBeenCalled(); expect(cdp.detach).toHaveBeenCalledTimes(1); @@ -83,32 +97,22 @@ test('closing during a CDP attachment releases it without capturing', async () = test('a failed attachment can be retried', async () => { attach.mockRejectedValueOnce(new Error('Tab detached')); - expect((await pw({ op: 'screenshot' })).ok).toBe(false); - expect((await pw({ op: 'screenshot' })).ok).toBe(true); + expect(await capture()).toBe(false); + expect(await capture()).toBe(true); expect(attach).toHaveBeenCalledTimes(2); }); -test('a viewer listener failure releases its browser connection', async () => { - vi.spyOn(Server.prototype, 'listen').mockImplementationOnce(function (this: Server) { - queueMicrotask(() => this.emit('error', new Error('Listener unavailable'))); - return this; - }); - const result = await pw({ op: 'attach' }); - expect(result.error).toBe('Listener unavailable'); - expect(browser.close).toHaveBeenCalledTimes(1); -}); - test('captures reuse recent tab state but refresh it when it expires', async () => { const now = vi.spyOn(Date, 'now').mockReturnValue(10000); browser.contexts = () => [{ pages: () => [page, Object.assign(new EventEmitter(), page)] }]; expect((await pw({ op: 'attach' })).ok).toBe(true); // No viewer yet, so attach left the tab state to the first capture. - expect((await pw({ op: 'screenshot' })).ok).toBe(true); + expect(await capture()).toBe(true); mocks.cli.mockClear(); - for (let i = 0; i < 10; i++) expect((await pw({ op: 'screenshot' })).ok).toBe(true); + for (let i = 0; i < 10; i++) expect(await capture()).toBe(true); expect(mocks.cli).not.toHaveBeenCalled(); now.mockReturnValue(10750); - expect((await pw({ op: 'screenshot' })).ok).toBe(true); + expect(await capture()).toBe(true); expect(mocks.cli).toHaveBeenCalledExactlyOnceWith('/tools/playwright-cli', ['--session=test', 'tab-list', '--json'], { cwd: binding.cwd, timeoutMs: 10_000 }); }); @@ -123,9 +127,9 @@ test('GUI tab selection refreshes immediately even with a recent capture', async if (args.includes('tab-list')) return { ok: true, exitCode: 0, stdout: JSON.stringify({ result: `- ${active}: (current) Test` }), stderr: '' }; return originalCli(binary, args, options); }); - expect((await pw({ op: 'screenshot' })).ok).toBe(true); + expect(await capture()).toBe(true); expect((await pw({ op: 'tab', action: 'select', tabId: '1' })).ok).toBe(true); - expect((await pw({ op: 'screenshot' })).ok).toBe(true); + expect(await capture()).toBe(true); expect(attach).toHaveBeenLastCalledWith(second); }); @@ -155,17 +159,6 @@ test('a single-page browser refreshes without asking the CLI for its selection', expect(mocks.cli.mock.calls.map(([, args]) => args[1])).toEqual(['list']); }); -test('hands the sidecar a fresh frame file per capture, so the next never rewrites one being read', async () => { - page.setViewportSize = vi.fn(async () => {}); - const file = (op: BrowserOp) => host.requestFile({ provider: 'playwright', binding, ...op }); - const first = await file({ op: 'screenshot' }); - cdp.send.mockResolvedValue({ data: Buffer.from('next').toString('base64') }); - const second = await file({ op: 'screenshot' }); - expect(second.path).not.toBe(first.path); - expect(readFileSync(first.path!, 'utf8')).toBe('hello'); - expect(readFileSync(second.path!, 'utf8')).toBe('next'); -}); - test('a GUI open launches its fresh session without closing it first; a named launch navigates it in its mode and relaunches it into the other', async () => { page.goto = vi.fn(async () => null); const opened = await pw({ op: 'launch', url: 'http://localhost/', headed: false }, { cwd: process.cwd() }); @@ -205,7 +198,7 @@ test('a relaunch without an http(s) page reopens blank; a GUI open still needs o }); test.each(['attach', 'startScreencast'] as const)('a screencast whose %s fails mid-navigation is retried on the next poll', async (failing) => { - const { wsPort } = await pw({ op: 'attach' }); + const { stream } = await pw({ op: 'attach' }); const starts = () => cdp.send.mock.calls.filter(([method]) => method === 'Page.startScreencast').length; if (failing === 'attach') attach.mockRejectedValueOnce(new Error('Target navigated')); else { @@ -214,15 +207,12 @@ test.each(['attach', 'startScreencast'] as const)('a screencast whose %s fails m return { data: 'aGVsbG8=' }; }); } - const { url } = await pw({ op: 'streamUrl', port: wsPort! }, {}); - const ws = new WebSocket(url!); - ws.on('error', () => {}); + const viewer = await openViewer((await pw({ op: 'view', stream: stream! })).url!); try { - await new Promise(resolve => ws.once('open', resolve)); // The next 750 ms poll attaches again and starts the screencast. await vi.waitFor(() => expect(starts()).toBe(failing === 'attach' ? 1 : 2), { timeout: 3000 }); } finally { - ws.terminate(); + viewer.socket.terminate(); } }); @@ -247,7 +237,7 @@ describe('attach', () => { test('a live session answers its viewer port without launching', async () => { running = true; const attached = await pw({ op: 'attach', url: 'http://localhost/' }); - expect(attached).toMatchObject({ ok: true, headed: false, wsPort: expect.any(Number) }); + expect(attached).toMatchObject({ ok: true, headed: false, stream: expect.any(Number) }); expect(attached).not.toHaveProperty('relaunched'); expect(verbs()).not.toContain('open'); }); @@ -258,7 +248,7 @@ describe('attach', () => { const attached = await pw({ op: 'attach', url: 'http://localhost/', headed: true }); // Opened at the page, so the caller has no navigation left to run. - expect(attached).toMatchObject({ ok: true, wsPort: expect.any(Number), relaunched: true }); + expect(attached).toMatchObject({ ok: true, stream: expect.any(Number), relaunched: true }); expect(mocks.cli.mock.calls.map(([, args]) => args)).toContainEqual(['--session=test', 'open', 'http://localhost/', '--browser=chromium', '--headed']); }); @@ -294,48 +284,38 @@ test('copy runs the shared edit script and never overwrites the clipboard with a }); test('viewers get tabs, url and status only when they change, and the current state when they connect', async () => { - const { wsPort } = await pw({ op: 'attach' }); - const sockets: WebSocket[] = []; + const viewers: Awaited>[] = []; const connectViewer = async () => { - const { url } = await pw({ op: 'streamUrl', port: wsPort! }, {}); - const types: string[] = []; - const ws = new WebSocket(url!); - sockets.push(ws); - ws.on('error', () => {}); - ws.on('message', raw => types.push(JSON.parse(String(raw)).type)); - await new Promise(resolve => ws.once('open', resolve)); - return types; + const viewer = await viewSession(); + viewers.push(viewer); + return () => viewer.states.map((state) => state.type); }; try { const first = await connectViewer(); - await vi.waitFor(() => expect(first).toEqual(['url', 'tabs', 'status'])); + await vi.waitFor(() => expect(first()).toEqual(['url', 'tabs', 'status'])); await pw({ op: 'attach' }); await pw({ op: 'attach' }); page.url = () => 'http://localhost/next'; await pw({ op: 'attach' }); // A repeated state message would have arrived ahead of the navigation. - await vi.waitFor(() => expect(first).toEqual(['url', 'tabs', 'status', 'url', 'tabs'])); + await vi.waitFor(() => expect(first()).toEqual(['url', 'tabs', 'status', 'url', 'tabs'])); const second = await connectViewer(); - await vi.waitFor(() => expect(second).toEqual(['url', 'tabs', 'status'])); - expect(first).toHaveLength(5); + await vi.waitFor(() => expect(second()).toEqual(['url', 'tabs', 'status'])); + expect(first()).toHaveLength(5); } finally { - for (const ws of sockets) ws.terminate(); + for (const viewer of viewers) viewer.socket.terminate(); } }); test('a long paste reaches the page whole without tripping the input backlog', async () => { - const { wsPort } = await pw({ op: 'attach' }); - const { url } = await pw({ op: 'streamUrl', port: wsPort! }, {}); - const ws = new WebSocket(url!); - ws.on('error', () => {}); + const viewer = await viewSession(); const closed = vi.fn(); - ws.on('close', closed); + viewer.socket.on('close', closed); try { - await new Promise(resolve => ws.once('open', resolve)); // Past 128 characters, a key pair per character overflowed the 256-message // queue. The first message would end between an emoji's two halves. - const text = `${'x'.repeat(PLAYWRIGHT_TEXT_INPUT_MAX - 1)}🙂${'é🙂\n'.repeat(40_000)}`; - for (const message of playwrightTextInputs(text)) ws.send(JSON.stringify(message)); + const text = `${'x'.repeat(VIEWER_TEXT_INPUT_MAX - 1)}🙂${'é🙂\n'.repeat(40_000)}`; + for (const message of viewerTextInputs(text)) viewer.send(message); const insertions = () => cdp.send.mock.calls.filter(([method]) => method === 'Input.insertText').map(([, params]) => params.text as string); const inserted = () => insertions().join(''); await vi.waitFor(() => expect(inserted()).toBe(text)); @@ -343,14 +323,47 @@ test('a long paste reaches the page whole without tripping the input backlog', a expect(closed).not.toHaveBeenCalled(); // Text past the per-message bound is refused, like oversized key input. cdp.send.mockClear(); - ws.send(JSON.stringify({ type: 'input_text', text: 'x'.repeat(PLAYWRIGHT_TEXT_INPUT_MAX + 1) })); - ws.send(JSON.stringify({ type: 'input_text', text: 'ok' })); + viewer.send({ type: 'input_text', text: 'x'.repeat(VIEWER_TEXT_INPUT_MAX + 1) }); + viewer.send({ type: 'input_text', text: 'ok' }); await vi.waitFor(() => expect(inserted()).toBe('ok')); } finally { - ws.terminate(); + viewer.socket.terminate(); } }); +test('frames reach the viewer as binary, decoded once, their acks paced to ~20 a second', async () => { + const handlers = new Map void>(); + cdp.on.mockImplementation((event: string, handler: (event: unknown) => void) => { handlers.set(event, handler); }); + const viewer = await viewSession(); + try { + await vi.waitFor(() => expect(handlers.has('Page.screencastFrame')).toBe(true)); + const acks = () => cdp.send.mock.calls.filter(([method]) => method === 'Page.screencastFrameAck'); + const screencastFrame = (n: number) => handlers.get('Page.screencastFrame')!({ + data: Buffer.from([0xff, 0xd8, n]).toString('base64'), sessionId: n, metadata: { deviceWidth: 640, deviceHeight: 480 }, + }); + screencastFrame(1); + await vi.waitFor(() => expect(acks()).toHaveLength(1)); + const start = Date.now(); + screencastFrame(2); + await vi.waitFor(() => expect(acks()).toHaveLength(2)); + // Chrome sends the next frame only once this one is acknowledged. + expect(Date.now() - start).toBeGreaterThanOrEqual(40); + await vi.waitFor(() => expect(viewer.frames.length).toBeGreaterThan(0)); + expect(viewer.frames[0]).toMatchObject({ kind: 'provisional', size: { width: 640, height: 480 } }); + expect([...viewer.frames[0].jpeg]).toEqual([0xff, 0xd8, 1]); + } finally { + viewer.socket.terminate(); + } +}); + +test('a browser that disconnects on its own tells its viewers it is gone', async () => { + const viewer = await viewSession(); + await vi.waitFor(() => expect(viewer.states.map((state) => state.type)).toContain('status')); + browser.emit('disconnected'); + expect(await viewer.closed).toBe(1000); + expect(viewer.states.at(-1)).toEqual({ type: 'status', connected: false, screencasting: false }); +}); + describe('a GUI launch that gives up', () => { const ok = { ok: true, exitCode: 0, stdout: '', stderr: '' }; let events: string[]; diff --git a/lib/src/host/playwright-host.test.ts b/lib/src/host/playwright-host.test.ts index fd08906ba..df852f9e0 100644 --- a/lib/src/host/playwright-host.test.ts +++ b/lib/src/host/playwright-host.test.ts @@ -4,15 +4,26 @@ import { createServer } from 'node:http'; import { mkdtemp, mkdir, rm } from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; -import { WebSocket } from 'ws'; import { spawnAndCapture } from 'dor-lib-common'; -import type { BrowserOp, BrowserRequestBinding } from '../lib/platform/browser-automation'; +import type { BrowserOp, BrowserRequestBinding, ViewerState } from '../lib/platform/browser-automation'; import { createBrowserHost } from './browser-host'; +import { openViewer, type TestViewer } from './browser-host-test-utils'; import { createPlaywrightProvider } from './playwright-host'; +/** A JPEG's pixel width, from its start-of-frame segment. */ +function jpegWidth(jpeg: Uint8Array): number { + for (let i = 2; i + 8 < jpeg.length;) { + if (jpeg[i] !== 0xff) return 0; + const marker = jpeg[i + 1]; + if (marker >= 0xc0 && marker <= 0xc3) return (jpeg[i + 7] << 8) | jpeg[i + 8]; + i += 2 + ((jpeg[i + 2] << 8) | jpeg[i + 3]); + } + return 0; +} + // Opt-in: tests the user's real CLI and matching Chromium, with a private session. const binaryPath = process.env.DORMOUSE_PLAYWRIGHT_TEST_BIN; -test.skipIf(!binaryPath)('real CLI: GUI launch, stream grants, native tabs, input, screenshots, relaunch and close', async () => { +test.skipIf(!binaryPath)('real CLI: GUI launch, viewer sockets, native tabs, input, crisp captures, relaunch and close', async () => { const cwd = await mkdtemp(path.join(os.tmpdir(), 'dor-pw-test-')); await mkdir(path.join(cwd, '.playwright')); await mkdir(path.join(cwd, 'nested')); @@ -23,8 +34,7 @@ test.skipIf(!binaryPath)('real CLI: GUI launch, stream grants, native tabs, inpu const host = createBrowserHost({ writeClipboardText: text => { clipboard = text; }, providers: { playwright: () => createPlaywrightProvider() } }); const pw = (op: BrowserOp, binding: BrowserRequestBinding) => host.request({ provider: 'playwright', binding, ...op }); let session = ''; - let socket: WebSocket | undefined; - const messages: any[] = []; + let viewer: TestViewer | undefined; const waitFor = async (predicate: () => boolean) => { const until = Date.now() + 10000; while (!predicate() && Date.now() < until) await new Promise(r => setTimeout(r, 50)); @@ -37,43 +47,38 @@ test.skipIf(!binaryPath)('real CLI: GUI launch, stream grants, native tabs, inpu session = opened.session!; const req = { cwd, binaryPath, session }; const nested = await pw({ op: 'attach' }, { ...req, cwd: path.join(cwd, 'nested') }); - expect(nested.wsPort).toBe(opened.wsPort); + expect(nested.stream).toBe(opened.stream); expect(nested.nativeIdentity).toBe(opened.nativeIdentity); - const stream = await pw({ op: 'streamUrl', port: opened.wsPort! }, {}); - socket = new WebSocket(stream.url!); - socket.on('message', raw => messages.push(JSON.parse(String(raw)))); - await waitFor(() => messages.some(m => m.type === 'frame') && messages.some(m => m.type === 'tabs')); - const replay = new WebSocket(stream.url!); - await new Promise(resolve => replay.on('unexpected-response', (_req, res) => { expect(res.statusCode).toBe(403); res.resume(); replay.terminate(); resolve(); }).on('error', () => {})); + const { url: viewUrl } = await pw({ op: 'view', stream: opened.stream! }, req); + viewer = await openViewer(viewUrl!); + await waitFor(() => viewer!.frames.length > 0 && viewer!.states.some(m => m.type === 'tabs')); + await expect(openViewer(viewUrl!)).rejects.toThrow('403'); // Parking drops the viewer socket, then reconnects to the same CLI browser. - socket.close(); - await new Promise(resolve => socket!.once('close', () => resolve())); - expect((await pw({ op: 'attach' }, req)).wsPort).toBe(opened.wsPort); - messages.length = 0; - const resumedStream = await pw({ op: 'streamUrl', port: opened.wsPort! }, {}); - socket = new WebSocket(resumedStream.url!); - socket.on('message', raw => messages.push(JSON.parse(String(raw)))); - await waitFor(() => messages.some(m => m.type === 'frame')); + viewer.socket.close(); + await viewer.closed; + expect((await pw({ op: 'attach' }, req)).stream).toBe(opened.stream); + viewer = await openViewer((await pw({ op: 'view', stream: opened.stream! }, req)).url!); + await waitFor(() => viewer!.frames.length > 0); expect((await pw({ op: 'edit', edit: 'selectAll' }, req)).ok).toBe(true); expect((await pw({ op: 'edit', edit: 'copy' }, req)).ok).toBe(true); expect(clipboard).toBe('hello'); - socket.send(JSON.stringify({ type: 'input_keyboard', eventType: 'keyDown', key: 'x', code: 'KeyX', text: 'x', windowsVirtualKeyCode: 88 })); + viewer.send({ type: 'input_keyboard', eventType: 'keyDown', key: 'x', code: 'KeyX', text: 'x', windowsVirtualKeyCode: 88 }); await new Promise(r => setTimeout(r, 200)); await pw({ op: 'edit', edit: 'selectAll' }, req); await pw({ op: 'edit', edit: 'copy' }, req); expect(clipboard).toBe('x'); - const shot = await pw({ op: 'screenshot', format: 'png' }, req); - expect(Buffer.isBuffer(shot.bytes)).toBe(false); - expect(Buffer.from(shot.bytes!).subarray(1, 4).toString()).toBe('PNG'); + // The typed key changed the page, so the host sharpened its stream frame. + await waitFor(() => viewer!.frames.some(f => f.kind === 'crisp')); const native = await spawnAndCapture(binaryPath!, [`--session=${session}`, 'tab-new', `${url}/second`], { cwd }); expect(native.ok && native.exitCode).toBe(0); - await waitFor(() => messages.some(m => m.type === 'tabs' && m.tabs.length === 2 && m.tabs[1].active)); + await waitFor(() => viewer!.states.some(m => m.type === 'tabs' && m.tabs.length === 2 && m.tabs[1].active)); expect((await pw({ op: 'tab', action: 'select', tabId: '0' }, req)).ok).toBe(true); - await waitFor(() => messages.at(-1)?.type === 'status' && [...messages].reverse().find(m => m.type === 'tabs')?.tabs[0].active); + const lastTabs = () => [...viewer!.states].reverse().find((m): m is Extract => m.type === 'tabs'); + await waitFor(() => viewer!.states.at(-1)?.type === 'status' && !!lastTabs()?.tabs[0].active); const viewport = await pw({ op: 'viewport', width: 640, height: 480, dpr: 2 }, req); expect(viewport.ok).toBe(true); - const sized = await pw({ op: 'screenshot', format: 'png' }, req); - expect(Buffer.from(sized.bytes!).readUInt32BE(16)).toBe(1280); + // Captured at device resolution, unlike the CSS-resolution stream. + await waitFor(() => viewer!.frames.some(f => f.kind === 'crisp' && jpegWidth(f.jpeg) === 1280)); // No operation outside the typed set reaches the CLI. expect((await host.request({ provider: 'playwright', binding: req, op: 'eval', script: 'process.exit()' })).ok).toBe(false); const popped = await pw({ op: 'launch', url, headed: true }, req); @@ -85,12 +90,15 @@ test.skipIf(!binaryPath)('real CLI: GUI launch, stream grants, native tabs, inpu expect(popTabs.ok && popTabs.stdout.match(/\d+:/g)).toHaveLength(1); const relaunched = await pw({ op: 'launch', url, headed: false }, req); expect(relaunched.ok, relaunched.error).toBe(true); - expect(relaunched.wsPort).not.toBe(opened.wsPort); - expect((await pw({ op: 'streamUrl', port: opened.wsPort! }, {})).ok).toBe(false); + expect(relaunched.stream).not.toBe(opened.stream); + // The relaunch ended the old browser's viewer, and nothing views it again. + expect(await viewer.closed).toBe(1001); + const stale = await openViewer((await pw({ op: 'view', stream: opened.stream! }, req)).url!); + expect(await stale.closed).toBe(1011); expect((await pw({ op: 'close' }, req)).ok).toBe(true); expect((await pw({ op: 'attach' }, req)).ok).toBe(false); } finally { - socket?.terminate(); + viewer?.socket.terminate(); if (session) await pw({ op: 'close' }, { cwd, binaryPath, session }); await host.close(); server.closeAllConnections(); diff --git a/lib/src/host/playwright-host.ts b/lib/src/host/playwright-host.ts index c102158d7..d82042a5b 100644 --- a/lib/src/host/playwright-host.ts +++ b/lib/src/host/playwright-host.ts @@ -2,23 +2,26 @@ * The Playwright provider beneath the shared browser host (`browser-host.ts`; * docs/specs/dor-browser.md → "Playwright"). The installed Playwright * CLI owns browsers; this provider owns what is genuinely Playwright's — the - * install and registry discovery, and the Dormouse viewer each browser is - * streamed through over CDP. The host owns everything the providers share. + * install and registry discovery, and the CDP connection each browser's + * frames, state and input travel over to the host's viewer sockets. The host + * owns everything the providers share. */ -import { createServer, type Server } from 'node:http'; import { realpathSync } from 'node:fs'; -import { WebSocketServer, WebSocket } from 'ws'; import type { Browser, Page, CDPSession } from 'playwright-core'; import { BROWSER_PROVIDERS, spawnAndCapture } from 'dor-lib-common'; import { messageOf } from '../lib/errors'; -import { PLAYWRIGHT_TEXT_INPUT_MAX, type BrowserResult } from '../lib/platform/browser-automation'; +import { CAPTURE_JPEG_QUALITY, type BrowserResult, type ViewerInput, type ViewerState } from '../lib/platform/browser-automation'; import type { BrowserProvider, LiveBrowser } from './browser-host'; +import type { ViewerSink } from './browser-viewer'; import { resolvePlaywrightInstall, playwrightWorkspace, type PlaywrightInstall } from './playwright-install'; -import { isLoopbackHost } from './loopback-guard'; -import { BrowserStreamGrants } from './browser-stream-guard'; const TAB_REFRESH_INTERVAL_MS = 750; const CONNECT_TIMEOUT_MS = 8_000; +// Chrome sends the next screencast frame only once the last is acknowledged, +// so pacing the acks caps the stream at ~20 frames a second. +const FRAME_INTERVAL_MS = 50; +// Input waiting on CDP past this closes the viewer socket. +const INPUT_BACKLOG = 256; // Every CLI call but `open` ends here at the latest, so a wedged playwright-cli // cannot hold a viewer refresh or a host operation forever. `open` alone runs // unbounded: it lasts as long as the page load, nothing waits on it past a @@ -48,15 +51,13 @@ function bind(session: string, cwd: string, install: PlaywrightInstall): Binding const workspace = playwrightWorkspace(cwd); return { session, cwd, install, workspace, key: JSON.stringify([install.libraryPath, workspace ?? '', session]) }; } -type StateMessage = - | { type: 'url'; url: string } - | { type: 'tabs'; tabs: { tabId: string; url: string; title: string; active: boolean }[] } - | { type: 'status'; connected: true; screencasting: boolean; viewportWidth?: number; viewportHeight?: number }; +/** One viewer socket's hold on a browser: headed ones get no frames. */ +type Subscriber = { sink: ViewerSink; headed: boolean }; type Viewer = Binding & { browser: Browser; - server: Server; - sockets: Set; - port: number; + /** This connection's number, which the host hands back as its stream. */ + instance: number; + subscribers: Set; controls: Map>; page?: Page; cdp?: CDPSession; @@ -67,20 +68,21 @@ type Viewer = Binding & { queue: Promise; queued: number; headed: boolean; - /** The last payload published per state message type, replayed to each viewer that connects. */ - sent: Map; + /** The last state published per message type, replayed to each viewer that connects. */ + sent: Map; }; const pagesOf = (v: Viewer) => v.browser.contexts().flatMap(context => context.pages()); const tabsOf = (v: Viewer) => Promise.all(pagesOf(v).map(async (page, index) => ({ tabId: String(index), url: page.url(), title: await page.title().catch(() => ''), active: page === v.page, }))); +const watching = (v: Viewer) => [...v.subscribers].some(s => !s.headed); export function createPlaywrightProvider(deps: { log?(text: string): void } = {}): BrowserProvider { const viewers = new Map(); const connecting = new Map>(); // Bumped whenever the host releases a binding's viewer: a connect begun // before must not publish the viewer it brings back. const generations = new Map(); - const grants = new BrowserStreamGrants(); + let instances = 0; let closed = false; const log = (e: unknown) => deps.log?.(`[playwright] ${messageOf(e)}`); /** One CLI call, ended after `timeoutMs` (none for `null`); throws when it could not run or finish. */ @@ -92,27 +94,20 @@ export function createPlaywrightProvider(deps: { log?(text: string): void } = {} if (!r.ok) throw new Error(r.error.message); return { exitCode: r.exitCode, stdout: r.stdout, stderr: r.stderr }; } - // A frame is superseded by the next one, so a socket backed up past 2 MB - // skips it. State is published only on change, so it is never skipped. - function broadcast(v: Viewer, payload: string, frame: boolean) { - for (const ws of v.sockets) { - if (ws.readyState === WebSocket.OPEN && (!frame || ws.bufferedAmount < 2_000_000)) ws.send(payload); - } - } // Every state message re-renders the pane, so the poll publishes only // changes; a connecting viewer is sent the latest state instead. - function publish(v: Viewer, data: StateMessage) { - const payload = JSON.stringify(data); - if (v.sent.get(data.type) === payload) return; - v.sent.set(data.type, payload); - broadcast(v, payload, false); + function publish(v: Viewer, message: ViewerState) { + const json = JSON.stringify(message); + if (v.sent.get(message.type)?.json === json) return; + v.sent.set(message.type, { json, message }); + for (const { sink } of v.subscribers) sink.state(message); } + /** Release the connection; its viewers are the host's to end. */ async function dispose(v: Viewer) { if (v.disposed) return; v.disposed = true; if (v.timer) clearTimeout(v.timer); - for (const ws of v.sockets) ws.terminate(); - v.server.close(); + v.subscribers.clear(); await v.cdp?.detach().catch(() => {}); await Promise.allSettled([...v.controls.values()].map(async pending => { await (await pending).detach(); })); v.controls.clear(); @@ -156,7 +151,7 @@ export function createPlaywrightProvider(deps: { log?(text: string): void } = {} await v.cdp?.detach().catch(() => {}); v.cdp = undefined; v.page = page; - if (page && !v.headed && v.sockets.size) await startFrames(v); + if (page) await startFrames(v); } const tabs = await tabsOf(v); if (v.disposed) return; @@ -172,16 +167,26 @@ export function createPlaywrightProvider(deps: { log?(text: string): void } = {} } catch (e) { log(e); } } async function startFrames(v: Viewer) { - if (!v.page || v.cdp || v.disposed || v.headed) return; + if (!v.page || v.cdp || v.disposed || v.headed || !watching(v)) return; const page = v.page; try { const cdp = await page.context().newCDPSession(page); // Another refresh, tab change, or parking may finish while CDP attaches. - if (v.disposed || !v.sockets.size || v.page !== page || v.cdp) { await cdp.detach().catch(() => {}); return; } + if (v.disposed || !watching(v) || v.page !== page || v.cdp) { await cdp.detach().catch(() => {}); return; } v.cdp = cdp; + let acked = 0; cdp.on('Page.screencastFrame', event => { - void cdp.send('Page.screencastFrameAck', { sessionId: event.sessionId }).catch(() => {}); - if (!v.disposed && v.cdp === cdp) broadcast(v, JSON.stringify({ type: 'frame', data: event.data, metadata: event.metadata }), true); + if (v.disposed || v.cdp !== cdp) return; + // Decoded once, here; the viewer sockets carry it as binary. + const jpeg = Buffer.from(event.data, 'base64'); + const { deviceWidth: width, deviceHeight: height } = event.metadata; + const size = width > 0 && height > 0 ? { width, height } : undefined; + for (const { sink, headed } of v.subscribers) if (!headed) sink.frame(jpeg, size); + const ack = () => { + acked = Date.now(); + void cdp.send('Page.screencastFrameAck', { sessionId: event.sessionId }).catch(() => {}); + }; + setTimeout(ack, Math.max(0, acked + FRAME_INTERVAL_MS - Date.now())).unref(); }); await cdp.send('Page.startScreencast', { format: 'jpeg', quality: 70 }); } catch (e) { @@ -194,7 +199,7 @@ export function createPlaywrightProvider(deps: { log?(text: string): void } = {} } } function schedule(v: Viewer) { - if (v.disposed || !v.sockets.size || v.timer) return; + if (v.disposed || !v.subscribers.size || v.timer) return; v.timer = setTimeout(() => { v.timer = undefined; void refresh(v).finally(() => schedule(v)); @@ -220,28 +225,18 @@ export function createPlaywrightProvider(deps: { log?(text: string): void } = {} void pending.catch(() => { if (v.controls.get(page) === pending) v.controls.delete(page); }); return pending; } - async function input(v: Viewer, raw: string) { - if (v.disposed || raw.length > 65536 || !v.page) return; - const data = JSON.parse(raw); - const page = v.page; - if (data.type === 'input_mouse' && ['mouseMoved', 'mousePressed', 'mouseReleased', 'mouseWheel'].includes(data.eventType)) { - if (![data.x, data.y].every(n => typeof n === 'number' && Number.isFinite(n) && Math.abs(n) <= 1e6)) return; - const cdp = await control(v, page); - await cdp.send('Input.dispatchMouseEvent', { type: data.eventType, x: data.x, y: data.y, - button: ['left', 'right', 'middle', 'none'].includes(data.button) ? data.button : 'none', - buttons: Number.isInteger(data.buttons) ? data.buttons & 31 : 0, - modifiers: Number.isInteger(data.modifiers) ? data.modifiers & 15 : 0, - clickCount: Math.min(3, Math.max(0, Number(data.clickCount) || 0)), - ...(data.eventType === 'mouseWheel' ? { deltaX: Number(data.deltaX) || 0, deltaY: Number(data.deltaY) || 0 } : {}) }); - } else if (data.type === 'input_keyboard' && ['keyDown', 'keyUp'].includes(data.eventType) && typeof data.key === 'string' && data.key.length <= 100) { - const cdp = await control(v, page); - await cdp.send('Input.dispatchKeyEvent', { type: data.eventType, key: data.key, code: typeof data.code === 'string' ? data.code.slice(0, 100) : '', - text: typeof data.text === 'string' ? data.text.slice(0, 1000) : '', - windowsVirtualKeyCode: Number.isInteger(data.windowsVirtualKeyCode) ? data.windowsVirtualKeyCode : 0, - modifiers: Number.isInteger(data.modifiers) ? data.modifiers & 15 : 0 }); - } else if (data.type === 'input_text' && typeof data.text === 'string' && data.text.length <= PLAYWRIGHT_TEXT_INPUT_MAX) { - // A paste arrives as text, not a key pair per character (`playwrightTextInputs`). - const cdp = await control(v, page); + /** One input message, validated by the host (`parseViewerInput`). */ + async function input(v: Viewer, data: Exclude) { + if (v.disposed || !v.page) return; + const cdp = await control(v, v.page); + if (data.type === 'input_mouse') { + const { eventType: type, x, y, button, buttons, modifiers, clickCount, deltaX, deltaY } = data; + await cdp.send('Input.dispatchMouseEvent', { type, x, y, button, buttons, modifiers, clickCount, ...(type === 'mouseWheel' ? { deltaX, deltaY } : {}) }); + } else if (data.type === 'input_keyboard') { + const { eventType: type, key, code, text, windowsVirtualKeyCode, modifiers } = data; + await cdp.send('Input.dispatchKeyEvent', { type, key, code, text, windowsVirtualKeyCode, modifiers }); + } else { + // A paste arrives as text, not a key pair per character (`viewerTextInputs`). await cdp.send('Input.insertText', { text: data.text }); } } @@ -278,14 +273,11 @@ export function createPlaywrightProvider(deps: { log?(text: string): void } = {} if (remaining <= 0) throw new Error('Playwright browser connection timed out'); const browser = await b.install.library.chromium.connect(endpoint, { timeout: Math.min(CONNECT_TIMEOUT_MS, remaining) }); if (closed || gen !== (generations.get(key) ?? 0)) { await browser.close(); throw new Error('Browser launch superseded'); } - const server = createServer((_req, res) => { res.writeHead(403); res.end(); }); - const wss = new WebSocketServer({ noServer: true, maxPayload: 65536 }); const v: Viewer = { ...b, browser, - server, - sockets: new Set(), - port: 0, + instance: ++instances, + subscribers: new Set(), controls: new Map(), disposed: false, queue: Promise.resolve(), @@ -293,42 +285,12 @@ export function createPlaywrightProvider(deps: { log?(text: string): void } = {} headed: descriptor.browser.launchOptions?.headless === false, sent: new Map(), }; - server.on('upgrade', (req, socket, head) => { - const token = /^\/stream\/([a-f0-9]{64})$/.exec(req.url ?? '')?.[1]; - if (!isLoopbackHost(req.headers.host, v.port) || !token || !grants.consume(token, v.port)) { socket.end('HTTP/1.1 403 Forbidden\r\nConnection: close\r\n\r\n'); return; } - wss.handleUpgrade(req, socket, head, ws => { - v.sockets.add(ws); - // Earlier viewers already hold this state; the refresh below sends only what changed. - for (const payload of v.sent.values()) ws.send(payload); - ws.on('error', log); - ws.on('message', raw => { - if (v.queued >= 256) { ws.close(1008, 'Input backlog exceeded'); return; } - v.queued++; - v.queue = v.queue.then(() => input(v, raw.toString())).catch(log).finally(() => { v.queued--; }); - }); - ws.on('close', () => { - v.sockets.delete(ws); - if (!v.sockets.size) { - if (v.timer) clearTimeout(v.timer); - v.timer = undefined; - void v.cdp?.detach().catch(() => {}); - v.cdp = undefined; - } - }); - void refresh(v).then(() => startFrames(v)).catch(log); - if (v.sockets.size === 1) schedule(v); - }); + browser.on('disconnected', () => { + // Gone on its own: the CLI closed it, or its window closed. + if (!v.disposed) for (const { sink } of v.subscribers) sink.gone(); + if (viewers.get(key) === v) viewers.delete(key); + void dispose(v); }); - try { - await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve); }); - } catch (error) { - await dispose(v); - throw error; - } - server.unref(); - v.port = (server.address() as { port: number }).port; - browser.on('disconnected', () => { if (viewers.get(key) === v) viewers.delete(key); void dispose(v); }); - if (closed || gen !== (generations.get(key) ?? 0)) { await dispose(v); throw new Error('Browser launch superseded'); } viewers.set(key, v); return v; })(); @@ -344,7 +306,7 @@ export function createPlaywrightProvider(deps: { log?(text: string): void } = {} return { v, page: v.page }; } const exited = (r: { exitCode: number; stderr: string }) => r.stderr.trim() || `playwright-cli exited ${r.exitCode}`; - const live = (v: Viewer): LiveBrowser => ({ wsPort: v.port, headed: v.headed }); + const live = (v: Viewer): LiveBrowser => ({ stream: v.instance, headed: v.headed }); return { pollMs: 200, @@ -361,7 +323,7 @@ export function createPlaywrightProvider(deps: { log?(text: string): void } = {} try { const v = await connect(b); // A connecting viewer is sent the current state; only live ones need it now. - if (v.sockets.size) await refresh(v); + if (v.subscribers.size) await refresh(v); return live(v); } catch (error) { if (!(error instanceof SessionNotOpenError)) throw error; @@ -450,18 +412,40 @@ export function createPlaywrightProvider(deps: { log?(text: string): void } = {} }, // CDP capture in-process. Captures share the viewer's polling cadence. - async screenshot(b, { format, quality }) { + async screenshot(b) { const { v, page } = await livePage(b, false); const cdp = await control(v, page); - const { data } = await cdp.send('Page.captureScreenshot', { format, ...(format === 'jpeg' ? { quality } : {}), captureBeyondViewport: false }); - // Keep the cross-host contract a plain typed array, including VS Code's message transport. - return { bytes: new Uint8Array(Buffer.from(data, 'base64')) }; + const { data } = await cdp.send('Page.captureScreenshot', { format: 'jpeg', quality: CAPTURE_JPEG_QUALITY, captureBeyondViewport: false }); + return { bytes: Buffer.from(data, 'base64') }; }, - async streamUrl(port) { - const v = [...viewers.values()].find(v => v.port === port && !v.disposed); - if (!v) throw new Error('Playwright stream is no longer live'); - return `ws://127.0.0.1:${v.port}/stream/${grants.issue(v.port)}`; + async view(b, stream, { headed }, sink) { + const v = await connect(b); + if (v.instance !== stream || v.disposed) throw new Error('Playwright stream is no longer live'); + const subscriber: Subscriber = { sink, headed }; + v.subscribers.add(subscriber); + // Earlier viewers already hold this state; the refresh below sends only what changed. + for (const { message } of v.sent.values()) sink.state(message); + void refresh(v).then(() => startFrames(v)).catch(log); + if (v.subscribers.size === 1) schedule(v); + return { + capturable: true, + input(message) { + if (v.queued >= INPUT_BACKLOG) return false; + v.queued++; + v.queue = v.queue.then(() => input(v, message)).catch(log).finally(() => { v.queued--; }); + return true; + }, + close() { + if (!v.subscribers.delete(subscriber) || watching(v)) return; + // Nobody sees its frames now; with no viewer left, nobody its tabs. + void v.cdp?.detach().catch(() => {}); + v.cdp = undefined; + if (v.subscribers.size) return; + if (v.timer) clearTimeout(v.timer); + v.timer = undefined; + }, + }; }, async dispose() { diff --git a/lib/src/host/private-capture-dir.ts b/lib/src/host/private-capture-dir.ts index da3aa27dd..2d8dec6a2 100644 --- a/lib/src/host/private-capture-dir.ts +++ b/lib/src/host/private-capture-dir.ts @@ -1,7 +1,6 @@ /** - * The private per-process directory a browser host writes screenshot frames - * into (docs/specs/dor-browser.md → "Browser Host"), shared - * by the agent-browser and Playwright hosts. + * The private per-process directory the browser host's captures are written + * into (docs/specs/dor-browser.md → "Viewer Socket"; `./browser-capture.ts`). * * A frame is a picture of the user's authenticated browser, so the *directory* * is the control: one `mkdtemp` per host, which is `0700` and unguessable. A diff --git a/lib/src/lib/platform/browser-automation.ts b/lib/src/lib/platform/browser-automation.ts index cd929e3ba..01875d641 100644 --- a/lib/src/lib/platform/browser-automation.ts +++ b/lib/src/lib/platform/browser-automation.ts @@ -5,6 +5,7 @@ * operations — no CLI argv, script or CDP method crosses it. */ import type { BrowserAutomationProvider, BrowserBinding } from 'dor-lib-common/browser-providers'; +import type { AgentBrowserTab } from '../agent-browser-tab'; export type { BrowserAutomationProvider }; export { BROWSER_REQUEST_TIMEOUT_MS } from 'dor-lib-common/browser-providers'; @@ -28,19 +29,17 @@ export type BrowserOp = /** Where the session streams now, found without starting a browser; one * that is gone relaunches at `url` when the caller names one. */ | { op: 'attach'; url?: string; headed?: boolean; requestId?: string } - /** The URL the webview connects to for a stream port. */ - | { op: 'streamUrl'; port: number } - /** One device-resolution frame. */ - | { op: 'screenshot'; format?: 'jpeg' | 'png'; quality?: number } + /** A single-use URL for one viewer socket on the browser at `stream` (what + * a launch or attach answered): its frames, state and input. `headed`: the + * Surface shows it as its own window, so no frame is sent. `debug` logs the + * socket's rates host-side. */ + | { op: 'view'; stream: number; headed?: boolean; debug?: boolean } | { op: 'edit'; edit: BrowserEditOp } | { op: 'navigate'; url: string } | { op: 'history'; dir: 'back' | 'forward' | 'reload' } | { op: 'tab'; action: 'select' | 'close'; tabId: string } | { op: 'viewport'; width: number; height: number; dpr: number } | { op: 'device'; name: string } - /** agent-browser only: the browser's CDP endpoint, for the popped-out URL - * observer. */ - | { op: 'cdpUrl' } /** Close the session — after the launch or attach of it running now — and * cancel `cancels`: requests the closing Surface sent that can bring the * browser up, by their `requestId`, however late the transport delivers @@ -59,45 +58,122 @@ export interface BrowserResult { cwd?: string; binaryPath?: string; nativeIdentity?: string; - /** `launch` / `attach`: the stream port, and whether the browser runs - * headed, when the host knows. */ - wsPort?: number; + /** `launch` / `attach`: the live browser's stream — agent-browser's daemon + * stream port, or the host's number for a Playwright connection — which + * `view` takes back; and whether it runs headed, when the host knows. */ + stream?: number; headed?: boolean; /** `attach`: the session was gone, and the host started a browser at the * caller's `url`, so that page is already open. */ relaunched?: boolean; - /** `streamUrl` / `cdpUrl`. */ + /** `view`: the viewer socket's URL. */ url?: string; /** `edit`: the text copy/cut placed on the OS clipboard. */ text?: string; - /** `screenshot`: the image bytes, or — to the standalone sidecar's caller — - * the private file holding them. */ - bytes?: Uint8Array; - path?: string; - mime?: string; } /** The most requests one `close` may cancel; the host refuses a longer list. */ export const BROWSER_CLOSE_MAX_CANCELS = 32; -/** The most characters one Playwright viewer `input_text` message carries. A - * paste takes as many messages as it needs, each under the viewer socket's - * 64 KiB payload cap even when every character JSON-escapes to six bytes. */ -export const PLAYWRIGHT_TEXT_INPUT_MAX = 8192; +/** The JPEG quality of a crisp capture, for either provider. */ +export const CAPTURE_JPEG_QUALITY = 85; + +// --- the viewer socket (docs/specs/dor-browser.md → "Viewer Socket") --- + +/** Host → webview, as JSON text: the browser's state, sent only on change. */ +export type ViewerState = + /** Whether the browser is up, and its viewport's CSS size when known. */ + | { type: 'status'; connected: boolean; screencasting: boolean; viewportWidth?: number; viewportHeight?: number } + | { type: 'tabs'; tabs: AgentBrowserTab[] } + /** The active tab committed a navigation (before its load completes). */ + | { type: 'url'; url: string } + /** A popped-out window's page as its browser reports it: URL and title. */ + | { type: 'page'; url: string; title: string | null }; + +/** Webview → host, as JSON text: native input, and a request to resend the + * last frame to a canvas that mounted blank. */ +export type ViewerInput = + | { + type: 'input_mouse'; + eventType: 'mouseMoved' | 'mousePressed' | 'mouseReleased' | 'mouseWheel'; + x: number; + y: number; + button: 'left' | 'right' | 'middle' | 'none'; + buttons: number; + clickCount: number; + modifiers: number; + deltaX?: number; + deltaY?: number; + } + | { type: 'input_keyboard'; eventType: 'keyDown' | 'keyUp'; key: string; code: string; text: string; windowsVirtualKeyCode: number; modifiers: number } + /** A paste, inserted whole. */ + | { type: 'input_text'; text: string } + | { type: 'repaint' }; + +/** A frame: a CSS-resolution stream frame painted at once, or the + * device-resolution capture that replaces it. */ +export type ViewerFrameKind = 'provisional' | 'crisp'; + +export interface ViewerFrame { + kind: ViewerFrameKind; + jpeg: Uint8Array; + /** The viewport's CSS size, when the host knows it. */ + size?: { width: number; height: number }; +} + +/** A frame travels as one binary message: this header, then the JPEG. Byte 0 + * is the kind, bytes 4-11 the viewport's CSS width and height (u32 LE, 0 when + * unknown). */ +export const VIEWER_FRAME_HEADER_BYTES = 12; +const FRAME_KIND_CODE: Record = { provisional: 1, crisp: 2 }; + +export function encodeViewerFrame({ kind, jpeg, size }: ViewerFrame): Uint8Array { + const message = new Uint8Array(VIEWER_FRAME_HEADER_BYTES + jpeg.byteLength); + const header = new DataView(message.buffer); + header.setUint8(0, FRAME_KIND_CODE[kind]); + header.setUint32(4, size?.width ?? 0, true); + header.setUint32(8, size?.height ?? 0, true); + message.set(jpeg, VIEWER_FRAME_HEADER_BYTES); + return message; +} + +/** The frame a binary message carries — its JPEG a view, not a copy — or + * null for one that is not a frame. */ +export function decodeViewerFrame(message: ArrayBuffer): ViewerFrame | null { + if (message.byteLength <= VIEWER_FRAME_HEADER_BYTES) return null; + const header = new DataView(message, 0, VIEWER_FRAME_HEADER_BYTES); + const code = header.getUint8(0); + const kind = code === FRAME_KIND_CODE.provisional ? 'provisional' : code === FRAME_KIND_CODE.crisp ? 'crisp' : null; + if (!kind) return null; + const width = header.getUint32(4, true); + const height = header.getUint32(8, true); + return { + kind, + jpeg: new Uint8Array(message, VIEWER_FRAME_HEADER_BYTES), + ...(width > 0 && height > 0 ? { size: { width, height } } : {}), + }; +} + +/** The most characters one `input_text` message carries. A paste takes as + * many messages as it needs, each under the viewer socket's 64 KiB payload + * cap even when every character JSON-escapes to six bytes. */ +export const VIEWER_TEXT_INPUT_MAX = 8192; /** - * A paste as Playwright viewer messages, each inserted whole by the host (CDP - * `Input.insertText`): one queued message per `PLAYWRIGHT_TEXT_INPUT_MAX` - * characters, not a key down and up per character, which a long paste would - * push past the host's input queue cap. Line endings become `\n`, and a - * message never ends inside a surrogate pair. + * A paste as viewer socket messages, each inserted whole by the host + * (Playwright: CDP `Input.insertText`; agent-browser: a key pair per + * character, sent to its daemon host-side): one message per + * `VIEWER_TEXT_INPUT_MAX` characters, never a key down and up per character + * from the webview, which a long paste would push past the host's input queue + * cap. Line endings become `\n`, and a message never ends inside a surrogate + * pair. */ -export function playwrightTextInputs(text: string): { type: 'input_text'; text: string }[] { +export function viewerTextInputs(text: string): { type: 'input_text'; text: string }[] { const normalized = text.replace(/\r\n?/g, '\n'); const messages: { type: 'input_text'; text: string }[] = []; for (let start = 0; start < normalized.length;) { - let end = Math.min(start + PLAYWRIGHT_TEXT_INPUT_MAX, normalized.length); + let end = Math.min(start + VIEWER_TEXT_INPUT_MAX, normalized.length); const last = normalized.charCodeAt(end - 1); if (end < normalized.length && last >= 0xd800 && last <= 0xdbff) end -= 1; messages.push({ type: 'input_text', text: normalized.slice(start, end) }); diff --git a/lib/src/lib/platform/vscode-adapter.test.ts b/lib/src/lib/platform/vscode-adapter.test.ts index 25d0ebc8c..543286f00 100644 --- a/lib/src/lib/platform/vscode-adapter.test.ts +++ b/lib/src/lib/platform/vscode-adapter.test.ts @@ -468,7 +468,7 @@ describe('VSCodeAdapter browser requests', () => { expect(new VSCodeAdapter().browserProviders).toEqual(['agent-browser', 'playwright']); }); - // A capture queued behind a page-loading `open` answers only after the + // A command queued behind a page-loading `open` answers only after the // CLI's 25s action timeout, and a launch the host bounds to answer inside // the same wait; giving up sooner makes the webview ask again. it('waits out a daemon command held behind a page load', async () => { @@ -476,7 +476,7 @@ describe('VSCodeAdapter browser requests', () => { const adapter = new VSCodeAdapter(); const binding = { session: 'sess' }; for (const request of [ - () => adapter.browser({ provider: 'agent-browser', binding, op: 'screenshot', format: 'jpeg' }), + () => adapter.browser({ provider: 'agent-browser', binding, op: 'launch', url: 'https://example.com/', headed: true }), () => adapter.browser({ provider: 'agent-browser', binding, op: 'history', dir: 'reload' }), () => adapter.browser({ provider: 'agent-browser', binding, op: 'edit', edit: 'copy' }), ]) { @@ -495,9 +495,9 @@ describe('VSCodeAdapter browser requests', () => { const request = postMessage.mock.calls.map(([message]) => message).find((message) => message.type === 'browser:request'); expect(request.request).toEqual({ provider: 'agent-browser', binding: { session: 'sess' }, op: 'attach', url: 'https://example.com/' }); windowTarget.dispatchEvent(hostMessage({ - type: 'browser:result', requestId: request.requestId, result: { ok: true, wsPort: 4321, relaunched: true, headed: true, nativeIdentity: 'id' }, + type: 'browser:result', requestId: request.requestId, result: { ok: true, stream: 4321, relaunched: true, headed: true, nativeIdentity: 'id' }, })); - expect(await attached).toEqual({ ok: true, wsPort: 4321, relaunched: true, headed: true, nativeIdentity: 'id' }); + expect(await attached).toEqual({ ok: true, stream: 4321, relaunched: true, headed: true, nativeIdentity: 'id' }); }); }); diff --git a/scripts/loopback-lint.mjs b/scripts/loopback-lint.mjs index f0dc9a701..aab9a4291 100644 --- a/scripts/loopback-lint.mjs +++ b/scripts/loopback-lint.mjs @@ -88,12 +88,6 @@ const ALLOWED = { + 'and unbundled — it ships in nothing. Both controls are checked by ' + 'standalone/scripts/dev-agent-browser.test.mjs; see ' + 'standalone/scripts/dev-host-guard.mjs for the bridge beside it.', - 'vscode-ext/src/agent-browser-host.ts': - 'The stream relay authenticates with a single-use 64-hex token (60s TTL, ' - + 'pinned to one target port) and drops Origin rather than rewriting it, so ' - + 'it vouches for no one. It skips the Host check on purpose: rebinding ' - + 'exists to make same-origin-looking requests, which buys nothing against ' - + 'an unguessable one-shot secret. See lib/src/host/loopback-guard.ts.', }; const GUARD_REFERENCES = ['loopback-guard', 'dev-host-guard']; diff --git a/scripts/spec-word-budgets.json b/scripts/spec-word-budgets.json index b3ec7a1f6..c7f2ae177 100644 --- a/scripts/spec-word-budgets.json +++ b/scripts/spec-word-budgets.json @@ -5,7 +5,7 @@ "docs/specs/alert.md": 8150, "docs/specs/auto-update.md": 1150, "docs/specs/deploy.md": 1900, - "docs/specs/dor-browser.md": 7350, + "docs/specs/dor-browser.md": 7800, "docs/specs/dor-cli.md": 6300, "docs/specs/dor-tool.md": 4150, "docs/specs/glossary.md": 2950, @@ -21,7 +21,7 @@ "docs/specs/security-audit.md": 2100, "docs/specs/security-ci.md": 2950, "docs/specs/security-hosted.md": 600, - "docs/specs/security-local.md": 3150, + "docs/specs/security-local.md": 3200, "docs/specs/security-remote.md": 5750, "docs/specs/security-supply-chain.md": 1150, "docs/specs/security.md": 1900, diff --git a/standalone/scripts/dev-agent-browser.mjs b/standalone/scripts/dev-agent-browser.mjs index 94a8ca592..70f3df3c6 100644 --- a/standalone/scripts/dev-agent-browser.mjs +++ b/standalone/scripts/dev-agent-browser.mjs @@ -2,7 +2,6 @@ import http from 'node:http'; import os from 'node:os'; import path from 'node:path'; -import { readFile, rm } from 'node:fs/promises'; import { randomBytes } from 'node:crypto'; import { sessionForKey } from 'dor-lib-common/browser-providers'; // cross-spawn, not node:child_process: this script spawns `dor` and @@ -124,25 +123,9 @@ const fireAndForget = { kill_sidecar_now: () => shutdown(), }; -// The sidecar answers a screenshot with a temp-file PATH (bytes stay off the -// stdio pipe). Production reads that file in Rust; this dev bridge has no Rust, -// so read it in Node and re-encode to the base64 the browser-sidecar adapter -// expects — the base64 travels in the HTTP invoke response, outside the event -// stream. Like Rust, it deletes the file once read: each capture is its own. -async function readCapture(result) { - if (!result?.ok || typeof result.path !== 'string') return result; - try { - return { ok: true, mime: result.mime, bytesBase64: (await readFile(result.path)).toString('base64') }; - } finally { - await rm(result.path, { force: true }); - } -} - const invokeMap = { // BROWSER_REQUEST_TIMEOUT_MS in dor-lib-common/src/browser-providers.ts. - browser_request: async ({ request }) => readCapture( - await requestSidecar('browser:request', { request }, 'browser:result', (data) => data.result, 40000), - ), + browser_request: ({ request }) => requestSidecar('browser:request', { request }, 'browser:result', (data) => data.result, 40000), get_available_shells: (_args) => requestSidecar('pty:getShells', {}, 'pty:shells', (data) => data.shells ?? []), pty_get_cwd: ({ id }) => requestSidecar('pty:getCwd', { id }, 'pty:cwd', (data) => data.cwd ?? null), pty_get_cwds: ({ ids }) => requestSidecar('pty:getCwds', { ids }, 'pty:cwds', (data) => data.cwds ?? {}), diff --git a/standalone/sidecar/main.js b/standalone/sidecar/main.js index ed8cc7f0c..eaa4f73b7 100644 --- a/standalone/sidecar/main.js +++ b/standalone/sidecar/main.js @@ -215,13 +215,11 @@ function handleLine(line) { })); break; case 'browser:request': - // A screenshot answers with its temp-file PATH, not the bytes: a - // ~100-700KB base64 line would otherwise ride the JSON-lines stdio pipe - // shared with all PTY traffic (head-of-line blocking terminal output on - // every frame). Rust reads the file itself and returns a raw - // tauri::ipc::Response for the webview. + // Frames never ride this JSON-lines stdio, which PTY traffic shares: + // the webview takes them over the host's viewer socket + // (docs/specs/dor-browser.md → "Viewer Socket"). respondAsync('browser:result', data.requestId, async () => ({ - result: await browserHost.requestFile(data.request), + result: await browserHost.request(data.request), })); break; case 'clipboard:readFiles': diff --git a/standalone/src-tauri/src/lib.rs b/standalone/src-tauri/src/lib.rs index accb53137..449616fd2 100644 --- a/standalone/src-tauri/src/lib.rs +++ b/standalone/src-tauri/src/lib.rs @@ -1510,38 +1510,6 @@ fn browser_request( Ok(response.get("result").cloned().unwrap_or(JsonValue::Null)) } -// The sidecar answers a screenshot with its temp-file PATH (the bytes never ride -// the JSON-lines stdio shared with PTY traffic). Read the file here and return a -// raw tauri::ipc::Response, so the webview gets an ArrayBuffer (what the panel -// decodes with createImageBitmap). Each capture is a fresh file handed to this -// reader alone, so it is deleted once read: a frame of the user's page must not -// wait on disk for the host's shutdown. -#[tauri::command(async)] -fn browser_screenshot( - state: tauri::State<'_, SidecarState>, - request: JsonValue, -) -> Result { - if request.get("op").and_then(JsonValue::as_str) != Some("screenshot") { - return Err("Expected screenshot operation".to_string()); - } - let result = browser_request(state, request)?; - if result.get("ok").and_then(JsonValue::as_bool) != Some(true) { - return Err(result - .get("error") - .and_then(JsonValue::as_str) - .unwrap_or("screenshot failed") - .to_string()); - } - let path = result - .get("path") - .and_then(JsonValue::as_str) - .ok_or("screenshot returned no path")?; - let bytes = std::fs::read(path) - .map_err(|err| format!("could not read screenshot file '{path}': {err}")); - let _ = std::fs::remove_file(path); - bytes.map(tauri::ipc::Response::new) -} - // Clipboard reads run natively on Windows (see clipboard_win) to avoid the // console-window flicker of shelling out to PowerShell; other platforms keep the // sidecar path (pbpaste/xclip never pop a console window). @@ -4430,7 +4398,6 @@ pub fn run() { save_notepad_archive, reset_notepad_archive, browser_request, - browser_screenshot, ]) .build(tauri::generate_context!()) .expect("error while building Dormouse") @@ -6164,12 +6131,10 @@ mod tests { } } - // Match direct callers of the blocking helper *and* - // `browser_screenshot`, which reaches it transitively through - // `browser_request` (its body never names `request_from_sidecar` - // directly). That pair carries the longest timeout - // (BROWSER_REQUEST_TIMEOUT = 40s), so it's the worst case to let - // slip plain-sync. + // Match direct callers of the blocking helper *and* anything that + // reaches it transitively through `browser_request`, which carries + // the longest timeout (BROWSER_REQUEST_TIMEOUT = 40s), so it's the + // worst case to let slip plain-sync. let reaches_blocking = [ "request_from_sidecar", "browser_request", "ARRIVAL_DISK_LOCK", "record_arrival_on_disk", "mark_arrival_adopted_on_disk", "return_arrival_on_disk", diff --git a/standalone/src/browser-sidecar-adapter.test.ts b/standalone/src/browser-sidecar-adapter.test.ts index e7116dd88..4216c799a 100644 --- a/standalone/src/browser-sidecar-adapter.test.ts +++ b/standalone/src/browser-sidecar-adapter.test.ts @@ -88,20 +88,18 @@ describe("BrowserSidecarAdapter session persistence", () => { }); }); -// The harness has no Rust to read a capture file, so its bridge answers a -// screenshot with base64 (`readCapture` in standalone/scripts/dev-agent-browser.mjs). describe("BrowserSidecarAdapter browser requests", () => { - it("forwards every request through one browser_request, decoding a screenshot's base64", async () => { + it("forwards every request through one browser_request, and answers a failure as a result", async () => { const host = new BrowserSidecarHost("http://localhost:1234"); - const invoke = vi.spyOn(host, "invoke").mockResolvedValueOnce({ ok: true, mime: "image/jpeg", bytesBase64: "/9gB" }); + const invoke = vi.spyOn(host, "invoke").mockResolvedValueOnce({ ok: true, url: "ws://127.0.0.1:9/view/abc" }); const adapter = new BrowserSidecarAdapter(host); - const request = { provider: "agent-browser" as const, binding: { session: "sess" }, op: "screenshot" as const }; + const request = { provider: "agent-browser" as const, binding: { session: "sess" }, op: "view" as const, stream: 4321 }; - expect(await adapter.browser(request)).toEqual({ ok: true, mime: "image/jpeg", bytes: Uint8Array.from([0xff, 0xd8, 0x01]) }); + expect(await adapter.browser(request)).toEqual({ ok: true, url: "ws://127.0.0.1:9/view/abc" }); expect(invoke).toHaveBeenCalledWith("browser_request", { request }); invoke.mockRejectedValueOnce(new Error("bridge down")); - expect(await adapter.browser({ ...request, op: "close" })).toEqual({ ok: false, error: "bridge down" }); + expect(await adapter.browser({ provider: "agent-browser", binding: { session: "sess" }, op: "close" })).toEqual({ ok: false, error: "bridge down" }); }); }); diff --git a/standalone/src/browser-sidecar-adapter.ts b/standalone/src/browser-sidecar-adapter.ts index 9a86813e0..10e6cdfff 100644 --- a/standalone/src/browser-sidecar-adapter.ts +++ b/standalone/src/browser-sidecar-adapter.ts @@ -53,13 +53,6 @@ import { BrowserSidecarHost } from "./browser-sidecar-host"; const errMessage = (err: unknown): string => err instanceof Error ? err.message : String(err); -function decodeBase64Bytes(base64: string): Uint8Array { - const binary = atob(base64); - const bytes = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i); - return bytes; -} - /** The `alert*` platform methods, taken from the shared client in the constructor. */ export interface BrowserSidecarAdapter extends AlertClientMethods {} @@ -261,10 +254,7 @@ export class BrowserSidecarAdapter implements PlatformAdapter { async browser(request: BrowserRequest): Promise { try { - // A screenshot's bytes arrive as base64: this bridge has no Rust to read - // the capture file (`readCapture` in standalone/scripts/dev-agent-browser.mjs). - const { bytesBase64, ...result } = await this.host.invoke("browser_request", { request }); - return bytesBase64 ? { ...result, bytes: decodeBase64Bytes(bytesBase64) } : result; + return await this.host.invoke("browser_request", { request }); } catch (err) { return { ok: false, error: errMessage(err) }; } diff --git a/standalone/src/tauri-adapter.test.ts b/standalone/src/tauri-adapter.test.ts index 60e8769d6..fe76d9c2a 100644 --- a/standalone/src/tauri-adapter.test.ts +++ b/standalone/src/tauri-adapter.test.ts @@ -109,23 +109,16 @@ describe("TauriAdapter cwd probing", () => { }); describe("TauriAdapter browser requests", () => { - it("sends a screenshot for raw bytes, every other request as JSON, and answers a failure as a result", async () => { - // Image bytes must never ride the JSON-lines pipe: only `browser_screenshot` - // reads the capture file in Rust and answers an ArrayBuffer. + it("sends every request through one browser_request, and answers a failure as a result", async () => { + // Frames never cross this IPC: the webview takes them over the host's + // viewer socket. const adapter = new TauriAdapter(); - vi.mocked(rawInvoke).mockImplementation(async (cmd: string) => { - if (cmd === "browser_screenshot") return Uint8Array.from([0xff, 0xd8]).buffer; - if (cmd === "browser_request") return { ok: true, wsPort: 4321 }; - return undefined; - }); + vi.mocked(rawInvoke).mockImplementation(async (cmd: string) => (cmd === "browser_request" ? { ok: true, stream: 4321 } : undefined)); const binding = { session: "sess" }; - const shot = await adapter.browser({ provider: "agent-browser", binding, op: "screenshot", format: "png" }); - expect(shot).toEqual({ ok: true, bytes: Uint8Array.from([0xff, 0xd8]), mime: "image/png" }); const attached = await adapter.browser({ provider: "playwright", binding, op: "attach" }); - expect(attached).toEqual({ ok: true, wsPort: 4321 }); + expect(attached).toEqual({ ok: true, stream: 4321 }); expect(vi.mocked(rawInvoke).mock.calls.filter(([cmd]) => cmd.startsWith("browser_"))).toEqual([ - ["browser_screenshot", { request: { provider: "agent-browser", binding, op: "screenshot", format: "png" } }], ["browser_request", { request: { provider: "playwright", binding, op: "attach" } }], ]); diff --git a/standalone/src/tauri-adapter.ts b/standalone/src/tauri-adapter.ts index 58293effa..9b7b69f4d 100644 --- a/standalone/src/tauri-adapter.ts +++ b/standalone/src/tauri-adapter.ts @@ -462,18 +462,13 @@ export class TauriAdapter implements PlatformAdapter { } // --- browser automation (docs/specs/dor-browser.md → "Browser Host"). - // One Rust command forwards every request to the sidecar's - // shared host; a screenshot takes `browser_screenshot`, which answers raw - // bytes (tauri::ipc::Response) rather than JSON. --- + // One Rust command forwards every request to the sidecar's shared host; + // frames reach the webview over the host's viewer socket, never this IPC. --- readonly browserProviders = BROWSER_PROVIDER_IDS; async browser(request: BrowserRequest): Promise { try { - if (request.op === "screenshot") { - const buffer = await rawInvoke("browser_screenshot", { request }); - return { ok: true, bytes: new Uint8Array(buffer), mime: request.format === "png" ? "image/png" : "image/jpeg" }; - } return await rawInvoke("browser_request", { request }); } catch (err) { return { ok: false, error: errMessage(err) }; diff --git a/vscode-ext/src/agent-browser-host.ts b/vscode-ext/src/agent-browser-host.ts index 1265d1625..8fd0825a1 100644 --- a/vscode-ext/src/agent-browser-host.ts +++ b/vscode-ext/src/agent-browser-host.ts @@ -3,23 +3,14 @@ * (docs/specs/dor-browser.md → "Browser Host"). * * The host itself is host-agnostic and lives in `lib/src/host/browser-host.ts` - * (shared verbatim with the standalone Node sidecar). This file only: - * 1. instantiates it with the VS-Code-specific bits — writing the OS - * clipboard, logging, and the relay below; and - * 2. owns the agent-browser **stream relay**, which is genuinely VS-Code-only: - * the agent-browser stream server returns 403 for `vscode-webview://` - * origins (only localhost or absent origins are accepted), so the webview - * cannot connect directly. It connects to a short-lived tokenized relay URL - * and the relay pipes bytes only to the authorized 127.0.0.1:. - * (The standalone webview's `tauri://localhost` origin is accepted, so it - * connects directly and needs no relay.) + * (shared verbatim with the standalone Node sidecar), viewer sockets included; + * this file only instantiates it with the VS-Code-specific bits — writing the + * OS clipboard, and logging. */ import * as vscode from 'vscode'; -import * as net from 'net'; import { log } from './log'; import { createAgentBrowserProvider } from '../../lib/src/host/agent-browser-host'; import { createBrowserHost } from '../../lib/src/host/browser-host'; -import { BrowserStreamGrants } from '../../lib/src/host/browser-stream-guard'; import { createPlaywrightProvider } from '../../lib/src/host/playwright-host'; const logInfo = (message: string) => log.info(message); @@ -29,7 +20,7 @@ const host = createBrowserHost({ writeClipboardText: async (text) => { await vscode.env.clipboard.writeText(text); }, log: logInfo, providers: { - 'agent-browser': () => createAgentBrowserProvider({ log: logInfo, streamUrl: createStreamRelayUrl }), + 'agent-browser': () => createAgentBrowserProvider({ log: logInfo }), playwright: () => createPlaywrightProvider({ log: logInfo }), }, }); @@ -49,90 +40,3 @@ export async function closeBrowserSessions(): Promise { new Promise((resolve) => { deadline = setTimeout(resolve, CLOSE_DEADLINE_MS); }), ]).finally(() => clearTimeout(deadline)); } - -let relayPortPromise: Promise | null = null; -// The same single-use, 60s, port-bound grants that guard the Playwright viewer. -const streamRelayGrants = new BrowserStreamGrants(); - -async function createStreamRelayUrl(streamPort: number): Promise { - const relayPort = await ensureStreamRelayPort(); - return `ws://127.0.0.1:${relayPort}/stream/${streamPort}/${streamRelayGrants.issue(streamPort)}`; -} - -function ensureStreamRelayPort(): Promise { - if (!relayPortPromise) { - relayPortPromise = new Promise((resolve, reject) => { - const server = net.createServer(handleRelayClient); - server.on('error', (err) => { - log.info(`[agent-browser] stream relay error: ${err.message}`); - relayPortPromise = null; - reject(err); - }); - server.unref(); - server.listen(0, '127.0.0.1', () => { - const address = server.address(); - if (address && typeof address === 'object') { - log.info(`[agent-browser] stream relay listening on 127.0.0.1:${address.port}`); - resolve(address.port); - } else { - reject(new Error('stream relay failed to bind')); - } - }); - }); - } - return relayPortPromise; -} - -// The relay is loopback-only on both sides: it accepts connections from -// 127.0.0.1 and dials only an explicitly granted 127.0.0.1:. It rewrites -// the upgrade request head (path → "/", Origin dropped, Host rewritten) and -// from then on is a dumb byte pipe in both directions. -function handleRelayClient(client: net.Socket): void { - let head = Buffer.alloc(0); - client.on('error', () => {}); - - const onData = (chunk: Buffer) => { - head = Buffer.concat([head, chunk]); - const headEnd = head.indexOf('\r\n\r\n'); - if (headEnd === -1) { - if (head.length > 16384) client.destroy(); - return; - } - client.off('data', onData); - client.pause(); - - const headText = head.subarray(0, headEnd).toString('latin1'); - const remainder = head.subarray(headEnd + 4); - const requestMatch = /^GET \/stream\/(\d{1,5})\/([a-f0-9]{64}) HTTP\/1\.1\r\n/i.exec(headText); - const targetPort = requestMatch ? Number(requestMatch[1]) : 0; - if (!targetPort || targetPort > 65535) { - client.end('HTTP/1.1 400 Bad Request\r\n\r\n'); - return; - } - const token = requestMatch?.[2] ?? ''; - if (!streamRelayGrants.consume(token, targetPort)) { - client.end('HTTP/1.1 403 Forbidden\r\n\r\n'); - return; - } - - const headerLines = headText.split('\r\n').slice(1).filter((line) => { - const name = line.slice(0, line.indexOf(':')).trim().toLowerCase(); - return name !== 'origin'; - }).map((line) => ( - line.toLowerCase().startsWith('host:') ? `Host: 127.0.0.1:${targetPort}` : line - )); - const rewritten = `GET / HTTP/1.1\r\n${headerLines.join('\r\n')}\r\n\r\n`; - - const upstream = net.connect(targetPort, '127.0.0.1', () => { - upstream.write(rewritten); - if (remainder.length) upstream.write(remainder); - client.pipe(upstream); - upstream.pipe(client); - client.resume(); - }); - upstream.on('error', () => client.destroy()); - client.on('close', () => upstream.destroy()); - }; - - client.on('data', onData); -} From ceaa1bb570913480578db991cb9edc467219a975 Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Thu, 24 Sep 2026 04:00:20 -0700 Subject: [PATCH 3/8] Pin that a host editing op paints the stream at once Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NCV5Uq6FeZbBfzfpk3vrpe --- lib/src/host/browser-host.test.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/lib/src/host/browser-host.test.ts b/lib/src/host/browser-host.test.ts index b69de1902..546860adb 100644 --- a/lib/src/host/browser-host.test.ts +++ b/lib/src/host/browser-host.test.ts @@ -159,6 +159,30 @@ describe('createBrowserHost', () => { } }); + it('paints the stream at once after an editing op, as after input', async () => { + const fake = fakeProvider(); + const host = createBrowserHost({ writeClipboardText: vi.fn(), providers: { 'agent-browser': () => fake.provider } }); + const s1 = { provider: 'agent-browser', binding: { session: 's1' } } as const; + try { + const viewer = await openViewer((await host.request({ ...s1, op: 'view', stream: 4321 })).url!); + await vi.waitFor(() => expect(fake.views).toHaveLength(1)); + const { sink } = fake.views[0]; + sink.frame(new Uint8Array([0xff, 0xd8, 1])); + await vi.waitFor(() => expect(viewer.frames.map((frame) => frame.kind)).toEqual(['provisional', 'crisp'])); + await new Promise((resolve) => setTimeout(resolve, 300)); + // At rest, a changed frame only pulses the crisp loop. + sink.frame(new Uint8Array([0xff, 0xd8, 2])); + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(viewer.frames.filter((frame) => frame.kind === 'provisional')).toHaveLength(1); + // Select-all changed the page without any input on the socket. + await host.request({ ...s1, op: 'edit', edit: 'selectAll' }); + sink.frame(new Uint8Array([0xff, 0xd8, 3])); + await vi.waitFor(() => expect(viewer.frames.filter((frame) => frame.kind === 'provisional')).toHaveLength(2)); + } finally { + await host.close(); + } + }); + it('joins no capture of the browser a relaunch replaced', async () => { const fake = fakeProvider(); const host = createBrowserHost({ writeClipboardText: vi.fn(), providers: { 'agent-browser': () => fake.provider } }); From 071b37b8b25e5a181a256032ff8d6dd5d4abb238 Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Thu, 24 Sep 2026 04:03:42 -0700 Subject: [PATCH 4/8] Pin the viewer socket's remaining rules, and drop a redundant port check A headed view must still receive a tab list or URL large enough to pass for a frame, a Playwright viewer whose input backs up behind CDP closes, a socket backed up past 2 MB skips provisional frames only, and a repaint resends the last frame. A view's input window now starts closed rather than at time zero. The agent-browser stream's explicit port-range check is gone: a port the URL cannot hold already fails the dial. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NCV5Uq6FeZbBfzfpk3vrpe --- lib/src/host/agent-browser-host.test.ts | 4 ++-- lib/src/host/agent-browser-host.ts | 1 - lib/src/host/browser-viewer.test.ts | 13 ++++++++++++- lib/src/host/browser-viewer.ts | 2 +- lib/src/host/playwright-host.lifecycle.test.ts | 8 ++++++++ 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/lib/src/host/agent-browser-host.test.ts b/lib/src/host/agent-browser-host.test.ts index 7492c6e1d..92dc38ee9 100644 --- a/lib/src/host/agent-browser-host.test.ts +++ b/lib/src/host/agent-browser-host.test.ts @@ -700,11 +700,11 @@ describe('agent-browser host viewer', () => { expect([...viewer.frames.find((f) => f.kind === 'crisp')!.jpeg]).toEqual([0xff, 0xd8, 0x99]); }); - it('routes a tab list or URL too large to tell from a frame by size as state', async () => { + it.each([false, true])('routes a tab list or URL too large to tell from a frame by size as state (headed: %s)', async (headed) => { running(session); const daemon = await fakeServer(); writeState(session, 'stream', daemon.port); - const viewer = await view(daemon.port); + const viewer = await view(daemon.port, headed); await daemon.connected(); const tabs = Array.from({ length: 80 }, (_, i) => ({ tabId: `t${i}`, diff --git a/lib/src/host/agent-browser-host.ts b/lib/src/host/agent-browser-host.ts index c402426c5..4ef6060df 100644 --- a/lib/src/host/agent-browser-host.ts +++ b/lib/src/host/agent-browser-host.ts @@ -431,7 +431,6 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): * messages come back from it, and only validated input goes to it. */ async function viewStream(b: ProviderBinding, port: number, headed: boolean, sink: ViewerSink): Promise { - if (!Number.isInteger(port) || port < 1 || port > 65535) throw new Error('an agent-browser stream is a TCP port'); // Only a daemon in the host's own socket directory is one it can capture. const capturable = (await readStateNumber(b.session, 'stream')) === port; const socket = new WebSocket(`ws://127.0.0.1:${port}`, { handshakeTimeout: STREAM_CONNECT_TIMEOUT_MS, perMessageDeflate: false }); diff --git a/lib/src/host/browser-viewer.test.ts b/lib/src/host/browser-viewer.test.ts index 76cd761b5..2e19c5920 100644 --- a/lib/src/host/browser-viewer.test.ts +++ b/lib/src/host/browser-viewer.test.ts @@ -96,7 +96,7 @@ describe('a viewer socket', () => { // A canvas that mounted blank gets the last frame back. socket.input({ type: 'repaint' }); - expect(socket.kinds().at(-1)).toBe('crisp 9'); + expect(socket.kinds()).toEqual(['provisional 1', 'crisp 9', 'provisional 3', 'crisp 9', 'crisp 9']); }); it('keeps a capture a provisional paint superseded owed, with nothing left to pulse it', async () => { @@ -245,6 +245,17 @@ describe('a viewer socket', () => { expect(socket.kinds()).toEqual(['provisional 1']); }); + it('skips a provisional frame for a socket backed up past 2 MB, never a crisp one', async () => { + const { socket, view, captures } = makeView(); + socket.bufferedAmount = 3_000_000; + view.frame(jpeg(1)); + expect(socket.frames()).toEqual([]); + await vi.advanceTimersByTimeAsync(100); + captures[0](jpeg(9)); + await vi.advanceTimersByTimeAsync(0); + expect(socket.kinds()).toEqual(['crisp 9']); + }); + it('tells the webview a browser that went away on its own is gone', () => { const { socket, view } = makeView(); view.gone(); diff --git a/lib/src/host/browser-viewer.ts b/lib/src/host/browser-viewer.ts index c6ee9b0e2..643de3690 100644 --- a/lib/src/host/browser-viewer.ts +++ b/lib/src/host/browser-viewer.ts @@ -150,7 +150,7 @@ export class BrowserView implements ViewerSink { // What the socket last carried, which is what the webview's canvas shows. private last: { kind: ViewerFrameKind; jpeg: Uint8Array; message: Uint8Array } | null = null; private size: { width: number; height: number } | undefined; - private provisionalUntil = 0; + private provisionalUntil = -Infinity; private activeTab: string | undefined; // Counts the provisional paints that supersede a capture in flight — not // those made only because one is overdue, which it is newer than. diff --git a/lib/src/host/playwright-host.lifecycle.test.ts b/lib/src/host/playwright-host.lifecycle.test.ts index 9443fa18f..838075dac 100644 --- a/lib/src/host/playwright-host.lifecycle.test.ts +++ b/lib/src/host/playwright-host.lifecycle.test.ts @@ -331,6 +331,14 @@ test('a long paste reaches the page whole without tripping the input backlog', a } }); +test('closes a viewer socket whose input backs up behind CDP', async () => { + const viewer = await viewSession(); + // CDP stops answering input. + cdp.send.mockImplementation((method: string) => method.startsWith('Input.') ? new Promise(() => {}) : Promise.resolve({ data: 'aGVsbG8=' })); + for (let i = 0; i < 300; i++) viewer.send({ type: 'input_mouse', eventType: 'mouseMoved', x: i, y: 1 }); + expect(await viewer.closed).toBe(1008); +}); + test('frames reach the viewer as binary, decoded once, their acks paced to ~20 a second', async () => { const handlers = new Map void>(); cdp.on.mockImplementation((event: string, handler: (event: unknown) => void) => { handlers.set(event, handler); }); From 8d3c45441efdeaf41edcfc7ea577fa934043f5d5 Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Thu, 24 Sep 2026 04:07:16 -0700 Subject: [PATCH 5/8] Simplify the viewer socket after review Declare the host's viewer server, its views and its captures up front, drop a daemon stream frame's second dedup path, keep the paste expansion private to the agent-browser provider, and give the controller one setter for the viewport size a status or a frame reports. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NCV5Uq6FeZbBfzfpk3vrpe --- docs/specs/dor-browser.md | 5 +++-- .../wall/agent-browser-surface-controller.ts | 18 ++++++++++-------- lib/src/host/agent-browser-host.ts | 15 +++++---------- lib/src/host/browser-host.ts | 10 ++++------ 4 files changed, 22 insertions(+), 26 deletions(-) diff --git a/docs/specs/dor-browser.md b/docs/specs/dor-browser.md index 3538cbf52..ee2ce06ff 100644 --- a/docs/specs/dor-browser.md +++ b/docs/specs/dor-browser.md @@ -591,8 +591,9 @@ match. **A refused path is dropped, never fatal**, so the host's own candidates run. The webview applies the same predicate before sending one (`browserHandle`) or storing one. -**Crisp captures are written into a private per-process directory, read back -and removed at once, and the directory is removed at shutdown** (rationale). +**A crisp capture a CLI writes lands in a private per-process directory, is +read back and removed at once, and the directory is removed at shutdown** +(rationale). **One capture per browser is in flight**: a viewer socket asking meanwhile joins it — never one from before the browser's close or relaunch — and an agent-browser capture's spawn is killed past 30s. **A tmpdir that cannot be diff --git a/lib/src/components/wall/agent-browser-surface-controller.ts b/lib/src/components/wall/agent-browser-surface-controller.ts index 554bcb3e5..f5886b4c7 100644 --- a/lib/src/components/wall/agent-browser-surface-controller.ts +++ b/lib/src/components/wall/agent-browser-surface-controller.ts @@ -974,9 +974,7 @@ export class AgentBrowserSurfaceController { const lost = !event.status.connected && phase.seen; if (event.status.connected) phase.seen = true; if (typeof event.status.viewportWidth === 'number' && typeof event.status.viewportHeight === 'number') { - this.device = { width: event.status.viewportWidth, height: event.status.viewportHeight }; - this.maybeDisengageSync(); - this.publishScreen(); + this.setDeviceSize(event.status.viewportWidth, event.status.viewportHeight); } if (lost) this.streamLost(); } else if (event.type === 'url') { @@ -990,11 +988,7 @@ export class AgentBrowserSurfaceController { } else if (event.type === 'tabs') { this.setTabs(event.tabs); } else if (event.type === 'frame') { - if (event.size) { - this.device = { width: event.size.width, height: event.size.height }; - this.maybeDisengageSync(); - this.publishScreen(); - } + if (event.size) this.setDeviceSize(event.size.width, event.size.height); this.paintFrame(event); } }); @@ -1010,6 +1004,14 @@ export class AgentBrowserSurfaceController { } } + /** The browser's viewport, as its stream reports it: the screen indicator + * and sync-to-pane follow it. */ + private setDeviceSize(width: number, height: number): void { + this.device = { width, height }; + this.maybeDisengageSync(); + this.publishScreen(); + } + // --- painting --- /** Paint a frame from the viewer socket, latest-only: one decodes at a diff --git a/lib/src/host/agent-browser-host.ts b/lib/src/host/agent-browser-host.ts index 4ef6060df..12e8de48e 100644 --- a/lib/src/host/agent-browser-host.ts +++ b/lib/src/host/agent-browser-host.ts @@ -98,7 +98,7 @@ export interface AgentBrowserProviderDeps { * A paste for the daemon's stream, which takes only key and mouse events: a * key down and up per character, a newline as Enter. */ -export function keyPairTextInputs(text: string): Extract[] { +function keyPairTextInputs(text: string): Extract[] { const messages: Extract[] = []; for (const ch of text) { if (ch === '\r') continue; @@ -453,11 +453,9 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): }); socket.on('message', (data: Buffer, isBinary) => { if (isBinary) return; + // A re-broadcast frame is dropped before it is parsed. const large = data.length > FRAME_THRESHOLD_BYTES && !CONTROL_MARKERS.some((marker) => data.includes(marker)); - if (large) { - if (headed || lastFrame?.equals(data)) return; - lastFrame = data; - } + if (large && (headed || lastFrame?.equals(data))) return; const text = data.toString(); let message: { type?: unknown; data?: unknown; metadata?: { deviceWidth?: unknown; deviceHeight?: unknown }; connected?: unknown; screencasting?: unknown; viewportWidth?: unknown; viewportHeight?: unknown; tabs?: unknown; url?: unknown }; try { @@ -466,11 +464,8 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): return; } if (message.type === 'frame' && typeof message.data === 'string') { - if (headed) return; - if (!large) { - if (lastFrame?.equals(data)) return; - lastFrame = data; - } + if (headed || lastFrame?.equals(data)) return; + lastFrame = data; sink.frame(Buffer.from(message.data, 'base64'), frameSize(message.metadata)); return; } diff --git a/lib/src/host/browser-host.ts b/lib/src/host/browser-host.ts index c1645f237..3a3fe4f5e 100644 --- a/lib/src/host/browser-host.ts +++ b/lib/src/host/browser-host.ts @@ -263,6 +263,10 @@ type Bound = { p: BrowserProvider; b: unknown; id: string }; export function createBrowserHost(deps: BrowserHostDeps) { const log = (error: unknown) => deps.log?.(`[browser-host] ${messageOf(error)}`); const providers = new Map>(); + const viewers = createViewerServer(); + // The viewer sockets open on each browser: a launch or close ends them. + const views = new Map>(); + const captures = createBrowserCaptures(); let closed = false; function providerFor(id: BrowserAutomationProvider): BrowserProvider { @@ -506,10 +510,6 @@ export function createBrowserHost(deps: BrowserHostDeps) { // --- viewer sockets and their captures --- - const viewers = createViewerServer(); - // The viewer sockets open on each browser: a launch or close ends them. - const views = new Map>(); - /** Open one viewer socket on the browser at `stream`, unless a launch or * close of it began since its URL was granted (`generation`). */ function openView(socket: WebSocket, bound: Bound, stream: number, isHeaded: boolean, debug: boolean, generation: number): void { @@ -536,8 +536,6 @@ export function createBrowserHost(deps: BrowserHostDeps) { ); } - const captures = createBrowserCaptures(); - /** One device-resolution JPEG of `bound`'s browser, for its viewer sockets' * crisp paint; undefined when none can be taken. A launch or close ends * every viewer socket of the browser first, so none asks mid-relaunch. */ From 0f84128eb5d750d615283e9eae9d30276d04bb7c Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Thu, 24 Sep 2026 04:23:12 -0700 Subject: [PATCH 6/8] Re-home the #777 review fixes into the viewer socket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - One proof of an agent-browser daemon, `liveDaemon` — a pid file from this boot naming a live process beside a stream port that accepts — now gates a stop's signal, every operation and capture, and the headed window's `get cdp-url`, where operations had checked only the pid. - `dor ab` hands over the port it read after every command, so the viewer dials a handed-over loopback port for any bind; it captures that browser only when the same proof names that port, and otherwise sends every changed frame, never reading the caller's socket directory. - Every capture writes a fresh, randomly named file, read into memory and deleted however the capture ends — a failed or killed one included — so no frame waits on disk; with the file transport gone, the per-reader copies and the pruning of unread frames are moot. - The viewer's upstream dials end by 5 s and `get cdp-url` by 10 s, and the viewer listener closes every connection at shutdown. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NCV5Uq6FeZbBfzfpk3vrpe --- docs/specs/dor-browser.md | 31 ++++---- docs/specs/dor-browser.rationale.md | 2 +- lib/src/host/agent-browser-host.test.ts | 95 ++++++++++++++++++++----- lib/src/host/agent-browser-host.ts | 71 +++++++++++------- lib/src/host/browser-capture.ts | 47 ++++++------ lib/src/host/browser-viewer.ts | 3 + scripts/spec-word-budgets.json | 2 +- 7 files changed, 164 insertions(+), 87 deletions(-) diff --git a/docs/specs/dor-browser.md b/docs/specs/dor-browser.md index ee2ce06ff..bc8d6b33f 100644 --- a/docs/specs/dor-browser.md +++ b/docs/specs/dor-browser.md @@ -455,18 +455,19 @@ replaces it (rationale): again** (rationale). - **A byte-identical capture is resent only after a provisional frame or a `repaint`.** -- **An active-tab change owes a capture**; a browser the host cannot capture (a - daemon in a socket directory it does not share) has every changed frame sent - as provisional. +- **An active-tab change owes a capture**; a browser the host cannot prove + live at the stream viewed ([agent-browser](#agent-browser)) has every changed + frame sent as provisional. Pinned by `lib/src/host/browser-viewer.test.ts`. **Upstreams.** agent-browser: the daemon's stream, dialed on `127.0.0.1` only; **the host must drop its ~20 Hz re-broadcast by raw comparison against the last frame and tab list**, decoding a changed frame once (rationale); **a tab list or -URL is state at any size**, never taken for a frame; a paste goes as key pairs. **A headed window's page is followed over its browser's -CDP, held host-side and dialed on loopback only** (rationale). Playwright: its -CDP screencast ([Playwright](#playwright)). +URL is state at any size**, never taken for a frame; a paste goes as key pairs. +**A headed window's page is followed over its browser's CDP, held host-side and +dialed on loopback only** (rationale). **Every upstream dial ends by 5 s**, and +`get cdp-url` by 10 s. Playwright: its CDP screencast ([Playwright](#playwright)). Source of truth: `createViewerServer`, `BrowserView` and `parseViewerInput` in `lib/src/host/browser-viewer.ts`; `BrowserStreamGrants` in @@ -591,14 +592,13 @@ match. **A refused path is dropped, never fatal**, so the host's own candidates run. The webview applies the same predicate before sending one (`browserHandle`) or storing one. -**A crisp capture a CLI writes lands in a private per-process directory, is -read back and removed at once, and the directory is removed at shutdown** -(rationale). +**A crisp capture a CLI writes lands in a fresh, randomly named file in a +private per-process directory, is read into memory and deleted — read or not, +failed or killed — and the directory is removed at shutdown** (rationale). **One capture per browser is in flight**: a viewer socket asking meanwhile joins it — never one from before the browser's close or relaunch — and an agent-browser capture's spawn is killed past 30s. **A tmpdir that cannot be -created fails that capture and is retried on the next, never memoized.** The -file name is reused per browser, with a fresh one after its close or relaunch. +created fails that capture and is retried on the next, never memoized.** Source of truth: `lib/src/host/browser-host.ts` (`parseBrowserRequest`, `createBrowserHost`, `BrowserProvider`), `BROWSER_PROVIDERS` (`isSessionName`, @@ -623,16 +623,17 @@ relaunch changes the mode. daemon up but not streaming is left alone; its native identity is its session. **A state file written before this boot reads as absent.** - **Never signal a pid its state files do not prove to be the session's live - daemon**: named by a pid file from this boot, alive, beside a stream port that - accepts, checked before `close` (rationale). + daemon** (`liveDaemon`): named by a pid file from this boot, alive, beside a + stream port that accepts, checked before `close` (rationale). - **A launch runs `open` in the binding's project directory while it exists**, so a relaunch reads the same `./agent-browser.json` the `dor ab` there did; every other call runs in the host's. - **Every spawn passes the `binaryPath` gate in `runWithBinaryFallback`**, the host's `DORMOUSE_AGENT_BROWSER_BIN` being the exact-match override. - **Only a launch's own steps may run a CLI verb with no daemon up**: an - operation runs only while `.pid` names a live process, since any - verb starts a daemon at `about:blank` to answer. + operation, a capture or `get cdp-url` runs only on that same proof, since + any verb starts a daemon at `about:blank` to answer; **a stream is captured + only when the proof names its port**. - **`dor ab` must read the stream port itself after a command that may bind** (`stream status --json`, safe once the command made the daemon) and hand it over; the host views it on loopback without reading the caller's socket diff --git a/docs/specs/dor-browser.rationale.md b/docs/specs/dor-browser.rationale.md index 05ffe9d52..a0c339f4b 100644 --- a/docs/specs/dor-browser.rationale.md +++ b/docs/specs/dor-browser.rationale.md @@ -134,7 +134,7 @@ A post-open blank-tab sweep can become such a query when a later relaunch, expli **Why a named launch into a live browser navigates.** A Tool re-announcing — its dev server moved — sends a named launch into the session it already has. Relaunching it stopped the daemon (`close`, then SIGTERM and SIGKILL), so an agent driving that Tool lost its tabs, page state and CDP clients on every move, and a `dor ab` command in flight failed or started a daemon mid-relaunch (review of #777, 2026-09). Only a change of mode needs a new browser. -**Why every capture writes fresh files.** A capture file reused per browser was answered before its reader read it, so a second capture of that browser in the gap — a second pane, or the Display modal beside the loop — rewrote it under the read: a torn or empty frame. A name rotated on close without deleting its file left the user's last page on disk until shutdown (review of #777, 2026-09). A joined capture's callers each get a copy because each reader deletes what it read. +**Why every capture writes a fresh file, deleted however it ends.** A capture file reused per browser was answered before its reader read it, so a second capture of that browser in the gap — a second pane, or the Display modal beside the loop — rewrote it under the read: a torn or empty frame. A name rotated on close without deleting its file left the user's last page on disk until shutdown (review of #777, 2026-09). The host now reads each frame into memory itself, so nothing waits on disk for a reader; a capture that failed or was killed at 30 s can still have written its file, so the delete does not depend on the read. **Why every call in a browser's queue is bounded.** A close ran inside the browser's lifecycle queue with no time limit, and agent-browser's `close` queues behind an `open` stalled on a slow page: a hung daemon held the close forever, every later attach, relaunch and pop of that session waited behind it, each webview request gave up at 40 s, and shutdown never settled (review of #777, 2026-09). A launch already bounded its stop by its deadline. diff --git a/lib/src/host/agent-browser-host.test.ts b/lib/src/host/agent-browser-host.test.ts index 92dc38ee9..5fa4d7139 100644 --- a/lib/src/host/agent-browser-host.test.ts +++ b/lib/src/host/agent-browser-host.test.ts @@ -3,7 +3,7 @@ import { existsSync, mkdtempSync, promises as fsp, statSync, utimesSync, writeFi import { createServer, type Server } from 'net'; import { tmpdir } from 'os'; import { dirname, join } from 'path'; -import { afterEach, beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest'; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, onTestFinished, vi } from 'vitest'; import type { BrowserOp, BrowserRequestBinding } from '../lib/platform/browser-automation'; import { WebSocketServer, type WebSocket } from 'ws'; import { createAgentBrowserProvider } from './agent-browser-host'; @@ -86,10 +86,20 @@ function writeState(session: string, ext: 'pid' | 'stream', value: number): void writeFileSync(join(process.env.AGENT_BROWSER_SOCKET_DIR!, `${session}.${ext}`), `${value}\n`); } -/** Give each session a running daemon, as its pid file says: the host drives - * no other. This test process stands in for it. */ +// A port something accepts on, standing in for a daemon's stream server. +let acceptingPort = 0; +let acceptingServer: Server | undefined; +beforeAll(async () => { ({ port: acceptingPort, server: acceptingServer } = await listen()); }); +afterAll(async () => { if (acceptingServer) await closeServer(acceptingServer); }); + +/** Give each session a daemon its state files prove live: a pid file naming + * this test process, beside a stream port that accepts (the host drives no + * other). A test that dials the stream writes its own port afterwards. */ function running(...sessions: string[]): void { - for (const session of sessions) writeState(session, 'pid', process.pid); + for (const session of sessions) { + writeState(session, 'pid', process.pid); + writeState(session, 'stream', acceptingPort); + } } // The host spawns through dor-lib-common's spawnAndCapture; mock just that @@ -521,7 +531,8 @@ describe('agent-browser host attach', () => { // Any verb starts a daemon to answer when none runs, at about:blank: an // operation on a daemon gone while its pane was hidden, or on one in a // socket directory the host does not share, must start nothing. - it('runs no operation on a session whose daemon is not running', async () => { + // The proof is the one a stop needs before it signals a pid. + it('runs no operation on a session whose state files do not prove its daemon live', async () => { const host = makeHost(); const ops: BrowserOp[] = [ { op: 'navigate', url: 'https://example.com/' }, @@ -532,15 +543,31 @@ describe('agent-browser host attach', () => { { op: 'edit', edit: 'copy' }, ]; const provider = createAgentBrowserProvider(); - for (const pid of [undefined, DEAD_PID]) { - if (pid !== undefined) writeState(session, 'pid', pid); + const cases: [string, () => Promise][] = [ + ['no state files', async () => {}], + ['a dead pid', async () => writeState(session, 'pid', DEAD_PID)], + ['a live pid with no stream that accepts', async () => { + writeState(session, 'pid', process.pid); + writeState(session, 'stream', await closedPort()); + }], + ['a live pid and stream from before this boot', async () => { + running(session); + for (const ext of ['pid', 'stream']) utimesSync(join(process.env.AGENT_BROWSER_SOCKET_DIR!, `${session}.${ext}`), 0, 0); + }], + ]; + for (const [name, arrange] of cases) { + await arrange(); for (const op of ops) { - expect(await ab(host, op, { session }), JSON.stringify(op)).toEqual({ ok: false, error: `agent-browser session '${session}' is not running` }); + expect(await ab(host, op, { session }), `${name}: ${JSON.stringify(op)}`).toEqual({ ok: false, error: `agent-browser session '${session}' is not running` }); } // Nor a capture for a viewer socket. - await expect(provider.screenshot({ session }, async () => '/nonexistent/shot.jpg')).rejects.toThrow('is not running'); + await expect(provider.screenshot({ session }, async () => '/nonexistent/shot.jpg'), name).rejects.toThrow('is not running'); } expect(spawnMock).not.toHaveBeenCalled(); + // Proven live, the same operation runs. + running(session); + enqueueSpawnResults([{}]); + expect(await ab(host, ops[1], { session })).toEqual({ ok: true }); }); it('relaunches a gone daemon at the page named, headed for a pop-out, and tracks that window for shutdown', async () => { @@ -758,9 +785,14 @@ describe('agent-browser host viewer', () => { expect(await offRange.closed).toBe(1011); }); - // `dor ab` under the caller's own AGENT_BROWSER_SOCKET_DIR hands over a port - // the host's socket directory knows nothing of. - it('watches a daemon in a socket directory it does not share: every changed frame, and no capture', async () => { + // `dor ab` hands over the port it read under its own environment; under + // the caller's own AGENT_BROWSER_SOCKET_DIR the host's state files know + // nothing of it, or name another daemon of the same session. + it.each([ + ['in a socket directory it does not share', false], + ['beside a daemon of the same session name in its own', true], + ])('watches a daemon it cannot prove live %s: every changed frame, and no capture', async (_name, sameName) => { + if (sameName) running(session); const daemon = await fakeServer(); const viewer = await view(daemon.port); await daemon.connected(); @@ -794,9 +826,25 @@ describe('agent-browser host viewer', () => { { type: 'page', url: 'https://two.example/', title: 'Two' }, ]); expect(viewer.frames).toEqual([]); - expect(spawnMock.mock.calls.map((call) => (call[1] as string[]).slice(2))).toEqual([['get', 'cdp-url']]); + // Bounded like every call a viewer makes. + expect(spawnMock.mock.calls.map((call) => [(call[1] as string[]).slice(2), call[2]])).toEqual([[['get', 'cdp-url'], { timeoutMs: 10_000 }]]); }); + it('ends a viewer whose stream accepts but never answers the upgrade, closing its connection', async () => { + const { port, server } = await listen(); + const hung: Promise[] = []; + // Read, so the host's end of the connection reaches this end. + server.on('connection', (tcp) => { tcp.resume(); hung.push(new Promise((resolve) => tcp.once('close', () => resolve()))); }); + try { + const viewer = await view(port); + expect(await viewer.closed).toBe(1011); + await vi.waitFor(() => expect(hung).toHaveLength(1)); + await hung[0]; + } finally { + await closeServer(server); + } + }, 10_000); + it('dials no CDP endpoint off loopback', async () => { running(session); const daemon = await fakeServer(); @@ -837,7 +885,7 @@ describe('agent-browser host captures', () => { // external process under the ambient umask. A derivable path straight in // os.tmpdir() let any other local account read every frame, or pre-create the // name as a symlink and have agent-browser clobber the target. - it('captures into a private, unguessable file, reads it back and removes it, bounded', async () => { + it('captures into a fresh private, unguessable file per capture, read back and deleted, bounded', async () => { writesFrames(); const captures = createBrowserCaptures(); expect([...await take(captures)]).toEqual([0xff, 0xd8, 1]); @@ -852,14 +900,29 @@ describe('agent-browser host captures', () => { const dir = dirname(file); expect(dir).not.toBe(tmpdir()); expect(statSync(dir).mode & 0o777).toBe(0o700); - // Reused per browser, so frames do not accumulate. + // Never a name another capture wrote, and nothing left behind. expect([...await take(captures)]).toEqual([0xff, 0xd8, 2]); - expect((spawnMock.mock.calls[1][1] as string[])[3]).toBe(file); + expect((spawnMock.mock.calls[1][1] as string[])[3]).not.toBe(file); + expect(await fsp.readdir(dir)).toEqual([]); // No frame of the user's browser outlives the host that took it. await captures.remove(); expect(existsSync(dir)).toBe(false); }); + it('deletes the frame of a capture that failed or was killed after writing it', async () => { + let file = ''; + spawnMock.mockImplementation(async (_binary: string, args: string[]) => { + file = args[3]; + writeFileSync(file, Uint8Array.from([0xff, 0xd8])); + return { ok: false, error: { code: 'ETIMEDOUT', message: 'screenshot timed out' } }; + }); + const captures = createBrowserCaptures(); + await expect(take(captures)).rejects.toThrow('timed out'); + expect(file).not.toBe(''); + expect(existsSync(file)).toBe(false); + await captures.remove(); + }); + it('joins a capture of the browser already running, and none it was told to forget', async () => { const frames = writesFrames(); const captures = createBrowserCaptures(); diff --git a/lib/src/host/agent-browser-host.ts b/lib/src/host/agent-browser-host.ts index 12e8de48e..b1e351105 100644 --- a/lib/src/host/agent-browser-host.ts +++ b/lib/src/host/agent-browser-host.ts @@ -83,7 +83,10 @@ const CAPTURE_TIMEOUT_MS = 30_000; const STREAM_PORT_READ_ATTEMPTS = 4; const STREAM_PORT_READ_DELAY_MS = 150; const PORT_PROBE_TIMEOUT_MS = 500; +// A viewer's upstream dials — the daemon's stream, a headed window's CDP — +// and the `get cdp-url` before one, each end here at the latest. const STREAM_CONNECT_TIMEOUT_MS = 5000; +const CDP_URL_TIMEOUT_MS = 10_000; // A stream message above this size is a frame (a base64 JPEG); status, tabs // and url are small — unless a long tab list or URL crosses it too. const FRAME_THRESHOLD_BYTES = 16384; @@ -247,25 +250,38 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): } } - /** - * One CLI call on a live browser. Any verb starts a daemon to answer when - * none runs, so it is refused unless the session's daemon runs as its pid - * file says: only a launch's own steps may start one (docs/specs/dor-browser.md - * → "Browser Host"). A session in a socket directory the host does not - * share has no pid file here, so it is refused too. - */ - async function drive(b: ProviderBinding, args: string[], options?: { timeoutMs?: number }): Promise { - const pid = await readStateNumber(b.session, 'pid'); - if (pid === undefined || !processAlive(pid)) throw new Error(`agent-browser session '${b.session}' is not running`); - return run(b, args, options); - } - /** The port `.stream` names, when something accepts on it. */ async function acceptingStreamPort(session: string): Promise { const port = await readStateNumber(session, 'stream'); return port !== undefined && await portAccepts(port) ? port : undefined; } + /** + * The session's daemon as its state files prove it: a pid file from this + * boot naming a live process, beside a stream port that accepts. The one + * proof the host acts on — to signal a pid, to run a verb, to capture — since + * a pid file alone may name any process, and any verb run with no daemon up + * starts one to answer. + */ + async function liveDaemon(session: string): Promise<{ pid: number; stream: number } | undefined> { + const pid = await readStateNumber(session, 'pid'); + if (pid === undefined || !processAlive(pid)) return undefined; + const stream = await acceptingStreamPort(session); + return stream === undefined ? undefined : { pid, stream }; + } + + /** + * One CLI call on a live browser, refused unless `liveDaemon` proves the + * session's daemon up: only a launch's own steps may start one + * (docs/specs/dor-browser.md → "agent-browser"). A session in a socket + * directory the host does not share has no state files here, so it is + * refused too. + */ + async function drive(b: ProviderBinding, args: string[], options?: { timeoutMs?: number }): Promise { + if (!await liveDaemon(b.session)) throw new Error(`agent-browser session '${b.session}' is not running`); + return run(b, args, options); + } + /** Terminate `session`'s daemon `pid`, proven live by the caller, and wait * for it to exit. */ async function killDaemon(session: string, pid: number): Promise { @@ -302,12 +318,10 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): // to answer. One up but not streaming is left alone: relaunching would // compete with it. async find(b) { + const live = await liveDaemon(b.session); + if (live) return { stream: live.stream }; const pid = await readStateNumber(b.session, 'pid'); - if (pid !== undefined && processAlive(pid)) { - const port = await acceptingStreamPort(b.session); - if (port !== undefined) return { stream: port }; - throw new Error(`agent-browser session '${b.session}' is not streaming`); - } + if (pid !== undefined && processAlive(pid)) throw new Error(`agent-browser session '${b.session}' is not streaming`); return { gone: `agent-browser session '${b.session}' is not running`, named: pid !== undefined }; }, @@ -320,9 +334,9 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): // files the old daemon left. async stop(b, timeoutMs) { const pid = await readStateNumber(b.session, 'pid'); - const proven = pid !== undefined && processAlive(pid) && await acceptingStreamPort(b.session) !== undefined; + const proven = await liveDaemon(b.session); await run(b, ['close'], { timeoutMs }); - if (proven) await killDaemon(b.session, pid); + if (proven) await killDaemon(b.session, proven.pid); return pid; }, @@ -426,13 +440,16 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): * CDP (`observePage`), which the stream does not report for navigations * made in the window itself. * - * `port` may be one `dor ab` read under a socket directory the host does - * not share: it is only ever dialed on loopback, only the stream's own - * messages come back from it, and only validated input goes to it. + * `port` is the one `dor ab` read after its command, or a launch or attach + * answered: it is only ever dialed on loopback, only the stream's own + * messages come back from it, and only validated input goes to it. The + * host captures the browser only when its own state files prove that port + * the session's live daemon (`liveDaemon`); a daemon in a socket directory + * it does not share is watched, never captured, and that directory never + * read. */ async function viewStream(b: ProviderBinding, port: number, headed: boolean, sink: ViewerSink): Promise { - // Only a daemon in the host's own socket directory is one it can capture. - const capturable = (await readStateNumber(b.session, 'stream')) === port; + const capturable = (await liveDaemon(b.session))?.stream === port; const socket = new WebSocket(`ws://127.0.0.1:${port}`, { handshakeTimeout: STREAM_CONNECT_TIMEOUT_MS, perMessageDeflate: false }); let opened = false; let closing = false; @@ -521,7 +538,7 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): const page = (url: unknown, title: unknown) => { if (typeof url === 'string') sink.state({ type: 'page', url, title: typeof title === 'string' ? title : null }); }; - void drive(b, ['get', 'cdp-url']).then((result) => { + void drive(b, ['get', 'cdp-url'], { timeoutMs: CDP_URL_TIMEOUT_MS }).then((result) => { const url = result.exitCode === 0 ? parseCdpUrl(result.stdout) : null; if (closed || !url) { if (!url) log(`[agent-browser] no CDP endpoint for ${b.session}: ${cliError(result)}`); @@ -532,7 +549,7 @@ export function createAgentBrowserProvider(deps: AgentBrowserProviderDeps = {}): log(`[agent-browser] refused a CDP endpoint off loopback: ${url}`); return; } - const socket = cdp = new WebSocket(url, { perMessageDeflate: false }); + const socket = cdp = new WebSocket(url, { handshakeTimeout: STREAM_CONNECT_TIMEOUT_MS, perMessageDeflate: false }); let nextId = 1; const send = (method: string, params?: Record) => socket.send(JSON.stringify({ id: nextId++, method, ...(params ? { params } : {}) })); socket.on('open', () => { diff --git a/lib/src/host/browser-capture.ts b/lib/src/host/browser-capture.ts index 69233833f..323641d33 100644 --- a/lib/src/host/browser-capture.ts +++ b/lib/src/host/browser-capture.ts @@ -1,8 +1,8 @@ /** * The crisp captures behind the viewer sockets (docs/specs/dor-browser.md → - * "Viewer Socket"): one device-resolution JPEG of a browser per ask, which a + * "Browser Host"): one device-resolution JPEG of a browser per ask, which a * CLI writes into the host's private capture directory and the host reads - * back and removes at once, so no frame waits on disk. + * back into memory and deletes, so no frame waits on disk. */ import { randomBytes } from 'crypto'; import * as path from 'path'; @@ -15,9 +15,8 @@ export type Shoot = (file: () => Promise) => Promise<{ path: string } | export interface BrowserCaptures { /** A JPEG of browser `id`, joining one of it already running. */ take(id: string, shoot: Shoot): Promise; - /** Join none of `id`'s running captures, and give its next a fresh file, - * so one still running cannot overwrite it: its browser was closed or - * replaced. */ + /** Join none of `id`'s running captures: its browser was closed or + * replaced, and one still running deletes its own file when it ends. */ forget(id: string): void; /** Drop the directory and every frame in it. */ remove(): Promise; @@ -28,43 +27,37 @@ export function createBrowserCaptures(): BrowserCaptures { // external process under the ambient umask — which is why the private // directory, not the file mode, is the control. const dir = privateCaptureDir('dormouse-browser-'); - // One file per browser, so frames don't litter; one capture of it at a time - // (below), so reusing the name is safe. The random name keeps it unguessable - // from the session alone. - const names = new Map(); // Surfaces can share a session, so a capture another viewer asks for // meanwhile joins rather than repeats. const inFlight = new Map>(); - async function file(id: string): Promise { - let name = names.get(id); - if (name === undefined) names.set(id, name = randomBytes(12).toString('hex')); - return path.join(await dir.get(), `shot-${name}.jpg`); - } - return { take(id, shoot) { const pending = inFlight.get(id); if (pending) return pending; - // Joined whole, read and unlink included: a caller joining only the - // capture would read a file the first caller has already removed. + // Joined whole, read and delete included: every caller gets the bytes, + // none a file another has already removed. const taking: Promise = (async () => { - const shot = await shoot(() => file(id)); - if ('bytes' in shot) return shot.bytes; - const buffer = await fs.readFile(shot.path); - await fs.unlink(shot.path).catch(() => {}); - return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + // A fresh random name per capture: unguessable, and never one a + // capture still running writes. + let written: string | undefined; + try { + const shot = await shoot(async () => (written = path.join(await dir.get(), `shot-${randomBytes(12).toString('hex')}.jpg`))); + if ('bytes' in shot) return shot.bytes; + const buffer = await fs.readFile(shot.path); + return new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength); + } finally { + // Read or not — a capture that failed or was killed may have + // written it anyway — the frame never outlives its capture. + if (written !== undefined) await fs.unlink(written).catch(() => {}); + } })().finally(() => { if (inFlight.get(id) === taking) inFlight.delete(id); }); inFlight.set(id, taking); return taking; }, forget(id) { inFlight.delete(id); - names.delete(id); - }, - async remove() { - await dir.remove(); - names.clear(); }, + remove: () => dir.remove(), }; } diff --git a/lib/src/host/browser-viewer.ts b/lib/src/host/browser-viewer.ts index 643de3690..358666b4a 100644 --- a/lib/src/host/browser-viewer.ts +++ b/lib/src/host/browser-viewer.ts @@ -104,6 +104,9 @@ export function createViewerServer(): ViewerServer { closed = true; for (const socket of wss.clients) socket.terminate(); const bound = await listening?.catch(() => null); + // Every connection too, so a client holding one open cannot hold + // shutdown. + bound?.server.closeAllConnections(); await new Promise((resolve) => (bound ? bound.server.close(() => resolve()) : resolve())); }, }; diff --git a/scripts/spec-word-budgets.json b/scripts/spec-word-budgets.json index c7f2ae177..69f47d809 100644 --- a/scripts/spec-word-budgets.json +++ b/scripts/spec-word-budgets.json @@ -5,7 +5,7 @@ "docs/specs/alert.md": 8150, "docs/specs/auto-update.md": 1150, "docs/specs/deploy.md": 1900, - "docs/specs/dor-browser.md": 7800, + "docs/specs/dor-browser.md": 7850, "docs/specs/dor-cli.md": 6300, "docs/specs/dor-tool.md": 4150, "docs/specs/glossary.md": 2950, From 67b745bf940c2c48f79b75702ac0dd730982b45e Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Thu, 24 Sep 2026 04:32:49 -0700 Subject: [PATCH 7/8] Pin that a stream frame of any size is deduplicated A stream frame small enough to pass for a control message is compared against the last like any other. The viewer listener's closeAllConnections at shutdown goes: Node's own server.close already ends a connection holding a half-sent request, so nothing observed it. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NCV5Uq6FeZbBfzfpk3vrpe --- lib/src/host/agent-browser-host.test.ts | 6 +++++- lib/src/host/browser-viewer.ts | 3 --- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/src/host/agent-browser-host.test.ts b/lib/src/host/agent-browser-host.test.ts index 5fa4d7139..11eea1206 100644 --- a/lib/src/host/agent-browser-host.test.ts +++ b/lib/src/host/agent-browser-host.test.ts @@ -711,6 +711,10 @@ describe('agent-browser host viewer', () => { daemon.send(frame(1)); daemon.send(frame(1)); daemon.send(frame(2, 900)); + // A frame small enough to pass for a control message is deduplicated too. + const small = { type: 'frame', data: Buffer.alloc(100, 3).toString('base64'), metadata: { deviceWidth: 900, deviceHeight: 600 } }; + daemon.send(small); + daemon.send(small); // A commit edge is never deduplicated: a reload commits the same URL. daemon.send({ type: 'url', url: 'https://example.com/' }); daemon.send({ type: 'url', url: 'https://example.com/' }); @@ -723,7 +727,7 @@ describe('agent-browser host viewer', () => { ]); await vi.waitFor(() => expect(viewer.frames.filter((f) => f.kind === 'crisp').length).toBeGreaterThan(0)); const provisional = viewer.frames.filter((f) => f.kind === 'provisional'); - expect(provisional.map((f) => [f.jpeg[0], f.jpeg.byteLength, f.size?.width])).toEqual([[1, 13_000, 800], [2, 13_000, 900]]); + expect(provisional.map((f) => [f.jpeg[0], f.jpeg.byteLength, f.size?.width])).toEqual([[1, 13_000, 800], [2, 13_000, 900], [3, 100, 900]]); expect([...viewer.frames.find((f) => f.kind === 'crisp')!.jpeg]).toEqual([0xff, 0xd8, 0x99]); }); diff --git a/lib/src/host/browser-viewer.ts b/lib/src/host/browser-viewer.ts index 358666b4a..643de3690 100644 --- a/lib/src/host/browser-viewer.ts +++ b/lib/src/host/browser-viewer.ts @@ -104,9 +104,6 @@ export function createViewerServer(): ViewerServer { closed = true; for (const socket of wss.clients) socket.terminate(); const bound = await listening?.catch(() => null); - // Every connection too, so a client holding one open cannot hold - // shutdown. - bound?.server.closeAllConnections(); await new Promise((resolve) => (bound ? bound.server.close(() => resolve()) : resolve())); }, }; From 7754d61143cccf8ed62065c97ff5b6508e7b8daa Mon Sep 17 00:00:00 2001 From: Ned Twigg Date: Thu, 24 Sep 2026 05:53:37 -0700 Subject: [PATCH 8/8] Close viewer sockets with any reason, and let no teardown reject MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A viewer socket whose provider could not subscribe was closed with that error as its reason, cut to 120 characters. ws caps a close reason at 123 UTF-8 bytes and throws a RangeError, synchronously, past that — a Playwright CLI's stderr, or a path under a non-ASCII home, is well past it. The throw came after the view had disposed itself, so the webview's socket stayed open with nothing behind it, and it rejected the promise the rejection handler returned, which nothing handled: on Node's default, the sidecar would exit with every terminal in it. Every close the host makes with a reason now goes through `closeSocket`, which cuts the reason to the longest whole-character prefix of 123 bytes and terminates the socket rather than throw. `openView`'s subscription chain ends in a catch, so a throw anywhere in attaching or dropping a subscription is logged, never an unhandled rejection. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01NCV5Uq6FeZbBfzfpk3vrpe --- lib/src/host/browser-host-test-utils.ts | 10 +++++-- lib/src/host/browser-host.test.ts | 39 +++++++++++++++++++++++++ lib/src/host/browser-host.ts | 8 +++-- lib/src/host/browser-viewer.test.ts | 13 ++++++++- lib/src/host/browser-viewer.ts | 34 ++++++++++++++++++++- 5 files changed, 97 insertions(+), 7 deletions(-) diff --git a/lib/src/host/browser-host-test-utils.ts b/lib/src/host/browser-host-test-utils.ts index 2ef29bdaa..5c926da75 100644 --- a/lib/src/host/browser-host-test-utils.ts +++ b/lib/src/host/browser-host-test-utils.ts @@ -88,6 +88,8 @@ export interface TestViewer { send(message: object): void; /** Settles with the close code once the host ends the socket. */ closed: Promise; + /** The reason the host closed it with, once it has. */ + reason?: string; } /** Connect to a viewer socket URL as the webview does, once it is open. */ @@ -104,10 +106,14 @@ export async function openViewer(url: string): Promise { const frame = decodeViewerFrame(data.buffer.slice(data.byteOffset, data.byteOffset + data.byteLength) as ArrayBuffer); if (frame) frames.push(frame); }); - const closed = new Promise((resolve) => socket.once('close', (code) => resolve(code))); + const viewer: TestViewer = { socket, frames, states, send: (message) => socket.send(JSON.stringify(message)), closed: Promise.resolve(0) }; + viewer.closed = new Promise((resolve) => socket.once('close', (code, reason) => { + viewer.reason = reason.toString(); + resolve(code); + })); await new Promise((resolve, reject) => { socket.once('open', () => resolve()); socket.once('unexpected-response', (_req, res) => reject(new Error(`viewer refused: ${res.statusCode}`))); }); - return { socket, frames, states, send: (message) => socket.send(JSON.stringify(message)), closed }; + return viewer; } diff --git a/lib/src/host/browser-host.test.ts b/lib/src/host/browser-host.test.ts index 546860adb..d8c66da2f 100644 --- a/lib/src/host/browser-host.test.ts +++ b/lib/src/host/browser-host.test.ts @@ -183,6 +183,45 @@ describe('createBrowserHost', () => { } }); + // ws refuses a close reason past 123 UTF-8 bytes by throwing; a provider's + // error — CLI stderr, a path under a non-ASCII home — can be any length. + it('ends a viewer whose provider cannot subscribe, whatever its error says', async () => { + const fake = fakeProvider(); + const said = `Échec: ${'é'.repeat(200)}`; + fake.provider.view = async () => { throw new Error(said); }; + const host = createBrowserHost({ writeClipboardText: vi.fn(), providers: { 'agent-browser': () => fake.provider } }); + try { + const viewer = await openViewer((await host.request({ provider: 'agent-browser', binding: { session: 's1' }, op: 'view', stream: 4321 })).url!); + expect(await viewer.closed).toBe(1011); + expect(Buffer.byteLength(viewer.reason!)).toBeLessThanOrEqual(123); + expect(said.startsWith(viewer.reason!)).toBe(true); + } finally { + await host.close(); + } + }); + + it('rejects nothing when a subscription that lands after its viewer left throws as it is dropped', async () => { + const fake = fakeProvider(); + let land!: () => void; + const landed = new Promise((resolve) => { land = resolve; }); + fake.provider.view = async () => { + await landed; + return { capturable: true, input: () => true, close: () => { throw new Error('upstream already gone'); } }; + }; + const log = vi.fn(); + const host = createBrowserHost({ writeClipboardText: vi.fn(), log, providers: { 'agent-browser': () => fake.provider } }); + try { + const viewer = await openViewer((await host.request({ provider: 'agent-browser', binding: { session: 's1' }, op: 'view', stream: 4321 })).url!); + viewer.socket.close(); + await viewer.closed; + await new Promise((resolve) => setTimeout(resolve, 20)); + land(); + await vi.waitFor(() => expect(log).toHaveBeenCalledWith('[browser-host] upstream already gone')); + } finally { + await host.close(); + } + }); + it('joins no capture of the browser a relaunch replaced', async () => { const fake = fakeProvider(); const host = createBrowserHost({ writeClipboardText: vi.fn(), providers: { 'agent-browser': () => fake.provider } }); diff --git a/lib/src/host/browser-host.ts b/lib/src/host/browser-host.ts index 3a3fe4f5e..1f9b724a6 100644 --- a/lib/src/host/browser-host.ts +++ b/lib/src/host/browser-host.ts @@ -34,7 +34,7 @@ import { } from '../lib/platform/browser-automation'; import { createBrowserCaptures } from './browser-capture'; import type { WebSocket } from 'ws'; -import { BrowserView, createViewerServer, type Upstream, type ViewerSink } from './browser-viewer'; +import { BrowserView, closeSocket, createViewerServer, type Upstream, type ViewerSink } from './browser-viewer'; /** An operation on a live browser that each provider maps to its own call: * a fixed agent-browser argv, or a Playwright client call. */ @@ -514,7 +514,7 @@ export function createBrowserHost(deps: BrowserHostDeps) { * close of it began since its URL was granted (`generation`). */ function openView(socket: WebSocket, bound: Bound, stream: number, isHeaded: boolean, debug: boolean, generation: number): void { if (closed || (generations.get(bound.id) ?? 0) !== generation) { - socket.close(1001, 'the browser was relaunched or closed'); + closeSocket(socket, 1001, 'the browser was relaunched or closed', log); return; } const view = new BrowserView(socket, { @@ -530,10 +530,12 @@ export function createBrowserHost(deps: BrowserHostDeps) { let open = views.get(bound.id); if (!open) views.set(bound.id, open = new Set()); open.add(view); + // Terminal: nothing here may reject, or the host process would go down + // with an unhandled rejection. bound.p.view(bound.b, stream, { headed: isHeaded }, view).then( (upstream) => view.attach(upstream), (error: unknown) => view.close(1011, messageOf(error)), - ); + ).catch(log); } /** One device-resolution JPEG of `bound`'s browser, for its viewer sockets' diff --git a/lib/src/host/browser-viewer.test.ts b/lib/src/host/browser-viewer.test.ts index 2e19c5920..87cf51558 100644 --- a/lib/src/host/browser-viewer.test.ts +++ b/lib/src/host/browser-viewer.test.ts @@ -4,7 +4,7 @@ import { request as httpRequest } from 'node:http'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { WebSocket } from 'ws'; import { decodeViewerFrame, type ViewerFrame, type ViewerState } from '../lib/platform/browser-automation'; -import { BrowserView, PROVISIONAL_INPUT_WINDOW_MS, createViewerServer, parseViewerInput, type Upstream } from './browser-viewer'; +import { BrowserView, PROVISIONAL_INPUT_WINDOW_MS, closeReason, createViewerServer, parseViewerInput, type Upstream } from './browser-viewer'; import { openViewer } from './browser-host-test-utils'; /** The webview's socket as a view sees it: what was sent, and input to send. */ @@ -264,6 +264,17 @@ describe('a viewer socket', () => { }); }); +describe('closeReason', () => { + it('keeps the longest whole-character prefix a close frame holds, 123 UTF-8 bytes', () => { + expect(closeReason('Input backlog exceeded')).toBe('Input backlog exceeded'); + // Three bytes each: 41 fit exactly, and none is ever split. + expect(closeReason('€'.repeat(60))).toBe('€'.repeat(41)); + const emoji = closeReason(`x${'🙂'.repeat(40)}`); + expect(emoji).toBe(`x${'🙂'.repeat(30)}`); + expect(emoji.isWellFormed()).toBe(true); + }); +}); + describe('parseViewerInput', () => { it('rebuilds each shape field by field, bounded, and refuses the rest', () => { expect(parseViewerInput(JSON.stringify({ type: 'input_mouse', eventType: 'mouseWheel', x: 1, y: 2, button: 'evil', buttons: 255, clickCount: 9, modifiers: 255, deltaX: 'x', deltaY: 5 }))) diff --git a/lib/src/host/browser-viewer.ts b/lib/src/host/browser-viewer.ts index 643de3690..b20e2b5b9 100644 --- a/lib/src/host/browser-viewer.ts +++ b/lib/src/host/browser-viewer.ts @@ -109,6 +109,38 @@ export function createViewerServer(): ViewerServer { }; } +// A WebSocket close reason is at most 123 UTF-8 bytes; `ws` throws a +// RangeError, synchronously, for a longer one. +const MAX_CLOSE_REASON_BYTES = 123; + +/** `reason`, cut to the longest whole-character prefix a close frame holds. */ +export function closeReason(reason: string): string { + if (Buffer.byteLength(reason) <= MAX_CLOSE_REASON_BYTES) return reason; + let bytes = 0; + let kept = ''; + for (const char of reason) { + bytes += Buffer.byteLength(char); + if (bytes > MAX_CLOSE_REASON_BYTES) break; + kept += char; + } + return kept; +} + +/** + * Close `socket` with `code` and a reason of any length — a provider's + * error, a path under a non-ASCII home — without ever throwing: every close + * the host makes runs in a handler or a promise callback, where a throw + * would leave the webview's socket open and reject with nothing to catch it. + */ +export function closeSocket(socket: WebSocket, code: number, reason: string, log?: (message: string) => void): void { + try { + socket.close(code, closeReason(reason)); + } catch (error) { + log?.(`[browser-viewer] close failed: ${error instanceof Error ? error.message : String(error)}`); + socket.terminate(); + } +} + // --- one socket --- /** Continued input keeps the stream painting this long after the last. */ @@ -192,7 +224,7 @@ export class BrowserView implements ViewerSink { * while the webview has yet to answer the close. */ close(code: number, reason: string): void { this.dispose(); - this.socket.close(code, reason.slice(0, 120)); + closeSocket(this.socket, code, reason, this.deps.log); } /** Paint the stream for a while: input reached the page another way (a