From c79ebaaa8b418f74420e45032a0476061be1cdc8 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 23:37:00 +0800 Subject: [PATCH 01/26] feat(sim): stream frames + interactive ops in the @linkcode/sim client Add STREAM_FRAME decode, tap/swipe/button/streamStart/streamStop, and a per-udid onFrame subscription so the engine can consume the sidecar's live framebuffer. Foundation for the CODE-397 simulator panel. --- .../host/sim/src/__tests__/client.test.ts | 44 ++++++++- packages/host/sim/src/__tests__/codec.test.ts | 28 +++++- packages/host/sim/src/client.ts | 92 ++++++++++++++++++- packages/host/sim/src/codec.ts | 13 +++ packages/host/sim/src/index.ts | 10 +- packages/host/sim/src/schema.ts | 10 ++ 6 files changed, 192 insertions(+), 5 deletions(-) diff --git a/packages/host/sim/src/__tests__/client.test.ts b/packages/host/sim/src/__tests__/client.test.ts index 7e2577f7..7bf7f08e 100644 --- a/packages/host/sim/src/__tests__/client.test.ts +++ b/packages/host/sim/src/__tests__/client.test.ts @@ -1,7 +1,7 @@ import { PassThrough } from 'node:stream'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { SimSidecarClient, SimSidecarError } from '../client'; -import { REQUEST, SCREENSHOT } from '../codec'; +import { REQUEST, SCREENSHOT, STREAM_FRAME } from '../codec'; const mocks = vi.hoisted(() => ({ spawn: vi.fn(), @@ -22,6 +22,11 @@ function resultFrame(payload: unknown): Buffer { return frame(0x81, Buffer.from(JSON.stringify(payload))); } +function streamFrameBytes(type: number, udid: string, image: number[]): Buffer { + const u = Buffer.from(udid); + return frame(type, Buffer.concat([Buffer.from([u.length, 0]), u, Buffer.from(image)])); +} + type FakeChild = PassThrough & { stdin: PassThrough; stdout: PassThrough; @@ -141,6 +146,43 @@ describe('SimSidecarClient', () => { expect(mocks.spawn).toHaveBeenCalledTimes(2); }); + it('sends interactive coordinates as JSON numbers', async () => { + const child = fakeChild(); + mocks.spawn.mockReturnValue(child); + const client = new SimSidecarClient('/bin/sim'); + + void client.tap('U-1', 0.25, 0.5); + await tick(); + const chunk = child.stdin.read() as Buffer; + const request = JSON.parse(chunk.subarray(5).toString('utf8')) as { + op: { type: string; x: unknown; y: unknown }; + }; + expect(request.op).toMatchObject({ type: 'tap', x: 0.25, y: 0.5 }); + expect(typeof request.op.x).toBe('number'); + }); + + it('fans STREAM_FRAMEs out to per-udid listeners until unsubscribed', async () => { + const child = fakeChild(); + mocks.spawn.mockReturnValue(child); + const client = new SimSidecarClient('/bin/sim'); + // Touch the child so the sidecar's stdout is wired up. + void client.streamStart('U-1'); + await tick(); + + const seen: Buffer[] = []; + const unsubscribe = client.onFrame('U-1', (image) => seen.push(image)); + + child.stdout.write(streamFrameBytes(STREAM_FRAME, 'U-1', [0xff, 0xd8, 0x01])); + child.stdout.write(streamFrameBytes(STREAM_FRAME, 'U-2', [0x00])); // other device: ignored + await tick(); + expect(seen.map((b) => [...b])).toEqual([[0xff, 0xd8, 0x01]]); + + unsubscribe(); + child.stdout.write(streamFrameBytes(STREAM_FRAME, 'U-1', [0x02])); + await tick(); + expect(seen).toHaveLength(1); + }); + it('rejects calls without a configured binary and after close', async () => { const unconfigured = new SimSidecarClient(''); await expect(unconfigured.probe()).rejects.toThrow('sim sidecar not configured'); diff --git a/packages/host/sim/src/__tests__/codec.test.ts b/packages/host/sim/src/__tests__/codec.test.ts index 8d031494..a79ad4ca 100644 --- a/packages/host/sim/src/__tests__/codec.test.ts +++ b/packages/host/sim/src/__tests__/codec.test.ts @@ -1,5 +1,12 @@ import { describe, expect, it } from 'vitest'; -import { decodeScreenshotFrame, FrameDecoder, MAX_FRAME_LEN, RESULT, writeFrame } from '../codec'; +import { + decodeScreenshotFrame, + decodeStreamFrame, + FrameDecoder, + MAX_FRAME_LEN, + RESULT, + writeFrame, +} from '../codec'; function frame(type: number, body: Buffer): Buffer { const header = Buffer.allocUnsafe(5); @@ -54,3 +61,22 @@ describe('decodeScreenshotFrame', () => { expect(() => decodeScreenshotFrame(Buffer.from([9, 0, 0x78]))).toThrow('truncated request id'); }); }); + +describe('decodeStreamFrame', () => { + it('splits the udid from the image bytes', () => { + const udid = Buffer.from('U-1'); + const body = Buffer.concat([ + Buffer.from([udid.length, 0]), + udid, + Buffer.from([0xff, 0xd8, 0xff]), + ]); + const decoded = decodeStreamFrame(body); + expect(decoded.udid).toBe('U-1'); + expect([...decoded.image]).toEqual([0xff, 0xd8, 0xff]); + }); + + it('rejects truncated bodies', () => { + expect(() => decodeStreamFrame(Buffer.from([9]))).toThrow('short stream frame'); + expect(() => decodeStreamFrame(Buffer.from([9, 0, 0x78]))).toThrow('truncated stream udid'); + }); +}); diff --git a/packages/host/sim/src/client.ts b/packages/host/sim/src/client.ts index e7227319..56b40015 100644 --- a/packages/host/sim/src/client.ts +++ b/packages/host/sim/src/client.ts @@ -5,20 +5,40 @@ import { extractErrorMessage } from 'foxts/extract-error-message'; import type { Frame } from './codec'; import { decodeScreenshotFrame, + decodeStreamFrame, FrameDecoder, REQUEST, RESULT, SCREENSHOT, + STREAM_FRAME, writeFrame, } from './codec'; -import type { SimDevice, SimImageFormat, SimProbe } from './schema'; +import type { + SimButton, + SimDevice, + SimImageFormat, + SimProbe, + SimStreamStartResult, +} from './schema'; import { SimLaunchResultSchema, SimListResultSchema, SimProbeSchema, SimResultSchema, + SimStreamStartResultSchema, } from './schema'; +/** A live framebuffer frame: JPEG bytes for one device. */ +export type SimFrameListener = (image: Buffer) => void; + +/** Options for {@link SimSidecarClient.streamStart}; omitted fields take the sidecar defaults. */ +export interface SimStreamOptions { + fps?: number; + quality?: number; + /** Downscale factor before encode (0..1; 1.0 = native). Lower trades resolution for rate/bandwidth. */ + scale?: number; +} + /** The sidecar child: piped stdin/stdout, inherited stderr (its logs go to the host's stderr). */ type SidecarChild = ChildProcessByStdio; @@ -62,6 +82,8 @@ export class SimSidecarClient { private child: SidecarChild | null = null; private readonly decoder = new FrameDecoder(); private readonly pending = new Map(); + /** Per-udid framebuffer listeners; `STREAM_FRAME`s are fanned out to the matching set. */ + private readonly frameListeners = new Map>(); private seq = 0; private closed = false; @@ -106,6 +128,65 @@ export class SimSidecarClient { return image; } + /** Tap at a normalized (0..1) point (private HID; macOS only). */ + async tap(udid: string, x: number, y: number): Promise { + await this.call('tap', { udid, x, y }); + } + + /** Swipe between two normalized (0..1) points over `durationMs` (private HID; macOS only). */ + async swipe( + udid: string, + from: { x: number; y: number }, + to: { x: number; y: number }, + durationMs?: number, + ): Promise { + await this.call('swipe', { + udid, + x0: from.x, + y0: from.y, + x1: to.x, + y1: to.y, + ...(durationMs !== undefined && { durationMs }), + }); + } + + /** Press a hardware button (private HID; macOS only). */ + async button(udid: string, button: SimButton): Promise { + await this.call('button', { udid, button }); + } + + /** + * Start streaming `udid`'s framebuffer as JPEG frames delivered to {@link onFrame} listeners. + * Frames are pushed until {@link streamStop} (or the client closes). + */ + async streamStart(udid: string, options: SimStreamOptions = {}): Promise { + return SimStreamStartResultSchema.parse(await this.call('streamStart', { udid, ...options })); + } + + /** Stop a running framebuffer stream. */ + async streamStop(udid: string): Promise { + await this.call('streamStop', { udid }); + } + + /** + * Subscribe to `udid`'s framebuffer frames; returns an unsubscribe function. Subscribing does not + * itself start the stream — pair it with {@link streamStart}. + */ + onFrame(udid: string, listener: SimFrameListener): () => void { + let set = this.frameListeners.get(udid); + if (!set) { + set = new Set(); + this.frameListeners.set(udid, set); + } + set.add(listener); + return () => { + const current = this.frameListeners.get(udid); + if (!current) return; + current.delete(listener); + if (current.size === 0) this.frameListeners.delete(udid); + }; + } + /** Release the client and its sidecar. Booted devices keep running server-side. */ close(): void { if (this.closed) return; @@ -113,12 +194,13 @@ export class SimSidecarClient { const child = this.child; this.child = null; this.decoder.reset(); + this.frameListeners.clear(); this.failAll(new Error('sim client closed')); // Close stdin (EOF) rather than kill: the sidecar drains queued replies before exiting. child?.stdin.end(); } - private call(type: string, params: Record): Promise { + private call(type: string, params: Record): Promise { if (this.closed) return Promise.reject(new Error('sim client closed')); // Unconfigured binary (see the daemon's `resolveSimSidecarPath`): fail with a clear, stable // message instead of `spawn('')`, which would surface as a confusing crash on every call. @@ -199,6 +281,12 @@ export class SimSidecarClient { this.take(requestId)?.resolve(image); break; } + case STREAM_FRAME: { + const { udid, image } = decodeStreamFrame(frame.body); + const listeners = this.frameListeners.get(udid); + if (listeners) for (const listener of listeners) listener(image); + break; + } default: break; } diff --git a/packages/host/sim/src/codec.ts b/packages/host/sim/src/codec.ts index dab5c68f..0378a369 100644 --- a/packages/host/sim/src/codec.ts +++ b/packages/host/sim/src/codec.ts @@ -11,6 +11,8 @@ export const REQUEST = 0x01; // Sidecar → daemon. export const RESULT = 0x81; export const SCREENSHOT = 0x82; +/** Unsolicited JPEG frame pushed while a `streamStart` stream runs. */ +export const STREAM_FRAME = 0x83; export const MAX_FRAME_LEN = 16 * 1024 * 1024; @@ -66,3 +68,14 @@ export function decodeScreenshotFrame(body: Buffer): { requestId: string; image: image: body.subarray(2 + idLen), }; } + +/** Decode a `STREAM_FRAME` body: `[u16 LE udid_len][udid][image bytes]`. */ +export function decodeStreamFrame(body: Buffer): { udid: string; image: Buffer } { + if (body.length < 2) throw new Error('short stream frame'); + const udidLen = body.readUInt16LE(0); + if (body.length < 2 + udidLen) throw new Error('truncated stream udid'); + return { + udid: body.subarray(2, 2 + udidLen).toString('utf8'), + image: body.subarray(2 + udidLen), + }; +} diff --git a/packages/host/sim/src/index.ts b/packages/host/sim/src/index.ts index 9c08f513..0097c7fe 100644 --- a/packages/host/sim/src/index.ts +++ b/packages/host/sim/src/index.ts @@ -1,2 +1,10 @@ +export type { SimFrameListener, SimStreamOptions } from './client'; export { SimSidecarClient, SimSidecarError } from './client'; -export type { SimDevice, SimErrorCode, SimImageFormat, SimProbe } from './schema'; +export type { + SimButton, + SimDevice, + SimErrorCode, + SimImageFormat, + SimProbe, + SimStreamStartResult, +} from './schema'; diff --git a/packages/host/sim/src/schema.ts b/packages/host/sim/src/schema.ts index 84683bcd..68e04a7c 100644 --- a/packages/host/sim/src/schema.ts +++ b/packages/host/sim/src/schema.ts @@ -43,4 +43,14 @@ export type SimProbe = z.infer; export const SimLaunchResultSchema = z.object({ pid: z.number().int().nullable() }); +/** `streamStart` reply: the accepted stream, or a no-op when one is already running. */ +export const SimStreamStartResultSchema = z.union([ + z.object({ streaming: z.literal(true), fps: z.number().int(), scale: z.number() }), + z.object({ alreadyStreaming: z.literal(true) }), +]); +export type SimStreamStartResult = z.infer; + +/** A hardware button the private HID layer can press. */ +export type SimButton = 'home' | 'lock'; + export type SimImageFormat = 'jpeg' | 'png'; From 92570716f00c3a43df762c5e35089fc462588d59 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 23:45:03 +0800 Subject: [PATCH 02/26] feat(engine): simulator tap/swipe/button + framebuffer stream in the backend port and service Extend SimulatorBackend and SimulatorService with claim-gated interactive ops and streamStart/streamStop plus a per-udid onFrame fan-out; dropping a device stops its stream idempotently. SimSidecarClient satisfies the widened port structurally. Transport wiring (wire variants, Hub targeting, client channel) lands next. --- .../src/__tests__/sim-mcp-endpoint.test.ts | 6 ++ .../simulator-request-handler.test.ts | 6 ++ .../src/__tests__/simulator-service.test.ts | 12 ++++ packages/host/engine/src/simulator/backend.ts | 37 ++++++++++++ packages/host/engine/src/simulator/service.ts | 58 ++++++++++++++++++- 5 files changed, 118 insertions(+), 1 deletion(-) diff --git a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts index fd176913..99c29e2c 100644 --- a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts +++ b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts @@ -33,6 +33,12 @@ function fakeBackend(): SimulatorBackend { terminate: vi.fn(asyncNoop), openUrl: vi.fn(asyncNoop), screenshot: vi.fn(() => Promise.resolve(new Uint8Array([0xff, 0xd8, 0x02]))), + tap: vi.fn(asyncNoop), + swipe: vi.fn(asyncNoop), + button: vi.fn(asyncNoop), + streamStart: vi.fn(() => Promise.resolve({ streaming: true as const, fps: 60, scale: 1 })), + streamStop: vi.fn(asyncNoop), + onFrame: vi.fn(() => noop), close: vi.fn(noop), }; } diff --git a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts index 4782ed00..5e8ea2cf 100644 --- a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts +++ b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts @@ -32,6 +32,12 @@ function fakeBackend(): SimulatorBackend { terminate: vi.fn(asyncNoop), openUrl: vi.fn(asyncNoop), screenshot: vi.fn(() => Promise.resolve(new Uint8Array([0xff, 0xd8, 0x01]))), + tap: vi.fn(asyncNoop), + swipe: vi.fn(asyncNoop), + button: vi.fn(asyncNoop), + streamStart: vi.fn(() => Promise.resolve({ streaming: true as const, fps: 60, scale: 1 })), + streamStop: vi.fn(asyncNoop), + onFrame: vi.fn(() => noop), close: vi.fn(noop), }; } diff --git a/packages/host/engine/src/__tests__/simulator-service.test.ts b/packages/host/engine/src/__tests__/simulator-service.test.ts index 6ab30d8c..41385ffc 100644 --- a/packages/host/engine/src/__tests__/simulator-service.test.ts +++ b/packages/host/engine/src/__tests__/simulator-service.test.ts @@ -29,6 +29,18 @@ function fakeBackend(devices: SimulatorDeviceInfo[]) { terminate: vi.fn(asyncNoop), openUrl: vi.fn(asyncNoop), screenshot: vi.fn(() => Promise.resolve(new Uint8Array([0xff, 0xd8]))), + tap: vi.fn(asyncNoop), + swipe: vi.fn(asyncNoop), + button: vi.fn(asyncNoop), + streamStart: vi.fn(() => + Promise.resolve>>({ + streaming: true, + fps: 60, + scale: 1, + }), + ), + streamStop: vi.fn(asyncNoop), + onFrame: vi.fn(() => noop), close: vi.fn(noop), } satisfies SimulatorBackend; } diff --git a/packages/host/engine/src/simulator/backend.ts b/packages/host/engine/src/simulator/backend.ts index fec04d57..ae54d361 100644 --- a/packages/host/engine/src/simulator/backend.ts +++ b/packages/host/engine/src/simulator/backend.ts @@ -27,6 +27,31 @@ export interface SimulatorProbe { export type SimulatorImageFormat = 'jpeg' | 'png'; +/** A hardware button the private HID layer can press. */ +export type SimulatorButton = 'home' | 'lock'; + +/** A normalized (0..1) point on the device screen. */ +export interface SimulatorPoint { + x: number; + y: number; +} + +/** Framebuffer stream tuning; omitted fields take the sidecar defaults. */ +export interface SimulatorStreamOptions { + fps?: number; + quality?: number; + /** Downscale factor before encode (0..1; 1.0 = native). Lower trades resolution for rate/bandwidth. */ + scale?: number; +} + +/** `streamStart` outcome: the accepted stream, or a no-op when one is already running. */ +export type SimulatorStreamStartResult = + | { streaming: true; fps: number; scale: number } + | { alreadyStreaming: true }; + +/** A live framebuffer frame: JPEG bytes for one device. */ +export type SimulatorFrameListener = (image: Uint8Array) => void; + export interface SimulatorBackend { probe(): Promise; list(): Promise; @@ -39,6 +64,18 @@ export interface SimulatorBackend { terminate(udid: string, bundleId: string): Promise; openUrl(udid: string, url: string): Promise; screenshot(udid: string, format?: SimulatorImageFormat): Promise; + /** Tap at a normalized (0..1) point (private HID; macOS only). */ + tap(udid: string, x: number, y: number): Promise; + /** Swipe between two normalized (0..1) points over `durationMs` (private HID; macOS only). */ + swipe(udid: string, from: SimulatorPoint, to: SimulatorPoint, durationMs?: number): Promise; + /** Press a hardware button (private HID; macOS only). */ + button(udid: string, button: SimulatorButton): Promise; + /** Start streaming `udid`'s framebuffer; frames arrive via {@link onFrame} listeners. */ + streamStart(udid: string, options?: SimulatorStreamOptions): Promise; + /** Stop a running framebuffer stream. */ + streamStop(udid: string): Promise; + /** Subscribe to `udid`'s framebuffer frames; returns an unsubscribe function. */ + onFrame(udid: string, listener: SimulatorFrameListener): () => void; /** Release the backend process (engine shutdown). Booted devices keep running host-side. */ close(): void; } diff --git a/packages/host/engine/src/simulator/service.ts b/packages/host/engine/src/simulator/service.ts index f348ab06..407d7230 100644 --- a/packages/host/engine/src/simulator/service.ts +++ b/packages/host/engine/src/simulator/service.ts @@ -3,7 +3,16 @@ import { Effect } from 'effect'; import { extractErrorMessage } from 'foxts/extract-error-message'; import { noop } from 'foxts/noop'; import { RequestError } from '../failure'; -import type { SimulatorBackend, SimulatorDeviceInfo, SimulatorImageFormat } from './backend'; +import type { + SimulatorBackend, + SimulatorButton, + SimulatorDeviceInfo, + SimulatorFrameListener, + SimulatorImageFormat, + SimulatorPoint, + SimulatorStreamOptions, + SimulatorStreamStartResult, +} from './backend'; /** Lazily-probed host capability; mirrors the wire's `SimulatorStatus` shape structurally. */ export interface SimulatorHostStatus { @@ -154,6 +163,51 @@ export class SimulatorService { return this.withClaim(sessionId, udid, () => this.backend.screenshot(udid, format)); } + async tap(sessionId: SessionId, udid: string, x: number, y: number): Promise { + this.claim(sessionId, udid); + return this.backend.tap(udid, x, y); + } + + async swipe( + sessionId: SessionId, + udid: string, + from: SimulatorPoint, + to: SimulatorPoint, + durationMs?: number, + ): Promise { + this.claim(sessionId, udid); + return this.backend.swipe(udid, from, to, durationMs); + } + + async button(sessionId: SessionId, udid: string, button: SimulatorButton): Promise { + this.claim(sessionId, udid); + return this.backend.button(udid, button); + } + + /** Start streaming a device's framebuffer for a session, claiming it. */ + async streamStart( + sessionId: SessionId, + udid: string, + options?: SimulatorStreamOptions, + ): Promise { + this.claim(sessionId, udid); + return this.backend.streamStart(udid, options); + } + + /** Stop a device's framebuffer stream. Requires the session to hold the device. */ + async streamStop(sessionId: SessionId, udid: string): Promise { + this.claim(sessionId, udid); + return this.backend.streamStop(udid); + } + + /** + * Subscribe to a device's framebuffer frames; returns an unsubscribe function. Ownership is + * enforced by {@link streamStart}, not here — this only wires the transport-level fan-out. + */ + onFrame(udid: string, listener: SimulatorFrameListener): () => void { + return this.backend.onFrame(udid, listener); + } + /** Release every device a session holds (the session-stop hook). */ releaseSession(sessionId: SessionId): void { for (const [udid, claim] of this.claims) { @@ -263,6 +317,8 @@ export class SimulatorService { const claim = this.claims.get(udid); if (claim?.idleTimer) clearTimeout(claim.idleTimer); this.claims.delete(udid); + // A released device must not keep streaming; `streamStop` is idempotent (a no-op if idle). + this.backend.streamStop(udid).catch(noop); } private async reclaimAll(): Promise { From 4f9ddf2e67e57ec91289dce637d592b60f4c56e6 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 00:11:25 +0800 Subject: [PATCH 03/26] feat(schema,transport,engine,client-core): simulator interactive + stream wire (wire 45) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add tap/swipe/button/stream.start/stream.stop commands, a stream.started reply, and a session-scoped simulator.stream.frame push (base64 JPEG) — bump wire 44->45 (Invariant 1). The engine routes the commands and fans a device's frames out via the transport; the Hub scopes frames to attached sessions like agent.event (never a global broadcast); client-core gains the methods and a per-udid subscribeSimulatorFrames. Frames ride base64 in JSON like screenshots; a binary side-channel stays a remote/high-fps concern. --- packages/client/core/src/client.ts | 63 +++++++++++++ .../client/core/src/client/control-channel.ts | 71 ++++++++++++++ .../core/src/client/pending-registry.ts | 2 + .../foundation/schema/src/wire/message.ts | 3 +- .../foundation/schema/src/wire/simulator.ts | 66 +++++++++++++ .../transport/src/__tests__/hub.test.ts | 36 ++++++++ packages/foundation/transport/src/hub.ts | 4 +- .../simulator-request-handler.test.ts | 46 +++++++++- .../engine/src/simulator/request-handler.ts | 92 ++++++++++++++++++- .../host/engine/src/wire/request-router.ts | 7 +- 10 files changed, 382 insertions(+), 8 deletions(-) diff --git a/packages/client/core/src/client.ts b/packages/client/core/src/client.ts index 88c272da..9bad8f37 100644 --- a/packages/client/core/src/client.ts +++ b/packages/client/core/src/client.ts @@ -113,6 +113,8 @@ type SimulatorActivityCb = (activity: { tool: string; phase: 'started' | 'settled'; }) => void; +/** A live framebuffer frame: base64-encoded JPEG bytes for one device. */ +type SimulatorFrameCb = (frame: { udid: string; data: string }) => void; type ConnectionCloseCb = (error: Error) => void; /** A broadcast about a schedule's or its runs' state — the three `schedule.*` push variants. */ @@ -154,6 +156,8 @@ export class LinkCodeClient { private readonly agentRuntimesChangedSubs = new Set(); private readonly simulatorDevicesChangedSubs = new Set(); private readonly simulatorActivitySubs = new Set(); + /** Framebuffer-frame listeners keyed by udid, so each panel tab only sees its device's frames. */ + private readonly simulatorFrameSubs = new Map>(); private readonly connectionCloseSubs = new Set(); private unsub: Unsubscribe | null = null; private offClose: Unsubscribe | null = null; @@ -339,6 +343,14 @@ export class LinkCodeClient { cb({ sessionId: p.sessionId, udid: p.udid, tool: p.tool, phase: p.phase }); } break; + case 'simulator.stream.started': + this.pending.resolve('simulatorStreamStart', p.replyTo, { fps: p.fps, scale: p.scale }); + break; + case 'simulator.stream.frame': + for (const cb of this.simulatorFrameSubs.get(p.udid) ?? []) { + cb({ udid: p.udid, data: p.data }); + } + break; case 'asset.listed': this.pending.resolve('assetList', p.replyTo, p.assets); break; @@ -675,6 +687,57 @@ export class LinkCodeClient { return this.control.simulatorScreenshot(sessionId, udid, format); } + simulatorTap(sessionId: SessionId, udid: string, x: number, y: number): Promise { + return this.control.simulatorTap(sessionId, udid, x, y); + } + + simulatorSwipe( + sessionId: SessionId, + udid: string, + from: { x: number; y: number }, + to: { x: number; y: number }, + durationMs?: number, + ): Promise { + return this.control.simulatorSwipe(sessionId, udid, from, to, durationMs); + } + + simulatorButton( + sessionId: SessionId, + udid: string, + button: 'home' | 'lock', + ): Promise { + return this.control.simulatorButton(sessionId, udid, button); + } + + /** Start streaming a device's framebuffer; frames arrive via {@link subscribeSimulatorFrames}. */ + simulatorStreamStart( + sessionId: SessionId, + udid: string, + options?: { fps?: number; quality?: number; scale?: number }, + ): Promise<{ fps: number; scale: number }> { + return this.control.simulatorStreamStart(sessionId, udid, options); + } + + simulatorStreamStop(sessionId: SessionId, udid: string): Promise { + return this.control.simulatorStreamStop(sessionId, udid); + } + + /** Subscribe to a device's live framebuffer frames; pair with {@link simulatorStreamStart}. */ + subscribeSimulatorFrames(udid: string, cb: SimulatorFrameCb): Unsubscribe { + let set = this.simulatorFrameSubs.get(udid); + if (!set) { + set = new Set(); + this.simulatorFrameSubs.set(udid, set); + } + set.add(cb); + return () => { + const current = this.simulatorFrameSubs.get(udid); + if (!current) return; + current.delete(cb); + if (current.size === 0) this.simulatorFrameSubs.delete(udid); + }; + } + /** Pushed after a state-changing simulator command (boot/shutdown) with a fresh device list. */ subscribeSimulatorDevicesChanged(cb: SimulatorDevicesChangedCb): Unsubscribe { this.simulatorDevicesChangedSubs.add(cb); diff --git a/packages/client/core/src/client/control-channel.ts b/packages/client/core/src/client/control-channel.ts index ec677a6e..9bd90d74 100644 --- a/packages/client/core/src/client/control-channel.ts +++ b/packages/client/core/src/client/control-channel.ts @@ -658,6 +658,77 @@ export class ControlChannel { })); } + simulatorTap(sessionId: SessionId, udid: string, x: number, y: number): Promise { + return this.sendCorrelated('ack', (clientReqId) => ({ + kind: 'simulator.tap', + clientReqId, + sessionId, + udid, + x, + y, + })); + } + + simulatorSwipe( + sessionId: SessionId, + udid: string, + from: { x: number; y: number }, + to: { x: number; y: number }, + durationMs?: number, + ): Promise { + return this.sendCorrelated('ack', (clientReqId) => ({ + kind: 'simulator.swipe', + clientReqId, + sessionId, + udid, + x0: from.x, + y0: from.y, + x1: to.x, + y1: to.y, + durationMs, + })); + } + + simulatorButton( + sessionId: SessionId, + udid: string, + button: 'home' | 'lock', + ): Promise { + return this.sendCorrelated('ack', (clientReqId) => ({ + kind: 'simulator.button', + clientReqId, + sessionId, + udid, + button, + })); + } + + /** Resolves with the accepted `{ fps, scale }`; frames then arrive as `simulator.stream.frame`. */ + simulatorStreamStart( + sessionId: SessionId, + udid: string, + options?: { fps?: number; quality?: number; scale?: number }, + ): Promise<{ fps: number; scale: number }> { + return this.sendCorrelated('simulatorStreamStart', (clientReqId) => ({ + kind: 'simulator.stream.start', + clientReqId, + sessionId, + udid, + fps: options?.fps, + quality: options?.quality, + scale: options?.scale, + })); + } + + simulatorStreamStop(sessionId: SessionId, udid: string): Promise { + return this.sendCorrelated('ack', (clientReqId) => ({ + kind: 'simulator.stream.stop', + clientReqId, + sessionId, + udid, + })); + } + private sendCorrelated( kind: K, makePayload: (clientReqId: string) => WirePayload, diff --git a/packages/client/core/src/client/pending-registry.ts b/packages/client/core/src/client/pending-registry.ts index 18d895ba..50afb6e4 100644 --- a/packages/client/core/src/client/pending-registry.ts +++ b/packages/client/core/src/client/pending-registry.ts @@ -98,6 +98,7 @@ export interface PendingValueMap { simulatorList: SimulatorDevice[]; simulatorLaunch: number | null; simulatorScreenshot: { format: SimulatorImageFormat; data: string }; + simulatorStreamStart: { fps: number; scale: number }; } type PendingMaps = { [K in keyof PendingValueMap]: Map> }; @@ -146,6 +147,7 @@ export class PendingRegistry { simulatorList: new Map(), simulatorLaunch: new Map(), simulatorScreenshot: new Map(), + simulatorStreamStart: new Map(), }; private readonly randomUUID: RandomUUID; diff --git a/packages/foundation/schema/src/wire/message.ts b/packages/foundation/schema/src/wire/message.ts index c303d324..c97195fe 100644 --- a/packages/foundation/schema/src/wire/message.ts +++ b/packages/foundation/schema/src/wire/message.ts @@ -15,7 +15,8 @@ import { WirePayloadSchema } from './payload'; // file.host/file.hosted, keeping every distinct schema on a distinct protocol version. // 44 adds the simulator.* variants (CODE-394). // 45 adds the simulator.activity broadcast (CODE-395). -export const WIRE_PROTOCOL_VERSION = 45 as const; +// 46 adds the simulator interactive + framebuffer-stream variants (CODE-397). +export const WIRE_PROTOCOL_VERSION = 46 as const; /** Complete wire message: version + unique id + timestamp + payload. */ export const WireMessageSchema = z.object({ diff --git a/packages/foundation/schema/src/wire/simulator.ts b/packages/foundation/schema/src/wire/simulator.ts index 001e244c..9018f414 100644 --- a/packages/foundation/schema/src/wire/simulator.ts +++ b/packages/foundation/schema/src/wire/simulator.ts @@ -8,6 +8,9 @@ import { import { WireRequestIdSchema } from './request'; const udid = z.string().min(1); +/** A normalized screen coordinate, 0..1 from the top-left. */ +const coord = z.number().min(0).max(1); +const SimulatorButtonSchema = z.enum(['home', 'lock']); /** * iOS Simulator wire variants. Commands are session-scoped: the engine's simulator service @@ -102,4 +105,67 @@ export const simulatorWireVariants = [ /** Base64-encoded image bytes. */ data: z.string(), }), + + // ── Interactive control + framebuffer streaming (CODE-397; private-API, macOS host only) ── + // Void commands reply with the generic `request.succeeded`/`request.failed`. Coordinates are + // normalized 0..1, so a downscaled stream needs no adjustment. + z.object({ + kind: z.literal('simulator.tap'), + clientReqId: WireRequestIdSchema, + sessionId: SessionIdSchema, + udid, + x: coord, + y: coord, + }), + z.object({ + kind: z.literal('simulator.swipe'), + clientReqId: WireRequestIdSchema, + sessionId: SessionIdSchema, + udid, + x0: coord, + y0: coord, + x1: coord, + y1: coord, + durationMs: z.number().int().positive().optional(), + }), + z.object({ + kind: z.literal('simulator.button'), + clientReqId: WireRequestIdSchema, + sessionId: SessionIdSchema, + udid, + button: SimulatorButtonSchema, + }), + z.object({ + kind: z.literal('simulator.stream.start'), + clientReqId: WireRequestIdSchema, + sessionId: SessionIdSchema, + udid, + fps: z.number().int().positive().optional(), + quality: z.number().min(0).max(1).optional(), + scale: z.number().min(0).max(1).optional(), + }), + z.object({ + kind: z.literal('simulator.stream.started'), + replyTo: WireRequestIdSchema, + udid, + fps: z.number().int(), + scale: z.number(), + }), + z.object({ + kind: z.literal('simulator.stream.stop'), + clientReqId: WireRequestIdSchema, + sessionId: SessionIdSchema, + udid, + }), + /** An unsolicited framebuffer frame while a stream runs. Routed session-scoped (like + * `agent.event`) so only connections attached to `sessionId` receive it — never a global + * broadcast, since frames are high-frequency. Base64 rides in JSON like `simulator.screenshotted`; + * a binary side-channel is a remote/high-fps concern, not v1's desktop-local path. */ + z.object({ + kind: z.literal('simulator.stream.frame'), + sessionId: SessionIdSchema, + udid, + /** Base64-encoded JPEG bytes. */ + data: z.string(), + }), ] as const; diff --git a/packages/foundation/transport/src/__tests__/hub.test.ts b/packages/foundation/transport/src/__tests__/hub.test.ts index e4b90766..6baabb58 100644 --- a/packages/foundation/transport/src/__tests__/hub.test.ts +++ b/packages/foundation/transport/src/__tests__/hub.test.ts @@ -44,6 +44,15 @@ function agentEvent(sessionId: SessionId): ValidatedWireMessage { }); } +function streamFrame(sessionId: SessionId): ValidatedWireMessage { + return createWireMessage({ + kind: 'simulator.stream.frame', + sessionId, + udid: 'U-1', + data: 'AAA=', + }); +} + describe('Hub subscriptions', () => { it('broadcasts agent.event to every connection by default', () => { const hub = new Hub(); @@ -80,6 +89,33 @@ describe('Hub subscriptions', () => { expect(normal.sent.map((m) => m.payload.kind)).toEqual(['agent.event', 'agent.event']); }); + it('scopes simulator.stream.frame to attached sessions (never a global broadcast)', () => { + const hub = new Hub(); + const scoped = new FakeConn(); + const normal = new FakeConn(); + hub.addConnection(scoped); + hub.addConnection(normal); + + scoped.emit( + createWireMessage({ kind: 'subscription.set', clientReqId: 'r1', mode: 'attached' }), + ); + scoped.emit(createWireMessage({ kind: 'session.attach', sessionId: S1 })); + + hub.send(streamFrame(S1)); + hub.send(streamFrame(S2)); + + // Scoped connection: the subscription ack plus only its attached session's frame. + expect(scoped.sent.map((m) => m.payload.kind)).toEqual([ + 'request.succeeded', + 'simulator.stream.frame', + ]); + // A default (mode 'all') connection still receives both frames. + expect(normal.sent.map((m) => m.payload.kind)).toEqual([ + 'simulator.stream.frame', + 'simulator.stream.frame', + ]); + }); + it('session.detach removes a session from a scoped subscription', () => { const hub = new Hub(); const scoped = new FakeConn(); diff --git a/packages/foundation/transport/src/hub.ts b/packages/foundation/transport/src/hub.ts index f66b5171..02347685 100644 --- a/packages/foundation/transport/src/hub.ts +++ b/packages/foundation/transport/src/hub.ts @@ -181,7 +181,9 @@ export class Hub implements Transport { } for (const conn of this.conns) { - if (p.kind === 'agent.event') { + // Session-scoped events (agent activity, high-frequency framebuffer frames) reach only + // connections that carry that session — never a global broadcast. + if (p.kind === 'agent.event' || p.kind === 'simulator.stream.frame') { const subscription = this.subscriptions.get(conn); if (subscription?.mode === 'attached' && !subscription.attached.has(p.sessionId)) { continue; diff --git a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts index 5e8ea2cf..317f2eae 100644 --- a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts +++ b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts @@ -9,7 +9,7 @@ import { SimulatorService } from '../simulator/service'; import { FakeAdapter, settleEngineTasks, startedSessionId } from './fixtures/session-harness'; import { createTestEngine } from './fixtures/test-engine'; -function fakeBackend(): SimulatorBackend { +function fakeBackend() { const devices = [ { udid: 'U-1', @@ -37,9 +37,9 @@ function fakeBackend(): SimulatorBackend { button: vi.fn(asyncNoop), streamStart: vi.fn(() => Promise.resolve({ streaming: true as const, fps: 60, scale: 1 })), streamStop: vi.fn(asyncNoop), - onFrame: vi.fn(() => noop), + onFrame: vi.fn((_udid: string, _listener: (image: Uint8Array) => void) => noop), close: vi.fn(noop), - }; + } satisfies SimulatorBackend; } function harness(backend?: SimulatorBackend) { @@ -169,4 +169,44 @@ describe('simulator wire requests', () => { expect(h.reply('steal')).toMatchObject({ kind: 'request.failed', code: 'conflict' }); await h.engine.stop(); }); + + it('start subscribes frames and fans them out session-scoped; stop unsubscribes', async () => { + const backend = fakeBackend(); + const h = harness(backend); + await h.engine.start(); + + await h.inject({ + kind: 'simulator.stream.start', + clientReqId: 'ss', + sessionId: 'session-1' as never, + udid: 'U-1', + scale: 0.5, + }); + expect(h.reply('ss')).toMatchObject({ + kind: 'simulator.stream.started', + udid: 'U-1', + fps: 60, + scale: 1, + }); + + // The handler subscribed a frame listener; a delivered frame becomes a session-scoped push. + const listener = backend.onFrame.mock.calls.at(-1)?.[1]; + listener?.(new Uint8Array([0xff, 0xd8, 0x01])); + expect(h.sent.find((p) => p.kind === 'simulator.stream.frame')).toMatchObject({ + kind: 'simulator.stream.frame', + sessionId: 'session-1', + udid: 'U-1', + data: Buffer.from([0xff, 0xd8, 0x01]).toString('base64'), + }); + + await h.inject({ + kind: 'simulator.stream.stop', + clientReqId: 'sx', + sessionId: 'session-1' as never, + udid: 'U-1', + }); + expect(h.reply('sx')).toMatchObject({ kind: 'request.succeeded' }); + expect(backend.streamStop).toHaveBeenCalledWith('U-1'); + await h.engine.stop(); + }); }); diff --git a/packages/host/engine/src/simulator/request-handler.ts b/packages/host/engine/src/simulator/request-handler.ts index 0b3f36f4..062de56e 100644 --- a/packages/host/engine/src/simulator/request-handler.ts +++ b/packages/host/engine/src/simulator/request-handler.ts @@ -1,4 +1,4 @@ -import type { WirePayload } from '@linkcode/schema'; +import type { SessionId, WirePayload } from '@linkcode/schema'; import type { Transport } from '@linkcode/transport'; import { createWireMessage } from '@linkcode/transport'; import { Effect } from 'effect'; @@ -19,7 +19,12 @@ type SimulatorRequest = Extract< | 'simulator.launch' | 'simulator.terminate' | 'simulator.open-url' - | 'simulator.screenshot'; + | 'simulator.screenshot' + | 'simulator.tap' + | 'simulator.swipe' + | 'simulator.button' + | 'simulator.stream.start' + | 'simulator.stream.stop'; } >; @@ -30,6 +35,9 @@ type SimulatorRequest = Extract< * watcher, so its own commands are the only change source it can observe). */ export class SimulatorRequestHandler { + /** Active framebuffer fan-out subscriptions, keyed by udid; the value unsubscribes it. */ + private readonly frameSubs = new Map void>(); + constructor( private readonly simulators: SimulatorService | undefined, private readonly transport: Transport, @@ -134,11 +142,91 @@ export class SimulatorRequestHandler { ); }), ); + case 'simulator.tap': + return this.withSimulators(payload.clientReqId, (simulators) => + simulatorOperation('simulator.tap', 'Failed to tap', async () => { + await simulators.tap(payload.sessionId, payload.udid, payload.x, payload.y); + this.responder.sendSuccess(payload.clientReqId); + }), + ); + case 'simulator.swipe': + return this.withSimulators(payload.clientReqId, (simulators) => + simulatorOperation('simulator.swipe', 'Failed to swipe', async () => { + await simulators.swipe( + payload.sessionId, + payload.udid, + { x: payload.x0, y: payload.y0 }, + { x: payload.x1, y: payload.y1 }, + payload.durationMs, + ); + this.responder.sendSuccess(payload.clientReqId); + }), + ); + case 'simulator.button': + return this.withSimulators(payload.clientReqId, (simulators) => + simulatorOperation('simulator.button', 'Failed to press button', async () => { + await simulators.button(payload.sessionId, payload.udid, payload.button); + this.responder.sendSuccess(payload.clientReqId); + }), + ); + case 'simulator.stream.start': + return this.withSimulators(payload.clientReqId, (simulators) => + simulatorOperation('simulator.stream.start', 'Failed to start stream', async () => { + const result = await simulators.streamStart(payload.sessionId, payload.udid, { + fps: payload.fps, + quality: payload.quality, + scale: payload.scale, + }); + this.subscribeFrames(simulators, payload.sessionId, payload.udid); + // `alreadyStreaming` carries no params, so echo the request's (defaulted) values. + const fps = 'streaming' in result ? result.fps : (payload.fps ?? 60); + const scale = 'streaming' in result ? result.scale : (payload.scale ?? 1); + this.transport.send( + createWireMessage({ + kind: 'simulator.stream.started', + replyTo: payload.clientReqId, + udid: payload.udid, + fps, + scale, + }), + ); + }), + ); + case 'simulator.stream.stop': + return this.withSimulators(payload.clientReqId, (simulators) => + simulatorOperation('simulator.stream.stop', 'Failed to stop stream', async () => { + this.unsubscribeFrames(payload.udid); + await simulators.streamStop(payload.sessionId, payload.udid); + this.responder.sendSuccess(payload.clientReqId); + }), + ); default: return Effect.void; } } + /** Fan the device's frames out to the transport as session-scoped `simulator.stream.frame`s. + * Idempotent: a second `streamStart` for a device already fanning out keeps the one subscription. */ + private subscribeFrames(simulators: SimulatorService, sessionId: SessionId, udid: string): void { + if (this.frameSubs.has(udid)) return; + const unsubscribe = simulators.onFrame(udid, (image) => { + this.transport.send( + createWireMessage({ + kind: 'simulator.stream.frame', + sessionId, + udid, + data: Buffer.from(image).toString('base64'), + }), + ); + }); + this.frameSubs.set(udid, unsubscribe); + } + + private unsubscribeFrames(udid: string): void { + this.frameSubs.get(udid)?.(); + this.frameSubs.delete(udid); + } + /** Best-effort push after a state-changing command; the command itself already succeeded. */ private async broadcastDevices(simulators: SimulatorService): Promise { try { diff --git a/packages/host/engine/src/wire/request-router.ts b/packages/host/engine/src/wire/request-router.ts index 9cbd85a8..044fd5d5 100644 --- a/packages/host/engine/src/wire/request-router.ts +++ b/packages/host/engine/src/wire/request-router.ts @@ -137,7 +137,12 @@ export class WireRequestRouter { case 'simulator.launch': case 'simulator.terminate': case 'simulator.open-url': - case 'simulator.screenshot': { + case 'simulator.screenshot': + case 'simulator.tap': + case 'simulator.swipe': + case 'simulator.button': + case 'simulator.stream.start': + case 'simulator.stream.stop': { return this.handlers.simulator.handle(p); } case 'agent-login.start': From a6107782482752804aab849e74536a2c8b485605 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 00:36:44 +0800 Subject: [PATCH 04/26] feat(ui,i18n): simulator screen canvas + optional panel-section vocabulary --- packages/presentation/i18n/src/locales/en.ts | 13 ++ .../presentation/i18n/src/locales/zh-cn.ts | 13 ++ packages/presentation/ui/package.json | 1 + .../presentation/ui/src/shell/panels/index.ts | 8 +- .../ui/src/shell/panels/vocabulary.tsx | 16 +- .../ui/src/shell/simulator/index.ts | 2 + .../src/shell/simulator/simulator-screen.tsx | 138 ++++++++++++++++++ 7 files changed, 188 insertions(+), 3 deletions(-) create mode 100644 packages/presentation/ui/src/shell/simulator/index.ts create mode 100644 packages/presentation/ui/src/shell/simulator/simulator-screen.tsx diff --git a/packages/presentation/i18n/src/locales/en.ts b/packages/presentation/i18n/src/locales/en.ts index 2f56f9f0..029d6db4 100644 --- a/packages/presentation/i18n/src/locales/en.ts +++ b/packages/presentation/i18n/src/locales/en.ts @@ -349,6 +349,7 @@ export const en = { terminal: 'Terminal', browser: 'Browser', files: 'Files', + simulator: 'Simulator', }, openWindow: 'Open window', newTerminalTab: 'New terminal', @@ -372,6 +373,18 @@ export const en = { terminalTakeControlFailed: 'Could not take control; the terminal remains view-only', terminalReplayTruncated: 'Earlier terminal history was truncated', terminalTakeControl: 'Take control', + simulatorSelectDevice: 'Select a device', + simulatorNoDevices: 'No simulator devices found', + simulatorUnavailable: 'iOS Simulator is not available on this host', + simulatorNoSession: 'Select a thread to drive the simulator', + simulatorBoot: 'Boot', + simulatorBooting: 'Booting device…', + simulatorConnecting: 'Connecting to screen…', + simulatorStreamFailed: 'Screen stream failed to start', + simulatorRetry: 'Retry', + simulatorBusy: 'Device is in use by another thread', + simulatorHome: 'Home', + simulatorLock: 'Lock screen', }, git: { emptyTitle: 'No active thread', diff --git a/packages/presentation/i18n/src/locales/zh-cn.ts b/packages/presentation/i18n/src/locales/zh-cn.ts index 1dc24039..c90a412d 100644 --- a/packages/presentation/i18n/src/locales/zh-cn.ts +++ b/packages/presentation/i18n/src/locales/zh-cn.ts @@ -339,6 +339,7 @@ export const zhCN = { terminal: '终端', browser: '浏览器', files: '文件', + simulator: '模拟器', }, openWindow: '打开窗口', newTerminalTab: '新建终端', @@ -362,6 +363,18 @@ export const zhCN = { terminalTakeControlFailed: '接管失败；终端仍保持只读', terminalReplayTruncated: '较早的终端历史已被截断', terminalTakeControl: '接管', + simulatorSelectDevice: '选择设备', + simulatorNoDevices: '未发现模拟器设备', + simulatorUnavailable: '此主机不支持 iOS 模拟器', + simulatorNoSession: '选择一个线程后即可操控模拟器', + simulatorBoot: '启动', + simulatorBooting: '正在启动设备…', + simulatorConnecting: '正在连接画面…', + simulatorStreamFailed: '画面流启动失败', + simulatorRetry: '重试', + simulatorBusy: '设备正被其他线程使用', + simulatorHome: '主屏幕', + simulatorLock: '锁屏', }, git: { emptyTitle: '暂无活跃线程', diff --git a/packages/presentation/ui/package.json b/packages/presentation/ui/package.json index 2b9b3b59..581ab1fc 100644 --- a/packages/presentation/ui/package.json +++ b/packages/presentation/ui/package.json @@ -13,6 +13,7 @@ "./shell/git": "./src/shell/git/index.ts", "./shell/panels": "./src/shell/panels/index.ts", "./shell/scripts": "./src/shell/scripts/index.ts", + "./shell/simulator": "./src/shell/simulator/index.ts", "./shell/terminal": "./src/shell/terminal/index.ts", "./styles.css": "./src/styles.css" }, diff --git a/packages/presentation/ui/src/shell/panels/index.ts b/packages/presentation/ui/src/shell/panels/index.ts index 62f6af0e..bcae0392 100644 --- a/packages/presentation/ui/src/shell/panels/index.ts +++ b/packages/presentation/ui/src/shell/panels/index.ts @@ -13,6 +13,7 @@ export { PanelRegion } from './panel-region'; export type { SectionPanelRegionProps, SectionPanelState } from './section-panel'; export { SectionPanelRegion } from './section-panel'; export type { + OptionalPanelSection, PanelControl, PanelSection, PanelSectionTab, @@ -20,4 +21,9 @@ export type { PanelTab, PanelWindowType, } from './vocabulary'; -export { PANEL_SECTIONS, PANEL_WINDOW_ICONS, PANEL_WINDOW_TYPES } from './vocabulary'; +export { + OPTIONAL_PANEL_SECTIONS, + PANEL_SECTIONS, + PANEL_WINDOW_ICONS, + PANEL_WINDOW_TYPES, +} from './vocabulary'; diff --git a/packages/presentation/ui/src/shell/panels/vocabulary.tsx b/packages/presentation/ui/src/shell/panels/vocabulary.tsx index c1e41df6..1625671b 100644 --- a/packages/presentation/ui/src/shell/panels/vocabulary.tsx +++ b/packages/presentation/ui/src/shell/panels/vocabulary.tsx @@ -1,4 +1,10 @@ -import { CompassIcon, FileDiffIcon, FolderCodeIcon, SquareTerminalIcon } from 'lucide-react'; +import { + CompassIcon, + FileDiffIcon, + FolderCodeIcon, + SmartphoneIcon, + SquareTerminalIcon, +} from 'lucide-react'; export type PanelSide = 'right' | 'bottom'; @@ -9,6 +15,11 @@ export type PanelWindowType = (typeof PANEL_WINDOW_TYPES)[number]; /** The right panel's fixed sections — a subset of {@link PanelWindowType}. */ export const PANEL_SECTIONS = ['diff', 'terminal', 'browser', 'files'] as const; +/** On-demand right-panel sections, added (and removed) via the section strip's + menu. */ +export const OPTIONAL_PANEL_SECTIONS = ['simulator'] as const; + +export type OptionalPanelSection = (typeof OPTIONAL_PANEL_SECTIONS)[number]; + export type PanelSection = (typeof PANEL_SECTIONS)[number]; export interface PanelTab { @@ -30,11 +41,12 @@ export interface PanelControl { } /** Window labels are translated at the render site (`workbench.panel.window.*`); only icons live here. */ -export const PANEL_WINDOW_ICONS: Record = { +export const PANEL_WINDOW_ICONS: Record = { diff: , terminal: , browser: , files: , + simulator: , }; /** Shared tab recipe: the active/inactive halves of every panel tab button's className. */ diff --git a/packages/presentation/ui/src/shell/simulator/index.ts b/packages/presentation/ui/src/shell/simulator/index.ts new file mode 100644 index 00000000..eef044a4 --- /dev/null +++ b/packages/presentation/ui/src/shell/simulator/index.ts @@ -0,0 +1,2 @@ +export type { SimulatorScreenPoint, SimulatorScreenProps } from './simulator-screen'; +export { SimulatorScreen } from './simulator-screen'; diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx new file mode 100644 index 00000000..3d5e7c2d --- /dev/null +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -0,0 +1,138 @@ +import { useEffect as useAbortableEffect } from 'foxact/use-abortable-effect'; +import { clamp } from 'foxts/clamp'; +import { noop } from 'foxts/noop'; +import { useRef, useState } from 'react'; +import { cn } from '../../lib/cn'; + +export interface SimulatorScreenPoint { + x: number; + y: number; +} + +/** Below this pointer travel (in element px) a press counts as a tap, not a swipe. */ +const SWIPE_THRESHOLD_PX = 8; + +export interface SimulatorScreenProps { + /** Feed of JPEG frames (base64, no data-URL prefix); returns the unsubscribe. */ + subscribeFrames: (onFrame: (jpegBase64: string) => void) => () => void; + /** Press in normalized [0,1] device coordinates. */ + onTap?: (point: SimulatorScreenPoint) => void; + /** Drag in normalized [0,1] device coordinates with its real duration. */ + onSwipe?: (from: SimulatorScreenPoint, to: SimulatorScreenPoint, durationMs: number) => void; + /** Shown centered until the first frame arrives. */ + placeholder?: React.ReactNode; + className?: string; +} + +/** + * Live device screen: paints an MJPEG-style frame feed onto a canvas (latest-wins — a frame + * arriving while the previous one decodes replaces it) and maps pointer presses back to + * normalized tap/swipe callbacks. Pure presentation: the feed and the input sinks are props. + * Key this component by device so a device switch resets the painted frame. + */ +export function SimulatorScreen({ + subscribeFrames, + onTap, + onSwipe, + placeholder, + className, +}: SimulatorScreenProps): React.ReactNode { + const canvasRef = useRef(null); + const pressRef = useRef<{ pointerId: number; start: SimulatorScreenPoint; at: number } | null>( + null, + ); + const [hasFrame, setHasFrame] = useState(false); + + useAbortableEffect( + (signal) => { + let latest: string | null = null; + let decoding = false; + const drawNext = (): void => { + if (decoding || latest === null || signal.aborted) return; + const frame = latest; + latest = null; + decoding = true; + void decodeJpegFrame(frame) + .then((bitmap) => { + const canvas = canvasRef.current; + if (signal.aborted || canvas === null) { + bitmap.close(); + return; + } + if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) { + canvas.width = bitmap.width; + canvas.height = bitmap.height; + } + canvas.getContext('2d')?.drawImage(bitmap, 0, 0); + bitmap.close(); + setHasFrame(true); + }) + // A corrupt frame is dropped; the next one repaints. + .catch(noop) + .finally(() => { + decoding = false; + drawNext(); + }); + }; + return subscribeFrames((frame) => { + latest = frame; + drawNext(); + }); + }, + [subscribeFrames], + ); + + const handlePointerDown = (event: React.PointerEvent): void => { + if (event.button !== 0) return; + event.currentTarget.setPointerCapture(event.pointerId); + pressRef.current = { + pointerId: event.pointerId, + start: normalizedPoint(event), + at: performance.now(), + }; + }; + + const handlePointerUp = (event: React.PointerEvent): void => { + const press = pressRef.current; + if (press?.pointerId !== event.pointerId) return; + pressRef.current = null; + const end = normalizedPoint(event); + const rect = event.currentTarget.getBoundingClientRect(); + const travelPx = Math.hypot( + (end.x - press.start.x) * rect.width, + (end.y - press.start.y) * rect.height, + ); + if (travelPx < SWIPE_THRESHOLD_PX) onTap?.(end); + else onSwipe?.(press.start, end, Math.round(performance.now() - press.at)); + }; + + return ( +
+ { + pressRef.current = null; + }} + /> + {!hasFrame && placeholder} +
+ ); +} + +function normalizedPoint(event: React.PointerEvent): SimulatorScreenPoint { + const rect = event.currentTarget.getBoundingClientRect(); + return { + x: clamp((event.clientX - rect.left) / rect.width, 0, 1), + y: clamp((event.clientY - rect.top) / rect.height, 0, 1), + }; +} + +function decodeJpegFrame(base64: string): Promise { + const bytes = Uint8Array.from(atob(base64), (char) => char.codePointAt(0) ?? 0); + return createImageBitmap(new Blob([bytes], { type: 'image/jpeg' })); +} From 45b5aa5d7d11d79f63bd774cfe65b37bc15a3a26 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 00:42:49 +0800 Subject: [PATCH 05/26] feat(workbench): simulator stream registry + panel container --- packages/client/workbench/src/index.ts | 2 + .../__tests__/stream-registry.test.ts | 110 ++++++++ .../client/workbench/src/simulator/panel.tsx | 235 ++++++++++++++++++ .../src/simulator/stream-registry.ts | 139 +++++++++++ 4 files changed, 486 insertions(+) create mode 100644 packages/client/workbench/src/simulator/__tests__/stream-registry.test.ts create mode 100644 packages/client/workbench/src/simulator/panel.tsx create mode 100644 packages/client/workbench/src/simulator/stream-registry.ts diff --git a/packages/client/workbench/src/index.ts b/packages/client/workbench/src/index.ts index 98cacff5..92311cf1 100644 --- a/packages/client/workbench/src/index.ts +++ b/packages/client/workbench/src/index.ts @@ -49,6 +49,8 @@ export * from './settings/search'; export * from './settings/terminal-prefs-store'; export * from './settings/terminal-settings'; export * from './sidebar/runtime-thread-im-menu'; +export * from './simulator/panel'; +export * from './simulator/stream-registry'; export * from './surface/selection-store'; export * from './surface/shell'; export * from './surface/use-workbench-sessions'; diff --git a/packages/client/workbench/src/simulator/__tests__/stream-registry.test.ts b/packages/client/workbench/src/simulator/__tests__/stream-registry.test.ts new file mode 100644 index 00000000..baa731f3 --- /dev/null +++ b/packages/client/workbench/src/simulator/__tests__/stream-registry.test.ts @@ -0,0 +1,110 @@ +import type { SessionId } from '@linkcode/schema'; +import { noop } from 'foxts/noop'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { SimulatorStreamClient } from '../stream-registry'; +import { acquireSimulatorStream, peekSimulatorStream } from '../stream-registry'; + +interface FakeClient extends SimulatorStreamClient { + started: Array<{ sessionId: SessionId; udid: string }>; + stopped: Array<{ sessionId: SessionId; udid: string }>; + failNextStart: boolean; +} + +function createFakeClient(): FakeClient { + const started: Array<{ sessionId: SessionId; udid: string }> = []; + const stopped: Array<{ sessionId: SessionId; udid: string }> = []; + return { + started, + stopped, + failNextStart: false, + simulatorStreamStart(this: FakeClient, sessionId, udid) { + if (this.failNextStart) { + this.failNextStart = false; + return Promise.reject(new Error('conflict')); + } + started.push({ sessionId, udid }); + return Promise.resolve({ fps: 30, scale: 0.5 }); + }, + simulatorStreamStop(sessionId, udid) { + stopped.push({ sessionId, udid }); + return Promise.resolve({ ok: true }); + }, + }; +} + +const S1 = 'session-1' as SessionId; +const S2 = 'session-2' as SessionId; + +describe('simulator stream registry', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it('starts one stream per device and stops it after the last release', async () => { + const client = createFakeClient(); + const lease = acquireSimulatorStream(client, 'U-1', S1); + expect(lease.getSnapshot().phase).toBe('starting'); + await vi.advanceTimersByTimeAsync(0); + expect(lease.getSnapshot().phase).toBe('streaming'); + expect(client.started).toEqual([{ sessionId: S1, udid: 'U-1' }]); + + lease.release(); + await vi.advanceTimersByTimeAsync(1000); + expect(client.stopped).toEqual([{ sessionId: S1, udid: 'U-1' }]); + expect(peekSimulatorStream(client, 'U-1')).toBeNull(); + }); + + it('shares the entry across a release/acquire handoff and keeps the owner session', async () => { + const client = createFakeClient(); + const first = acquireSimulatorStream(client, 'U-1', S1); + await vi.advanceTimersByTimeAsync(0); + + // Docked → maximized remount, meanwhile the active thread switched to another session. + first.release(); + const second = acquireSimulatorStream(client, 'U-1', S2); + await vi.advanceTimersByTimeAsync(1000); + expect(client.started).toHaveLength(1); + expect(client.stopped).toEqual([]); + // The claim holder stays the session that started the stream. + expect(second.getSnapshot().sessionId).toBe(S1); + + second.release(); + await vi.advanceTimersByTimeAsync(1000); + expect(client.stopped).toEqual([{ sessionId: S1, udid: 'U-1' }]); + }); + + it('surfaces a failed start and restarts on demand', async () => { + const client = createFakeClient(); + client.failNextStart = true; + const lease = acquireSimulatorStream(client, 'U-1', S1); + const phases: string[] = []; + lease.subscribe(() => phases.push(lease.getSnapshot().phase)); + await vi.advanceTimersByTimeAsync(0); + expect(lease.getSnapshot().phase).toBe('failed'); + + lease.restart(); + await vi.advanceTimersByTimeAsync(0); + expect(lease.getSnapshot().phase).toBe('streaming'); + expect(phases).toEqual(['failed', 'starting', 'streaming']); + lease.release(); + await vi.advanceTimersByTimeAsync(1000); + }); + + it('ignores a stale start resolution after the entry was released', async () => { + const client = createFakeClient(); + let resolveStart: () => void = noop; + client.simulatorStreamStart = () => + new Promise((resolve) => { + resolveStart = () => resolve({ fps: 30, scale: 0.5 }); + }); + const lease = acquireSimulatorStream(client, 'U-1', S1); + lease.release(); + await vi.advanceTimersByTimeAsync(1000); + resolveStart(); + await vi.advanceTimersByTimeAsync(0); + expect(peekSimulatorStream(client, 'U-1')).toBeNull(); + }); +}); diff --git a/packages/client/workbench/src/simulator/panel.tsx b/packages/client/workbench/src/simulator/panel.tsx new file mode 100644 index 00000000..afac227d --- /dev/null +++ b/packages/client/workbench/src/simulator/panel.tsx @@ -0,0 +1,235 @@ +import { useLinkCodeClient } from '@linkcode/client-core'; +import type { SessionId, SimulatorDevice, SimulatorStatus } from '@linkcode/schema'; +import type { SimulatorScreenPoint } from '@linkcode/ui/shell/simulator'; +import { SimulatorScreen } from '@linkcode/ui/shell/simulator'; +import { Button } from 'coss-ui/components/button'; +import { + Select, + SelectItem, + SelectPopup, + SelectTrigger, + SelectValue, +} from 'coss-ui/components/select'; +import { useEffect as useAbortableEffect } from 'foxact/use-abortable-effect'; +import { noop } from 'foxts/noop'; +import { useCallback, useRef, useState, useSyncExternalStore } from 'react'; +import { useTranslations } from 'use-intl'; +import type { SimulatorStreamLease } from './stream-registry'; +import { acquireSimulatorStream, peekSimulatorStream } from './stream-registry'; + +const BUSY_BANNER_MS = 3000; + +/** + * The right panel's Simulator section: device picker plus a live, touchable device screen. + * Interactions ride the session that started the stream (it holds the device claim), so the + * user co-drives the same device an agent is using; a claim conflict surfaces as a banner. + */ +export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): React.ReactNode { + const t = useTranslations('workbench.panel'); + const client = useLinkCodeClient(); + const [status, setStatus] = useState(null); + const [devices, setDevices] = useState(null); + const [selectedUdid, setSelectedUdid] = useState(null); + const [busy, setBusy] = useState(false); + const busyTimerRef = useRef | undefined>(undefined); + const leaseRef = useRef(null); + + useAbortableEffect( + (signal) => { + void client + .simulatorStatus() + .then((value) => { + if (!signal.aborted) setStatus(value); + }) + .catch(() => { + if (!signal.aborted) setStatus({ available: false }); + }); + void client + .simulatorList() + .then((value) => { + if (!signal.aborted) setDevices(value); + }) + .catch(() => { + if (!signal.aborted) setDevices([]); + }); + const unsubscribe = client.subscribeSimulatorDevicesChanged(setDevices); + return () => { + unsubscribe(); + clearTimeout(busyTimerRef.current); + }; + }, + [client], + ); + + const device = pickDevice(devices, selectedUdid); + const udid = device?.udid ?? null; + const booted = device?.state === 'Booted'; + const canStream = sessionId !== null && udid !== null && booted; + + const subscribe = useCallback( + (onStoreChange: () => void) => { + if (sessionId === null || udid === null || !booted) return noop; + const lease = acquireSimulatorStream(client, udid, sessionId); + leaseRef.current = lease; + const unsubscribe = lease.subscribe(onStoreChange); + return () => { + unsubscribe(); + lease.release(); + if (leaseRef.current === lease) leaseRef.current = null; + }; + }, + [client, sessionId, udid, booted], + ); + const snapshot = useSyncExternalStore(subscribe, () => + canStream ? peekSimulatorStream(client, udid) : null, + ); + /** The claim-holding session every interaction must ride. */ + const ownerSessionId = snapshot?.sessionId ?? null; + + const subscribeFrames = useCallback( + (onFrame: (jpegBase64: string) => void) => + udid === null ? noop : client.subscribeSimulatorFrames(udid, (frame) => onFrame(frame.data)), + [client, udid], + ); + + const flagBusy = useCallback(() => { + setBusy(true); + clearTimeout(busyTimerRef.current); + busyTimerRef.current = setTimeout(() => setBusy(false), BUSY_BANNER_MS); + }, []); + + if (status !== null && !status.available) { + return {t('simulatorUnavailable')}; + } + + const handleTap = (point: SimulatorScreenPoint): void => { + if (ownerSessionId === null || udid === null) return; + void client.simulatorTap(ownerSessionId, udid, point.x, point.y).catch(flagBusy); + }; + const handleSwipe = ( + from: SimulatorScreenPoint, + to: SimulatorScreenPoint, + durationMs: number, + ): void => { + if (ownerSessionId === null || udid === null) return; + void client.simulatorSwipe(ownerSessionId, udid, from, to, durationMs).catch(flagBusy); + }; + const pressButton = (button: 'home' | 'lock'): void => { + if (ownerSessionId === null || udid === null) return; + void client.simulatorButton(ownerSessionId, udid, button).catch(flagBusy); + }; + const bootDevice = (): void => { + if (sessionId === null || udid === null) return; + void client.simulatorBoot(sessionId, udid).catch(flagBusy); + }; + + const deviceItems = (devices ?? []).map((item) => ({ + value: item.udid, + label: item.runtimeName === undefined ? item.name : `${item.name} · ${item.runtimeName}`, + })); + + return ( +
+
+ {deviceItems.length > 0 && udid !== null && ( + + )} +
+ + +
+
+
+ {devices !== null && devices.length === 0 && ( + {t('simulatorNoDevices')} + )} + {device !== null && !booted && ( +
+ {device.state === 'Booting' ? ( + {t('simulatorBooting')} + ) : sessionId === null ? ( + {t('simulatorNoSession')} + ) : ( + + )} +
+ )} + {device !== null && booted && sessionId === null && ( + {t('simulatorNoSession')} + )} + {canStream && ( + {t('simulatorConnecting')} + } + className="p-2" + /> + )} + {snapshot?.phase === 'failed' && ( +
+ {t('simulatorStreamFailed')} + +
+ )} + {busy && ( +
+
+ {t('simulatorBusy')} +
+
+ )} +
+
+ ); +} + +function CenteredHint({ children }: React.PropsWithChildren): React.ReactNode { + return ( +
+ {children} +
+ ); +} + +function pickDevice( + devices: SimulatorDevice[] | null, + selectedUdid: string | null, +): SimulatorDevice | null { + if (!devices || devices.length === 0) return null; + const picked = + selectedUdid === null ? undefined : devices.find((item) => item.udid === selectedUdid); + return picked ?? devices.find((item) => item.state === 'Booted') ?? devices[0]; +} diff --git a/packages/client/workbench/src/simulator/stream-registry.ts b/packages/client/workbench/src/simulator/stream-registry.ts new file mode 100644 index 00000000..de29040d --- /dev/null +++ b/packages/client/workbench/src/simulator/stream-registry.ts @@ -0,0 +1,139 @@ +import type { LinkCodeClient } from '@linkcode/client-core'; +import type { SessionId } from '@linkcode/schema'; +import { noop } from 'foxts/noop'; + +/** The slice of `LinkCodeClient` the stream registry needs. */ +export type SimulatorStreamClient = Pick< + LinkCodeClient, + 'simulatorStreamStart' | 'simulatorStreamStop' +>; + +/** Panel-facing stream parameters: plenty of fidelity for a right-panel pane without flooding the wire. */ +const STREAM_OPTIONS = { fps: 30, scale: 0.5 } as const; + +/** Bridges the unmount→mount gap of the docked↔maximized handoff without stopping the stream. */ +const CLOSE_DELAY_MS = 250; + +export interface SimulatorStreamSnapshot { + phase: 'starting' | 'streaming' | 'failed'; + /** The session that started the stream — it holds the device claim, so interactions must ride it. */ + sessionId: SessionId; +} + +export interface SimulatorStreamLease { + getSnapshot: () => SimulatorStreamSnapshot; + subscribe: (listener: () => void) => () => void; + /** Retry the stream start after `failed`. */ + restart: () => void; + release: () => void; +} + +interface RegistryEntry { + refCount: number; + closeTimer: ReturnType | null; + snapshot: SimulatorStreamSnapshot; + listeners: Set<() => void>; +} + +const registries = new WeakMap>(); + +function getRegistry(client: SimulatorStreamClient): Map { + let registry = registries.get(client); + if (!registry) { + registry = new Map(); + registries.set(client, registry); + } + return registry; +} + +function notify(entry: RegistryEntry): void { + for (const listener of entry.listeners) listener(); +} + +function startStream( + client: SimulatorStreamClient, + registry: Map, + udid: string, + entry: RegistryEntry, +): void { + client + .simulatorStreamStart(entry.snapshot.sessionId, udid, STREAM_OPTIONS) + .then(() => { + if (registry.get(udid) !== entry) return; + entry.snapshot = { ...entry.snapshot, phase: 'streaming' }; + notify(entry); + }) + .catch(() => { + if (registry.get(udid) !== entry) return; + entry.snapshot = { ...entry.snapshot, phase: 'failed' }; + notify(entry); + }); +} + +/** + * Stream lifetime is keyed by device, not component mount: the deferred stop keeps one daemon + * stream running across panel remounts, and the last lease out stops it. Frames themselves flow + * through `client.subscribeSimulatorFrames` independently of this registry. + */ +export function acquireSimulatorStream( + client: SimulatorStreamClient, + udid: string, + sessionId: SessionId, +): SimulatorStreamLease { + const registry = getRegistry(client); + let entry = registry.get(udid); + + if (entry) { + if (entry.closeTimer !== null) { + clearTimeout(entry.closeTimer); + entry.closeTimer = null; + } + entry.refCount += 1; + } else { + const created: RegistryEntry = { + refCount: 1, + closeTimer: null, + snapshot: { phase: 'starting', sessionId }, + listeners: new Set(), + }; + registry.set(udid, created); + entry = created; + startStream(client, registry, udid, created); + } + + const leased = entry; + let released = false; + + return { + getSnapshot: () => leased.snapshot, + subscribe(listener) { + leased.listeners.add(listener); + return () => leased.listeners.delete(listener); + }, + restart() { + if (registry.get(udid) !== leased || leased.snapshot.phase !== 'failed') return; + leased.snapshot = { ...leased.snapshot, phase: 'starting' }; + notify(leased); + startStream(client, registry, udid, leased); + }, + release() { + if (released) return; + released = true; + leased.refCount -= 1; + if (leased.refCount > 0) return; + leased.closeTimer = setTimeout(() => { + if (registry.get(udid) !== leased || leased.refCount > 0) return; + registry.delete(udid); + // Best-effort: the daemon also stops the stream when the owning session drops. + void client.simulatorStreamStop(leased.snapshot.sessionId, udid).catch(noop); + }, CLOSE_DELAY_MS); + }, + }; +} + +export function peekSimulatorStream( + client: SimulatorStreamClient, + udid: string, +): SimulatorStreamSnapshot | null { + return registries.get(client)?.get(udid)?.snapshot ?? null; +} From c4772289b13b06e172efd897b07fab3d062a6f5c Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 00:50:15 +0800 Subject: [PATCH 06/26] feat(desktop,ui): simulator as an on-demand right-panel section --- .../src/__tests__/shell-state.test.ts | 78 +++++++++++++++-- .../src/renderer/src/shell/desktop-shell.tsx | 7 ++ .../src/shell/layout/right-panel-region.tsx | 15 +++- .../src/renderer/src/shell/store/model.ts | 34 +++++--- .../src/renderer/src/shell/store/store.ts | 34 +++++++- .../presentation/ui/src/shell/panels/index.ts | 1 + .../ui/src/shell/panels/section-panel.tsx | 84 ++++++++++++++++++- .../ui/src/shell/panels/vocabulary.tsx | 5 +- 8 files changed, 233 insertions(+), 25 deletions(-) diff --git a/apps/desktop/src/renderer/src/__tests__/shell-state.test.ts b/apps/desktop/src/renderer/src/__tests__/shell-state.test.ts index 1637480b..1cdcf704 100644 --- a/apps/desktop/src/renderer/src/__tests__/shell-state.test.ts +++ b/apps/desktop/src/renderer/src/__tests__/shell-state.test.ts @@ -56,9 +56,30 @@ describe('desktop shell state persistence', () => { expect(panelTypes(state.bottomPanel)).toEqual(['terminal']); }); - it('clamps latest layout values', () => { + it('falls back to defaults for a stale v2 payload instead of migrating it', () => { const state = parsePersistedDesktopShellState({ version: 2, + sidebarOpen: false, + layout: DEFAULT_LAYOUT, + expansionStack: [], + rightPanel: { + open: true, + activeSection: 'terminal', + terminalTabCount: 2, + activeTerminalTabIndex: 0, + }, + bottomPanel: { open: true, tabs: ['terminal'], activeTabIndex: 0 }, + }); + + expect(state.sidebarOpen).toBe(true); + expect(state.expansionStack).toEqual([]); + expect(state.rightPanel).toEqual(createDefaultRightPanelState()); + expect(panelTypes(state.bottomPanel)).toEqual(['terminal']); + }); + + it('clamps latest layout values', () => { + const state = parsePersistedDesktopShellState({ + version: 3, sidebarOpen: true, layout: { sidebarW: 10, rightW: 10000, bottomH: 1 }, expansionStack: [], @@ -80,7 +101,7 @@ describe('desktop shell state persistence', () => { it('rejects invalid bottom tabs and falls back when none remain', () => { const state = parsePersistedDesktopShellState({ - version: 2, + version: 3, sidebarOpen: true, layout: { sidebarW: SIDEBAR_MAX_SIZE, @@ -103,7 +124,7 @@ describe('desktop shell state persistence', () => { it('restores the right panel section and terminal tab count, clamping the active index', () => { const state = parsePersistedDesktopShellState({ - version: 2, + version: 3, sidebarOpen: true, layout: DEFAULT_LAYOUT, expansionStack: [], @@ -124,7 +145,7 @@ describe('desktop shell state persistence', () => { it('seeds a first terminal tab when restoring an open panel showing an empty terminal section', () => { const state = parsePersistedDesktopShellState({ - version: 2, + version: 3, sidebarOpen: true, layout: DEFAULT_LAYOUT, expansionStack: [], @@ -143,7 +164,7 @@ describe('desktop shell state persistence', () => { it('does not seed a terminal tab when the restored panel is closed or on another section', () => { const closed = parsePersistedDesktopShellState({ - version: 2, + version: 3, sidebarOpen: true, layout: DEFAULT_LAYOUT, expansionStack: [], @@ -158,7 +179,7 @@ describe('desktop shell state persistence', () => { expect(closed.rightPanel.terminal.tabs).toEqual([]); const otherSection = parsePersistedDesktopShellState({ - version: 2, + version: 3, sidebarOpen: true, layout: DEFAULT_LAYOUT, expansionStack: [], @@ -175,7 +196,7 @@ describe('desktop shell state persistence', () => { it('rejects an invalid right panel section and falls back to diff', () => { const state = parsePersistedDesktopShellState({ - version: 2, + version: 3, sidebarOpen: true, layout: DEFAULT_LAYOUT, expansionStack: [], @@ -191,9 +212,37 @@ describe('desktop shell state persistence', () => { expect(state.rightPanel.activeSection).toBe('diff'); }); + it('falls the active section back to diff when the simulator section lost its membership', () => { + const payload = { + version: 3, + sidebarOpen: true, + layout: DEFAULT_LAYOUT, + expansionStack: [], + rightPanel: { + open: true, + activeSection: 'simulator', + simulatorAdded: false, + terminalTabCount: 0, + activeTerminalTabIndex: 0, + }, + bottomPanel: { open: false, tabs: ['terminal'], activeTabIndex: 0 }, + }; + + const dropped = parsePersistedDesktopShellState(payload); + expect(dropped.rightPanel.activeSection).toBe('diff'); + expect(dropped.rightPanel.simulatorAdded).toBe(false); + + const kept = parsePersistedDesktopShellState({ + ...payload, + rightPanel: { ...payload.rightPanel, simulatorAdded: true }, + }); + expect(kept.rightPanel.activeSection).toBe('simulator'); + expect(kept.rightPanel.simulatorAdded).toBe(true); + }); + it('caps a corrupted terminal tab count', () => { const state = parsePersistedDesktopShellState({ - version: 2, + version: 3, sidebarOpen: true, layout: DEFAULT_LAYOUT, expansionStack: [], @@ -211,7 +260,7 @@ describe('desktop shell state persistence', () => { it('filters expansion stack to open panels', () => { const state = parsePersistedDesktopShellState({ - version: 2, + version: 3, sidebarOpen: true, layout: DEFAULT_LAYOUT, expansionStack: ['right', 'bottom', 'bottom', 'invalid'], @@ -232,6 +281,7 @@ describe('desktop shell state persistence', () => { const rightPanel: RightPanelState = { open: true, activeSection: 'browser', + simulatorAdded: true, terminal: { tabs: [createRightTerminalTab(), createRightTerminalTab()], activeTabId: null }, files: { tabs: [fileTab, createRightFileTab('/w/report.pdf')], activeTabId: fileTab.id }, browser: { url: 'http://web--app-1a2b3c.localhost:19523' }, @@ -253,6 +303,7 @@ describe('desktop shell state persistence', () => { expect(parsed.layout).toEqual(source.layout); expect(parsed.expansionStack).toEqual(['right', 'bottom']); expect(parsed.rightPanel.activeSection).toBe('browser'); + expect(parsed.rightPanel.simulatorAdded).toBe(true); expect(parsed.rightPanel.terminal.tabs).toHaveLength(2); expect(parsed.rightPanel.files.tabs.map((tab) => tab.path)).toEqual([ '/w/PLAN.md', @@ -358,6 +409,15 @@ describe('revealSectionState', () => { expect(revealed.activeSection).toBe('files'); expect(revealed.terminal.tabs).toEqual([]); }); + + it('adds the simulator section to the strip when revealing it', () => { + const panel = createDefaultRightPanelState(); + + const revealed = revealSectionState(panel, 'simulator', true); + + expect(revealed.activeSection).toBe('simulator'); + expect(revealed.simulatorAdded).toBe(true); + }); }); describe('openFileTabState', () => { diff --git a/apps/desktop/src/renderer/src/shell/desktop-shell.tsx b/apps/desktop/src/renderer/src/shell/desktop-shell.tsx index 04bac5e3..356ee148 100644 --- a/apps/desktop/src/renderer/src/shell/desktop-shell.tsx +++ b/apps/desktop/src/renderer/src/shell/desktop-shell.tsx @@ -132,6 +132,8 @@ export function DesktopShell({ toggleMaxPanel: state.toggleMaxPanel, setActiveSection: state.setActiveSection, openRightPanelSection: state.openRightPanelSection, + addRightSection: state.addRightSection, + closeRightSection: state.closeRightSection, addRightTerminalTab: state.addRightTerminalTab, closeRightTerminalTab: state.closeRightTerminalTab, setActiveRightTerminalTab: state.setActiveRightTerminalTab, @@ -251,6 +253,8 @@ export function DesktopShell({ toggleMaxPanel, setActiveSection, openRightPanelSection, + addRightSection, + closeRightSection, addRightTerminalTab, closeRightTerminalTab, setActiveRightTerminalTab, @@ -406,6 +410,7 @@ export function DesktopShell({ void; onSelectSection: (section: PanelSection) => void; + onAddSection: (section: OptionalPanelSection) => void; + onCloseSection: (section: OptionalPanelSection) => void; onSelectTerminalTab: (id: string) => void; onCloseTerminalTab: (id: string) => void; onAddTerminalTab: () => void; @@ -71,9 +79,12 @@ export function DesktopRightPanelRegion({ onOpenFile={onOpenFileTab} /> ), + simulator: , }} terminalContentTargetRef={terminalContentTargetRef} onSelectSection={onSelectSection} + onAddSection={onAddSection} + onCloseSection={onCloseSection} onSelectTerminalTab={onSelectTerminalTab} onCloseTerminalTab={onCloseTerminalTab} onAddTerminalTab={onAddTerminalTab} diff --git a/apps/desktop/src/renderer/src/shell/store/model.ts b/apps/desktop/src/renderer/src/shell/store/model.ts index e28e74b9..1870dffb 100644 --- a/apps/desktop/src/renderer/src/shell/store/model.ts +++ b/apps/desktop/src/renderer/src/shell/store/model.ts @@ -5,7 +5,7 @@ import type { PanelTab, PanelWindowType, } from '@linkcode/ui/shell/panels'; -import { PANEL_SECTIONS, PANEL_WINDOW_TYPES } from '@linkcode/ui/shell/panels'; +import { ALL_PANEL_SECTIONS, PANEL_WINDOW_TYPES } from '@linkcode/ui/shell/panels'; import { clamp } from 'foxts/clamp'; import { createFixedArray } from 'foxts/create-fixed-array'; import { z } from 'zod'; @@ -42,10 +42,12 @@ export interface RightPanelBrowserState { } /** The right panel: fixed Diff/Terminal/Browser/Files sections, with per-instance - * sub-tabs for Terminal (PTYs) and Files (viewers). */ + * sub-tabs for Terminal (PTYs) and Files (viewers), plus the on-demand Simulator section. */ export interface RightPanelState { open: boolean; activeSection: PanelSection; + /** The on-demand Simulator section is present in the strip. */ + simulatorAdded: boolean; terminal: RightPanelTerminalState; files: RightPanelFilesState; browser: RightPanelBrowserState; @@ -66,7 +68,7 @@ export interface DesktopShellState { } export interface PersistedDesktopShellState { - version: 2; + version: 3; sidebarOpen: boolean; layout: LayoutState; expansionStack: PanelSide[]; @@ -77,6 +79,7 @@ export interface PersistedDesktopShellState { export interface PersistedRightPanelState { open: boolean; activeSection: PanelSection; + simulatorAdded: boolean; terminalTabCount: number; activeTerminalTabIndex: number; fileTabPaths: string[]; @@ -90,7 +93,7 @@ export interface PersistedPanelState { activeTabIndex: number; } -export const DESKTOP_SHELL_STORAGE_KEY = 'linkcode.desktop.shell-state:v2'; +export const DESKTOP_SHELL_STORAGE_KEY = 'linkcode.desktop.shell-state:v3'; export const SIDEBAR_MIN_SIZE = 240; export const SIDEBAR_MAX_SIZE = 520; @@ -125,7 +128,7 @@ const MAX_PERSISTED_RIGHT_FILE_TABS = 20; let tabSequence = 0; const PanelSideSchema = z.enum(['right', 'bottom']); -const PanelSectionSchema = z.enum(PANEL_SECTIONS); +const PanelSectionSchema = z.enum(ALL_PANEL_SECTIONS); const PanelWindowTypeSchema = z.enum(PANEL_WINDOW_TYPES); const FiniteNumberSchema = z.number(); @@ -162,6 +165,7 @@ export function createDefaultRightPanelState(): RightPanelState { return { open: false, activeSection: 'diff', + simulatorAdded: false, terminal: { tabs: [], activeTabId: null }, files: { tabs: [], activeTabId: null }, browser: { url: null }, @@ -195,7 +199,8 @@ export function seedTerminalSection(terminal: RightPanelTerminalState): RightPan return { tabs: [tab], activeTabId: tab.id }; } -/** Brings `section` forward, seeding the terminal section's first tab when it becomes visible. */ +/** Brings `section` forward, seeding the terminal section's first tab when it becomes visible; + * revealing the on-demand simulator section also adds it to the strip. */ export function revealSectionState( panel: RightPanelState, section: PanelSection, @@ -205,6 +210,7 @@ export function revealSectionState( ...panel, open, activeSection: section, + simulatorAdded: panel.simulatorAdded || section === 'simulator', terminal: open && section === 'terminal' ? seedTerminalSection(panel.terminal) : panel.terminal, }; } @@ -301,7 +307,7 @@ export function parsePersistedDesktopShellState(value: unknown): DesktopShellSta export function serializeDesktopShellState(state: DesktopShellState): PersistedDesktopShellState { return { - version: 2, + version: 3, sidebarOpen: state.sidebarOpen, layout: normalizeLayout(state.layout), expansionStack: normalizeExpansionStack( @@ -325,7 +331,7 @@ function createPersistedShellStateSchema(): z.ZodType { return z .object({ - version: z.literal(2), + version: z.literal(3), sidebarOpen: z.boolean().catch(fallback.sidebarOpen), layout: PersistedLayoutSchema, expansionStack: PersistedExpansionStackSchema, @@ -388,6 +394,7 @@ function createPersistedRightPanelSchema(): z.ZodType { .object({ open: z.boolean().catch(fallback.open), activeSection: PanelSectionSchema.catch(fallback.activeSection), + simulatorAdded: z.boolean().catch(false), terminalTabCount: FiniteNumberSchema.int().nonnegative().catch(0), activeTerminalTabIndex: FiniteNumberSchema.int().catch(0), // Absent in pre-files persisted payloads; the catches make the section start empty. @@ -398,6 +405,7 @@ function createPersistedRightPanelSchema(): z.ZodType { .catch({ open: fallback.open, activeSection: fallback.activeSection, + simulatorAdded: false, terminalTabCount: 0, activeTerminalTabIndex: 0, fileTabPaths: [], @@ -408,6 +416,7 @@ function createPersistedRightPanelSchema(): z.ZodType { ({ open, activeSection, + simulatorAdded, terminalTabCount, activeTerminalTabIndex, fileTabPaths, @@ -426,11 +435,15 @@ function createPersistedRightPanelSchema(): z.ZodType { tabs, activeTabId: tabs.length > 0 ? tabs[activeIndex].id : null, }; + // A persisted active on-demand section without its membership falls back to the default. + const section = + activeSection === 'simulator' && !simulatorAdded ? fallback.activeSection : activeSection; return { open, - activeSection, - terminal: open && activeSection === 'terminal' ? seedTerminalSection(terminal) : terminal, + activeSection: section, + simulatorAdded, + terminal: open && section === 'terminal' ? seedTerminalSection(terminal) : terminal, files: { tabs: fileTabs, activeTabId: fileTabs.length > 0 ? fileTabs[activeFileIndex].id : null, @@ -445,6 +458,7 @@ function serializeRightPanel(panel: RightPanelState): PersistedRightPanelState { return { open: panel.open, activeSection: panel.activeSection, + simulatorAdded: panel.simulatorAdded, terminalTabCount: panel.terminal.tabs.length, activeTerminalTabIndex: clamp( panel.terminal.tabs.findIndex((tab) => tab.id === panel.terminal.activeTabId), diff --git a/apps/desktop/src/renderer/src/shell/store/store.ts b/apps/desktop/src/renderer/src/shell/store/store.ts index c619c62a..b8d19087 100644 --- a/apps/desktop/src/renderer/src/shell/store/store.ts +++ b/apps/desktop/src/renderer/src/shell/store/store.ts @@ -1,5 +1,9 @@ import { zodPersist } from '@linkcode/common/zustand'; -import type { PanelSection, PanelWindowType } from '@linkcode/ui/shell/panels'; +import type { + OptionalPanelSection, + PanelSection, + PanelWindowType, +} from '@linkcode/ui/shell/panels'; import { clamp } from 'foxts/clamp'; import { create } from 'zustand'; import type { @@ -43,6 +47,10 @@ interface DesktopShellActions { setActiveSection: (section: PanelSection) => void; /** Opens the right panel (if closed) and switches it to `section` in one step. */ openRightPanelSection: (section: PanelSection) => void; + /** Adds an on-demand section to the strip (via its + menu) and brings it forward. */ + addRightSection: (section: OptionalPanelSection) => void; + /** Removes an on-demand section, falling the active section back to the default. */ + closeRightSection: (section: OptionalPanelSection) => void; addRightTerminalTab: () => void; closeRightTerminalTab: (id: string) => void; setActiveRightTerminalTab: (id: string) => void; @@ -221,6 +229,28 @@ export const useDesktopShellStore = create()( })); }, + addRightSection(section) { + updateShellState((current) => ({ + ...current, + rightPanel: revealSectionState(current.rightPanel, section, true), + })); + }, + + closeRightSection(section) { + updateShellState((current) => { + const panel = current.rightPanel; + return { + ...current, + rightPanel: { + ...panel, + // 'simulator' is the only on-demand section today. + simulatorAdded: false, + activeSection: panel.activeSection === section ? 'diff' : panel.activeSection, + }, + }; + }); + }, + addRightTerminalTab() { const tab = createRightTerminalTab(); updateShellState((current) => ({ @@ -340,7 +370,7 @@ export const useDesktopShellStore = create()( }, { name: DESKTOP_SHELL_STORAGE_KEY, - version: 2, + version: 3, schema: PersistedDesktopShellStateSchema, partialize: serializeDesktopShellState, }, diff --git a/packages/presentation/ui/src/shell/panels/index.ts b/packages/presentation/ui/src/shell/panels/index.ts index bcae0392..b6e397f2 100644 --- a/packages/presentation/ui/src/shell/panels/index.ts +++ b/packages/presentation/ui/src/shell/panels/index.ts @@ -22,6 +22,7 @@ export type { PanelWindowType, } from './vocabulary'; export { + ALL_PANEL_SECTIONS, OPTIONAL_PANEL_SECTIONS, PANEL_SECTIONS, PANEL_WINDOW_ICONS, diff --git a/packages/presentation/ui/src/shell/panels/section-panel.tsx b/packages/presentation/ui/src/shell/panels/section-panel.tsx index cbef2219..36cd3f0b 100644 --- a/packages/presentation/ui/src/shell/panels/section-panel.tsx +++ b/packages/presentation/ui/src/shell/panels/section-panel.tsx @@ -1,11 +1,23 @@ +import { + Menu, + MenuGroup, + MenuGroupLabel, + MenuItem, + MenuPopup, + MenuTrigger, +} from 'coss-ui/components/menu'; +import { PlusIcon } from 'lucide-react'; import { useTranslations } from 'use-intl'; import { cn } from '../../lib/cn'; +import { ShellIconButton } from '../shell-control'; import type { ChromeSurface, PanelChromePortalProps } from './chrome-portal'; import { getPanelChromePlacement, PanelContextualChromePortal } from './chrome-portal'; import { PanelContextualControls } from './panel-controls'; +import { PanelTabCloseButton } from './panel-tab-close-button'; import { SectionTerminalTabStrip } from './section-terminal-tabs'; -import type { PanelSection, PanelSectionTab } from './vocabulary'; +import type { OptionalPanelSection, PanelSection, PanelSectionTab } from './vocabulary'; import { + OPTIONAL_PANEL_SECTIONS, PANEL_SECTIONS, PANEL_TAB_ACTIVE_CLASSNAME, PANEL_TAB_INACTIVE_CLASSNAME, @@ -15,6 +27,8 @@ import { export interface SectionPanelState { open: boolean; activeSection: PanelSection; + /** The on-demand Simulator section is currently present in the strip. */ + simulatorAdded: boolean; terminal: { tabs: PanelSectionTab[]; activeTabId: string | null; @@ -38,6 +52,10 @@ export interface SectionPanelRegionProps { * host portals the terminal tab stack into, so PTY instances survive docked ↔ maximized moves. */ terminalContentTargetRef: (element: HTMLDivElement | null) => void; onSelectSection: (section: PanelSection) => void; + /** Adds an on-demand section from the strip's + menu and brings it forward. */ + onAddSection: (section: OptionalPanelSection) => void; + /** Removes an on-demand section from the strip. */ + onCloseSection: (section: OptionalPanelSection) => void; onSelectTerminalTab: (id: string) => void; onCloseTerminalTab: (id: string) => void; onAddTerminalTab: () => void; @@ -59,6 +77,8 @@ export function SectionPanelRegion({ sectionContent, terminalContentTargetRef, onSelectSection, + onAddSection, + onCloseSection, onSelectTerminalTab, onCloseTerminalTab, onAddTerminalTab, @@ -105,7 +125,10 @@ export function SectionPanelRegion({ > void; + onAddSection: (section: OptionalPanelSection) => void; + onCloseSection: (section: OptionalPanelSection) => void; }): React.ReactNode { + const t = useTranslations('workbench.panel'); const tWindow = useTranslations('workbench.panel.window'); + const added: readonly OptionalPanelSection[] = simulatorAdded ? OPTIONAL_PANEL_SECTIONS : []; + const addable = OPTIONAL_PANEL_SECTIONS.filter((section) => !added.includes(section)); return (
@@ -160,6 +192,56 @@ function SectionTabStrip({ ); })} + {added.map((section) => { + const label = tWindow(section); + return ( +
+ + onCloseSection(section)} + /> +
+ ); + })} + {addable.length > 0 && ( + + + + + } + /> + + + {t('openWindow')} + {addable.map((section) => ( + onAddSection(section)}> + {PANEL_WINDOW_ICONS[section]} + {tWindow(section)} + + ))} + + + + )}
); } diff --git a/packages/presentation/ui/src/shell/panels/vocabulary.tsx b/packages/presentation/ui/src/shell/panels/vocabulary.tsx index 1625671b..f3f9addd 100644 --- a/packages/presentation/ui/src/shell/panels/vocabulary.tsx +++ b/packages/presentation/ui/src/shell/panels/vocabulary.tsx @@ -20,7 +20,10 @@ export const OPTIONAL_PANEL_SECTIONS = ['simulator'] as const; export type OptionalPanelSection = (typeof OPTIONAL_PANEL_SECTIONS)[number]; -export type PanelSection = (typeof PANEL_SECTIONS)[number]; +/** Every section the right panel can show — the fixed strip plus the on-demand extras. */ +export const ALL_PANEL_SECTIONS = [...PANEL_SECTIONS, ...OPTIONAL_PANEL_SECTIONS] as const; + +export type PanelSection = (typeof ALL_PANEL_SECTIONS)[number]; export interface PanelTab { id: string; From 15579094b4092809776b6eb14adcaf3e26fe1fdc Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 01:18:02 +0800 Subject: [PATCH 07/26] test(desktop): simulator-panel e2e (summon, device list, live stream) --- apps/desktop/e2e/simulator-panel.e2e.mts | 337 +++++++++++++++++++++++ apps/desktop/package.json | 1 + 2 files changed, 338 insertions(+) create mode 100644 apps/desktop/e2e/simulator-panel.e2e.mts diff --git a/apps/desktop/e2e/simulator-panel.e2e.mts b/apps/desktop/e2e/simulator-panel.e2e.mts new file mode 100644 index 00000000..af7e7060 --- /dev/null +++ b/apps/desktop/e2e/simulator-panel.e2e.mts @@ -0,0 +1,337 @@ +/** + * Simulator-panel E2E (CODE-397): boots an isolated daemon (with the linkcode-sim sidecar) + the + * built desktop app, summons the on-demand Simulator section from the right panel's + menu, and + * asserts the device picker reflects the host's real device list. When a booted simulator exists, + * it also starts a pi session and asserts live frames paint the canvas end-to-end. Run + * `pnpm -F @linkcode/desktop e2e:simulator` after building daemon and desktop; macOS only. + */ + +import type { ChildProcess } from 'node:child_process'; +import { execFileSync, spawn } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { noop } from 'foxts/noop'; +import { wait } from 'foxts/wait'; +import type { ElectronApplication, Page } from 'playwright-core'; +import { _electron } from 'playwright-core'; +import { io } from 'socket.io-client'; + +const require = createRequire(import.meta.url); +const desktopDir = resolve(import.meta.dirname, '..'); +const daemonDir = resolve(desktopDir, '../daemon'); +const repoRoot = resolve(desktopDir, '../..'); +const simSidecar = join(repoRoot, 'target', 'release', 'linkcode-sim'); +const electronBinary = require('electron') as unknown as string; + +const PORT = 43000 + (process.pid % 1000); + +/** Must match `WIRE_PROTOCOL_VERSION` (node can't load the raw-TS schema barrel); a mismatch is + * silently discarded by the daemon, surfacing here as the session.start timeout. */ +const WIRE_VERSION = 45; + +function fail(message: string): never { + console.error(`FAIL: ${message}`); + process.exit(1); +} + +async function waitForDaemon(): Promise { + const deadline = Date.now() + 30000; + while (Date.now() < deadline) { + try { + await fetch(`http://127.0.0.1:${PORT}/socket.io/?EIO=4&transport=polling`); + return; + } catch { + await wait(250); + } + } + fail(`daemon did not come up on port ${PORT}`); +} + +function bootedDeviceCount(): number { + try { + const raw = execFileSync('xcrun', ['simctl', 'list', 'devices', 'booted', '-j'], { + encoding: 'utf8', + }); + const parsed = JSON.parse(raw) as { devices: Record }; + return Object.values(parsed.devices).reduce((sum, list) => sum + list.length, 0); + } catch { + return 0; + } +} + +function piOnPath(): boolean { + try { + execFileSync('which', ['pi'], { stdio: 'ignore' }); + return true; + } catch { + return false; + } +} + +/** Seed the stream-claim session over the wire: the panel binds to the active thread. */ +function seedPiSession(cwd: string): Promise { + const socket = io(`http://127.0.0.1:${PORT}`, { transports: ['websocket'] }); + return new Promise((_resolve, reject) => { + const timer = setTimeout(() => { + reject(new Error('session.start timed out (stale WIRE_VERSION pin?)')); + }, 60000); + socket.on('frame', (raw: unknown) => { + const payload = (raw as { payload?: Record }).payload; + if (payload?.replyTo !== 'e2e-session') return; + clearTimeout(timer); + if (payload.kind === 'session.started' && typeof payload.sessionId === 'string') { + _resolve(payload.sessionId); + } else { + reject(new Error(`session.start failed: ${JSON.stringify(payload)}`)); + } + }); + socket.on('connect', () => { + socket.emit('frame', { + v: WIRE_VERSION, + id: `e2e-${Date.now().toString(36)}`, + ts: Date.now(), + payload: { + kind: 'session.start', + clientReqId: 'e2e-session', + opts: { kind: 'pi', cwd }, + }, + }); + }); + socket.on('connect_error', (error: Error) => { + clearTimeout(timer); + reject(error); + }); + }).finally(() => socket.close()); +} + +async function run(win: Page, chatRoot: string, deepPass: boolean): Promise { + await win + .locator('button[aria-label="Toggle side panel"]:visible') + .first() + .waitFor({ state: 'visible', timeout: 30000 }); + await win.waitForTimeout(2000); + + // The right panel starts closed on a fresh profile, leaving its chrome strip hit-testing + // behind the main pane. Toggle it open until the + trigger is actually clickable (a trial + // click probes actionability without clicking) — the open animation makes one wait racy. + const toggle = win.locator('button[aria-label="Toggle side panel"]:visible').first(); + const plus = win.locator('button[aria-label="Open window"]:visible'); + if ((await toggle.getAttribute('aria-pressed')) !== 'true') { + await toggle.click(); + await win.waitForTimeout(600); + if ((await toggle.getAttribute('aria-pressed')) !== 'true') { + fail('the right panel did not open from its chrome toggle'); + } + } + // Both panels label their + "Open window" and locator order is not stable, so probe every + // match with a trial click and keep the one that is actually actionable. + let plusTarget = null as Awaited>[number] | null; + const plusDeadline = Date.now() + 15000; + while (plusTarget === null && Date.now() < plusDeadline) { + for (const candidate of await plus.all()) { + try { + await candidate.click({ trial: true, timeout: 1000 }); + plusTarget = candidate; + break; + } catch { + // Covered instance (collapsed panel); try the next match. + } + } + if (plusTarget === null) await wait(500); + } + if (plusTarget === null) { + const shot = join(tmpdir(), `linkcode-e2e-simulator-plus-${process.pid}.png`); + await win.screenshot({ path: shot }); + console.error(`screenshot: ${shot}`); + console.error(`toggle aria-pressed: ${await toggle.getAttribute('aria-pressed')}`); + for (const candidate of await plus.all()) { + console.error(`plus box: ${JSON.stringify(await candidate.boundingBox())}`); + } + fail('the section strip + trigger never became clickable'); + } + + const plusBefore = await plus.count(); + await plusTarget.click(); + await win.getByRole('menuitem', { name: 'Simulator' }).click(); + await win.waitForTimeout(2000); + + const sectionTab = win.locator('button[aria-label="Simulator"][aria-pressed="true"]:visible'); + if ((await sectionTab.count()) === 0) fail('adding Simulator did not activate its section tab'); + console.log('simulator section summoned and active'); + + // The right panel's + trigger disappears once the only optional section is added. + const plusAfter = await win.locator('button[aria-label="Open window"]:visible').count(); + if (plusAfter !== plusBefore - 1) { + fail('the section strip + menu should hide while every optional section is added'); + } + + // The panel body must reflect the daemon's real device probe. + const picker = win.locator('[aria-label="Select a device"]'); + const noDevices = win.getByText('No simulator devices found'); + const deadline = Date.now() + 15000; + let bodyReady = false; + while (Date.now() < deadline) { + if ((await picker.count()) > 0 || (await noDevices.count()) > 0) { + bodyReady = true; + break; + } + await wait(500); + } + if (!bodyReady) fail('the panel showed neither a device picker nor the empty-list hint'); + if ((await picker.count()) > 0) { + console.log(`device picker: ${JSON.stringify(await picker.first().textContent())}`); + } else { + console.log('daemon reported no simulator devices'); + } + + const summonShot = join(tmpdir(), `linkcode-e2e-simulator-summon-${process.pid}.png`); + await win.screenshot({ path: summonShot }); + console.log(`screenshot: ${summonShot}`); + + if (deepPass && (await picker.count()) > 0) { + // Deep pass: with a session claim the booted device streams; frames must paint the canvas. + const sessionId = await seedPiSession(chatRoot); + console.log(`pi session seeded for the stream claim: ${sessionId}`); + // The wire-seeded session came from another connection; reload so the fresh session.list + // includes it. This doubles as the persisted-panel restore check: the Simulator section + // must come back open after the reload (shell-state v3). + await win.reload(); + await win + .locator('button[aria-label="Toggle side panel"]:visible') + .first() + .waitFor({ state: 'visible', timeout: 30000 }); + await win.waitForTimeout(2000); + if ((await sectionTab.count()) === 0) { + fail('the Simulator section did not restore from persisted shell state after reload'); + } + // The untitled row reads " in "; clicking it makes the thread active. + const row = win.getByText(/ in LinkCode$/).first(); + await row.waitFor({ state: 'visible', timeout: 15000 }); + await row.click(); + await win.waitForTimeout(1500); + + const canvas = win.locator('canvas:visible'); + const frameDeadline = Date.now() + 30000; + let painted: { width: number; height: number } | null = null; + while (Date.now() < frameDeadline) { + if ((await canvas.count()) > 0) { + const size = await canvas.first().evaluate((el) => { + const c = el as HTMLCanvasElement; + return { width: c.width, height: c.height }; + }); + if (size.width > 0 && size.height > 0) { + painted = size; + break; + } + } + await wait(500); + } + if (!painted) fail('no frame painted the simulator canvas within 30s'); + console.log(`canvas painted at ${painted.width}x${painted.height}`); + + const streamShot = join(tmpdir(), `linkcode-e2e-simulator-stream-${process.pid}.png`); + await win.screenshot({ path: streamShot }); + console.log(`screenshot: ${streamShot}`); + } else { + console.log('no booted simulator — skipping the live-stream pass'); + } + + // Closing the section removes it from the strip and falls back to the default section. + await win.locator('button[aria-label="Close Simulator"]:visible').first().click(); + await win.waitForTimeout(1000); + if ((await win.locator('button[aria-label="Simulator"]:visible').count()) !== 0) { + fail('closing the Simulator section did not remove its tab'); + } + if ((await win.locator('button[aria-label="Open window"]:visible').count()) !== plusBefore) { + fail('the + menu did not return after removing the section'); + } + console.log('simulator section closed; + menu restored'); +} + +async function main(): Promise { + if (process.platform !== 'darwin') fail('simulator E2E is macOS-only'); + if (!existsSync(join(daemonDir, 'dist/index.js'))) { + fail('apps/daemon/dist is missing — run `pnpm -F @linkcode/daemon build` first'); + } + if (!existsSync(join(desktopDir, 'out/main/index.js'))) { + fail('apps/desktop/out is missing — run `pnpm -F @linkcode/desktop build` first'); + } + if (!existsSync(simSidecar)) { + fail('target/release/linkcode-sim is missing — run `cargo build --release -p linkcode-sim`'); + } + + const home = mkdtempSync(join(tmpdir(), 'linkcode-e2e-home-')); + const userData = mkdtempSync(join(tmpdir(), 'linkcode-e2e-userdata-')); + const chatRoot = join(home, 'LinkCode'); + mkdirSync(chatRoot, { recursive: true }); + // A fake $HOME cannot move Electron's appData on macOS, so localStorage (the persisted shell + // state) would leak across runs through the real ~/Library. A per-run profile isolates the + // app-name universe instead — the daemon must carry the SAME profile to be discovered. + const profile = `e2e-sim-${process.pid}`; + const appSupport = join( + process.env.HOME ?? home, + 'Library', + 'Application Support', + `LinkCode Development (${profile})`, + ); + // Locale-stable text locators + no first-install import prompt over the shell. + mkdirSync(appSupport, { recursive: true }); + writeFileSync( + join(appSupport, 'settings.json'), + `${JSON.stringify({ locale: 'en', historyImportOnboardingHandled: true }, null, 2)}\n`, + ); + const deepPass = bootedDeviceCount() > 0 && piOnPath(); + console.log(`live-stream pass (booted device + pi CLI): ${deepPass ? 'yes' : 'no'}`); + + let daemon: ChildProcess | null = null; + let app: ElectronApplication | null = null; + let passed = false; + try { + daemon = spawn(process.execPath, ['dist/index.js'], { + cwd: daemonDir, + env: { + ...process.env, + HOME: home, + LINKCODE_PORT: String(PORT), + LINKCODE_PROFILE: profile, + LINKCODE_SIM_SIDECAR_PATH: simSidecar, + }, + stdio: 'ignore', + }); + await waitForDaemon(); + console.log(`daemon up on :${PORT} (HOME=${home})`); + + app = await _electron.launch({ + executablePath: electronBinary, + args: [desktopDir, `--user-data-dir=${userData}`, '--use-mock-keychain'], + env: { ...process.env, HOME: home, LINKCODE_PROFILE: profile }, + }); + + const win = await app.firstWindow(); + try { + await run(win, chatRoot, deepPass); + } catch (error) { + const shot = join(tmpdir(), `linkcode-e2e-simulator-${process.pid}.png`); + await win.screenshot({ path: shot }).catch(noop); + console.error(`screenshot: ${shot}`); + throw error; + } + passed = true; + console.log('PASS'); + } finally { + await app?.close().catch(noop); + daemon?.kill('SIGTERM'); + if (passed) { + rmSync(home, { recursive: true, force: true }); + rmSync(userData, { recursive: true, force: true }); + rmSync(appSupport, { recursive: true, force: true }); + } else { + console.error(`kept for debugging: HOME=${home} userData=${userData} appData=${appSupport}`); + process.exitCode = 1; + } + } +} + +void main(); diff --git a/apps/desktop/package.json b/apps/desktop/package.json index fb43d653..7f382333 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -19,6 +19,7 @@ "e2e:packaged": "pnpm run package:devshell && pnpm run e2e:packaged:smoke", "e2e:notifications": "node e2e/notifications.e2e.mts", "e2e:file-tree": "node e2e/file-tree.e2e.mts", + "e2e:simulator": "node e2e/simulator-panel.e2e.mts", "e2e:window-bounds": "node e2e/window-bounds.e2e.mts", "lint": "pnpm --dir ../.. exec eslint --format=sukka apps/desktop" }, From 1ed803316963eefaaa34518a3ee76baaf8ab193e Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 01:22:56 +0800 Subject: [PATCH 08/26] feat(ui): device-style bezel around the simulator screen --- .../src/shell/simulator/simulator-screen.tsx | 70 ++++++++++++++----- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index 3d5e7c2d..709f3fb3 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -41,7 +41,8 @@ export function SimulatorScreen({ const pressRef = useRef<{ pointerId: number; start: SimulatorScreenPoint; at: number } | null>( null, ); - const [hasFrame, setHasFrame] = useState(false); + /** `w / h` of the latest frame; sizes the bezel box so the screen fits without letterboxing. */ + const [frameAspect, setFrameAspect] = useState(null); useAbortableEffect( (signal) => { @@ -64,8 +65,9 @@ export function SimulatorScreen({ canvas.height = bitmap.height; } canvas.getContext('2d')?.drawImage(bitmap, 0, 0); + const aspect = `${bitmap.width} / ${bitmap.height}`; bitmap.close(); - setHasFrame(true); + setFrameAspect((previous) => (previous === aspect ? previous : aspect)); }) // A corrupt frame is dropped; the next one repaints. .catch(noop) @@ -97,10 +99,10 @@ export function SimulatorScreen({ if (press?.pointerId !== event.pointerId) return; pressRef.current = null; const end = normalizedPoint(event); - const rect = event.currentTarget.getBoundingClientRect(); + const drawn = drawnScreenRect(event.currentTarget); const travelPx = Math.hypot( - (end.x - press.start.x) * rect.width, - (end.y - press.start.y) * rect.height, + (end.x - press.start.x) * drawn.width, + (end.y - press.start.y) * drawn.height, ); if (travelPx < SWIPE_THRESHOLD_PX) onTap?.(end); else onSwipe?.(press.start, end, Math.round(performance.now() - press.at)); @@ -110,25 +112,57 @@ export function SimulatorScreen({
- { - pressRef.current = null; - }} - /> - {!hasFrame && placeholder} + {/* Device-style bezel around the live screen; appears once the first frame reports its + aspect. Deliberately theme-independent — a hardware frame is black in both themes. + The aspect-ratio + max constraints contain-fit the box inside the pane. */} +
+ { + pressRef.current = null; + }} + /> +
+ {frameAspect === null && placeholder}
); } +/** The screen bitmap's on-page box: the canvas is `object-contain`, so the drawn frame can be + * letterboxed inside the element (the bezel's fixed padding skews the content-box aspect). */ +function drawnScreenRect(canvas: HTMLCanvasElement): { + left: number; + top: number; + width: number; + height: number; +} { + const rect = canvas.getBoundingClientRect(); + if (canvas.width === 0 || canvas.height === 0) return rect; + const scale = Math.min(rect.width / canvas.width, rect.height / canvas.height); + const width = canvas.width * scale; + const height = canvas.height * scale; + return { + left: rect.left + (rect.width - width) / 2, + top: rect.top + (rect.height - height) / 2, + width, + height, + }; +} + function normalizedPoint(event: React.PointerEvent): SimulatorScreenPoint { - const rect = event.currentTarget.getBoundingClientRect(); + const drawn = drawnScreenRect(event.currentTarget); return { - x: clamp((event.clientX - rect.left) / rect.width, 0, 1), - y: clamp((event.clientY - rect.top) / rect.height, 0, 1), + x: clamp((event.clientX - drawn.left) / drawn.width, 0, 1), + y: clamp((event.clientY - drawn.top) / drawn.height, 0, 1), }; } From 9af3b834797250075ac41e47ece6f9b96beab742 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 01:43:00 +0800 Subject: [PATCH 09/26] feat(sim-sidecar): screenMask op rendering the devicetype framebuffer mask --- crates/linkcode-sim/PROTOCOL.md | 3 + crates/linkcode-sim/src/main.rs | 38 +++++ crates/linkcode-sim/src/mask.rs | 229 ++++++++++++++++++++++++++++++ crates/linkcode-sim/src/rpc.rs | 3 + crates/linkcode-sim/src/simctl.rs | 82 +++++++++++ 5 files changed, 355 insertions(+) create mode 100644 crates/linkcode-sim/src/mask.rs diff --git a/crates/linkcode-sim/PROTOCOL.md b/crates/linkcode-sim/PROTOCOL.md index ac2a2b34..d9422885 100644 --- a/crates/linkcode-sim/PROTOCOL.md +++ b/crates/linkcode-sim/PROTOCOL.md @@ -56,6 +56,9 @@ Unknown frame types are ignored. A malformed `REQUEST` fails only that request | `terminate` | `udid`, `bundleId` | `{}` | | `openUrl` | `udid`, `url` | `{}` | | `screenshot` | `udid`, `format?` (`jpeg` default, `png`) | bytes on a [`SCREENSHOT`](#screenshot-body) frame | +| `screenMask` | `udid` | transparent PNG bytes on a [`SCREENSHOT`](#screenshot-body) frame | + +`screenMask` rasterizes the devicetype bundle's `framebufferMask` PDF — the exact screen outline (corner curvature, sensor island) at the device's native pixel size — so clients can clip the framebuffer to the real device shape. Resolved at runtime from the local Xcode install (`simctl list devicetypes` → `bundlePath`); nothing Apple-owned ships with Link Code. Devicetypes without a mask (and non-macOS hosts) fail with a normal `RESULT` error and clients fall back to a generic rounding. Per-op deadlines are enforced sidecar-side (`boot` 180s, `install` 120s, `screenshot` 30s, others 60s); a child that outlives its deadline is killed and reported as `timeout`. diff --git a/crates/linkcode-sim/src/main.rs b/crates/linkcode-sim/src/main.rs index 843f3ca6..63f9a14e 100644 --- a/crates/linkcode-sim/src/main.rs +++ b/crates/linkcode-sim/src/main.rs @@ -14,6 +14,7 @@ mod capture; mod interactive; +mod mask; #[cfg(target_os = "macos")] mod private; mod proto; @@ -115,6 +116,11 @@ fn main() { bench_encode(); return; } + // Diagnostic: render a device's screen mask to a PNG file: `linkcode-sim diag-mask [out]`. + if subcommand.as_deref() == Some("diag-mask") { + diag_mask(); + return; + } let (tx, rx) = channel::(); @@ -229,6 +235,18 @@ fn serve(request: Request, tx: &Sender) { Err(e) => Err(e), } } + Op::ScreenMask { udid } => { + match mask::screen_mask(&udid).and_then(|image| { + encode_screenshot(&request_id, &image) + .map_err(|e| OpError::new(ErrorCode::Io, e.to_string())) + }) { + Ok(body) => { + send(tx, SCREENSHOT, body); + return; + } + Err(e) => Err(e), + } + } Op::Tap { udid, x, y } => interactive::tap(&udid, x, y), Op::Swipe { udid, @@ -266,6 +284,26 @@ fn send(tx: &Sender, type_byte: u8, body: Vec) { let _ = tx.send(OutMsg::Frame { type_byte, body }); } +/// Diagnostic entry: render the screen mask for a device to a PNG file. +fn diag_mask() { + let udid = std::env::args() + .nth(2) + .expect("usage: diag-mask [out.png]"); + let out = std::env::args() + .nth(3) + .unwrap_or_else(|| "mask.png".to_owned()); + match mask::screen_mask(&udid) { + Ok(bytes) => { + std::fs::write(&out, &bytes).expect("write mask png"); + eprintln!("wrote {} bytes to {out}", bytes.len()); + } + Err(error) => { + eprintln!("mask failed: {}", error.message); + std::process::exit(1); + } + } +} + fn send_error(tx: &Sender, request_id: &str, error: &OpError) { send(tx, RESULT, error_body(request_id, error)); } diff --git a/crates/linkcode-sim/src/mask.rs b/crates/linkcode-sim/src/mask.rs new file mode 100644 index 00000000..d91babcf --- /dev/null +++ b/crates/linkcode-sim/src/mask.rs @@ -0,0 +1,229 @@ +//! Per-device screen-mask rendering: the devicetype bundle ships a `framebufferMask` PDF (the +//! exact screen outline — corner curvature, sensor island), which this module rasterizes to a +//! transparent PNG so clients can clip the framebuffer to the real device shape. Public surface +//! only: `simctl list` supplies the bundle path, `plutil` reads the profile, CoreGraphics renders. + +#[cfg(not(target_os = "macos"))] +mod imp { + use crate::rpc::{ErrorCode, OpError}; + + pub fn screen_mask(_udid: &str) -> Result, OpError> { + Err(OpError::new( + ErrorCode::XcodeMissing, + "screen masks require macOS", + )) + } +} + +#[cfg(target_os = "macos")] +mod imp { + use std::ffi::c_void; + use std::path::Path; + use std::process::Command; + + use serde::Deserialize; + + use crate::rpc::{ErrorCode, OpError}; + use crate::simctl; + + /// Apple's OS-provided plist tool by absolute path (same rationale as `simctl.rs`'s XCRUN). + const PLUTIL: &str = "/usr/bin/plutil"; + + /// The devicetype profile keys the mask render needs; the profile is a binary plist. + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct Profile { + framebuffer_mask: Option, + } + + pub fn screen_mask(udid: &str) -> Result, OpError> { + let bundle = simctl::device_type_bundle_path(udid)?; + let resources = bundle.join("Contents/Resources"); + let profile = read_profile(&resources.join("profile.plist"))?; + let mask_name = profile.framebuffer_mask.ok_or_else(|| { + OpError::new( + ErrorCode::SimctlFailed, + "devicetype profile has no framebufferMask", + ) + })?; + let pdf = std::fs::read(resources.join(format!("{mask_name}.pdf"))) + .map_err(|e| OpError::new(ErrorCode::Io, format!("read framebufferMask pdf: {e}")))?; + render_pdf_to_png(&pdf) + } + + fn read_profile(path: &Path) -> Result { + let output = Command::new(PLUTIL) + .args(["-convert", "json", "-o", "-"]) + .arg(path) + .output() + .map_err(|e| OpError::new(ErrorCode::Io, format!("spawn {PLUTIL}: {e}")))?; + if !output.status.success() { + return Err(OpError::new( + ErrorCode::Io, + format!( + "{PLUTIL} failed on {}: {}", + path.display(), + String::from_utf8_lossy(&output.stderr).trim() + ), + )); + } + serde_json::from_slice(&output.stdout) + .map_err(|e| OpError::new(ErrorCode::Io, format!("parse devicetype profile: {e}"))) + } + + /// Rasterize page 1 of the mask PDF into a transparent RGBA PNG at the page's native size. + fn render_pdf_to_png(pdf: &[u8]) -> Result, OpError> { + let fail = |message: &str| OpError::new(ErrorCode::Io, message.to_owned()); + + // SAFETY: plain CF/CG object construction and drawing; every created object is released + // on all paths below, and the PDF bytes are copied into the CFData up front. + unsafe { + let data = CFDataCreate(std::ptr::null(), pdf.as_ptr(), pdf.len() as isize); + let provider = CGDataProviderCreateWithCFData(data); + let document = CGPDFDocumentCreateWithProvider(provider); + CGDataProviderRelease(provider); + CFRelease(data.cast_const()); + if document.is_null() { + return Err(fail("framebufferMask is not a readable PDF")); + } + let page = CGPDFDocumentGetPage(document, 1); + if page.is_null() { + CGPDFDocumentRelease(document); + return Err(fail("framebufferMask PDF has no pages")); + } + let media = CGPDFPageGetBoxRect(page, KCG_PDF_MEDIA_BOX); + let width = media.size.width.round().max(1.0) as usize; + let height = media.size.height.round().max(1.0) as usize; + + let space = CGColorSpaceCreateDeviceRGB(); + let context = CGBitmapContextCreate( + std::ptr::null_mut(), + width, + height, + 8, + width * 4, + space, + KCG_ALPHA_PREMULTIPLIED_LAST, + ); + CGColorSpaceRelease(space); + if context.is_null() { + CGPDFDocumentRelease(document); + return Err(fail("could not create the mask bitmap context")); + } + CGContextDrawPDFPage(context, page); + let image = CGBitmapContextCreateImage(context); + CGContextRelease(context); + CGPDFDocumentRelease(document); + if image.is_null() { + return Err(fail("could not snapshot the mask bitmap")); + } + + let png = encode_png(image); + CGImageRelease(image); + png.ok_or_else(|| fail("could not encode the mask PNG")) + } + } + + /// Encode a CGImage as PNG via ImageIO; `None` on any ImageIO failure. + unsafe fn encode_png(image: *mut c_void) -> Option> { + // SAFETY: `image` is a live CGImage owned by the caller; all CF objects created here are + // released before returning, and the destination's bytes are copied out first. + unsafe { + let out = CFDataCreateMutable(std::ptr::null(), 0); + let png_type = + CFStringCreateWithCString(std::ptr::null(), c"public.png".as_ptr(), KCF_UTF8); + let dest = CGImageDestinationCreateWithData(out, png_type, 1, std::ptr::null()); + CFRelease(png_type); + if dest.is_null() { + CFRelease(out.cast_const()); + return None; + } + CGImageDestinationAddImage(dest, image, std::ptr::null()); + let finalized = CGImageDestinationFinalize(dest); + CFRelease(dest.cast_const()); + let bytes = finalized.then(|| { + let len = CFDataGetLength(out.cast_const()) as usize; + std::slice::from_raw_parts(CFDataGetBytePtr(out.cast_const()), len).to_vec() + }); + CFRelease(out.cast_const()); + bytes + } + } + + #[repr(C)] + struct CGPoint { + x: f64, + y: f64, + } + #[repr(C)] + struct CGSize { + width: f64, + height: f64, + } + #[repr(C)] + struct CGRect { + origin: CGPoint, + size: CGSize, + } + + const KCG_PDF_MEDIA_BOX: i32 = 0; + const KCG_ALPHA_PREMULTIPLIED_LAST: u32 = 1; + const KCF_UTF8: u32 = 0x0800_0100; + + #[link(name = "CoreGraphics", kind = "framework")] + unsafe extern "C" { + fn CGColorSpaceCreateDeviceRGB() -> *mut c_void; + fn CGColorSpaceRelease(space: *mut c_void); + fn CGBitmapContextCreate( + data: *mut c_void, + width: usize, + height: usize, + bits_per_component: usize, + bytes_per_row: usize, + space: *mut c_void, + bitmap_info: u32, + ) -> *mut c_void; + fn CGBitmapContextCreateImage(context: *mut c_void) -> *mut c_void; + fn CGContextRelease(context: *mut c_void); + fn CGImageRelease(image: *mut c_void); + fn CGDataProviderCreateWithCFData(data: *mut c_void) -> *mut c_void; + fn CGDataProviderRelease(provider: *mut c_void); + fn CGPDFDocumentCreateWithProvider(provider: *mut c_void) -> *mut c_void; + fn CGPDFDocumentRelease(document: *mut c_void); + fn CGPDFDocumentGetPage(document: *mut c_void, page: usize) -> *mut c_void; + fn CGPDFPageGetBoxRect(page: *mut c_void, box_: i32) -> CGRect; + fn CGContextDrawPDFPage(context: *mut c_void, page: *mut c_void); + } + + #[link(name = "CoreFoundation", kind = "framework")] + unsafe extern "C" { + fn CFRelease(cf: *const c_void); + fn CFDataCreate(allocator: *const c_void, bytes: *const u8, length: isize) -> *mut c_void; + fn CFDataCreateMutable(allocator: *const c_void, capacity: isize) -> *mut c_void; + fn CFDataGetLength(data: *const c_void) -> isize; + fn CFDataGetBytePtr(data: *const c_void) -> *const u8; + fn CFStringCreateWithCString( + allocator: *const c_void, + cstr: *const i8, + encoding: u32, + ) -> *const c_void; + } + + #[link(name = "ImageIO", kind = "framework")] + unsafe extern "C" { + fn CGImageDestinationCreateWithData( + data: *mut c_void, + type_: *const c_void, + count: usize, + options: *const c_void, + ) -> *mut c_void; + fn CGImageDestinationAddImage( + dest: *mut c_void, + image: *mut c_void, + properties: *const c_void, + ); + fn CGImageDestinationFinalize(dest: *mut c_void) -> bool; + } +} + +pub use imp::screen_mask; diff --git a/crates/linkcode-sim/src/rpc.rs b/crates/linkcode-sim/src/rpc.rs index cd148452..91f852fe 100644 --- a/crates/linkcode-sim/src/rpc.rs +++ b/crates/linkcode-sim/src/rpc.rs @@ -53,6 +53,9 @@ pub enum Op { #[serde(default)] format: ImageFormat, }, + /// Render the devicetype's framebuffer-mask PDF (the exact screen outline) as a transparent + /// PNG; bytes come back on a `SCREENSHOT` frame. + ScreenMask { udid: String }, /// Single-finger tap at a normalised (0..1) point (private API; P1). Tap { udid: String, x: f64, y: f64 }, /// Swipe between two normalised (0..1) points over `duration_ms` (private API; P1). diff --git a/crates/linkcode-sim/src/simctl.rs b/crates/linkcode-sim/src/simctl.rs index befc5167..8e8f6208 100644 --- a/crates/linkcode-sim/src/simctl.rs +++ b/crates/linkcode-sim/src/simctl.rs @@ -144,6 +144,22 @@ pub fn screenshot(udid: &str, format: ImageFormat) -> Result, OpError> { read } +/// Resolve a device's devicetype bundle directory (`…/DeviceTypes/.simdevicetype`), +/// joining `list devices` (udid → devicetype identifier) with `list devicetypes` (identifier → +/// `bundlePath`). Public simctl surface only. +pub fn device_type_bundle_path(udid: &str) -> Result { + let devices_raw = run_ok( + simctl(["list", "-j", "devices", "available"]), + DEFAULT_TIMEOUT, + )?; + let identifier = parse_device_type_identifier(&devices_raw, udid) + .map_err(|message| OpError::new(ErrorCode::SimctlFailed, message))?; + let types_raw = run_ok(simctl(["list", "-j", "devicetypes"]), DEFAULT_TIMEOUT)?; + parse_device_type_bundle_path(&types_raw, &identifier) + .map(std::path::PathBuf::from) + .map_err(|message| OpError::new(ErrorCode::SimctlFailed, message)) +} + fn simctl<'a>(args: impl IntoIterator) -> Command { let mut cmd = apple_tool(XCRUN); cmd.arg("simctl").args(args); @@ -240,6 +256,41 @@ fn parse_launch_pid(stdout: &str) -> Option { stdout.trim().rsplit(": ").next()?.trim().parse().ok() } +fn parse_device_type_identifier(devices_json: &str, udid: &str) -> Result { + let raw: RawDeviceList = + serde_json::from_str(devices_json).map_err(|e| format!("parse device list: {e}"))?; + raw.devices + .into_values() + .flatten() + .find(|device| device.udid == udid) + .ok_or_else(|| format!("no device {udid}"))? + .device_type_identifier + .ok_or_else(|| format!("device {udid} reports no devicetype identifier")) +} + +#[derive(Deserialize)] +struct RawDeviceTypeList { + devicetypes: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct RawDeviceType { + identifier: String, + bundle_path: Option, +} + +fn parse_device_type_bundle_path(types_json: &str, identifier: &str) -> Result { + let raw: RawDeviceTypeList = + serde_json::from_str(types_json).map_err(|e| format!("parse devicetype list: {e}"))?; + raw.devicetypes + .into_iter() + .find(|entry| entry.identifier == identifier) + .ok_or_else(|| format!("no devicetype {identifier}"))? + .bundle_path + .ok_or_else(|| format!("devicetype {identifier} reports no bundle path")) +} + #[derive(Deserialize)] struct RawDeviceList { devices: std::collections::HashMap>, @@ -346,4 +397,35 @@ mod tests { assert_eq!(parse_launch_pid("com.example.app: 4242\n"), Some(4242)); assert_eq!(parse_launch_pid("garbage"), None); } + + #[test] + fn resolves_a_device_type_identifier_by_udid() { + assert_eq!( + parse_device_type_identifier(DEVICES, "AAAA").unwrap(), + "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro" + ); + assert!(parse_device_type_identifier(DEVICES, "missing").is_err()); + } + + #[test] + fn resolves_a_device_type_bundle_path() { + let types = r#"{ + "devicetypes": [ + { + "identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro", + "bundlePath": "/Library/Developer/CoreSimulator/Profiles/DeviceTypes/iPhone 17 Pro.simdevicetype", + "name": "iPhone 17 Pro" + } + ] + }"#; + assert_eq!( + parse_device_type_bundle_path( + types, + "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro" + ) + .unwrap(), + "/Library/Developer/CoreSimulator/Profiles/DeviceTypes/iPhone 17 Pro.simdevicetype" + ); + assert!(parse_device_type_bundle_path(types, "other").is_err()); + } } From e4e1faf683a9b8df1856291564a27f5f55a2f4a4 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 01:46:18 +0800 Subject: [PATCH 10/26] feat(schema,engine,client-core): simulator screen-mask wire (wire 46) --- apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts | 1 + apps/desktop/e2e/simulator-panel.e2e.mts | 2 +- packages/client/core/src/client.ts | 8 ++++++++ packages/client/core/src/client/control-channel.ts | 9 +++++++++ .../client/core/src/client/pending-registry.ts | 2 ++ packages/foundation/schema/src/wire/message.ts | 3 ++- packages/foundation/schema/src/wire/simulator.ts | 13 +++++++++++++ .../__tests__/simulator-request-handler.test.ts | 1 + .../engine/src/__tests__/simulator-service.test.ts | 1 + packages/host/engine/src/simulator/backend.ts | 2 ++ .../host/engine/src/simulator/request-handler.ts | 14 ++++++++++++++ packages/host/engine/src/simulator/service.ts | 5 +++++ packages/host/engine/src/wire/request-router.ts | 1 + packages/host/sim/src/client.ts | 7 +++++++ 14 files changed, 67 insertions(+), 2 deletions(-) diff --git a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts index 99c29e2c..0baed637 100644 --- a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts +++ b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts @@ -33,6 +33,7 @@ function fakeBackend(): SimulatorBackend { terminate: vi.fn(asyncNoop), openUrl: vi.fn(asyncNoop), screenshot: vi.fn(() => Promise.resolve(new Uint8Array([0xff, 0xd8, 0x02]))), + screenMask: vi.fn(() => Promise.resolve(new Uint8Array([0x89, 0x50]))), tap: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), diff --git a/apps/desktop/e2e/simulator-panel.e2e.mts b/apps/desktop/e2e/simulator-panel.e2e.mts index af7e7060..7a3e3c63 100644 --- a/apps/desktop/e2e/simulator-panel.e2e.mts +++ b/apps/desktop/e2e/simulator-panel.e2e.mts @@ -29,7 +29,7 @@ const PORT = 43000 + (process.pid % 1000); /** Must match `WIRE_PROTOCOL_VERSION` (node can't load the raw-TS schema barrel); a mismatch is * silently discarded by the daemon, surfacing here as the session.start timeout. */ -const WIRE_VERSION = 45; +const WIRE_VERSION = 46; function fail(message: string): never { console.error(`FAIL: ${message}`); diff --git a/packages/client/core/src/client.ts b/packages/client/core/src/client.ts index 9bad8f37..69dc196c 100644 --- a/packages/client/core/src/client.ts +++ b/packages/client/core/src/client.ts @@ -332,6 +332,9 @@ export class LinkCodeClient { case 'simulator.launched': this.pending.resolve('simulatorLaunch', p.replyTo, p.pid); break; + case 'simulator.screen-masked': + this.pending.resolve('simulatorScreenMask', p.replyTo, p.data); + break; case 'simulator.screenshotted': this.pending.resolve('simulatorScreenshot', p.replyTo, { format: p.format, data: p.data }); break; @@ -687,6 +690,11 @@ export class LinkCodeClient { return this.control.simulatorScreenshot(sessionId, udid, format); } + /** The device's screen-outline mask as base64 PNG — clip the stream to the real screen shape. */ + simulatorScreenMask(udid: string): Promise { + return this.control.simulatorScreenMask(udid); + } + simulatorTap(sessionId: SessionId, udid: string, x: number, y: number): Promise { return this.control.simulatorTap(sessionId, udid, x, y); } diff --git a/packages/client/core/src/client/control-channel.ts b/packages/client/core/src/client/control-channel.ts index 9bd90d74..99e35935 100644 --- a/packages/client/core/src/client/control-channel.ts +++ b/packages/client/core/src/client/control-channel.ts @@ -658,6 +658,15 @@ export class ControlChannel { })); } + /** Resolves with the device's screen-outline mask as base64 PNG (no session claim). */ + simulatorScreenMask(udid: string): Promise { + return this.sendCorrelated('simulatorScreenMask', (clientReqId) => ({ + kind: 'simulator.screen-mask', + clientReqId, + udid, + })); + } + simulatorTap(sessionId: SessionId, udid: string, x: number, y: number): Promise { return this.sendCorrelated('ack', (clientReqId) => ({ kind: 'simulator.tap', diff --git a/packages/client/core/src/client/pending-registry.ts b/packages/client/core/src/client/pending-registry.ts index 50afb6e4..ce47c51c 100644 --- a/packages/client/core/src/client/pending-registry.ts +++ b/packages/client/core/src/client/pending-registry.ts @@ -98,6 +98,7 @@ export interface PendingValueMap { simulatorList: SimulatorDevice[]; simulatorLaunch: number | null; simulatorScreenshot: { format: SimulatorImageFormat; data: string }; + simulatorScreenMask: string; simulatorStreamStart: { fps: number; scale: number }; } @@ -147,6 +148,7 @@ export class PendingRegistry { simulatorList: new Map(), simulatorLaunch: new Map(), simulatorScreenshot: new Map(), + simulatorScreenMask: new Map(), simulatorStreamStart: new Map(), }; diff --git a/packages/foundation/schema/src/wire/message.ts b/packages/foundation/schema/src/wire/message.ts index c97195fe..c4494bea 100644 --- a/packages/foundation/schema/src/wire/message.ts +++ b/packages/foundation/schema/src/wire/message.ts @@ -16,7 +16,8 @@ import { WirePayloadSchema } from './payload'; // 44 adds the simulator.* variants (CODE-394). // 45 adds the simulator.activity broadcast (CODE-395). // 46 adds the simulator interactive + framebuffer-stream variants (CODE-397). -export const WIRE_PROTOCOL_VERSION = 46 as const; +// 47 adds the simulator screen-mask wire (CODE-397). +export const WIRE_PROTOCOL_VERSION = 47 as const; /** Complete wire message: version + unique id + timestamp + payload. */ export const WireMessageSchema = z.object({ diff --git a/packages/foundation/schema/src/wire/simulator.ts b/packages/foundation/schema/src/wire/simulator.ts index 9018f414..f8ba700b 100644 --- a/packages/foundation/schema/src/wire/simulator.ts +++ b/packages/foundation/schema/src/wire/simulator.ts @@ -105,6 +105,19 @@ export const simulatorWireVariants = [ /** Base64-encoded image bytes. */ data: z.string(), }), + /** Read-only devicetype metadata (no session claim): the device's screen-outline mask, + * rendered host-side from the local Xcode's devicetype bundle. */ + z.object({ + kind: z.literal('simulator.screen-mask'), + clientReqId: WireRequestIdSchema, + udid, + }), + z.object({ + kind: z.literal('simulator.screen-masked'), + replyTo: WireRequestIdSchema, + /** Base64-encoded transparent PNG. */ + data: z.string(), + }), // ── Interactive control + framebuffer streaming (CODE-397; private-API, macOS host only) ── // Void commands reply with the generic `request.succeeded`/`request.failed`. Coordinates are diff --git a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts index 317f2eae..e247652a 100644 --- a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts +++ b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts @@ -32,6 +32,7 @@ function fakeBackend() { terminate: vi.fn(asyncNoop), openUrl: vi.fn(asyncNoop), screenshot: vi.fn(() => Promise.resolve(new Uint8Array([0xff, 0xd8, 0x01]))), + screenMask: vi.fn(() => Promise.resolve(new Uint8Array([0x89, 0x50]))), tap: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), diff --git a/packages/host/engine/src/__tests__/simulator-service.test.ts b/packages/host/engine/src/__tests__/simulator-service.test.ts index 41385ffc..61e92586 100644 --- a/packages/host/engine/src/__tests__/simulator-service.test.ts +++ b/packages/host/engine/src/__tests__/simulator-service.test.ts @@ -29,6 +29,7 @@ function fakeBackend(devices: SimulatorDeviceInfo[]) { terminate: vi.fn(asyncNoop), openUrl: vi.fn(asyncNoop), screenshot: vi.fn(() => Promise.resolve(new Uint8Array([0xff, 0xd8]))), + screenMask: vi.fn(() => Promise.resolve(new Uint8Array([0x89, 0x50]))), tap: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), diff --git a/packages/host/engine/src/simulator/backend.ts b/packages/host/engine/src/simulator/backend.ts index ae54d361..8b7ce3c6 100644 --- a/packages/host/engine/src/simulator/backend.ts +++ b/packages/host/engine/src/simulator/backend.ts @@ -64,6 +64,8 @@ export interface SimulatorBackend { terminate(udid: string, bundleId: string): Promise; openUrl(udid: string, url: string): Promise; screenshot(udid: string, format?: SimulatorImageFormat): Promise; + /** The device's screen-outline mask as a transparent PNG (rendered from the local Xcode). */ + screenMask(udid: string): Promise; /** Tap at a normalized (0..1) point (private HID; macOS only). */ tap(udid: string, x: number, y: number): Promise; /** Swipe between two normalized (0..1) points over `durationMs` (private HID; macOS only). */ diff --git a/packages/host/engine/src/simulator/request-handler.ts b/packages/host/engine/src/simulator/request-handler.ts index 062de56e..408bc371 100644 --- a/packages/host/engine/src/simulator/request-handler.ts +++ b/packages/host/engine/src/simulator/request-handler.ts @@ -20,6 +20,7 @@ type SimulatorRequest = Extract< | 'simulator.terminate' | 'simulator.open-url' | 'simulator.screenshot' + | 'simulator.screen-mask' | 'simulator.tap' | 'simulator.swipe' | 'simulator.button' @@ -142,6 +143,19 @@ export class SimulatorRequestHandler { ); }), ); + case 'simulator.screen-mask': + return this.withSimulators(payload.clientReqId, (simulators) => + simulatorOperation('simulator.screen-mask', 'Failed to load screen mask', async () => { + const mask = await simulators.screenMask(payload.udid); + this.transport.send( + createWireMessage({ + kind: 'simulator.screen-masked', + replyTo: payload.clientReqId, + data: Buffer.from(mask).toString('base64'), + }), + ); + }), + ); case 'simulator.tap': return this.withSimulators(payload.clientReqId, (simulators) => simulatorOperation('simulator.tap', 'Failed to tap', async () => { diff --git a/packages/host/engine/src/simulator/service.ts b/packages/host/engine/src/simulator/service.ts index 407d7230..bc2a7f3a 100644 --- a/packages/host/engine/src/simulator/service.ts +++ b/packages/host/engine/src/simulator/service.ts @@ -96,6 +96,11 @@ export class SimulatorService { return this.backend.list(); } + /** Read-only devicetype metadata (screen outline PNG); claims are not required. */ + screenMask(udid: string): Promise { + return this.backend.screenMask(udid); + } + /** Boot a device for a session, claiming it. Booting a device the user already booted claims * it without marking it reclaimable. */ async boot(sessionId: SessionId, udid: string): Promise { diff --git a/packages/host/engine/src/wire/request-router.ts b/packages/host/engine/src/wire/request-router.ts index 044fd5d5..24f682cc 100644 --- a/packages/host/engine/src/wire/request-router.ts +++ b/packages/host/engine/src/wire/request-router.ts @@ -138,6 +138,7 @@ export class WireRequestRouter { case 'simulator.terminate': case 'simulator.open-url': case 'simulator.screenshot': + case 'simulator.screen-mask': case 'simulator.tap': case 'simulator.swipe': case 'simulator.button': diff --git a/packages/host/sim/src/client.ts b/packages/host/sim/src/client.ts index 56b40015..96c58fe7 100644 --- a/packages/host/sim/src/client.ts +++ b/packages/host/sim/src/client.ts @@ -128,6 +128,13 @@ export class SimSidecarClient { return image; } + /** The device's screen-outline mask as a transparent PNG (rendered from the local Xcode). */ + async screenMask(udid: string): Promise { + const image = await this.call('screenMask', { udid }); + if (!Buffer.isBuffer(image)) throw new Error('sim sidecar sent a non-binary screen mask'); + return image; + } + /** Tap at a normalized (0..1) point (private HID; macOS only). */ async tap(udid: string, x: number, y: number): Promise { await this.call('tap', { udid, x, y }); From cbcc70eaed823c5b69b5f1b47bb7fc55e558d7f5 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 01:50:28 +0800 Subject: [PATCH 11/26] feat(workbench,ui): clip the simulator screen with the real device mask --- .../client/workbench/src/simulator/panel.tsx | 16 ++++++++++++++++ .../ui/src/shell/simulator/simulator-screen.tsx | 17 ++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/client/workbench/src/simulator/panel.tsx b/packages/client/workbench/src/simulator/panel.tsx index afac227d..7aea3d38 100644 --- a/packages/client/workbench/src/simulator/panel.tsx +++ b/packages/client/workbench/src/simulator/panel.tsx @@ -30,6 +30,8 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): const [status, setStatus] = useState(null); const [devices, setDevices] = useState(null); const [selectedUdid, setSelectedUdid] = useState(null); + /** Screen-outline masks by udid; `null` = the host has none (fall back to generic rounding). */ + const [masks, setMasks] = useState>>({}); const [busy, setBusy] = useState(false); const busyTimerRef = useRef | undefined>(undefined); const leaseRef = useRef(null); @@ -66,6 +68,19 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): const booted = device?.state === 'Booted'; const canStream = sessionId !== null && udid !== null && booted; + // Fetch bookkeeping lives in a ref (not `masks`) so the effect never loops on its own writes; + // the cache write itself is deliberately not abort-gated — a udid switch mid-fetch must still + // land the result for the next switch back. + const maskFetchedRef = useRef(new Set()); + useAbortableEffect(() => { + if (udid === null || maskFetchedRef.current.has(udid)) return; + maskFetchedRef.current.add(udid); + void client + .simulatorScreenMask(udid) + .then((data) => setMasks((prev) => ({ ...prev, [udid]: `data:image/png;base64,${data}` }))) + .catch(() => setMasks((prev) => ({ ...prev, [udid]: null }))); + }, [client, udid]); + const subscribe = useCallback( (onStoreChange: () => void) => { if (sessionId === null || udid === null || !booted) return noop; @@ -190,6 +205,7 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): subscribeFrames={subscribeFrames} onTap={handleTap} onSwipe={handleSwipe} + maskUrl={masks[udid] ?? null} placeholder={ {t('simulatorConnecting')} } diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index 709f3fb3..b78e5e97 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -19,6 +19,10 @@ export interface SimulatorScreenProps { onTap?: (point: SimulatorScreenPoint) => void; /** Drag in normalized [0,1] device coordinates with its real duration. */ onSwipe?: (from: SimulatorScreenPoint, to: SimulatorScreenPoint, durationMs: number) => void; + /** The device's real screen-outline mask (image URL); clips the stream to the exact screen + * shape. Shares the frame's aspect, so `mask-size: contain` tracks the contained bitmap. + * Absent → a generic corner rounding. */ + maskUrl?: string | null; /** Shown centered until the first frame arrives. */ placeholder?: React.ReactNode; className?: string; @@ -34,6 +38,7 @@ export function SimulatorScreen({ subscribeFrames, onTap, onSwipe, + maskUrl, placeholder, className, }: SimulatorScreenProps): React.ReactNode { @@ -124,7 +129,17 @@ export function SimulatorScreen({ > { From f5489c65f3333fe34310c1e0aebb308cd00e325c Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 02:05:51 +0800 Subject: [PATCH 12/26] feat(ui): composite a realistic device chassis in canvas native space --- apps/desktop/e2e/simulator-panel.e2e.mts | 71 ++-- .../src/shell/simulator/simulator-screen.tsx | 302 ++++++++++++++---- 2 files changed, 289 insertions(+), 84 deletions(-) diff --git a/apps/desktop/e2e/simulator-panel.e2e.mts b/apps/desktop/e2e/simulator-panel.e2e.mts index 7a3e3c63..21b9846a 100644 --- a/apps/desktop/e2e/simulator-panel.e2e.mts +++ b/apps/desktop/e2e/simulator-panel.e2e.mts @@ -49,15 +49,17 @@ async function waitForDaemon(): Promise { fail(`daemon did not come up on port ${PORT}`); } -function bootedDeviceCount(): number { +function bootedUdids(): string[] { try { const raw = execFileSync('xcrun', ['simctl', 'list', 'devices', 'booted', '-j'], { encoding: 'utf8', }); - const parsed = JSON.parse(raw) as { devices: Record }; - return Object.values(parsed.devices).reduce((sum, list) => sum + list.length, 0); + const parsed = JSON.parse(raw) as { devices: Record> }; + return Object.values(parsed.devices) + .flat() + .map((device) => device.udid); } catch { - return 0; + return []; } } @@ -70,33 +72,27 @@ function piOnPath(): boolean { } } -/** Seed the stream-claim session over the wire: the panel binds to the active thread. */ -function seedPiSession(cwd: string): Promise { +/** One correlated request over a fresh wire connection; resolves with the reply payload. */ +function wireRequest( + payload: Record & { clientReqId: string }, +): Promise> { const socket = io(`http://127.0.0.1:${PORT}`, { transports: ['websocket'] }); - return new Promise((_resolve, reject) => { + return new Promise>((_resolve, reject) => { const timer = setTimeout(() => { - reject(new Error('session.start timed out (stale WIRE_VERSION pin?)')); + reject(new Error(`${String(payload.kind)} timed out (stale WIRE_VERSION pin?)`)); }, 60000); socket.on('frame', (raw: unknown) => { - const payload = (raw as { payload?: Record }).payload; - if (payload?.replyTo !== 'e2e-session') return; + const reply = (raw as { payload?: Record }).payload; + if (reply?.replyTo !== payload.clientReqId) return; clearTimeout(timer); - if (payload.kind === 'session.started' && typeof payload.sessionId === 'string') { - _resolve(payload.sessionId); - } else { - reject(new Error(`session.start failed: ${JSON.stringify(payload)}`)); - } + _resolve(reply); }); socket.on('connect', () => { socket.emit('frame', { v: WIRE_VERSION, id: `e2e-${Date.now().toString(36)}`, ts: Date.now(), - payload: { - kind: 'session.start', - clientReqId: 'e2e-session', - opts: { kind: 'pi', cwd }, - }, + payload, }); }); socket.on('connect_error', (error: Error) => { @@ -106,6 +102,19 @@ function seedPiSession(cwd: string): Promise { }).finally(() => socket.close()); } +/** Seed the stream-claim session over the wire: the panel binds to the active thread. */ +async function seedPiSession(cwd: string): Promise { + const reply = await wireRequest({ + kind: 'session.start', + clientReqId: 'e2e-session', + opts: { kind: 'pi', cwd }, + }); + if (reply.kind === 'session.started' && typeof reply.sessionId === 'string') { + return reply.sessionId; + } + throw new Error(`session.start failed: ${JSON.stringify(reply)}`); +} + async function run(win: Page, chatRoot: string, deepPass: boolean): Promise { await win .locator('button[aria-label="Toggle side panel"]:visible') @@ -234,6 +243,26 @@ async function run(win: Page, chatRoot: string, deepPass: boolean): Promise { join(appSupport, 'settings.json'), `${JSON.stringify({ locale: 'en', historyImportOnboardingHandled: true }, null, 2)}\n`, ); - const deepPass = bootedDeviceCount() > 0 && piOnPath(); + const deepPass = bootedUdids().length > 0 && piOnPath(); console.log(`live-stream pass (booted device + pi CLI): ${deepPass ? 'yes' : 'no'}`); let daemon: ChildProcess | null = null; diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index b78e5e97..6317427e 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -12,6 +12,27 @@ export interface SimulatorScreenPoint { /** Below this pointer travel (in element px) a press counts as a tap, not a swipe. */ const SWIPE_THRESHOLD_PX = 8; +// Chassis proportions as fractions of the native screen width, matched against Simulator.app's +// iPhone chrome: a thin black display band inside a titanium rim, with side-button bumps. +/** Screen edge → chassis outer edge (black band + rim). */ +const PAD_FRACTION = 0.028; +/** The titanium band's thickness (outermost part of the pad). */ +const RIM_FRACTION = 0.011; +/** How far the side buttons protrude beyond the chassis. */ +const BUTTON_DEPTH_FRACTION = 0.009; +/** Side buttons as `[top, height]` fractions of the native screen height. */ +const LEFT_BUTTONS: ReadonlyArray = [ + [0.245, 0.045], + [0.315, 0.1], + [0.425, 0.1], +]; +const RIGHT_BUTTONS: ReadonlyArray = [[0.355, 0.175]]; +/** Fallback screen corner radius (fraction of width) when the host has no mask to measure. */ +const FALLBACK_CORNER_FRACTION = 0.11; + +const RIM_COLOR = '#3a3a3c'; +const BEZEL_COLOR = '#000000'; + export interface SimulatorScreenProps { /** Feed of JPEG frames (base64, no data-URL prefix); returns the unsubscribe. */ subscribeFrames: (onFrame: (jpegBase64: string) => void) => () => void; @@ -19,20 +40,31 @@ export interface SimulatorScreenProps { onTap?: (point: SimulatorScreenPoint) => void; /** Drag in normalized [0,1] device coordinates with its real duration. */ onSwipe?: (from: SimulatorScreenPoint, to: SimulatorScreenPoint, durationMs: number) => void; - /** The device's real screen-outline mask (image URL); clips the stream to the exact screen - * shape. Shares the frame's aspect, so `mask-size: contain` tracks the contained bitmap. - * Absent → a generic corner rounding. */ + /** The device's real screen-outline mask (image URL, framebuffer-sized): clips the stream to + * the exact screen shape, and its measured corner radius keeps the chassis curve concentric. + * Absent → a generic rounding. */ maskUrl?: string | null; /** Shown centered until the first frame arrives. */ placeholder?: React.ReactNode; className?: string; } +/** Everything the compositor knows about one device feed, in native framebuffer pixels. */ +interface PaintState { + mask: ImageBitmap | null; + /** Screen corner radius measured from the mask's alpha (native px); null until measured. */ + cornerRadius: number | null; + /** Last decoded frame, retained so a late-arriving mask can recomposite it. */ + frame: ImageBitmap | null; + /** Screen-sized scratch layer the mask is composited on before drawing into the chassis. */ + screenLayer: OffscreenCanvas | null; +} + /** - * Live device screen: paints an MJPEG-style frame feed onto a canvas (latest-wins — a frame - * arriving while the previous one decodes replaces it) and maps pointer presses back to - * normalized tap/swipe callbacks. Pure presentation: the feed and the input sinks are props. - * Key this component by device so a device switch resets the painted frame. + * Live device screen composited like the real machine: the chassis (bezel band + concentric + * outer corners) and the mask-clipped framebuffer are painted together in native pixel space, + * so every proportion scales with the panel exactly. Latest-wins frame decode; pointer presses + * map back to normalized tap/swipe callbacks. Key this component by device. */ export function SimulatorScreen({ subscribeFrames, @@ -43,36 +75,40 @@ export function SimulatorScreen({ className, }: SimulatorScreenProps): React.ReactNode { const canvasRef = useRef(null); + const paintRef = useRef({ + mask: null, + cornerRadius: null, + frame: null, + screenLayer: null, + }); const pressRef = useRef<{ pointerId: number; start: SimulatorScreenPoint; at: number } | null>( null, ); - /** `w / h` of the latest frame; sizes the bezel box so the screen fits without letterboxing. */ - const [frameAspect, setFrameAspect] = useState(null); + /** `w / h` of the whole painted device (screen + bezel); sizes the canvas box. */ + const [deviceAspect, setDeviceAspect] = useState(null); useAbortableEffect( (signal) => { + const state = paintRef.current; let latest: string | null = null; let decoding = false; const drawNext = (): void => { if (decoding || latest === null || signal.aborted) return; - const frame = latest; + const encoded = latest; latest = null; decoding = true; - void decodeJpegFrame(frame) + void createImageBitmap(new Blob([base64Bytes(encoded)], { type: 'image/jpeg' })) .then((bitmap) => { - const canvas = canvasRef.current; - if (signal.aborted || canvas === null) { + if (signal.aborted || canvasRef.current === null) { bitmap.close(); return; } - if (canvas.width !== bitmap.width || canvas.height !== bitmap.height) { - canvas.width = bitmap.width; - canvas.height = bitmap.height; - } - canvas.getContext('2d')?.drawImage(bitmap, 0, 0); - const aspect = `${bitmap.width} / ${bitmap.height}`; - bitmap.close(); - setFrameAspect((previous) => (previous === aspect ? previous : aspect)); + state.frame?.close(); + state.frame = bitmap; + paintDevice(canvasRef.current, state); + const canvas = canvasRef.current; + const aspect = `${canvas.width} / ${canvas.height}`; + setDeviceAspect((previous) => (previous === aspect ? previous : aspect)); }) // A corrupt frame is dropped; the next one repaints. .catch(noop) @@ -81,14 +117,48 @@ export function SimulatorScreen({ drawNext(); }); }; - return subscribeFrames((frame) => { + const unsubscribe = subscribeFrames((frame) => { latest = frame; drawNext(); }); + return () => { + unsubscribe(); + state.frame?.close(); + state.frame = null; + }; }, [subscribeFrames], ); + useAbortableEffect( + (signal) => { + if (maskUrl == null) return; + const state = paintRef.current; + void fetch(maskUrl, { signal }) + .then(decodeResponse) + .then((bitmap) => { + if (signal.aborted) { + bitmap.close(); + return; + } + state.mask?.close(); + state.mask = bitmap; + state.cornerRadius = measureCornerRadius(bitmap); + // Recomposite the held frame so the mask applies without waiting for the next one. + if (canvasRef.current !== null && state.frame !== null) { + paintDevice(canvasRef.current, state); + } + }) + .catch(noop); + return () => { + state.mask?.close(); + state.mask = null; + state.cornerRadius = null; + }; + }, + [maskUrl], + ); + const handlePointerDown = (event: React.PointerEvent): void => { if (event.button !== 0) return; event.currentTarget.setPointerCapture(event.pointerId); @@ -104,10 +174,10 @@ export function SimulatorScreen({ if (press?.pointerId !== event.pointerId) return; pressRef.current = null; const end = normalizedPoint(event); - const drawn = drawnScreenRect(event.currentTarget); + const screen = screenRectOnPage(event.currentTarget); const travelPx = Math.hypot( - (end.x - press.start.x) * drawn.width, - (end.y - press.start.y) * drawn.height, + (end.x - press.start.x) * screen.width, + (end.y - press.start.y) * screen.height, ); if (travelPx < SWIPE_THRESHOLD_PX) onTap?.(end); else onSwipe?.(press.start, end, Math.round(performance.now() - press.at)); @@ -117,44 +187,120 @@ export function SimulatorScreen({
- {/* Device-style bezel around the live screen; appears once the first frame reports its - aspect. Deliberately theme-independent — a hardware frame is black in both themes. - The aspect-ratio + max constraints contain-fit the box inside the pane. */} -
- { - pressRef.current = null; - }} - /> -
- {frameAspect === null && placeholder} + style={deviceAspect === null ? undefined : { aspectRatio: deviceAspect }} + onPointerDown={handlePointerDown} + onPointerUp={handlePointerUp} + onPointerCancel={() => { + pressRef.current = null; + }} + /> + {deviceAspect === null && placeholder}
); } -/** The screen bitmap's on-page box: the canvas is `object-contain`, so the drawn frame can be - * letterboxed inside the element (the bezel's fixed padding skews the content-box aspect). */ -function drawnScreenRect(canvas: HTMLCanvasElement): { +/** Paint the whole device in native pixels: titanium rim, black display band, side-button bumps, + * then the mask-clipped frame. The chassis corners stay concentric with the screen corners. */ +function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { + const frame = state.frame; + if (frame === null) return; + const pad = Math.round(frame.width * PAD_FRACTION); + const rim = Math.round(frame.width * RIM_FRACTION); + const buttonDepth = Math.round(frame.width * BUTTON_DEPTH_FRACTION); + const width = frame.width + 2 * pad + 2 * buttonDepth; + const height = frame.height + 2 * pad; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + const context = canvas.getContext('2d'); + if (context === null) return; + + const screenRadius = state.cornerRadius ?? frame.width * FALLBACK_CORNER_FRACTION; + context.clearRect(0, 0, width, height); + + // Side buttons first, so the chassis paints over their inner halves. + context.fillStyle = RIM_COLOR; + const buttonWidth = buttonDepth * 3; + for (const [top, size] of LEFT_BUTTONS) { + context.beginPath(); + context.roundRect(0, pad + top * frame.height, buttonWidth, size * frame.height, buttonDepth); + context.fill(); + } + for (const [top, size] of RIGHT_BUTTONS) { + context.beginPath(); + context.roundRect( + width - buttonWidth, + pad + top * frame.height, + buttonWidth, + size * frame.height, + buttonDepth, + ); + context.fill(); + } + + // Titanium rim, then the black display band inside it — both concentric with the screen. + context.beginPath(); + context.roundRect(buttonDepth, 0, width - 2 * buttonDepth, height, screenRadius + pad); + context.fillStyle = RIM_COLOR; + context.fill(); + context.beginPath(); + context.roundRect( + buttonDepth + rim, + rim, + width - 2 * buttonDepth - 2 * rim, + height - 2 * rim, + screenRadius + pad - rim, + ); + context.fillStyle = BEZEL_COLOR; + context.fill(); + + // Screen: composite the frame against the mask on a screen-sized layer, then inset it. + let layer = state.screenLayer; + if (layer?.width !== frame.width || layer.height !== frame.height) { + layer = new OffscreenCanvas(frame.width, frame.height); + state.screenLayer = layer; + } + const layerContext = layer.getContext('2d'); + if (layerContext === null) return; + layerContext.clearRect(0, 0, layer.width, layer.height); + if (state.mask === null) { + layerContext.save(); + layerContext.beginPath(); + layerContext.roundRect(0, 0, layer.width, layer.height, screenRadius); + layerContext.clip(); + layerContext.drawImage(frame, 0, 0); + layerContext.restore(); + } else { + layerContext.drawImage(frame, 0, 0); + layerContext.globalCompositeOperation = 'destination-in'; + layerContext.drawImage(state.mask, 0, 0, layer.width, layer.height); + layerContext.globalCompositeOperation = 'source-over'; + } + context.drawImage(layer, buttonDepth + pad, pad); +} + +/** First fully-opaque row along the mask's left edge = the screen's corner radius (native px). */ +function measureCornerRadius(mask: ImageBitmap): number | null { + const probe = new OffscreenCanvas(1, mask.height); + const context = probe.getContext('2d'); + if (context === null) return null; + context.drawImage(mask, 0, 0); + const alpha = context.getImageData(0, 0, 1, mask.height).data; + for (let y = 0; y < mask.height; y += 1) { + if (alpha[y * 4 + 3] > 127) return y; + } + return null; +} + +/** The painted device's on-page box (the canvas is `object-contain`, so it may be letterboxed). */ +function deviceRectOnPage(canvas: HTMLCanvasElement): { left: number; top: number; width: number; @@ -173,15 +319,45 @@ function drawnScreenRect(canvas: HTMLCanvasElement): { }; } +/** The screen's on-page box: the device box inset by the chassis band and button margin. */ +function screenRectOnPage(canvas: HTMLCanvasElement): { + left: number; + top: number; + width: number; + height: number; +} { + const device = deviceRectOnPage(canvas); + if (canvas.width === 0) return device; + // Recover the native insets the painter rounded from the screen width. + const approxScreenWidth = canvas.width / (1 + 2 * PAD_FRACTION + 2 * BUTTON_DEPTH_FRACTION); + const pad = Math.round(approxScreenWidth * PAD_FRACTION); + const buttonDepth = Math.round(approxScreenWidth * BUTTON_DEPTH_FRACTION); + const scale = device.width / canvas.width; + return { + left: device.left + (pad + buttonDepth) * scale, + top: device.top + pad * scale, + width: (canvas.width - 2 * pad - 2 * buttonDepth) * scale, + height: (canvas.height - 2 * pad) * scale, + }; +} + function normalizedPoint(event: React.PointerEvent): SimulatorScreenPoint { - const drawn = drawnScreenRect(event.currentTarget); + const screen = screenRectOnPage(event.currentTarget); return { - x: clamp((event.clientX - drawn.left) / drawn.width, 0, 1), - y: clamp((event.clientY - drawn.top) / drawn.height, 0, 1), + x: clamp((event.clientX - screen.left) / screen.width, 0, 1), + y: clamp((event.clientY - screen.top) / screen.height, 0, 1), }; } -function decodeJpegFrame(base64: string): Promise { - const bytes = Uint8Array.from(atob(base64), (char) => char.codePointAt(0) ?? 0); - return createImageBitmap(new Blob([bytes], { type: 'image/jpeg' })); +function base64Bytes(base64: string): Uint8Array { + const raw = atob(base64); + const bytes = new Uint8Array(raw.length); + for (let index = 0; index < raw.length; index += 1) { + bytes[index] = raw.codePointAt(index) ?? 0; + } + return bytes; +} + +function decodeResponse(response: Response): Promise { + return response.blob().then((blob) => createImageBitmap(blob)); } From 3b76326aca585a842524fcf287987525ace29131 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 05:37:36 +0800 Subject: [PATCH 13/26] fix(ui): grow the chassis from the real mask for even band and matching curvature --- .../src/shell/simulator/simulator-screen.tsx | 120 ++++++++++++------ 1 file changed, 84 insertions(+), 36 deletions(-) diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index 6317427e..e136759c 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -52,12 +52,13 @@ export interface SimulatorScreenProps { /** Everything the compositor knows about one device feed, in native framebuffer pixels. */ interface PaintState { mask: ImageBitmap | null; - /** Screen corner radius measured from the mask's alpha (native px); null until measured. */ - cornerRadius: number | null; /** Last decoded frame, retained so a late-arriving mask can recomposite it. */ frame: ImageBitmap | null; /** Screen-sized scratch layer the mask is composited on before drawing into the chassis. */ screenLayer: OffscreenCanvas | null; + /** Cached chassis artwork (rim + display band), rebuilt when the geometry key changes. */ + chassis: OffscreenCanvas | null; + chassisKey: string; } /** @@ -77,9 +78,10 @@ export function SimulatorScreen({ const canvasRef = useRef(null); const paintRef = useRef({ mask: null, - cornerRadius: null, frame: null, screenLayer: null, + chassis: null, + chassisKey: '', }); const pressRef = useRef<{ pointerId: number; start: SimulatorScreenPoint; at: number } | null>( null, @@ -143,7 +145,6 @@ export function SimulatorScreen({ } state.mask?.close(); state.mask = bitmap; - state.cornerRadius = measureCornerRadius(bitmap); // Recomposite the held frame so the mask applies without waiting for the next one. if (canvasRef.current !== null && state.frame !== null) { paintDevice(canvasRef.current, state); @@ -153,7 +154,6 @@ export function SimulatorScreen({ return () => { state.mask?.close(); state.mask = null; - state.cornerRadius = null; }; }, [maskUrl], @@ -205,13 +205,13 @@ export function SimulatorScreen({ ); } -/** Paint the whole device in native pixels: titanium rim, black display band, side-button bumps, - * then the mask-clipped frame. The chassis corners stay concentric with the screen corners. */ +/** Paint the whole device in native pixels: side-button bumps, then the cached chassis (rim + + * display band, both grown from the real mask so the band stays even around the corners), then + * the mask-clipped frame. */ function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { const frame = state.frame; if (frame === null) return; const pad = Math.round(frame.width * PAD_FRACTION); - const rim = Math.round(frame.width * RIM_FRACTION); const buttonDepth = Math.round(frame.width * BUTTON_DEPTH_FRACTION); const width = frame.width + 2 * pad + 2 * buttonDepth; const height = frame.height + 2 * pad; @@ -221,8 +221,6 @@ function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { } const context = canvas.getContext('2d'); if (context === null) return; - - const screenRadius = state.cornerRadius ?? frame.width * FALLBACK_CORNER_FRACTION; context.clearRect(0, 0, width, height); // Side buttons first, so the chassis paints over their inner halves. @@ -245,21 +243,7 @@ function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { context.fill(); } - // Titanium rim, then the black display band inside it — both concentric with the screen. - context.beginPath(); - context.roundRect(buttonDepth, 0, width - 2 * buttonDepth, height, screenRadius + pad); - context.fillStyle = RIM_COLOR; - context.fill(); - context.beginPath(); - context.roundRect( - buttonDepth + rim, - rim, - width - 2 * buttonDepth - 2 * rim, - height - 2 * rim, - screenRadius + pad - rim, - ); - context.fillStyle = BEZEL_COLOR; - context.fill(); + context.drawImage(buildChassis(state, frame), buttonDepth, 0); // Screen: composite the frame against the mask on a screen-sized layer, then inset it. let layer = state.screenLayer; @@ -273,7 +257,7 @@ function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { if (state.mask === null) { layerContext.save(); layerContext.beginPath(); - layerContext.roundRect(0, 0, layer.width, layer.height, screenRadius); + layerContext.roundRect(0, 0, layer.width, layer.height, frame.width * FALLBACK_CORNER_FRACTION); layerContext.clip(); layerContext.drawImage(frame, 0, 0); layerContext.restore(); @@ -286,17 +270,81 @@ function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { context.drawImage(layer, buttonDepth + pad, pad); } -/** First fully-opaque row along the mask's left edge = the screen's corner radius (native px). */ -function measureCornerRadius(mask: ImageBitmap): number | null { - const probe = new OffscreenCanvas(1, mask.height); - const context = probe.getContext('2d'); - if (context === null) return null; - context.drawImage(mask, 0, 0); - const alpha = context.getImageData(0, 0, 1, mask.height).data; - for (let y = 0; y < mask.height; y += 1) { - if (alpha[y * 4 + 3] > 127) return y; +/** + * The chassis artwork: the titanium rim and the black display band, built by morphologically + * dilating the real screen mask (stamping it along a circle of offsets). Growing the true shape + * keeps the band width even the whole way around the corner and the outer curvature in the same + * family as Apple's continuous-curvature screen corners — a `roundRect`'s circular arcs visibly + * diverge from them. Cached per frame-size + mask identity; without a mask a rounded rect stands + * in for the shape. + */ +function buildChassis(state: PaintState, frame: ImageBitmap): OffscreenCanvas { + const pad = Math.round(frame.width * PAD_FRACTION); + const rim = Math.round(frame.width * RIM_FRACTION); + const width = frame.width + 2 * pad; + const height = frame.height + 2 * pad; + const key = `${width}x${height}:${state.mask === null ? 'fallback' : 'mask'}`; + if (state.chassis !== null && state.chassisKey === key) return state.chassis; + + const chassis = new OffscreenCanvas(width, height); + const context = chassis.getContext('2d'); + if (context === null) return chassis; + context.clearRect(0, 0, width, height); + const rimShape = growScreenShape(state.mask, frame, pad); + const bandShape = growScreenShape(state.mask, frame, pad - rim); + colorize(rimShape, RIM_COLOR); + colorize(bandShape, BEZEL_COLOR); + context.drawImage(rimShape, 0, 0); + context.drawImage(bandShape, rim, rim); + state.chassis = chassis; + state.chassisKey = key; + return chassis; +} + +/** The screen shape expanded outward by `grow` px: the mask stamped along a circle of offsets + * (morphological dilation); a rounded rect when no mask exists. */ +function growScreenShape( + mask: ImageBitmap | null, + frame: ImageBitmap, + grow: number, +): OffscreenCanvas { + const shape = new OffscreenCanvas(frame.width + 2 * grow, frame.height + 2 * grow); + const context = shape.getContext('2d'); + if (context === null) return shape; + if (mask === null) { + context.beginPath(); + context.roundRect( + 0, + 0, + shape.width, + shape.height, + frame.width * FALLBACK_CORNER_FRACTION + grow, + ); + context.fill(); + return shape; } - return null; + const STAMPS = 24; + for (let step = 0; step < STAMPS; step += 1) { + const angle = (2 * Math.PI * step) / STAMPS; + context.drawImage( + mask, + grow + grow * Math.cos(angle), + grow + grow * Math.sin(angle), + frame.width, + frame.height, + ); + } + return shape; +} + +/** Replace every opaque pixel of `shape` with `color` in place. */ +function colorize(shape: OffscreenCanvas, color: string): void { + const context = shape.getContext('2d'); + if (context === null) return; + context.globalCompositeOperation = 'source-in'; + context.fillStyle = color; + context.fillRect(0, 0, shape.width, shape.height); + context.globalCompositeOperation = 'source-over'; } /** The painted device's on-page box (the canvas is `object-contain`, so it may be letterboxed). */ From c0aee6dfc4e794fd047bbc65d9737a3389f29fdb Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 05:52:36 +0800 Subject: [PATCH 14/26] feat(sim-sidecar): hardware H.264 streaming via VideoToolbox (zero-copy IOSurface) --- crates/linkcode-sim/PROTOCOL.md | 8 +- crates/linkcode-sim/src/capture.rs | 188 +++++++++-- crates/linkcode-sim/src/interactive.rs | 55 +++- crates/linkcode-sim/src/main.rs | 4 +- crates/linkcode-sim/src/private/mod.rs | 2 + crates/linkcode-sim/src/private/screen.rs | 36 ++ crates/linkcode-sim/src/private/vt.rs | 381 ++++++++++++++++++++++ crates/linkcode-sim/src/proto.rs | 18 + crates/linkcode-sim/src/rpc.rs | 25 +- 9 files changed, 686 insertions(+), 31 deletions(-) create mode 100644 crates/linkcode-sim/src/private/vt.rs diff --git a/crates/linkcode-sim/PROTOCOL.md b/crates/linkcode-sim/PROTOCOL.md index d9422885..ae240a4f 100644 --- a/crates/linkcode-sim/PROTOCOL.md +++ b/crates/linkcode-sim/PROTOCOL.md @@ -73,11 +73,17 @@ Off macOS, or when SimulatorKit is unavailable, every P1 op fails with `xcodeMis | `tap` | `udid`, `x`, `y` (normalised 0..1) | `{}` | | `swipe` | `udid`, `x0`, `y0`, `x1`, `y1`, `durationMs?` | `{}` | | `button` | `udid`, `button` (`home`/`lock`) | `{}` | -| `streamStart` | `udid`, `fps?` (60), `quality?` (0.6), `scale?` (1.0) | `{ streaming, fps, scale }` — JPEG frames then arrive on `STREAM_FRAME`s | +| `streamStart` | `udid`, `fps?` (60), `quality?` (0.6), `scale?` (1.0), `codec?` (`jpeg` default, `h264`) | `{ streaming, fps, scale, codec }` — frames then arrive on `STREAM_FRAME`s (jpeg) or `STREAM_FRAME_H264`s | | `streamStop` | `udid` | `{}` | `scale` (0.1..1.0) downscales each frame before JPEG encode: at native resolution the encode bounds the frame rate near ~55 fps, so `scale` below 1.0 both lifts the achievable rate toward the display's 60 Hz and cuts bandwidth (e.g. `0.5` ≈ one third the bytes). `tap`/`swipe` stay in normalized 0..1 coordinates, so a downscaled stream needs no coordinate adjustment. +`codec: h264` switches the stream to hardware H.264 (VideoToolbox): the retained framebuffer `IOSurface` is wrapped in a `CVPixelBuffer` and encoded on the media engine — pixels never enter CPU memory — at **native resolution** (`scale`/`quality` are ignored; bitrate carries the bandwidth budget, ~10-25× below the JPEG stream). Access units are Annex-B with SPS/PPS prepended on keyframes (≤2s apart) — exactly what a WebCodecs `VideoDecoder` configured without a `description` consumes. Delivery is **ordered and lossless** (deltas depend on each other); if the sidecar must drop (stalled consumer), it drops through the next keyframe. If the private worker gives up, the stream degrades to slow simctl JPEG `STREAM_FRAME`s — each wire frame declares its own encoding, so a mixed stream stays decodable. + +### `STREAM_FRAME_H264` body (`0x84`) + +`[u16 LE udid_len][udid][u8 key][Annex-B access unit]` — `key` is `1` on sync frames. + Input (`tap`/`swipe`/`button`) runs in the sidecar's main process via a per-udid warmed HID client and is stable. Framebuffer streaming runs in a **crash-isolated worker subprocess**: the sidecar spawns it, reads its frames, and respawns it on the intermittent hard crashes of the private framebuffer path. If the worker crash-loops and gives up, the stream degrades to `simctl io screenshot` frames — slower, but frames never stop and the sidecar never crashes. ## Result diff --git a/crates/linkcode-sim/src/capture.rs b/crates/linkcode-sim/src/capture.rs index 943dc82c..35d3ba38 100644 --- a/crates/linkcode-sim/src/capture.rs +++ b/crates/linkcode-sim/src/capture.rs @@ -12,16 +12,26 @@ //! This is the standard isolation pattern for fragile native code (GPU/plugin sandboxes): contain //! the crash, supervise, recover — defense in depth around a path that is now stable in steady state. +use std::collections::VecDeque; use std::io::{self, Read, Write}; use std::process::{Child, Command, Stdio}; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Condvar, Mutex}; use std::thread; use std::time::{Duration, Instant}; -/// One framebuffer JPEG frame. +use crate::rpc::StreamCodec; + +/// One encoded frame payload (JPEG image or H.264 access unit). pub type Frame = Arc>; +/// One H.264 access unit in delivery order. Unlike JPEG frames, deltas must not be dropped +/// individually — a gap desyncs the decoder until the next keyframe. +pub struct EncodedUnit { + pub data: Frame, + pub key: bool, +} + /// A drift-free, precise frame pacer (macOS). It advances an absolute deadline by a fixed interval /// each tick and waits to it with `mach_wait_until`, whose accuracy is well under a millisecond. /// `thread::sleep` on macOS coalesces timers and overshoots by several ms — enough that even a @@ -115,14 +125,76 @@ const MAX_FAST_CRASHES: u32 = 6; pub struct StreamParams { pub fps: u32, pub quality: f64, - /// Downscale factor applied before JPEG encode (0..1; 1.0 = native resolution). + /// Downscale factor applied before JPEG encode (0..1; 1.0 = native resolution). H.264 always + /// encodes at native resolution — bitrate, not resolution, carries its bandwidth budget. pub scale: f64, + pub codec: StreamCodec, +} + +/// Bounded H.264 delivery queue. On overflow (a stalled consumer) the whole queue is dropped and +/// delivery resumes at the next keyframe — the only safe resync point. +struct EncodedQueue { + state: Mutex, + ready: Condvar, } -/// A supervised framebuffer stream for one device. Holds the latest delivered frame; the worker -/// subprocess is spawned, read, and respawned by a background manager thread. +struct QueueState { + units: VecDeque, + need_key: bool, +} + +/// ~4s at 60 fps before the overflow resync kicks in. +const MAX_QUEUED_UNITS: usize = 240; + +impl EncodedQueue { + fn new() -> EncodedQueue { + EncodedQueue { + state: Mutex::new(QueueState { + units: VecDeque::new(), + need_key: false, + }), + ready: Condvar::new(), + } + } + + fn push(&self, data: Vec, key: bool) { + let mut state = self.state.lock().expect("encoded queue poisoned"); + if state.need_key { + if !key { + return; + } + state.need_key = false; + } + if state.units.len() >= MAX_QUEUED_UNITS { + state.units.clear(); + if !key { + state.need_key = true; + return; + } + } + state.units.push_back(EncodedUnit { + data: Arc::new(data), + key, + }); + self.ready.notify_one(); + } + + fn pop(&self, timeout: Duration) -> Option { + let state = self.state.lock().expect("encoded queue poisoned"); + let (mut state, _) = self + .ready + .wait_timeout_while(state, timeout, |state| state.units.is_empty()) + .expect("encoded queue poisoned"); + state.units.pop_front() + } +} + +/// A supervised framebuffer stream for one device. JPEG mode holds the latest delivered frame +/// (latest-wins); H.264 mode delivers ordered access units. The worker subprocess is spawned, +/// read, and respawned by a background manager thread. pub struct CaptureStream { latest: Arc>>, + encoded: Arc, stopped: Arc, /// Set true once the manager gives up after a crash loop, so callers can degrade. dead: Arc, @@ -136,18 +208,21 @@ impl CaptureStream { /// Start streaming device `udid` with `params` by supervising a capture worker. pub fn start(udid: String, params: StreamParams) -> CaptureStream { let latest = Arc::new(Mutex::new(None)); + let encoded = Arc::new(EncodedQueue::new()); let stopped = Arc::new(AtomicBool::new(false)); let dead = Arc::new(AtomicBool::new(false)); let worker_pid = Arc::new(AtomicU32::new(0)); let manager = thread::spawn({ let latest = Arc::clone(&latest); + let encoded = Arc::clone(&encoded); let stopped = Arc::clone(&stopped); let dead = Arc::clone(&dead); let worker_pid = Arc::clone(&worker_pid); - move || supervise(&udid, params, &latest, &stopped, &dead, &worker_pid) + move || supervise(&udid, params, &latest, &encoded, &stopped, &dead, &worker_pid) }); CaptureStream { latest, + encoded, stopped, dead, worker_pid, @@ -155,7 +230,7 @@ impl CaptureStream { } } - /// The most recently delivered frame, or `None` before the first frame (or once dead). + /// The most recently delivered JPEG frame, or `None` before the first frame (or once dead). pub fn latest(&self) -> Option { self.latest .lock() @@ -163,6 +238,11 @@ impl CaptureStream { .clone() } + /// The next H.264 access unit in order, waiting up to `timeout` for one to arrive. + pub fn next_encoded(&self, timeout: Duration) -> Option { + self.encoded.pop(timeout) + } + /// Whether the worker crash-looped and the stream gave up. pub fn is_dead(&self) -> bool { self.dead.load(Ordering::Relaxed) @@ -194,6 +274,7 @@ fn supervise( udid: &str, params: StreamParams, latest: &Arc>>, + encoded: &Arc, stopped: &Arc, dead: &Arc, worker_pid: &Arc, @@ -213,7 +294,7 @@ fn supervise( // SAFETY: our own just-spawned, un-reaped child pid. unsafe { libc::kill(child.id() as libc::pid_t, libc::SIGKILL) }; } - pump_worker(&mut child, latest, stopped); + pump_worker(&mut child, params.codec, latest, encoded, stopped); let _ = child.wait(); worker_pid.store(0, Ordering::Relaxed); } @@ -247,14 +328,22 @@ fn spawn_worker(udid: &str, params: StreamParams) -> io::Result { .arg(format!("{}", params.quality)) .arg(format!("{}", params.fps)) .arg(format!("{}", params.scale)) + .arg(params.codec.cli_name()) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::inherit()) .spawn() } -/// Read length-prefixed JPEG frames from the worker until EOF (worker exit/crash) or stop. -fn pump_worker(child: &mut Child, latest: &Arc>>, stopped: &Arc) { +/// Read `[u32 LE len][u8 flags][payload]` frames from the worker until EOF (worker exit/crash) or +/// stop. JPEG payloads replace the latest slot; H.264 payloads append to the ordered queue. +fn pump_worker( + child: &mut Child, + codec: StreamCodec, + latest: &Arc>>, + encoded: &Arc, + stopped: &Arc, +) { let Some(stdout) = child.stdout.take() else { return; }; @@ -265,24 +354,32 @@ fn pump_worker(child: &mut Child, latest: &Arc>>, stopped: & break; // worker gone } let len = u32::from_le_bytes(header) as usize; - if len == 0 || len > MAX_WORKER_FRAME { + if !(2..=MAX_WORKER_FRAME).contains(&len) { break; // corrupt stream } let mut frame = vec![0u8; len]; if reader.read_exact(&mut frame).is_err() { break; } - *latest.lock().expect("capture stream mutex poisoned") = Some(Arc::new(frame)); + let flags = frame[0]; + frame.drain(..1); + match codec { + StreamCodec::Jpeg => { + *latest.lock().expect("capture stream mutex poisoned") = Some(Arc::new(frame)); + } + StreamCodec::H264 => encoded.push(frame, flags & 1 != 0), + } } } -/// The worker entry point (`linkcode-sim capture-worker `): open the -/// device's framebuffer and stream length-prefixed JPEG frames to stdout at `fps`, each downscaled by -/// `scale`. Exits non-zero on any failure; the parent respawns. A hard `SIGABRT` from the private -/// path also just ends this process. +/// The worker entry point (`linkcode-sim capture-worker `): +/// open the device's framebuffer and stream `[u32 len][u8 flags][payload]` frames to stdout at +/// `fps` — JPEG images (downscaled by `scale`) or H.264 access units (native resolution, encoded +/// straight from the retained `IOSurface` by VideoToolbox). Exits non-zero on any failure; the +/// parent respawns. A hard `SIGABRT` from the private path also just ends this process. #[cfg(target_os = "macos")] pub fn run_worker() -> ! { - use crate::private::{Screen, SimDevice}; + use crate::private::{Screen, SimDevice, VtEncoder}; let mut args = std::env::args().skip(2); let udid = args.next().unwrap_or_default(); @@ -297,6 +394,10 @@ pub fn run_worker() -> ! { .and_then(|a| a.parse::().ok()) .unwrap_or(1.0) .clamp(0.1, 1.0); + let codec = match args.next().as_deref() { + Some("h264") => StreamCodec::H264, + _ => StreamCodec::Jpeg, + }; let Some(device) = SimDevice::resolve(&udid) else { eprintln!("sim capture-worker: device {udid} not found"); @@ -313,18 +414,55 @@ pub fn run_worker() -> ! { let mut clock = FrameClock::new(fps); let mut stdout = io::stdout().lock(); - loop { - if let Some(jpeg) = screen.capture_jpeg(quality, scale) { - let len = u32::try_from(jpeg.len()).unwrap_or(0); - if len == 0 - || stdout.write_all(&len.to_le_bytes()).is_err() - || stdout.write_all(&jpeg).is_err() - || stdout.flush().is_err() + match codec { + StreamCodec::Jpeg => loop { + if let Some(jpeg) = screen.capture_jpeg(quality, scale) + && !write_worker_frame(&mut stdout, 1, &jpeg) { break; // parent closed the pipe } + clock.tick(); + }, + StreamCodec::H264 => { + // Lazy per-dimension encoder: a rotation changes the surface size mid-stream. + let mut encoder: Option<(VtEncoder, usize, usize)> = None; + 'stream: loop { + if let Some(surface) = screen.capture_surface() { + let (width, height) = (surface.width(), surface.height()); + if encoder + .as_ref() + .is_none_or(|(_, w, h)| *w != width || *h != height) + { + encoder = VtEncoder::new(width, height, fps).map(|e| (e, width, height)); + if encoder.is_none() { + eprintln!("sim capture-worker: VideoToolbox session failed"); + std::process::exit(4); + } + } + if let Some((vt, _, _)) = encoder.as_mut() { + for unit in vt.encode(&surface) { + if !write_worker_frame(&mut stdout, u8::from(unit.key), &unit.data) { + break 'stream; // parent closed the pipe + } + } + } + } + clock.tick(); + } } - clock.tick(); } std::process::exit(0); } + +/// Write one `[u32 LE len][u8 flags][payload]` worker frame; false once the parent is gone. +#[cfg(target_os = "macos")] +fn write_worker_frame(stdout: &mut impl Write, flags: u8, payload: &[u8]) -> bool { + let Ok(len) = u32::try_from(payload.len() + 1) else { + return false; + }; + len != 1 + && stdout.write_all(&len.to_le_bytes()).is_ok() + && stdout.write_all(&[flags]).is_ok() + && stdout.write_all(payload).is_ok() + && stdout.flush().is_ok() +} diff --git a/crates/linkcode-sim/src/interactive.rs b/crates/linkcode-sim/src/interactive.rs index a6f6c505..b15e8639 100644 --- a/crates/linkcode-sim/src/interactive.rs +++ b/crates/linkcode-sim/src/interactive.rs @@ -51,6 +51,7 @@ mod stubs { _fps: u32, _quality: f64, _scale: f64, + _codec: crate::rpc::StreamCodec, _tx: &Sender, ) -> Result { Err(unsupported()) @@ -74,7 +75,10 @@ mod imp { use super::*; use crate::capture::{CaptureStream, Frame, FrameClock}; use crate::private::{self, Button, Input, SimDevice}; - use crate::proto::{STREAM_FRAME, encode_stream_frame}; + use crate::proto::{ + STREAM_FRAME, STREAM_FRAME_H264, encode_stream_frame, encode_stream_frame_h264, + }; + use crate::rpc::StreamCodec; /// A live-but-silent private worker (framebuffer registration produced no callbacks) is treated /// as unusable after this long with no new frame, and the pusher degrades to simctl. @@ -171,6 +175,7 @@ mod imp { fps: u32, quality: f64, scale: f64, + codec: StreamCodec, tx: &Sender, ) -> Result { if !available() { @@ -195,6 +200,7 @@ mod imp { fps, quality: quality.clamp(0.1, 1.0), scale, + codec, }, )); let stop = Arc::new(AtomicBool::new(false)); @@ -203,7 +209,10 @@ mod imp { let stop = Arc::clone(&stop); let tx = tx.clone(); let udid = udid.to_owned(); - move || push_frames(&udid, fps, &stream, &stop, &tx) + move || match codec { + StreamCodec::Jpeg => push_frames(&udid, fps, &stream, &stop, &tx), + StreamCodec::H264 => push_h264(&udid, &stream, &stop, &tx), + } }); reg.streams.insert( udid.to_owned(), @@ -213,7 +222,7 @@ mod imp { pusher: Some(pusher), }, ); - Ok(json!({ "streaming": true, "fps": fps, "scale": scale })) + Ok(json!({ "streaming": true, "fps": fps, "scale": scale, "codec": codec })) } pub fn stream_stop(udid: &str) -> Result { @@ -296,4 +305,44 @@ mod imp { } } } + + /// Push H.264 access units in order (deltas must not be dropped). If the private worker gives + /// up, degrades to slow simctl JPEG frames — each wire frame carries its codec, so a mixed + /// stream stays decodable client-side. + fn push_h264(udid: &str, stream: &CaptureStream, stop: &AtomicBool, tx: &Sender) { + let fallback_interval = Duration::from_millis(500); + while !stop.load(Ordering::Relaxed) { + if stream.is_dead() { + let tick = Instant::now(); + if let Ok(jpeg) = crate::simctl::screenshot(udid, crate::rpc::ImageFormat::Jpeg) + && let Ok(body) = encode_stream_frame(udid, &jpeg) + && tx + .send(OutMsg::Frame { + type_byte: STREAM_FRAME, + body, + }) + .is_err() + { + break; // daemon gone + } + if let Some(rest) = fallback_interval.checked_sub(tick.elapsed()) { + thread::sleep(rest); + } + continue; + } + let Some(unit) = stream.next_encoded(Duration::from_millis(250)) else { + continue; + }; + if let Ok(body) = encode_stream_frame_h264(udid, unit.key, &unit.data) + && tx + .send(OutMsg::Frame { + type_byte: STREAM_FRAME_H264, + body, + }) + .is_err() + { + break; // daemon gone + } + } + } } diff --git a/crates/linkcode-sim/src/main.rs b/crates/linkcode-sim/src/main.rs index 63f9a14e..b3a5329c 100644 --- a/crates/linkcode-sim/src/main.rs +++ b/crates/linkcode-sim/src/main.rs @@ -262,7 +262,8 @@ fn serve(request: Request, tx: &Sender) { fps, quality, scale, - } => interactive::stream_start(&udid, fps, quality, scale, tx), + codec, + } => interactive::stream_start(&udid, fps, quality, scale, codec, tx), Op::StreamStop { udid } => interactive::stream_stop(&udid), }; match outcome { @@ -341,6 +342,7 @@ fn diag_interactive() { fps: 12, quality: 0.6, scale: 1.0, + codec: rpc::StreamCodec::Jpeg, }, ); eprintln!("supervised capture stream started; driving input for ~5s"); diff --git a/crates/linkcode-sim/src/private/mod.rs b/crates/linkcode-sim/src/private/mod.rs index 8bcea54e..f5c60358 100644 --- a/crates/linkcode-sim/src/private/mod.rs +++ b/crates/linkcode-sim/src/private/mod.rs @@ -10,10 +10,12 @@ mod device; mod framework; mod input; mod screen; +mod vt; pub use device::SimDevice; pub use input::{Button, Input}; pub use screen::{Screen, bench_encode}; +pub use vt::VtEncoder; /// Whether this host can drive simulators interactively (private frameworks resolved). pub fn interactive_available() -> bool { diff --git a/crates/linkcode-sim/src/private/screen.rs b/crates/linkcode-sim/src/private/screen.rs index 61894a4d..e8c48d70 100644 --- a/crates/linkcode-sim/src/private/screen.rs +++ b/crates/linkcode-sim/src/private/screen.rs @@ -158,6 +158,33 @@ impl Held { unsafe { CFRetain(surface.cast_const()) }; Held(surface) } + + fn retained_clone(&self) -> Held { + Held::new(self.0) + } +} + +/// A retained `IOSurface` handed out of the sink for zero-copy H.264 encoding: the encoder wraps +/// it in a `CVPixelBuffer` so VideoToolbox reads the pixels on the media engine — they never cross +/// into CPU memory. The compositor updates the surface in place, so a frame encoded mid-write can +/// tear for one frame; that is the standard trade of every surface-fed capture pipeline. +pub struct CapturedSurface(Held); + +impl CapturedSurface { + /// The raw `IOSurfaceRef`, valid while this value lives. + pub fn as_ptr(&self) -> *mut c_void { + self.0.0 + } + + pub fn width(&self) -> usize { + // SAFETY: read-only dimension query on a retained live surface. + unsafe { IOSurfaceGetWidth(self.0.0) } + } + + pub fn height(&self) -> usize { + // SAFETY: read-only dimension query on a retained live surface. + unsafe { IOSurfaceGetHeight(self.0.0) } + } } impl Drop for Held { @@ -545,6 +572,15 @@ impl Screen { let frame = self.sink.latest()?; encode_bgra_jpeg(&frame, quality.clamp(0.1, 1.0), scale.clamp(0.1, 1.0)) } + + /// The current live surface, retained for the caller — the zero-copy input for the H.264 + /// encoder. `None` before the first surface delivery. + pub fn capture_surface(&self) -> Option { + let guard = self.sink.current.lock().expect("frame sink mutex poisoned"); + guard + .as_ref() + .map(|held| CapturedSurface(held.retained_clone())) + } } /// Encode an owned BGRA [`RawFrame`] to JPEG bytes via CoreGraphics + ImageIO, optionally downscaled diff --git a/crates/linkcode-sim/src/private/vt.rs b/crates/linkcode-sim/src/private/vt.rs new file mode 100644 index 00000000..fac7a1a1 --- /dev/null +++ b/crates/linkcode-sim/src/private/vt.rs @@ -0,0 +1,381 @@ +//! Hardware H.264 encoding via VideoToolbox (public framework). +//! +//! The capture worker feeds the retained framebuffer `IOSurface` straight into a +//! `VTCompressionSession` (wrapped in a `CVPixelBuffer` — zero-copy: the media engine reads the +//! surface, pixels never enter CPU memory). Output samples are converted from AVCC (length-prefixed +//! NALs) to Annex-B with SPS/PPS prepended on keyframes, which is what a WebCodecs `VideoDecoder` +//! configured without a `description` consumes. + +use std::ffi::c_void; +use std::sync::Mutex; + +use super::screen::CapturedSurface; + +/// One encoded H.264 access unit in Annex-B form. +pub struct EncodedFrame { + pub data: Vec, + pub key: bool, +} + +/// Frames the VT output callback has produced and the worker has not yet drained. +type OutputSink = Mutex>; + +/// A realtime, low-latency H.264 compression session for one stream. +pub struct VtEncoder { + session: *mut c_void, + /// Boxed so the address handed to the callback as refcon stays stable. + sink: Box, + fps: i32, + frame_index: i64, +} + +// SAFETY: the session is only messaged from the worker's single encode thread; the sink is +// internally synchronized against the VT callback queue. +unsafe impl Send for VtEncoder {} + +impl VtEncoder { + /// Create a session for `width`×`height` at `fps`. `None` on any VideoToolbox failure. + pub fn new(width: usize, height: usize, fps: u32) -> Option { + let sink: Box = Box::new(Mutex::new(Vec::new())); + let refcon = (&raw const *sink).cast_mut().cast::(); + let mut session: *mut c_void = std::ptr::null_mut(); + // SAFETY: plain session construction; the callback + refcon outlive the session because + // the sink box lives in this struct and the session is invalidated on drop. + let status = unsafe { + VTCompressionSessionCreate( + std::ptr::null(), + width as i32, + height as i32, + CODEC_H264, + std::ptr::null(), + std::ptr::null(), + std::ptr::null(), + output_callback as *const c_void, + refcon, + &mut session, + ) + }; + if status != 0 || session.is_null() { + return None; + } + let fps = fps.clamp(1, 60) as i32; + // SAFETY: property setters on a live session with CF values released after use. + unsafe { + set_bool(session, kVTCompressionPropertyKey_RealTime, true); + set_bool( + session, + kVTCompressionPropertyKey_AllowFrameReordering, + false, + ); + set_i32(session, kVTCompressionPropertyKey_AverageBitRate, BITRATE); + set_i32(session, kVTCompressionPropertyKey_ExpectedFrameRate, fps); + set_f64( + session, + kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, + KEYFRAME_INTERVAL_SECONDS, + ); + VTCompressionSessionPrepareToEncodeFrames(session); + } + Some(VtEncoder { + session, + sink, + fps, + frame_index: 0, + }) + } + + /// Encode one frame from the live surface; returns every access unit completed so far (the + /// realtime session normally answers one-in-one-out with ~a frame of latency). + pub fn encode(&mut self, surface: &CapturedSurface) -> Vec { + let mut pixel_buffer: *mut c_void = std::ptr::null_mut(); + // SAFETY: wraps the retained IOSurface without copying; the pixel buffer retains the + // surface for as long as VideoToolbox needs it, and our reference is released below. + let status = unsafe { + CVPixelBufferCreateWithIOSurface( + std::ptr::null(), + surface.as_ptr(), + std::ptr::null(), + &mut pixel_buffer, + ) + }; + if status == 0 && !pixel_buffer.is_null() { + let pts = CMTime { + value: self.frame_index, + timescale: self.fps, + flags: CMTIME_FLAGS_VALID, + epoch: 0, + }; + self.frame_index += 1; + // SAFETY: encode on a live session; invalid duration is the documented "unknown". + unsafe { + VTCompressionSessionEncodeFrame( + self.session, + pixel_buffer, + pts, + CMTime { + value: 0, + timescale: 0, + flags: 0, + epoch: 0, + }, + std::ptr::null(), + std::ptr::null_mut(), + std::ptr::null_mut(), + ); + CFRelease(pixel_buffer.cast_const()); + } + } + std::mem::take(&mut *self.sink.lock().expect("vt sink poisoned")) + } +} + +impl Drop for VtEncoder { + fn drop(&mut self) { + // SAFETY: invalidate stops the callback before the sink is freed, then release. + unsafe { + VTCompressionSessionInvalidate(self.session); + CFRelease(self.session.cast_const()); + } + } +} + +/// VT output callback (VideoToolbox's queue): convert the sample to Annex-B and park it in the +/// sink for the worker's encode thread to drain. +unsafe extern "C" fn output_callback( + refcon: *mut c_void, + _source: *mut c_void, + status: i32, + _flags: u32, + sample: *mut c_void, +) { + if status != 0 || sample.is_null() || refcon.is_null() { + return; + } + // SAFETY: refcon is the boxed sink owned by the live session (invalidated before drop). + let sink = unsafe { &*refcon.cast::() }; + // SAFETY: sample is a live CMSampleBuffer for the duration of this callback. + if let Some(frame) = unsafe { annex_b_frame(sample) } { + sink.lock().expect("vt sink poisoned").push(frame); + } +} + +/// Convert a CMSampleBuffer's AVCC payload to one Annex-B access unit (SPS/PPS prepended on +/// keyframes). +unsafe fn annex_b_frame(sample: *mut c_void) -> Option { + // SAFETY (whole body): read-only CoreMedia accessors on a live sample buffer; the block + // buffer's bytes are copied out before returning. + unsafe { + let key = is_keyframe(sample); + let block = CMSampleBufferGetDataBuffer(sample); + if block.is_null() { + return None; + } + let mut length: usize = 0; + let mut data: *mut u8 = std::ptr::null_mut(); + if CMBlockBufferGetDataPointer(block, 0, std::ptr::null_mut(), &mut length, &mut data) != 0 + || data.is_null() + { + return None; + } + let avcc = std::slice::from_raw_parts(data, length); + + let mut out = Vec::with_capacity(length + 256); + if key { + let format = CMSampleBufferGetFormatDescription(sample); + for index in 0..2 { + let mut set: *const u8 = std::ptr::null(); + let mut set_len: usize = 0; + if CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + format, + index, + &mut set, + &mut set_len, + std::ptr::null_mut(), + std::ptr::null_mut(), + ) == 0 + && !set.is_null() + { + out.extend_from_slice(&START_CODE); + out.extend_from_slice(std::slice::from_raw_parts(set, set_len)); + } + } + } + let mut offset = 0usize; + while offset + 4 <= avcc.len() { + let nal_len = u32::from_be_bytes([ + avcc[offset], + avcc[offset + 1], + avcc[offset + 2], + avcc[offset + 3], + ]) as usize; + offset += 4; + if nal_len == 0 || offset + nal_len > avcc.len() { + break; + } + out.extend_from_slice(&START_CODE); + out.extend_from_slice(&avcc[offset..offset + nal_len]); + offset += nal_len; + } + (!out.is_empty()).then_some(EncodedFrame { data: out, key }) + } +} + +/// A sample without the `NotSync` attachment is a sync (key) frame. +unsafe fn is_keyframe(sample: *mut c_void) -> bool { + // SAFETY: read-only attachment introspection on a live sample buffer. + unsafe { + let attachments = CMSampleBufferGetSampleAttachmentsArray(sample, false); + if attachments.is_null() || CFArrayGetCount(attachments) == 0 { + return true; + } + let first = CFArrayGetValueAtIndex(attachments, 0); + let not_sync = CFDictionaryGetValue(first.cast(), kCMSampleAttachmentKey_NotSync.cast()); + not_sync.is_null() || !CFBooleanGetValue(not_sync) + } +} + +const START_CODE: [u8; 4] = [0, 0, 0, 1]; +/// `'avc1'` as a FourCC. +const CODEC_H264: u32 = 0x6176_6331; +/// Streaming target; at simulator resolutions this lands well under a JPEG stream's bandwidth. +const BITRATE: i32 = 6_000_000; +const KEYFRAME_INTERVAL_SECONDS: f64 = 2.0; +const CMTIME_FLAGS_VALID: u32 = 1; + +#[repr(C)] +struct CMTime { + value: i64, + timescale: i32, + flags: u32, + epoch: i64, +} + +// SAFETY-adjacent helpers: each sets one CF-typed session property and releases the value. +unsafe fn set_bool(session: *mut c_void, key: *const c_void, value: bool) { + // SAFETY: kCFBooleanTrue/False are constants; SetProperty copies/retains as needed. + unsafe { + VTSessionSetProperty( + session, + key, + if value { + kCFBooleanTrue + } else { + kCFBooleanFalse + }, + ); + } +} + +unsafe fn set_i32(session: *mut c_void, key: *const c_void, value: i32) { + // SAFETY: creates a CFNumber, hands it to the session (which retains), then releases. + unsafe { + let number = CFNumberCreate( + std::ptr::null(), + KCF_NUMBER_SINT32, + (&raw const value).cast(), + ); + VTSessionSetProperty(session, key, number); + CFRelease(number); + } +} + +unsafe fn set_f64(session: *mut c_void, key: *const c_void, value: f64) { + // SAFETY: creates a CFNumber, hands it to the session (which retains), then releases. + unsafe { + let number = CFNumberCreate( + std::ptr::null(), + KCF_NUMBER_DOUBLE, + (&raw const value).cast(), + ); + VTSessionSetProperty(session, key, number); + CFRelease(number); + } +} + +const KCF_NUMBER_SINT32: isize = 3; +const KCF_NUMBER_DOUBLE: isize = 13; + +#[link(name = "VideoToolbox", kind = "framework")] +unsafe extern "C" { + fn VTCompressionSessionCreate( + allocator: *const c_void, + width: i32, + height: i32, + codec_type: u32, + encoder_specification: *const c_void, + source_image_buffer_attributes: *const c_void, + compressed_data_allocator: *const c_void, + output_callback: *const c_void, + refcon: *mut c_void, + session_out: *mut *mut c_void, + ) -> i32; + fn VTSessionSetProperty(session: *mut c_void, key: *const c_void, value: *const c_void) -> i32; + fn VTCompressionSessionPrepareToEncodeFrames(session: *mut c_void) -> i32; + fn VTCompressionSessionEncodeFrame( + session: *mut c_void, + image_buffer: *mut c_void, + presentation_timestamp: CMTime, + duration: CMTime, + frame_properties: *const c_void, + source_frame_refcon: *mut c_void, + info_flags_out: *mut u32, + ) -> i32; + fn VTCompressionSessionInvalidate(session: *mut c_void); + static kVTCompressionPropertyKey_RealTime: *const c_void; + static kVTCompressionPropertyKey_AllowFrameReordering: *const c_void; + static kVTCompressionPropertyKey_AverageBitRate: *const c_void; + static kVTCompressionPropertyKey_ExpectedFrameRate: *const c_void; + static kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration: *const c_void; +} + +#[link(name = "CoreMedia", kind = "framework")] +unsafe extern "C" { + fn CMSampleBufferGetDataBuffer(sample: *mut c_void) -> *mut c_void; + fn CMSampleBufferGetFormatDescription(sample: *mut c_void) -> *mut c_void; + fn CMSampleBufferGetSampleAttachmentsArray( + sample: *mut c_void, + create_if_necessary: bool, + ) -> *const c_void; + fn CMBlockBufferGetDataPointer( + buffer: *mut c_void, + offset: usize, + length_at_offset: *mut usize, + total_length: *mut usize, + data_pointer: *mut *mut u8, + ) -> i32; + fn CMVideoFormatDescriptionGetH264ParameterSetAtIndex( + format: *mut c_void, + index: usize, + parameter_set_out: *mut *const u8, + parameter_set_size_out: *mut usize, + parameter_set_count_out: *mut usize, + nal_unit_header_length_out: *mut i32, + ) -> i32; + static kCMSampleAttachmentKey_NotSync: *const c_void; +} + +#[link(name = "CoreVideo", kind = "framework")] +unsafe extern "C" { + fn CVPixelBufferCreateWithIOSurface( + allocator: *const c_void, + surface: *mut c_void, + pixel_buffer_attributes: *const c_void, + pixel_buffer_out: *mut *mut c_void, + ) -> i32; +} + +#[link(name = "CoreFoundation", kind = "framework")] +unsafe extern "C" { + fn CFRelease(cf: *const c_void); + fn CFNumberCreate( + allocator: *const c_void, + number_type: isize, + value_ptr: *const c_void, + ) -> *const c_void; + fn CFArrayGetCount(array: *const c_void) -> isize; + fn CFArrayGetValueAtIndex(array: *const c_void, index: isize) -> *const c_void; + fn CFDictionaryGetValue(dict: *const c_void, key: *const c_void) -> *const c_void; + fn CFBooleanGetValue(boolean: *const c_void) -> bool; + static kCFBooleanTrue: *const c_void; + static kCFBooleanFalse: *const c_void; +} diff --git a/crates/linkcode-sim/src/proto.rs b/crates/linkcode-sim/src/proto.rs index 5ffd5c66..1622154e 100644 --- a/crates/linkcode-sim/src/proto.rs +++ b/crates/linkcode-sim/src/proto.rs @@ -20,6 +20,7 @@ pub const RESULT: u8 = 0x81; pub const SCREENSHOT: u8 = 0x82; /// A streamed framebuffer frame: `[u16 LE udid_len][udid][jpeg]` (unsolicited, while a stream runs). pub const STREAM_FRAME: u8 = 0x83; +pub const STREAM_FRAME_H264: u8 = 0x84; /// Encode a `STREAM_FRAME` body: `[u16 LE udid_len][udid][jpeg bytes]`. pub fn encode_stream_frame(udid: &str, jpeg: &[u8]) -> io::Result> { @@ -37,6 +38,23 @@ pub fn encode_stream_frame(udid: &str, jpeg: &[u8]) -> io::Result> { Ok(out) } +/// Encode a `STREAM_FRAME_H264` body: `[u16 LE udid_len][udid][u8 key][Annex-B access unit]`. +pub fn encode_stream_frame_h264(udid: &str, key: bool, data: &[u8]) -> io::Result> { + let id = udid.as_bytes(); + if id.is_empty() || id.len() > MAX_REQUEST_ID_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid udid length", + )); + } + let mut out = Vec::with_capacity(3 + id.len() + data.len()); + out.extend_from_slice(&(id.len() as u16).to_le_bytes()); + out.extend_from_slice(id); + out.push(u8::from(key)); + out.extend_from_slice(data); + Ok(out) +} + /// Read one frame. Returns `Ok(None)` on a clean end-of-stream (the daemon closed the pipe). pub fn read_frame(reader: &mut impl Read) -> io::Result)>> { let mut len = [0u8; 4]; diff --git a/crates/linkcode-sim/src/rpc.rs b/crates/linkcode-sim/src/rpc.rs index 91f852fe..ced4d23d 100644 --- a/crates/linkcode-sim/src/rpc.rs +++ b/crates/linkcode-sim/src/rpc.rs @@ -70,7 +70,8 @@ pub enum Op { }, /// Press a hardware button (private API; P1). Button { udid: String, button: ButtonKind }, - /// Start streaming the device framebuffer as JPEG `FRAME`s at `fps` (private API; P1). + /// Start streaming the device framebuffer at `fps` (private API; P1). `codec` picks JPEG + /// `STREAM_FRAME`s (latest-wins) or H.264 `STREAM_FRAME_H264` access units (ordered). StreamStart { udid: String, #[serde(default = "default_fps")] @@ -79,6 +80,8 @@ pub enum Op { quality: f64, #[serde(default = "default_scale")] scale: f64, + #[serde(default)] + codec: StreamCodec, }, /// Stop a running framebuffer stream. StreamStop { udid: String }, @@ -102,6 +105,26 @@ pub enum ButtonKind { Lock, } +/// Framebuffer stream encodings. JPEG frames are independently decodable (latest-wins delivery); +/// H.264 access units are ordered and delta-dependent (hardware encode/decode, ~10× less bandwidth). +#[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum StreamCodec { + #[default] + Jpeg, + H264, +} + +impl StreamCodec { + /// The value passed to the capture worker on its command line. + pub fn cli_name(self) -> &'static str { + match self { + Self::Jpeg => "jpeg", + Self::H264 => "h264", + } + } +} + /// Screenshot encodings supported by `simctl io screenshot --type`. #[derive(Clone, Copy, Default, Deserialize, PartialEq, Eq)] #[serde(rename_all = "camelCase")] From 4a7e54ee50d87162f36518726cee77203171aee2 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 06:03:10 +0800 Subject: [PATCH 15/26] feat(schema,engine,client-core): H.264 stream codec plumbing (wire 47) --- .../src/__tests__/sim-mcp-endpoint.test.ts | 4 ++- apps/desktop/e2e/simulator-panel.e2e.mts | 2 +- packages/client/core/src/client.ts | 22 +++++++++++---- .../client/core/src/client/control-channel.ts | 8 ++++-- .../core/src/client/pending-registry.ts | 3 +- .../__tests__/stream-registry.test.ts | 4 +-- .../foundation/schema/src/model/simulator.ts | 5 ++++ .../foundation/schema/src/wire/message.ts | 3 +- .../foundation/schema/src/wire/simulator.ts | 8 +++++- .../transport/src/__tests__/hub.test.ts | 2 ++ .../simulator-request-handler.test.ts | 12 +++++--- .../src/__tests__/simulator-service.test.ts | 1 + packages/host/engine/src/simulator/backend.ts | 17 +++++++++-- .../engine/src/simulator/request-handler.ts | 9 ++++-- .../host/sim/src/__tests__/client.test.ts | 8 ++++-- packages/host/sim/src/client.ts | 28 ++++++++++++++++--- packages/host/sim/src/codec.ts | 14 ++++++++++ packages/host/sim/src/index.ts | 3 +- packages/host/sim/src/schema.ts | 13 +++++++-- 19 files changed, 131 insertions(+), 35 deletions(-) diff --git a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts index 0baed637..9c42b213 100644 --- a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts +++ b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts @@ -37,7 +37,9 @@ function fakeBackend(): SimulatorBackend { tap: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), - streamStart: vi.fn(() => Promise.resolve({ streaming: true as const, fps: 60, scale: 1 })), + streamStart: vi.fn(() => + Promise.resolve({ streaming: true as const, fps: 60, scale: 1, codec: 'jpeg' as const }), + ), streamStop: vi.fn(asyncNoop), onFrame: vi.fn(() => noop), close: vi.fn(noop), diff --git a/apps/desktop/e2e/simulator-panel.e2e.mts b/apps/desktop/e2e/simulator-panel.e2e.mts index 21b9846a..543de420 100644 --- a/apps/desktop/e2e/simulator-panel.e2e.mts +++ b/apps/desktop/e2e/simulator-panel.e2e.mts @@ -29,7 +29,7 @@ const PORT = 43000 + (process.pid % 1000); /** Must match `WIRE_PROTOCOL_VERSION` (node can't load the raw-TS schema barrel); a mismatch is * silently discarded by the daemon, surfacing here as the session.start timeout. */ -const WIRE_VERSION = 46; +const WIRE_VERSION = 47; function fail(message: string): never { console.error(`FAIL: ${message}`); diff --git a/packages/client/core/src/client.ts b/packages/client/core/src/client.ts index 69dc196c..6b73af04 100644 --- a/packages/client/core/src/client.ts +++ b/packages/client/core/src/client.ts @@ -41,6 +41,7 @@ import type { SimulatorDevice, SimulatorImageFormat, SimulatorStatus, + SimulatorStreamCodec, StartOptions, TerminalMetadata, TerminalReplayEvent, @@ -113,8 +114,13 @@ type SimulatorActivityCb = (activity: { tool: string; phase: 'started' | 'settled'; }) => void; -/** A live framebuffer frame: base64-encoded JPEG bytes for one device. */ -type SimulatorFrameCb = (frame: { udid: string; data: string }) => void; +/** A live stream frame: base64 bytes (JPEG image or Annex-B H.264 access unit) for one device. */ +type SimulatorFrameCb = (frame: { + udid: string; + codec: SimulatorStreamCodec; + key: boolean; + data: string; +}) => void; type ConnectionCloseCb = (error: Error) => void; /** A broadcast about a schedule's or its runs' state — the three `schedule.*` push variants. */ @@ -347,11 +353,15 @@ export class LinkCodeClient { } break; case 'simulator.stream.started': - this.pending.resolve('simulatorStreamStart', p.replyTo, { fps: p.fps, scale: p.scale }); + this.pending.resolve('simulatorStreamStart', p.replyTo, { + fps: p.fps, + scale: p.scale, + codec: p.codec, + }); break; case 'simulator.stream.frame': for (const cb of this.simulatorFrameSubs.get(p.udid) ?? []) { - cb({ udid: p.udid, data: p.data }); + cb({ udid: p.udid, codec: p.codec, key: p.key, data: p.data }); } break; case 'asset.listed': @@ -721,8 +731,8 @@ export class LinkCodeClient { simulatorStreamStart( sessionId: SessionId, udid: string, - options?: { fps?: number; quality?: number; scale?: number }, - ): Promise<{ fps: number; scale: number }> { + options?: { fps?: number; quality?: number; scale?: number; codec?: SimulatorStreamCodec }, + ): Promise<{ fps: number; scale: number; codec: SimulatorStreamCodec }> { return this.control.simulatorStreamStart(sessionId, udid, options); } diff --git a/packages/client/core/src/client/control-channel.ts b/packages/client/core/src/client/control-channel.ts index 99e35935..dc1bc8e9 100644 --- a/packages/client/core/src/client/control-channel.ts +++ b/packages/client/core/src/client/control-channel.ts @@ -38,6 +38,7 @@ import type { SimulatorDevice, SimulatorImageFormat, SimulatorStatus, + SimulatorStreamCodec, StartOptions, WirePayload, WorkspaceFile, @@ -712,12 +713,12 @@ export class ControlChannel { })); } - /** Resolves with the accepted `{ fps, scale }`; frames then arrive as `simulator.stream.frame`. */ + /** Resolves with the accepted `{ fps, scale, codec }`; frames then arrive as `simulator.stream.frame`. */ simulatorStreamStart( sessionId: SessionId, udid: string, - options?: { fps?: number; quality?: number; scale?: number }, - ): Promise<{ fps: number; scale: number }> { + options?: { fps?: number; quality?: number; scale?: number; codec?: SimulatorStreamCodec }, + ): Promise<{ fps: number; scale: number; codec: SimulatorStreamCodec }> { return this.sendCorrelated('simulatorStreamStart', (clientReqId) => ({ kind: 'simulator.stream.start', clientReqId, @@ -726,6 +727,7 @@ export class ControlChannel { fps: options?.fps, quality: options?.quality, scale: options?.scale, + codec: options?.codec, })); } diff --git a/packages/client/core/src/client/pending-registry.ts b/packages/client/core/src/client/pending-registry.ts index ce47c51c..6e289777 100644 --- a/packages/client/core/src/client/pending-registry.ts +++ b/packages/client/core/src/client/pending-registry.ts @@ -22,6 +22,7 @@ import type { SimulatorDevice, SimulatorImageFormat, SimulatorStatus, + SimulatorStreamCodec, TerminalMetadata, WirePayload, WorkspaceFile, @@ -99,7 +100,7 @@ export interface PendingValueMap { simulatorLaunch: number | null; simulatorScreenshot: { format: SimulatorImageFormat; data: string }; simulatorScreenMask: string; - simulatorStreamStart: { fps: number; scale: number }; + simulatorStreamStart: { fps: number; scale: number; codec: SimulatorStreamCodec }; } type PendingMaps = { [K in keyof PendingValueMap]: Map> }; diff --git a/packages/client/workbench/src/simulator/__tests__/stream-registry.test.ts b/packages/client/workbench/src/simulator/__tests__/stream-registry.test.ts index baa731f3..e0c51f31 100644 --- a/packages/client/workbench/src/simulator/__tests__/stream-registry.test.ts +++ b/packages/client/workbench/src/simulator/__tests__/stream-registry.test.ts @@ -23,7 +23,7 @@ function createFakeClient(): FakeClient { return Promise.reject(new Error('conflict')); } started.push({ sessionId, udid }); - return Promise.resolve({ fps: 30, scale: 0.5 }); + return Promise.resolve({ fps: 30, scale: 0.5, codec: 'h264' as const }); }, simulatorStreamStop(sessionId, udid) { stopped.push({ sessionId, udid }); @@ -98,7 +98,7 @@ describe('simulator stream registry', () => { let resolveStart: () => void = noop; client.simulatorStreamStart = () => new Promise((resolve) => { - resolveStart = () => resolve({ fps: 30, scale: 0.5 }); + resolveStart = () => resolve({ fps: 30, scale: 0.5, codec: 'h264' }); }); const lease = acquireSimulatorStream(client, 'U-1', S1); lease.release(); diff --git a/packages/foundation/schema/src/model/simulator.ts b/packages/foundation/schema/src/model/simulator.ts index a4a6605b..72321580 100644 --- a/packages/foundation/schema/src/model/simulator.ts +++ b/packages/foundation/schema/src/model/simulator.ts @@ -32,3 +32,8 @@ export type SimulatorStatus = z.infer; export const SimulatorImageFormatSchema = z.enum(['jpeg', 'png']); export type SimulatorImageFormat = z.infer; + +/** Framebuffer stream encodings: independently-decodable JPEG frames, or ordered hardware H.264 + * access units (Annex-B, native resolution — decoded client-side with WebCodecs). */ +export const SimulatorStreamCodecSchema = z.enum(['jpeg', 'h264']); +export type SimulatorStreamCodec = z.infer; diff --git a/packages/foundation/schema/src/wire/message.ts b/packages/foundation/schema/src/wire/message.ts index c4494bea..1b590376 100644 --- a/packages/foundation/schema/src/wire/message.ts +++ b/packages/foundation/schema/src/wire/message.ts @@ -17,7 +17,8 @@ import { WirePayloadSchema } from './payload'; // 45 adds the simulator.activity broadcast (CODE-395). // 46 adds the simulator interactive + framebuffer-stream variants (CODE-397). // 47 adds the simulator screen-mask wire (CODE-397). -export const WIRE_PROTOCOL_VERSION = 47 as const; +// 48 adds the H.264 stream codec plumbing (CODE-397). +export const WIRE_PROTOCOL_VERSION = 48 as const; /** Complete wire message: version + unique id + timestamp + payload. */ export const WireMessageSchema = z.object({ diff --git a/packages/foundation/schema/src/wire/simulator.ts b/packages/foundation/schema/src/wire/simulator.ts index f8ba700b..8949676e 100644 --- a/packages/foundation/schema/src/wire/simulator.ts +++ b/packages/foundation/schema/src/wire/simulator.ts @@ -4,6 +4,7 @@ import { SimulatorDeviceSchema, SimulatorImageFormatSchema, SimulatorStatusSchema, + SimulatorStreamCodecSchema, } from '../model/simulator'; import { WireRequestIdSchema } from './request'; @@ -156,6 +157,7 @@ export const simulatorWireVariants = [ fps: z.number().int().positive().optional(), quality: z.number().min(0).max(1).optional(), scale: z.number().min(0).max(1).optional(), + codec: SimulatorStreamCodecSchema.optional(), }), z.object({ kind: z.literal('simulator.stream.started'), @@ -163,6 +165,7 @@ export const simulatorWireVariants = [ udid, fps: z.number().int(), scale: z.number(), + codec: SimulatorStreamCodecSchema, }), z.object({ kind: z.literal('simulator.stream.stop'), @@ -178,7 +181,10 @@ export const simulatorWireVariants = [ kind: z.literal('simulator.stream.frame'), sessionId: SessionIdSchema, udid, - /** Base64-encoded JPEG bytes. */ + codec: SimulatorStreamCodecSchema, + /** Sync frame (always true for JPEG; H.264 deltas depend on every frame since the last key). */ + key: z.boolean(), + /** Base64-encoded frame bytes (JPEG image or Annex-B H.264 access unit). */ data: z.string(), }), ] as const; diff --git a/packages/foundation/transport/src/__tests__/hub.test.ts b/packages/foundation/transport/src/__tests__/hub.test.ts index 6baabb58..2af366b5 100644 --- a/packages/foundation/transport/src/__tests__/hub.test.ts +++ b/packages/foundation/transport/src/__tests__/hub.test.ts @@ -49,6 +49,8 @@ function streamFrame(sessionId: SessionId): ValidatedWireMessage { kind: 'simulator.stream.frame', sessionId, udid: 'U-1', + codec: 'jpeg', + key: true, data: 'AAA=', }); } diff --git a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts index e247652a..14916437 100644 --- a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts +++ b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts @@ -4,7 +4,7 @@ import { createWireMessage } from '@linkcode/transport'; import { nullthrow } from 'foxts/guard'; import { asyncNoop, noop } from 'foxts/noop'; import { describe, expect, it, vi } from 'vitest'; -import type { SimulatorBackend } from '../simulator/backend'; +import type { SimulatorBackend, SimulatorFrameListener } from '../simulator/backend'; import { SimulatorService } from '../simulator/service'; import { FakeAdapter, settleEngineTasks, startedSessionId } from './fixtures/session-harness'; import { createTestEngine } from './fixtures/test-engine'; @@ -36,9 +36,11 @@ function fakeBackend() { tap: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), - streamStart: vi.fn(() => Promise.resolve({ streaming: true as const, fps: 60, scale: 1 })), + streamStart: vi.fn(() => + Promise.resolve({ streaming: true as const, fps: 60, scale: 1, codec: 'jpeg' as const }), + ), streamStop: vi.fn(asyncNoop), - onFrame: vi.fn((_udid: string, _listener: (image: Uint8Array) => void) => noop), + onFrame: vi.fn((_udid: string, _listener: SimulatorFrameListener) => noop), close: vi.fn(noop), } satisfies SimulatorBackend; } @@ -192,11 +194,13 @@ describe('simulator wire requests', () => { // The handler subscribed a frame listener; a delivered frame becomes a session-scoped push. const listener = backend.onFrame.mock.calls.at(-1)?.[1]; - listener?.(new Uint8Array([0xff, 0xd8, 0x01])); + listener?.({ codec: 'jpeg', data: new Uint8Array([0xff, 0xd8, 0x01]), key: true }); expect(h.sent.find((p) => p.kind === 'simulator.stream.frame')).toMatchObject({ kind: 'simulator.stream.frame', sessionId: 'session-1', udid: 'U-1', + codec: 'jpeg', + key: true, data: Buffer.from([0xff, 0xd8, 0x01]).toString('base64'), }); diff --git a/packages/host/engine/src/__tests__/simulator-service.test.ts b/packages/host/engine/src/__tests__/simulator-service.test.ts index 61e92586..4b5e9c86 100644 --- a/packages/host/engine/src/__tests__/simulator-service.test.ts +++ b/packages/host/engine/src/__tests__/simulator-service.test.ts @@ -38,6 +38,7 @@ function fakeBackend(devices: SimulatorDeviceInfo[]) { streaming: true, fps: 60, scale: 1, + codec: 'jpeg', }), ), streamStop: vi.fn(asyncNoop), diff --git a/packages/host/engine/src/simulator/backend.ts b/packages/host/engine/src/simulator/backend.ts index 8b7ce3c6..fb6c32cc 100644 --- a/packages/host/engine/src/simulator/backend.ts +++ b/packages/host/engine/src/simulator/backend.ts @@ -36,21 +36,32 @@ export interface SimulatorPoint { y: number; } +/** Framebuffer stream encodings: independently-decodable JPEG frames, or ordered hardware H.264 + * access units (native resolution, ~10× less bandwidth). */ +export type SimulatorStreamCodec = 'jpeg' | 'h264'; + /** Framebuffer stream tuning; omitted fields take the sidecar defaults. */ export interface SimulatorStreamOptions { fps?: number; quality?: number; /** Downscale factor before encode (0..1; 1.0 = native). Lower trades resolution for rate/bandwidth. */ scale?: number; + codec?: SimulatorStreamCodec; } /** `streamStart` outcome: the accepted stream, or a no-op when one is already running. */ export type SimulatorStreamStartResult = - | { streaming: true; fps: number; scale: number } + | { streaming: true; fps: number; scale: number; codec: SimulatorStreamCodec } | { alreadyStreaming: true }; -/** A live framebuffer frame: JPEG bytes for one device. */ -export type SimulatorFrameListener = (image: Uint8Array) => void; +/** One live stream frame; H.264 units are ordered and delta-dependent (`key` on sync frames). */ +export interface SimulatorStreamFrame { + codec: SimulatorStreamCodec; + data: Uint8Array; + key: boolean; +} + +export type SimulatorFrameListener = (frame: SimulatorStreamFrame) => void; export interface SimulatorBackend { probe(): Promise; diff --git a/packages/host/engine/src/simulator/request-handler.ts b/packages/host/engine/src/simulator/request-handler.ts index 408bc371..7c78b96c 100644 --- a/packages/host/engine/src/simulator/request-handler.ts +++ b/packages/host/engine/src/simulator/request-handler.ts @@ -190,11 +190,13 @@ export class SimulatorRequestHandler { fps: payload.fps, quality: payload.quality, scale: payload.scale, + codec: payload.codec, }); this.subscribeFrames(simulators, payload.sessionId, payload.udid); // `alreadyStreaming` carries no params, so echo the request's (defaulted) values. const fps = 'streaming' in result ? result.fps : (payload.fps ?? 60); const scale = 'streaming' in result ? result.scale : (payload.scale ?? 1); + const codec = 'streaming' in result ? result.codec : (payload.codec ?? 'jpeg'); this.transport.send( createWireMessage({ kind: 'simulator.stream.started', @@ -202,6 +204,7 @@ export class SimulatorRequestHandler { udid: payload.udid, fps, scale, + codec, }), ); }), @@ -223,13 +226,15 @@ export class SimulatorRequestHandler { * Idempotent: a second `streamStart` for a device already fanning out keeps the one subscription. */ private subscribeFrames(simulators: SimulatorService, sessionId: SessionId, udid: string): void { if (this.frameSubs.has(udid)) return; - const unsubscribe = simulators.onFrame(udid, (image) => { + const unsubscribe = simulators.onFrame(udid, (frame) => { this.transport.send( createWireMessage({ kind: 'simulator.stream.frame', sessionId, udid, - data: Buffer.from(image).toString('base64'), + codec: frame.codec, + key: frame.key, + data: Buffer.from(frame.data).toString('base64'), }), ); }); diff --git a/packages/host/sim/src/__tests__/client.test.ts b/packages/host/sim/src/__tests__/client.test.ts index 7bf7f08e..07677e4a 100644 --- a/packages/host/sim/src/__tests__/client.test.ts +++ b/packages/host/sim/src/__tests__/client.test.ts @@ -169,13 +169,15 @@ describe('SimSidecarClient', () => { void client.streamStart('U-1'); await tick(); - const seen: Buffer[] = []; - const unsubscribe = client.onFrame('U-1', (image) => seen.push(image)); + const seen: Array<{ codec: string; key: boolean; bytes: number[] }> = []; + const unsubscribe = client.onFrame('U-1', (frame) => + seen.push({ codec: frame.codec, key: frame.key, bytes: [...frame.data] }), + ); child.stdout.write(streamFrameBytes(STREAM_FRAME, 'U-1', [0xff, 0xd8, 0x01])); child.stdout.write(streamFrameBytes(STREAM_FRAME, 'U-2', [0x00])); // other device: ignored await tick(); - expect(seen.map((b) => [...b])).toEqual([[0xff, 0xd8, 0x01]]); + expect(seen).toEqual([{ codec: 'jpeg', key: true, bytes: [0xff, 0xd8, 0x01] }]); unsubscribe(); child.stdout.write(streamFrameBytes(STREAM_FRAME, 'U-1', [0x02])); diff --git a/packages/host/sim/src/client.ts b/packages/host/sim/src/client.ts index 96c58fe7..0c3396fc 100644 --- a/packages/host/sim/src/client.ts +++ b/packages/host/sim/src/client.ts @@ -6,11 +6,13 @@ import type { Frame } from './codec'; import { decodeScreenshotFrame, decodeStreamFrame, + decodeStreamFrameH264, FrameDecoder, REQUEST, RESULT, SCREENSHOT, STREAM_FRAME, + STREAM_FRAME_H264, writeFrame, } from './codec'; import type { @@ -18,6 +20,7 @@ import type { SimDevice, SimImageFormat, SimProbe, + SimStreamCodec, SimStreamStartResult, } from './schema'; import { @@ -28,8 +31,14 @@ import { SimStreamStartResultSchema, } from './schema'; -/** A live framebuffer frame: JPEG bytes for one device. */ -export type SimFrameListener = (image: Buffer) => void; +/** One live stream frame: a JPEG image, or one ordered H.264 access unit (`key` on sync frames). */ +export interface SimStreamFrame { + codec: SimStreamCodec; + data: Buffer; + key: boolean; +} + +export type SimFrameListener = (frame: SimStreamFrame) => void; /** Options for {@link SimSidecarClient.streamStart}; omitted fields take the sidecar defaults. */ export interface SimStreamOptions { @@ -37,6 +46,8 @@ export interface SimStreamOptions { quality?: number; /** Downscale factor before encode (0..1; 1.0 = native). Lower trades resolution for rate/bandwidth. */ scale?: number; + /** `h264` streams ordered hardware-encoded access units at native resolution; default `jpeg`. */ + codec?: SimStreamCodec; } /** The sidecar child: piped stdin/stdout, inherited stderr (its logs go to the host's stderr). */ @@ -179,6 +190,11 @@ export class SimSidecarClient { * Subscribe to `udid`'s framebuffer frames; returns an unsubscribe function. Subscribing does not * itself start the stream — pair it with {@link streamStart}. */ + private fanOutFrame(udid: string, frame: SimStreamFrame): void { + const listeners = this.frameListeners.get(udid); + if (listeners) for (const listener of listeners) listener(frame); + } + onFrame(udid: string, listener: SimFrameListener): () => void { let set = this.frameListeners.get(udid); if (!set) { @@ -290,8 +306,12 @@ export class SimSidecarClient { } case STREAM_FRAME: { const { udid, image } = decodeStreamFrame(frame.body); - const listeners = this.frameListeners.get(udid); - if (listeners) for (const listener of listeners) listener(image); + this.fanOutFrame(udid, { codec: 'jpeg', data: image, key: true }); + break; + } + case STREAM_FRAME_H264: { + const { udid, key, data } = decodeStreamFrameH264(frame.body); + this.fanOutFrame(udid, { codec: 'h264', data, key }); break; } default: diff --git a/packages/host/sim/src/codec.ts b/packages/host/sim/src/codec.ts index 0378a369..20a124ba 100644 --- a/packages/host/sim/src/codec.ts +++ b/packages/host/sim/src/codec.ts @@ -13,6 +13,8 @@ export const RESULT = 0x81; export const SCREENSHOT = 0x82; /** Unsolicited JPEG frame pushed while a `streamStart` stream runs. */ export const STREAM_FRAME = 0x83; +/** Unsolicited H.264 access unit (Annex-B) pushed while an h264 stream runs. */ +export const STREAM_FRAME_H264 = 0x84; export const MAX_FRAME_LEN = 16 * 1024 * 1024; @@ -79,3 +81,15 @@ export function decodeStreamFrame(body: Buffer): { udid: string; image: Buffer } image: body.subarray(2 + udidLen), }; } + +/** Decode a `STREAM_FRAME_H264` body: `[u16 LE udid_len][udid][u8 key][Annex-B access unit]`. */ +export function decodeStreamFrameH264(body: Buffer): { udid: string; key: boolean; data: Buffer } { + if (body.length < 3) throw new Error('short h264 stream frame'); + const udidLen = body.readUInt16LE(0); + if (body.length < 3 + udidLen) throw new Error('truncated h264 stream udid'); + return { + udid: body.subarray(2, 2 + udidLen).toString('utf8'), + key: body[2 + udidLen] === 1, + data: body.subarray(3 + udidLen), + }; +} diff --git a/packages/host/sim/src/index.ts b/packages/host/sim/src/index.ts index 0097c7fe..5b62633f 100644 --- a/packages/host/sim/src/index.ts +++ b/packages/host/sim/src/index.ts @@ -1,4 +1,4 @@ -export type { SimFrameListener, SimStreamOptions } from './client'; +export type { SimFrameListener, SimStreamFrame, SimStreamOptions } from './client'; export { SimSidecarClient, SimSidecarError } from './client'; export type { SimButton, @@ -6,5 +6,6 @@ export type { SimErrorCode, SimImageFormat, SimProbe, + SimStreamCodec, SimStreamStartResult, } from './schema'; diff --git a/packages/host/sim/src/schema.ts b/packages/host/sim/src/schema.ts index 68e04a7c..7570b620 100644 --- a/packages/host/sim/src/schema.ts +++ b/packages/host/sim/src/schema.ts @@ -43,9 +43,18 @@ export type SimProbe = z.infer; export const SimLaunchResultSchema = z.object({ pid: z.number().int().nullable() }); -/** `streamStart` reply: the accepted stream, or a no-op when one is already running. */ +export const SimStreamCodecSchema = z.enum(['jpeg', 'h264']); +export type SimStreamCodec = z.infer; + +/** `streamStart` reply: the accepted stream, or a no-op when one is already running. `codec` is + * absent from sidecars predating h264 support — those always stream JPEG. */ export const SimStreamStartResultSchema = z.union([ - z.object({ streaming: z.literal(true), fps: z.number().int(), scale: z.number() }), + z.object({ + streaming: z.literal(true), + fps: z.number().int(), + scale: z.number(), + codec: SimStreamCodecSchema.default('jpeg'), + }), z.object({ alreadyStreaming: z.literal(true) }), ]); export type SimStreamStartResult = z.infer; From 29a27b86bc2dd93f69dfb00f6f7fa5d3c5c6887f Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 06:12:03 +0800 Subject: [PATCH 16/26] feat(ui,workbench): decode H.264 simulator streams with WebCodecs --- .../client/workbench/src/simulator/panel.tsx | 10 +- .../src/simulator/stream-registry.ts | 5 +- .../ui/src/shell/simulator/index.ts | 6 +- .../src/shell/simulator/simulator-screen.tsx | 168 ++++++++++++------ 4 files changed, 131 insertions(+), 58 deletions(-) diff --git a/packages/client/workbench/src/simulator/panel.tsx b/packages/client/workbench/src/simulator/panel.tsx index 7aea3d38..3808ba90 100644 --- a/packages/client/workbench/src/simulator/panel.tsx +++ b/packages/client/workbench/src/simulator/panel.tsx @@ -1,6 +1,6 @@ import { useLinkCodeClient } from '@linkcode/client-core'; import type { SessionId, SimulatorDevice, SimulatorStatus } from '@linkcode/schema'; -import type { SimulatorScreenPoint } from '@linkcode/ui/shell/simulator'; +import type { SimulatorScreenFrame, SimulatorScreenPoint } from '@linkcode/ui/shell/simulator'; import { SimulatorScreen } from '@linkcode/ui/shell/simulator'; import { Button } from 'coss-ui/components/button'; import { @@ -102,8 +102,12 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): const ownerSessionId = snapshot?.sessionId ?? null; const subscribeFrames = useCallback( - (onFrame: (jpegBase64: string) => void) => - udid === null ? noop : client.subscribeSimulatorFrames(udid, (frame) => onFrame(frame.data)), + (onFrame: (frame: SimulatorScreenFrame) => void) => + udid === null + ? noop + : client.subscribeSimulatorFrames(udid, (frame) => + onFrame({ codec: frame.codec, key: frame.key, data: frame.data }), + ), [client, udid], ); diff --git a/packages/client/workbench/src/simulator/stream-registry.ts b/packages/client/workbench/src/simulator/stream-registry.ts index de29040d..53664f9a 100644 --- a/packages/client/workbench/src/simulator/stream-registry.ts +++ b/packages/client/workbench/src/simulator/stream-registry.ts @@ -8,8 +8,9 @@ export type SimulatorStreamClient = Pick< 'simulatorStreamStart' | 'simulatorStreamStop' >; -/** Panel-facing stream parameters: plenty of fidelity for a right-panel pane without flooding the wire. */ -const STREAM_OPTIONS = { fps: 30, scale: 0.5 } as const; +/** Panel-facing stream parameters: hardware H.264 at native resolution — full 60 fps at a + * fraction of a JPEG stream's bandwidth. Hosts that cannot honor it fall back to JPEG frames. */ +const STREAM_OPTIONS = { fps: 60, codec: 'h264' } as const; /** Bridges the unmount→mount gap of the docked↔maximized handoff without stopping the stream. */ const CLOSE_DELAY_MS = 250; diff --git a/packages/presentation/ui/src/shell/simulator/index.ts b/packages/presentation/ui/src/shell/simulator/index.ts index eef044a4..8a7111e4 100644 --- a/packages/presentation/ui/src/shell/simulator/index.ts +++ b/packages/presentation/ui/src/shell/simulator/index.ts @@ -1,2 +1,6 @@ -export type { SimulatorScreenPoint, SimulatorScreenProps } from './simulator-screen'; +export type { + SimulatorScreenFrame, + SimulatorScreenPoint, + SimulatorScreenProps, +} from './simulator-screen'; export { SimulatorScreen } from './simulator-screen'; diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index e136759c..ed68be3a 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -33,9 +33,17 @@ const FALLBACK_CORNER_FRACTION = 0.11; const RIM_COLOR = '#3a3a3c'; const BEZEL_COLOR = '#000000'; +/** One stream frame: a base64 JPEG image, or one ordered base64 Annex-B H.264 access unit. */ +export interface SimulatorScreenFrame { + codec: 'jpeg' | 'h264'; + key: boolean; + data: string; +} + export interface SimulatorScreenProps { - /** Feed of JPEG frames (base64, no data-URL prefix); returns the unsubscribe. */ - subscribeFrames: (onFrame: (jpegBase64: string) => void) => () => void; + /** Feed of stream frames; returns the unsubscribe. H.264 units decode through WebCodecs + * (hardware, GPU-resident output); JPEG frames decode via `createImageBitmap`. */ + subscribeFrames: (onFrame: (frame: SimulatorScreenFrame) => void) => () => void; /** Press in normalized [0,1] device coordinates. */ onTap?: (point: SimulatorScreenPoint) => void; /** Drag in normalized [0,1] device coordinates with its real duration. */ @@ -49,11 +57,23 @@ export interface SimulatorScreenProps { className?: string; } +/** A decoded frame the compositor can draw: a bitmap (JPEG path) or a GPU-resident VideoFrame + * (H.264 path). Both close() and drawImage() uniformly; only the size accessors differ. */ +type DecodedFrame = ImageBitmap | VideoFrame; + +function frameWidth(frame: DecodedFrame): number { + return 'displayWidth' in frame ? frame.displayWidth : frame.width; +} + +function frameHeight(frame: DecodedFrame): number { + return 'displayHeight' in frame ? frame.displayHeight : frame.height; +} + /** Everything the compositor knows about one device feed, in native framebuffer pixels. */ interface PaintState { mask: ImageBitmap | null; /** Last decoded frame, retained so a late-arriving mask can recomposite it. */ - frame: ImageBitmap | null; + frame: DecodedFrame | null; /** Screen-sized scratch layer the mask is composited on before drawing into the chassis. */ screenLayer: OffscreenCanvas | null; /** Cached chassis artwork (rim + display band), rebuilt when the geometry key changes. */ @@ -92,39 +112,86 @@ export function SimulatorScreen({ useAbortableEffect( (signal) => { const state = paintRef.current; - let latest: string | null = null; + + const adopt = (frame: DecodedFrame): void => { + if (signal.aborted || canvasRef.current === null) { + frame.close(); + return; + } + state.frame?.close(); + state.frame = frame; + paintDevice(canvasRef.current, state); + const canvas = canvasRef.current; + const aspect = `${canvas.width} / ${canvas.height}`; + setDeviceAspect((previous) => (previous === aspect ? previous : aspect)); + }; + + // JPEG path: latest-wins decode via createImageBitmap. + let latestJpeg: string | null = null; let decoding = false; - const drawNext = (): void => { - if (decoding || latest === null || signal.aborted) return; - const encoded = latest; - latest = null; + const drawNextJpeg = (): void => { + if (decoding || latestJpeg === null || signal.aborted) return; + const encoded = latestJpeg; + latestJpeg = null; decoding = true; void createImageBitmap(new Blob([base64Bytes(encoded)], { type: 'image/jpeg' })) - .then((bitmap) => { - if (signal.aborted || canvasRef.current === null) { - bitmap.close(); - return; - } - state.frame?.close(); - state.frame = bitmap; - paintDevice(canvasRef.current, state); - const canvas = canvasRef.current; - const aspect = `${canvas.width} / ${canvas.height}`; - setDeviceAspect((previous) => (previous === aspect ? previous : aspect)); - }) + .then(adopt) // A corrupt frame is dropped; the next one repaints. .catch(noop) .finally(() => { decoding = false; - drawNext(); + drawNextJpeg(); }); }; + + // H.264 path: hardware decode via WebCodecs; output VideoFrames stay GPU-resident. The + // decoder configures lazily, consumes only from a keyframe, and resets (waiting for the + // next key, ≤2s away) on any decode error. + let decoder: VideoDecoder | null = null; + let awaitingKey = true; + let timestamp = 0; + const resetDecoder = (): void => { + if (decoder !== null && decoder.state !== 'closed') decoder.close(); + decoder = null; + awaitingKey = true; + }; + const decodeH264 = (frame: SimulatorScreenFrame): void => { + if (decoder === null) { + const created = new VideoDecoder({ + output: adopt, + error: resetDecoder, + }); + // High profile at a level comfortably above any simulator resolution; the in-band + // SPS/PPS on each keyframe governs the actual stream parameters. + created.configure({ codec: 'avc1.640034', optimizeForLatency: true }); + decoder = created; + awaitingKey = true; + } + if (awaitingKey && !frame.key) return; + awaitingKey = false; + decoder.decode( + new EncodedVideoChunk({ + type: frame.key ? 'key' : 'delta', + // Synthetic monotonic clock; frames present as they arrive, so only order matters. + timestamp: timestamp++, + data: base64Bytes(frame.data), + }), + ); + }; + const unsubscribe = subscribeFrames((frame) => { - latest = frame; - drawNext(); + if (frame.codec === 'h264') { + decodeH264(frame); + return; + } + // A JPEG frame amid an h264 stream means the host degraded; drop the decoder. + resetDecoder(); + latestJpeg = frame.data; + drawNextJpeg(); }); return () => { unsubscribe(); + resetDecoder(); state.frame?.close(); state.frame = null; }; @@ -211,10 +278,12 @@ export function SimulatorScreen({ function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { const frame = state.frame; if (frame === null) return; - const pad = Math.round(frame.width * PAD_FRACTION); - const buttonDepth = Math.round(frame.width * BUTTON_DEPTH_FRACTION); - const width = frame.width + 2 * pad + 2 * buttonDepth; - const height = frame.height + 2 * pad; + const screenW = frameWidth(frame); + const screenH = frameHeight(frame); + const pad = Math.round(screenW * PAD_FRACTION); + const buttonDepth = Math.round(screenW * BUTTON_DEPTH_FRACTION); + const width = screenW + 2 * pad + 2 * buttonDepth; + const height = screenH + 2 * pad; if (canvas.width !== width || canvas.height !== height) { canvas.width = width; canvas.height = height; @@ -228,27 +297,27 @@ function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { const buttonWidth = buttonDepth * 3; for (const [top, size] of LEFT_BUTTONS) { context.beginPath(); - context.roundRect(0, pad + top * frame.height, buttonWidth, size * frame.height, buttonDepth); + context.roundRect(0, pad + top * screenH, buttonWidth, size * screenH, buttonDepth); context.fill(); } for (const [top, size] of RIGHT_BUTTONS) { context.beginPath(); context.roundRect( width - buttonWidth, - pad + top * frame.height, + pad + top * screenH, buttonWidth, - size * frame.height, + size * screenH, buttonDepth, ); context.fill(); } - context.drawImage(buildChassis(state, frame), buttonDepth, 0); + context.drawImage(buildChassis(state, screenW, screenH), buttonDepth, 0); // Screen: composite the frame against the mask on a screen-sized layer, then inset it. let layer = state.screenLayer; - if (layer?.width !== frame.width || layer.height !== frame.height) { - layer = new OffscreenCanvas(frame.width, frame.height); + if (layer?.width !== screenW || layer.height !== screenH) { + layer = new OffscreenCanvas(screenW, screenH); state.screenLayer = layer; } const layerContext = layer.getContext('2d'); @@ -257,7 +326,7 @@ function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { if (state.mask === null) { layerContext.save(); layerContext.beginPath(); - layerContext.roundRect(0, 0, layer.width, layer.height, frame.width * FALLBACK_CORNER_FRACTION); + layerContext.roundRect(0, 0, layer.width, layer.height, screenW * FALLBACK_CORNER_FRACTION); layerContext.clip(); layerContext.drawImage(frame, 0, 0); layerContext.restore(); @@ -278,11 +347,11 @@ function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { * diverge from them. Cached per frame-size + mask identity; without a mask a rounded rect stands * in for the shape. */ -function buildChassis(state: PaintState, frame: ImageBitmap): OffscreenCanvas { - const pad = Math.round(frame.width * PAD_FRACTION); - const rim = Math.round(frame.width * RIM_FRACTION); - const width = frame.width + 2 * pad; - const height = frame.height + 2 * pad; +function buildChassis(state: PaintState, screenW: number, screenH: number): OffscreenCanvas { + const pad = Math.round(screenW * PAD_FRACTION); + const rim = Math.round(screenW * RIM_FRACTION); + const width = screenW + 2 * pad; + const height = screenH + 2 * pad; const key = `${width}x${height}:${state.mask === null ? 'fallback' : 'mask'}`; if (state.chassis !== null && state.chassisKey === key) return state.chassis; @@ -290,8 +359,8 @@ function buildChassis(state: PaintState, frame: ImageBitmap): OffscreenCanvas { const context = chassis.getContext('2d'); if (context === null) return chassis; context.clearRect(0, 0, width, height); - const rimShape = growScreenShape(state.mask, frame, pad); - const bandShape = growScreenShape(state.mask, frame, pad - rim); + const rimShape = growScreenShape(state.mask, screenW, screenH, pad); + const bandShape = growScreenShape(state.mask, screenW, screenH, pad - rim); colorize(rimShape, RIM_COLOR); colorize(bandShape, BEZEL_COLOR); context.drawImage(rimShape, 0, 0); @@ -305,21 +374,16 @@ function buildChassis(state: PaintState, frame: ImageBitmap): OffscreenCanvas { * (morphological dilation); a rounded rect when no mask exists. */ function growScreenShape( mask: ImageBitmap | null, - frame: ImageBitmap, + screenW: number, + screenH: number, grow: number, ): OffscreenCanvas { - const shape = new OffscreenCanvas(frame.width + 2 * grow, frame.height + 2 * grow); + const shape = new OffscreenCanvas(screenW + 2 * grow, screenH + 2 * grow); const context = shape.getContext('2d'); if (context === null) return shape; if (mask === null) { context.beginPath(); - context.roundRect( - 0, - 0, - shape.width, - shape.height, - frame.width * FALLBACK_CORNER_FRACTION + grow, - ); + context.roundRect(0, 0, shape.width, shape.height, screenW * FALLBACK_CORNER_FRACTION + grow); context.fill(); return shape; } @@ -330,8 +394,8 @@ function growScreenShape( mask, grow + grow * Math.cos(angle), grow + grow * Math.sin(angle), - frame.width, - frame.height, + screenW, + screenH, ); } return shape; From 09410ba00b1b970d64362874b1a736a32e202a34 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 07:14:21 +0800 Subject: [PATCH 17/26] feat(sim,schema,engine,client,ui): streamed touch, wheel scroll, HID keyboard (wire 48) --- .../src/__tests__/sim-mcp-endpoint.test.ts | 2 + apps/desktop/e2e/simulator-panel.e2e.mts | 61 ++++++++- crates/linkcode-sim/src/interactive.rs | 53 +++++++- crates/linkcode-sim/src/main.rs | 25 +++- crates/linkcode-sim/src/private/input.rs | 52 ++++++++ crates/linkcode-sim/src/private/mod.rs | 2 +- crates/linkcode-sim/src/rpc.rs | 25 ++++ packages/client/core/src/client.ts | 22 ++++ .../client/core/src/client/control-channel.ts | 35 +++++ .../client/workbench/src/simulator/panel.tsx | 25 ++-- .../foundation/schema/src/model/simulator.ts | 4 + .../foundation/schema/src/wire/message.ts | 3 +- .../foundation/schema/src/wire/simulator.ts | 22 ++++ .../simulator-request-handler.test.ts | 2 + .../src/__tests__/simulator-service.test.ts | 2 + packages/host/engine/src/simulator/backend.ts | 7 + .../engine/src/simulator/request-handler.ts | 22 ++++ packages/host/engine/src/simulator/service.ts | 17 +++ .../host/engine/src/wire/request-router.ts | 2 + packages/host/sim/src/client.ts | 11 ++ packages/host/sim/src/index.ts | 1 + packages/host/sim/src/schema.ts | 3 + .../shell/simulator/__tests__/keymap.test.ts | 54 ++++++++ .../ui/src/shell/simulator/index.ts | 2 + .../ui/src/shell/simulator/keymap.ts | 102 +++++++++++++++ .../src/shell/simulator/simulator-screen.tsx | 123 ++++++++++++++---- 26 files changed, 628 insertions(+), 51 deletions(-) create mode 100644 packages/presentation/ui/src/shell/simulator/__tests__/keymap.test.ts create mode 100644 packages/presentation/ui/src/shell/simulator/keymap.ts diff --git a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts index 9c42b213..79998839 100644 --- a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts +++ b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts @@ -35,6 +35,8 @@ function fakeBackend(): SimulatorBackend { screenshot: vi.fn(() => Promise.resolve(new Uint8Array([0xff, 0xd8, 0x02]))), screenMask: vi.fn(() => Promise.resolve(new Uint8Array([0x89, 0x50]))), tap: vi.fn(asyncNoop), + touch: vi.fn(asyncNoop), + key: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), streamStart: vi.fn(() => diff --git a/apps/desktop/e2e/simulator-panel.e2e.mts b/apps/desktop/e2e/simulator-panel.e2e.mts index 543de420..2a0189aa 100644 --- a/apps/desktop/e2e/simulator-panel.e2e.mts +++ b/apps/desktop/e2e/simulator-panel.e2e.mts @@ -29,7 +29,7 @@ const PORT = 43000 + (process.pid % 1000); /** Must match `WIRE_PROTOCOL_VERSION` (node can't load the raw-TS schema barrel); a mismatch is * silently discarded by the daemon, surfacing here as the session.start timeout. */ -const WIRE_VERSION = 47; +const WIRE_VERSION = 48; function fail(message: string): never { console.error(`FAIL: ${message}`); @@ -260,6 +260,65 @@ async function run(win: Page, chatRoot: string, deepPass: boolean): Promise => + deviceCanvas.evaluate((el) => { + const source = el as HTMLCanvasElement; + const probe = document.createElement('canvas'); + probe.width = 32; + probe.height = 64; + const ctx = probe.getContext('2d'); + if (ctx === null) return -1; + ctx.drawImage(source, 0, 0, 32, 64); + const data = ctx.getImageData(0, 0, 32, 64).data; + let hash = 0; + for (let i = 0; i < data.length; i += 16) hash = ((hash * 31 + data[i]) & 0xffffff) >>> 0; + return hash; + }); + const waitForHashChange = async (previous: number, label: string): Promise => { + const deadline = Date.now() + 10000; + while (Date.now() < deadline) { + const next = await canvasHash(); + if (next !== previous && next !== -1) return next; + await wait(500); + } + fail(`device screen did not change after ${label}`); + }; + + const onSafari = await canvasHash(); + await win.getByRole('button', { name: 'Home', exact: true }).click(); + const onHome = await waitForHashChange(onSafari, 'pressing Home'); + console.log('panel Home button returned the device to the home screen'); + + // Tap Safari in the dock through the canvas (device-normalized ≈ (0.41, 0.93), mapped + // into the canvas box across the painted chassis margins). + const box = await deviceCanvas.boundingBox(); + if (!box) fail('device canvas has no bounding box'); + await deviceCanvas.click({ + position: { x: box.width * 0.416, y: box.height * 0.919 }, + }); + const onSafariAgain = await waitForHashChange(onHome, 'tapping the dock'); + console.log('canvas tap reopened Safari from the dock'); + + // Streamed-touch drag: press, move upward in real time, release — the page must scroll. + await win.waitForTimeout(1500); + const dragX = box.x + box.width * 0.5; + const dragY = box.y + box.height * 0.6; + await win.mouse.move(dragX, dragY); + await win.mouse.down(); + for (let step = 1; step <= 10; step += 1) { + await win.mouse.move(dragX, dragY - step * box.height * 0.03); + await win.waitForTimeout(20); + } + await win.mouse.up(); + await waitForHashChange(onSafariAgain, 'dragging to scroll'); + console.log('streamed-touch drag scrolled the page'); + const tapShot = join(tmpdir(), `linkcode-e2e-simulator-tap-${process.pid}.png`); + await win.screenshot({ path: tapShot }); + console.log(`screenshot: ${tapShot}`); } console.log('holding the window open for 30s for manual inspection…'); await win.waitForTimeout(30000); diff --git a/crates/linkcode-sim/src/interactive.rs b/crates/linkcode-sim/src/interactive.rs index b15e8639..63ed69b2 100644 --- a/crates/linkcode-sim/src/interactive.rs +++ b/crates/linkcode-sim/src/interactive.rs @@ -11,7 +11,7 @@ use std::sync::mpsc::Sender; use serde_json::{Value, json}; use crate::OutMsg; -use crate::rpc::{ButtonKind, ErrorCode, OpError}; +use crate::rpc::{ButtonKind, ErrorCode, OpError, TouchPhase}; fn unsupported() -> OpError { OpError::new( @@ -21,7 +21,7 @@ fn unsupported() -> OpError { } #[cfg(target_os = "macos")] -pub use imp::{available, button, stream_start, stream_stop, swipe, tap}; +pub use imp::{available, button, key, stream_start, stream_stop, swipe, tap, touch}; #[cfg(not(target_os = "macos"))] mod stubs { @@ -33,6 +33,9 @@ mod stubs { pub fn tap(_udid: &str, _x: f64, _y: f64) -> Result { Err(unsupported()) } + pub fn touch(_udid: &str, _phase: TouchPhase, _x: f64, _y: f64) -> Result { + Err(unsupported()) + } pub fn swipe( _udid: &str, _x0: f64, @@ -46,6 +49,9 @@ mod stubs { pub fn button(_udid: &str, _button: ButtonKind) -> Result { Err(unsupported()) } + pub fn key(_udid: &str, _usage: u32, _modifiers: &[u32]) -> Result { + Err(unsupported()) + } pub fn stream_start( _udid: &str, _fps: u32, @@ -62,7 +68,7 @@ mod stubs { } #[cfg(not(target_os = "macos"))] -pub use stubs::{available, button, stream_start, stream_stop, swipe, tap}; +pub use stubs::{available, button, key, stream_start, stream_stop, swipe, tap, touch}; #[cfg(target_os = "macos")] mod imp { @@ -89,6 +95,8 @@ mod imp { struct Registry { inputs: HashMap>, streams: HashMap, + /// Active streamed-touch identifiers by udid (down allocates, up removes). + touches: HashMap, } struct StreamHandle { @@ -106,6 +114,7 @@ mod imp { Mutex::new(Registry { inputs: HashMap::new(), streams: HashMap::new(), + touches: HashMap::new(), }) }) } @@ -136,6 +145,36 @@ mod imp { } } + /// One phase of a streamed touch gesture. A `move`/`up` without an active stream is a benign + /// race (a duplicate up, a move after cancel) and no-ops successfully. + pub fn touch(udid: &str, phase: TouchPhase, x: f64, y: f64) -> Result { + let input = input_for(udid)?; + let mut reg = registry().lock().expect("interactive registry poisoned"); + let identifier = match phase { + TouchPhase::Down => { + let id = input.allocate_touch(); + reg.touches.insert(udid.to_owned(), id); + Some(id) + } + TouchPhase::Move => reg.touches.get(udid).copied(), + TouchPhase::Up => reg.touches.remove(udid), + }; + drop(reg); + let Some(identifier) = identifier else { + return Ok(json!({})); + }; + let hid_phase = match phase { + TouchPhase::Down => private::Phase::Down, + TouchPhase::Move => private::Phase::Move, + TouchPhase::Up => private::Phase::Up, + }; + if input.touch_phase(x, y, identifier, hid_phase) { + Ok(json!({})) + } else { + Err(OpError::new(ErrorCode::SimctlFailed, "touch failed")) + } + } + pub fn swipe( udid: &str, x0: f64, @@ -170,6 +209,14 @@ mod imp { } } + pub fn key(udid: &str, usage: u32, modifiers: &[u32]) -> Result { + if input_for(udid)?.key(usage, modifiers, Duration::from_millis(20)) { + Ok(json!({})) + } else { + Err(OpError::new(ErrorCode::SimctlFailed, "key press failed")) + } + } + pub fn stream_start( udid: &str, fps: u32, diff --git a/crates/linkcode-sim/src/main.rs b/crates/linkcode-sim/src/main.rs index b3a5329c..fe7db07a 100644 --- a/crates/linkcode-sim/src/main.rs +++ b/crates/linkcode-sim/src/main.rs @@ -162,13 +162,20 @@ fn main() { } match serde_json::from_slice::(&body) { Ok(request) => { - gate.acquire(); - let tx = tx.clone(); - let gate = Arc::clone(&gate); - thread::spawn(move || { + // Touch phases and key presses must keep stdio order — a per-request thread would + // race a gesture's down/move/up or reorder typed characters. Each is a handful of + // fast HID sends, so inline handling is cheap and stays off the inflight gate. + if matches!(request.op, Op::Touch { .. } | Op::Key { .. }) { serve(request, &tx); - gate.release(); - }); + } else { + gate.acquire(); + let tx = tx.clone(); + let gate = Arc::clone(&gate); + thread::spawn(move || { + serve(request, &tx); + gate.release(); + }); + } } Err(err) => { eprintln!("invalid REQUEST frame: {err}"); @@ -248,6 +255,7 @@ fn serve(request: Request, tx: &Sender) { } } Op::Tap { udid, x, y } => interactive::tap(&udid, x, y), + Op::Touch { udid, phase, x, y } => interactive::touch(&udid, phase, x, y), Op::Swipe { udid, x0, @@ -257,6 +265,11 @@ fn serve(request: Request, tx: &Sender) { duration_ms, } => interactive::swipe(&udid, x0, y0, x1, y1, duration_ms), Op::Button { udid, button } => interactive::button(&udid, button), + Op::Key { + udid, + usage, + modifiers, + } => interactive::key(&udid, usage, &modifiers), Op::StreamStart { udid, fps, diff --git a/crates/linkcode-sim/src/private/input.rs b/crates/linkcode-sim/src/private/input.rs index 4335a6f7..748d9d0d 100644 --- a/crates/linkcode-sim/src/private/input.rs +++ b/crates/linkcode-sim/src/private/input.rs @@ -61,6 +61,9 @@ type CreateFingerFn = unsafe extern "C" fn( type AppendEventFn = unsafe extern "C" fn(*mut c_void, *mut c_void, u32); type TrackpadWrapFn = unsafe extern "C" fn(*const c_void) -> *mut c_void; type ButtonFn = unsafe extern "C" fn(u32, u32, u32) -> *mut c_void; +/// `IndigoHIDMessageForHIDArbitrary(target, page, usage, operation)` — routes any HID +/// (page, usage) pair; operation 1=down, 2=up. No timestamp arg. +type HidArbitraryFn = unsafe extern "C" fn(u32, u32, u32, u32) -> *mut c_void; type ServiceFn = unsafe extern "C" fn() -> *mut c_void; unsafe extern "C" { @@ -114,10 +117,14 @@ struct Symbols { append_event: AppendEventFn, trackpad_wrap: TrackpadWrapFn, button: ButtonFn, + hid_arbitrary: Option, create_pointer_service: Option, create_mouse_service: Option, } +/// HID keyboard/keypad usage page (the page every key press below lives on). +const KEYBOARD_USAGE_PAGE: u32 = 7; + /// A warmed HID client bound to one device, plus the resolved private symbols. Created lazily; a /// resolution failure means this host cannot inject input (the caller degrades to view-only). pub struct Input { @@ -146,6 +153,17 @@ impl Input { Some(input) } + /// Allocate the shared identifier for one caller-driven touch stream (down → moves → up). + pub fn allocate_touch(&self) -> u32 { + self.next_identifier() + } + + /// Inject one phase of a caller-driven touch stream. The caller owns the cadence and the + /// down/move/up sequencing; `identifier` ties the phases into one gesture. + pub fn touch_phase(&self, x: f64, y: f64, identifier: u32, phase: Phase) -> bool { + self.send_touch(x, y, identifier, phase) + } + /// Single-finger tap at a normalised (0..1) point, holding for `hold`. pub fn tap(&self, x: f64, y: f64, hold: Duration) -> bool { let id = self.next_identifier(); @@ -177,6 +195,37 @@ impl Input { self.send_touch(x1, y1, id, Phase::Up) && ok >= steps / 2 } + /// Press one keyboard key (HID usage on page 7) with `modifiers` (usages `0xE0..=0xE7`) + /// bracketed around it: modifier-downs → key-down → hold → key-up → modifier-ups. + pub fn key(&self, usage: u32, modifiers: &[u32], hold: Duration) -> bool { + let Some(hid) = self.symbols.hid_arbitrary else { + return false; + }; + let send_op = |usage: u32, operation: u32| -> bool { + // SAFETY: symbol resolved above; returns a malloc'd message consumed by send. + let message = unsafe { hid(TOUCH_TARGET, KEYBOARD_USAGE_PAGE, usage, operation) }; + if message.is_null() { + return false; + } + self.send_message(message); + true + }; + for modifier in modifiers { + if !send_op(*modifier, 1) { + return false; + } + } + if !send_op(usage, 1) { + return false; + } + sleep(hold.max(Duration::from_millis(20))); + let mut ok = send_op(usage, 2); + for modifier in modifiers.iter().rev() { + ok = send_op(*modifier, 2) && ok; + } + ok + } + /// Press a hardware button (home/lock) for `hold`. pub fn button(&self, button: Button, hold: Duration) -> bool { let arg0 = match button { @@ -366,6 +415,7 @@ fn resolve_symbols() -> Option { let trackpad_wrap = framework::dlsym(kit, "IndigoHIDMessageForTrackpadEventFromHIDEventRef")?; let button = framework::dlsym(kit, "IndigoHIDMessageForButton")?; + let hid_arbitrary = framework::dlsym(kit, "IndigoHIDMessageForHIDArbitrary"); Some(Symbols { create_digitizer: std::mem::transmute::<*mut c_void, CreateDigitizerFn>( create_digitizer, @@ -374,6 +424,8 @@ fn resolve_symbols() -> Option { append_event: std::mem::transmute::<*mut c_void, AppendEventFn>(append_event), trackpad_wrap: std::mem::transmute::<*mut c_void, TrackpadWrapFn>(trackpad_wrap), button: std::mem::transmute::<*mut c_void, ButtonFn>(button), + hid_arbitrary: hid_arbitrary + .map(|p| std::mem::transmute::<*mut c_void, HidArbitraryFn>(p)), create_pointer_service: framework::dlsym(kit, "IndigoHIDMessageToCreatePointerService") .map(|p| std::mem::transmute::<*mut c_void, ServiceFn>(p)), create_mouse_service: framework::dlsym(kit, "IndigoHIDMessageToCreateMouseService") diff --git a/crates/linkcode-sim/src/private/mod.rs b/crates/linkcode-sim/src/private/mod.rs index f5c60358..29f3e96d 100644 --- a/crates/linkcode-sim/src/private/mod.rs +++ b/crates/linkcode-sim/src/private/mod.rs @@ -13,7 +13,7 @@ mod screen; mod vt; pub use device::SimDevice; -pub use input::{Button, Input}; +pub use input::{Button, Input, Phase}; pub use screen::{Screen, bench_encode}; pub use vt::VtEncoder; diff --git a/crates/linkcode-sim/src/rpc.rs b/crates/linkcode-sim/src/rpc.rs index ced4d23d..482876a3 100644 --- a/crates/linkcode-sim/src/rpc.rs +++ b/crates/linkcode-sim/src/rpc.rs @@ -58,6 +58,14 @@ pub enum Op { ScreenMask { udid: String }, /// Single-finger tap at a normalised (0..1) point (private API; P1). Tap { udid: String, x: f64, y: f64 }, + /// One phase of a caller-driven touch stream at a normalised point (private API; P1). + /// Sequencing (one `down`, moves, one `up`) is the caller's; phases keep stdio order. + Touch { + udid: String, + phase: TouchPhase, + x: f64, + y: f64, + }, /// Swipe between two normalised (0..1) points over `duration_ms` (private API; P1). Swipe { udid: String, @@ -70,6 +78,14 @@ pub enum Op { }, /// Press a hardware button (private API; P1). Button { udid: String, button: ButtonKind }, + /// Press one keyboard key: an HID usage on page 7 with modifier usages (`0xE0..`) bracketed + /// around it (private API; P1). Key order is preserved (handled inline like `touch`). + Key { + udid: String, + usage: u32, + #[serde(default)] + modifiers: Vec, + }, /// Start streaming the device framebuffer at `fps` (private API; P1). `codec` picks JPEG /// `STREAM_FRAME`s (latest-wins) or H.264 `STREAM_FRAME_H264` access units (ordered). StreamStart { @@ -105,6 +121,15 @@ pub enum ButtonKind { Lock, } +/// One phase of a streamed touch gesture. +#[derive(Clone, Copy, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum TouchPhase { + Down, + Move, + Up, +} + /// Framebuffer stream encodings. JPEG frames are independently decodable (latest-wins delivery); /// H.264 access units are ordered and delta-dependent (hardware encode/decode, ~10× less bandwidth). #[derive(Clone, Copy, Default, PartialEq, Eq, Deserialize, Serialize)] diff --git a/packages/client/core/src/client.ts b/packages/client/core/src/client.ts index 6b73af04..4f04b2df 100644 --- a/packages/client/core/src/client.ts +++ b/packages/client/core/src/client.ts @@ -42,6 +42,7 @@ import type { SimulatorImageFormat, SimulatorStatus, SimulatorStreamCodec, + SimulatorTouchPhase, StartOptions, TerminalMetadata, TerminalReplayEvent, @@ -709,6 +710,27 @@ export class LinkCodeClient { return this.control.simulatorTap(sessionId, udid, x, y); } + /** Press one keyboard key (HID usage on page 7) with modifier usages held around it. */ + simulatorKey( + sessionId: SessionId, + udid: string, + usage: number, + modifiers: number[], + ): Promise { + return this.control.simulatorKey(sessionId, udid, usage, modifiers); + } + + /** One phase of a streamed touch gesture; the caller owns the down/move/up sequencing. */ + simulatorTouch( + sessionId: SessionId, + udid: string, + phase: SimulatorTouchPhase, + x: number, + y: number, + ): Promise { + return this.control.simulatorTouch(sessionId, udid, phase, x, y); + } + simulatorSwipe( sessionId: SessionId, udid: string, diff --git a/packages/client/core/src/client/control-channel.ts b/packages/client/core/src/client/control-channel.ts index dc1bc8e9..44b6abbb 100644 --- a/packages/client/core/src/client/control-channel.ts +++ b/packages/client/core/src/client/control-channel.ts @@ -39,6 +39,7 @@ import type { SimulatorImageFormat, SimulatorStatus, SimulatorStreamCodec, + SimulatorTouchPhase, StartOptions, WirePayload, WorkspaceFile, @@ -679,6 +680,40 @@ export class ControlChannel { })); } + simulatorKey( + sessionId: SessionId, + udid: string, + usage: number, + modifiers: number[], + ): Promise { + return this.sendCorrelated('ack', (clientReqId) => ({ + kind: 'simulator.key', + clientReqId, + sessionId, + udid, + usage, + modifiers, + })); + } + + simulatorTouch( + sessionId: SessionId, + udid: string, + phase: SimulatorTouchPhase, + x: number, + y: number, + ): Promise { + return this.sendCorrelated('ack', (clientReqId) => ({ + kind: 'simulator.touch', + clientReqId, + sessionId, + udid, + phase, + x, + y, + })); + } + simulatorSwipe( sessionId: SessionId, udid: string, diff --git a/packages/client/workbench/src/simulator/panel.tsx b/packages/client/workbench/src/simulator/panel.tsx index 3808ba90..61fbf335 100644 --- a/packages/client/workbench/src/simulator/panel.tsx +++ b/packages/client/workbench/src/simulator/panel.tsx @@ -1,6 +1,11 @@ import { useLinkCodeClient } from '@linkcode/client-core'; import type { SessionId, SimulatorDevice, SimulatorStatus } from '@linkcode/schema'; -import type { SimulatorScreenFrame, SimulatorScreenPoint } from '@linkcode/ui/shell/simulator'; +import type { + SimulatorKeyPress, + SimulatorScreenFrame, + SimulatorScreenPoint, + SimulatorScreenTouchPhase, +} from '@linkcode/ui/shell/simulator'; import { SimulatorScreen } from '@linkcode/ui/shell/simulator'; import { Button } from 'coss-ui/components/button'; import { @@ -121,17 +126,15 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): return {t('simulatorUnavailable')}; } - const handleTap = (point: SimulatorScreenPoint): void => { + const handleTouch = (phase: SimulatorScreenTouchPhase, point: SimulatorScreenPoint): void => { if (ownerSessionId === null || udid === null) return; - void client.simulatorTap(ownerSessionId, udid, point.x, point.y).catch(flagBusy); + const request = client.simulatorTouch(ownerSessionId, udid, phase, point.x, point.y); + // Surface a claim conflict once per gesture, not per 60 Hz move. + void request.catch(phase === 'down' ? flagBusy : noop); }; - const handleSwipe = ( - from: SimulatorScreenPoint, - to: SimulatorScreenPoint, - durationMs: number, - ): void => { + const handleKey = (press: SimulatorKeyPress): void => { if (ownerSessionId === null || udid === null) return; - void client.simulatorSwipe(ownerSessionId, udid, from, to, durationMs).catch(flagBusy); + void client.simulatorKey(ownerSessionId, udid, press.usage, press.modifiers).catch(flagBusy); }; const pressButton = (button: 'home' | 'lock'): void => { if (ownerSessionId === null || udid === null) return; @@ -207,8 +210,8 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): {t('simulatorConnecting')} diff --git a/packages/foundation/schema/src/model/simulator.ts b/packages/foundation/schema/src/model/simulator.ts index 72321580..50a3f9aa 100644 --- a/packages/foundation/schema/src/model/simulator.ts +++ b/packages/foundation/schema/src/model/simulator.ts @@ -37,3 +37,7 @@ export type SimulatorImageFormat = z.infer; * access units (Annex-B, native resolution — decoded client-side with WebCodecs). */ export const SimulatorStreamCodecSchema = z.enum(['jpeg', 'h264']); export type SimulatorStreamCodec = z.infer; + +/** One phase of a streamed touch gesture (one `down`, moves, one `up` per gesture). */ +export const SimulatorTouchPhaseSchema = z.enum(['down', 'move', 'up']); +export type SimulatorTouchPhase = z.infer; diff --git a/packages/foundation/schema/src/wire/message.ts b/packages/foundation/schema/src/wire/message.ts index 1b590376..d260b212 100644 --- a/packages/foundation/schema/src/wire/message.ts +++ b/packages/foundation/schema/src/wire/message.ts @@ -18,7 +18,8 @@ import { WirePayloadSchema } from './payload'; // 46 adds the simulator interactive + framebuffer-stream variants (CODE-397). // 47 adds the simulator screen-mask wire (CODE-397). // 48 adds the H.264 stream codec plumbing (CODE-397). -export const WIRE_PROTOCOL_VERSION = 48 as const; +// 49 adds streamed touch, wheel scroll, and HID keyboard input (CODE-397). +export const WIRE_PROTOCOL_VERSION = 49 as const; /** Complete wire message: version + unique id + timestamp + payload. */ export const WireMessageSchema = z.object({ diff --git a/packages/foundation/schema/src/wire/simulator.ts b/packages/foundation/schema/src/wire/simulator.ts index 8949676e..2f00c912 100644 --- a/packages/foundation/schema/src/wire/simulator.ts +++ b/packages/foundation/schema/src/wire/simulator.ts @@ -5,6 +5,7 @@ import { SimulatorImageFormatSchema, SimulatorStatusSchema, SimulatorStreamCodecSchema, + SimulatorTouchPhaseSchema, } from '../model/simulator'; import { WireRequestIdSchema } from './request'; @@ -131,6 +132,17 @@ export const simulatorWireVariants = [ x: coord, y: coord, }), + /** One phase of a streamed touch gesture — the panel forwards pointer events in real time so + * the device sees the finger during a drag (long-press, rubber-banding, icon drags). */ + z.object({ + kind: z.literal('simulator.touch'), + clientReqId: WireRequestIdSchema, + sessionId: SessionIdSchema, + udid, + phase: SimulatorTouchPhaseSchema, + x: coord, + y: coord, + }), z.object({ kind: z.literal('simulator.swipe'), clientReqId: WireRequestIdSchema, @@ -149,6 +161,16 @@ export const simulatorWireVariants = [ udid, button: SimulatorButtonSchema, }), + /** One keyboard key press: an HID usage on page 7 with modifier usages (`0xE0..`) held + * around it. Clients decompose typed characters (US layout) before sending. */ + z.object({ + kind: z.literal('simulator.key'), + clientReqId: WireRequestIdSchema, + sessionId: SessionIdSchema, + udid, + usage: z.number().int().nonnegative(), + modifiers: z.array(z.number().int().nonnegative()).max(8), + }), z.object({ kind: z.literal('simulator.stream.start'), clientReqId: WireRequestIdSchema, diff --git a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts index 14916437..4f7df8cc 100644 --- a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts +++ b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts @@ -34,6 +34,8 @@ function fakeBackend() { screenshot: vi.fn(() => Promise.resolve(new Uint8Array([0xff, 0xd8, 0x01]))), screenMask: vi.fn(() => Promise.resolve(new Uint8Array([0x89, 0x50]))), tap: vi.fn(asyncNoop), + touch: vi.fn(asyncNoop), + key: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), streamStart: vi.fn(() => diff --git a/packages/host/engine/src/__tests__/simulator-service.test.ts b/packages/host/engine/src/__tests__/simulator-service.test.ts index 4b5e9c86..db618d42 100644 --- a/packages/host/engine/src/__tests__/simulator-service.test.ts +++ b/packages/host/engine/src/__tests__/simulator-service.test.ts @@ -31,6 +31,8 @@ function fakeBackend(devices: SimulatorDeviceInfo[]) { screenshot: vi.fn(() => Promise.resolve(new Uint8Array([0xff, 0xd8]))), screenMask: vi.fn(() => Promise.resolve(new Uint8Array([0x89, 0x50]))), tap: vi.fn(asyncNoop), + touch: vi.fn(asyncNoop), + key: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), streamStart: vi.fn(() => diff --git a/packages/host/engine/src/simulator/backend.ts b/packages/host/engine/src/simulator/backend.ts index fb6c32cc..f005aa37 100644 --- a/packages/host/engine/src/simulator/backend.ts +++ b/packages/host/engine/src/simulator/backend.ts @@ -30,6 +30,9 @@ export type SimulatorImageFormat = 'jpeg' | 'png'; /** A hardware button the private HID layer can press. */ export type SimulatorButton = 'home' | 'lock'; +/** One phase of a streamed touch gesture (one `down`, moves, one `up` per gesture). */ +export type SimulatorTouchPhase = 'down' | 'move' | 'up'; + /** A normalized (0..1) point on the device screen. */ export interface SimulatorPoint { x: number; @@ -79,10 +82,14 @@ export interface SimulatorBackend { screenMask(udid: string): Promise; /** Tap at a normalized (0..1) point (private HID; macOS only). */ tap(udid: string, x: number, y: number): Promise; + /** One phase of a streamed touch gesture; the caller owns the down/move/up sequencing. */ + touch(udid: string, phase: SimulatorTouchPhase, x: number, y: number): Promise; /** Swipe between two normalized (0..1) points over `durationMs` (private HID; macOS only). */ swipe(udid: string, from: SimulatorPoint, to: SimulatorPoint, durationMs?: number): Promise; /** Press a hardware button (private HID; macOS only). */ button(udid: string, button: SimulatorButton): Promise; + /** Press one keyboard key (HID usage on page 7) with modifier usages held around it. */ + key(udid: string, usage: number, modifiers: number[]): Promise; /** Start streaming `udid`'s framebuffer; frames arrive via {@link onFrame} listeners. */ streamStart(udid: string, options?: SimulatorStreamOptions): Promise; /** Stop a running framebuffer stream. */ diff --git a/packages/host/engine/src/simulator/request-handler.ts b/packages/host/engine/src/simulator/request-handler.ts index 7c78b96c..711c12c4 100644 --- a/packages/host/engine/src/simulator/request-handler.ts +++ b/packages/host/engine/src/simulator/request-handler.ts @@ -22,8 +22,10 @@ type SimulatorRequest = Extract< | 'simulator.screenshot' | 'simulator.screen-mask' | 'simulator.tap' + | 'simulator.touch' | 'simulator.swipe' | 'simulator.button' + | 'simulator.key' | 'simulator.stream.start' | 'simulator.stream.stop'; } @@ -163,6 +165,19 @@ export class SimulatorRequestHandler { this.responder.sendSuccess(payload.clientReqId); }), ); + case 'simulator.touch': + return this.withSimulators(payload.clientReqId, (simulators) => + simulatorOperation('simulator.touch', 'Failed to inject touch', async () => { + await simulators.touch( + payload.sessionId, + payload.udid, + payload.phase, + payload.x, + payload.y, + ); + this.responder.sendSuccess(payload.clientReqId); + }), + ); case 'simulator.swipe': return this.withSimulators(payload.clientReqId, (simulators) => simulatorOperation('simulator.swipe', 'Failed to swipe', async () => { @@ -183,6 +198,13 @@ export class SimulatorRequestHandler { this.responder.sendSuccess(payload.clientReqId); }), ); + case 'simulator.key': + return this.withSimulators(payload.clientReqId, (simulators) => + simulatorOperation('simulator.key', 'Failed to press key', async () => { + await simulators.key(payload.sessionId, payload.udid, payload.usage, payload.modifiers); + this.responder.sendSuccess(payload.clientReqId); + }), + ); case 'simulator.stream.start': return this.withSimulators(payload.clientReqId, (simulators) => simulatorOperation('simulator.stream.start', 'Failed to start stream', async () => { diff --git a/packages/host/engine/src/simulator/service.ts b/packages/host/engine/src/simulator/service.ts index bc2a7f3a..d9f6230e 100644 --- a/packages/host/engine/src/simulator/service.ts +++ b/packages/host/engine/src/simulator/service.ts @@ -12,6 +12,7 @@ import type { SimulatorPoint, SimulatorStreamOptions, SimulatorStreamStartResult, + SimulatorTouchPhase, } from './backend'; /** Lazily-probed host capability; mirrors the wire's `SimulatorStatus` shape structurally. */ @@ -173,6 +174,17 @@ export class SimulatorService { return this.backend.tap(udid, x, y); } + async touch( + sessionId: SessionId, + udid: string, + phase: SimulatorTouchPhase, + x: number, + y: number, + ): Promise { + this.claim(sessionId, udid); + return this.backend.touch(udid, phase, x, y); + } + async swipe( sessionId: SessionId, udid: string, @@ -189,6 +201,11 @@ export class SimulatorService { return this.backend.button(udid, button); } + async key(sessionId: SessionId, udid: string, usage: number, modifiers: number[]): Promise { + this.claim(sessionId, udid); + return this.backend.key(udid, usage, modifiers); + } + /** Start streaming a device's framebuffer for a session, claiming it. */ async streamStart( sessionId: SessionId, diff --git a/packages/host/engine/src/wire/request-router.ts b/packages/host/engine/src/wire/request-router.ts index 24f682cc..8bccfd85 100644 --- a/packages/host/engine/src/wire/request-router.ts +++ b/packages/host/engine/src/wire/request-router.ts @@ -140,6 +140,8 @@ export class WireRequestRouter { case 'simulator.screenshot': case 'simulator.screen-mask': case 'simulator.tap': + case 'simulator.touch': + case 'simulator.key': case 'simulator.swipe': case 'simulator.button': case 'simulator.stream.start': diff --git a/packages/host/sim/src/client.ts b/packages/host/sim/src/client.ts index 0c3396fc..3ce81be8 100644 --- a/packages/host/sim/src/client.ts +++ b/packages/host/sim/src/client.ts @@ -22,6 +22,7 @@ import type { SimProbe, SimStreamCodec, SimStreamStartResult, + SimTouchPhase, } from './schema'; import { SimLaunchResultSchema, @@ -151,6 +152,11 @@ export class SimSidecarClient { await this.call('tap', { udid, x, y }); } + /** One phase of a streamed touch gesture; the caller owns the down/move/up sequencing. */ + async touch(udid: string, phase: SimTouchPhase, x: number, y: number): Promise { + await this.call('touch', { udid, phase, x, y }); + } + /** Swipe between two normalized (0..1) points over `durationMs` (private HID; macOS only). */ async swipe( udid: string, @@ -173,6 +179,11 @@ export class SimSidecarClient { await this.call('button', { udid, button }); } + /** Press one keyboard key (HID usage on page 7) with modifier usages held around it. */ + async key(udid: string, usage: number, modifiers: number[]): Promise { + await this.call('key', { udid, usage, modifiers }); + } + /** * Start streaming `udid`'s framebuffer as JPEG frames delivered to {@link onFrame} listeners. * Frames are pushed until {@link streamStop} (or the client closes). diff --git a/packages/host/sim/src/index.ts b/packages/host/sim/src/index.ts index 5b62633f..af27fee6 100644 --- a/packages/host/sim/src/index.ts +++ b/packages/host/sim/src/index.ts @@ -8,4 +8,5 @@ export type { SimProbe, SimStreamCodec, SimStreamStartResult, + SimTouchPhase, } from './schema'; diff --git a/packages/host/sim/src/schema.ts b/packages/host/sim/src/schema.ts index 7570b620..d201a875 100644 --- a/packages/host/sim/src/schema.ts +++ b/packages/host/sim/src/schema.ts @@ -62,4 +62,7 @@ export type SimStreamStartResult = z.infer; /** A hardware button the private HID layer can press. */ export type SimButton = 'home' | 'lock'; +/** One phase of a streamed touch gesture (one `down`, moves, one `up` per gesture). */ +export type SimTouchPhase = 'down' | 'move' | 'up'; + export type SimImageFormat = 'jpeg' | 'png'; diff --git a/packages/presentation/ui/src/shell/simulator/__tests__/keymap.test.ts b/packages/presentation/ui/src/shell/simulator/__tests__/keymap.test.ts new file mode 100644 index 00000000..3d2e6d6c --- /dev/null +++ b/packages/presentation/ui/src/shell/simulator/__tests__/keymap.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from 'vitest'; +import { simulatorKeyPress } from '../keymap'; + +function event( + key: string, + mods: Partial> = {}, +) { + return { + key, + ctrlKey: mods.ctrl ?? false, + shiftKey: mods.shift ?? false, + altKey: mods.alt ?? false, + metaKey: mods.meta ?? false, + }; +} + +describe('simulatorKeyPress', () => { + it('maps letters, digits, and named keys to page-7 usages', () => { + expect(simulatorKeyPress(event('a'))).toEqual({ usage: 0x04, modifiers: [] }); + expect(simulatorKeyPress(event('z'))).toEqual({ usage: 0x1d, modifiers: [] }); + expect(simulatorKeyPress(event('1'))).toEqual({ usage: 0x1e, modifiers: [] }); + expect(simulatorKeyPress(event('0'))).toEqual({ usage: 0x27, modifiers: [] }); + expect(simulatorKeyPress(event('Enter'))).toEqual({ usage: 0x28, modifiers: [] }); + expect(simulatorKeyPress(event('Backspace'))).toEqual({ usage: 0x2a, modifiers: [] }); + expect(simulatorKeyPress(event(' '))).toEqual({ usage: 0x2c, modifiers: [] }); + }); + + it('adds shift for uppercase and shifted punctuation even without shiftKey (caps lock)', () => { + expect(simulatorKeyPress(event('A'))).toEqual({ usage: 0x04, modifiers: [0xe1] }); + expect(simulatorKeyPress(event('?', { shift: true }))).toEqual({ + usage: 0x38, + modifiers: [0xe1], + }); + // Shift is not duplicated when already held. + expect(simulatorKeyPress(event('A', { shift: true }))).toEqual({ + usage: 0x04, + modifiers: [0xe1], + }); + }); + + it('leaves command/option combos and non-US characters to the app', () => { + expect(simulatorKeyPress(event('k', { meta: true }))).toBeNull(); + expect(simulatorKeyPress(event('é', { alt: true }))).toBeNull(); + expect(simulatorKeyPress(event('你'))).toBeNull(); + expect(simulatorKeyPress(event('F1'))).toBeNull(); + }); + + it('keeps control as a held modifier', () => { + expect(simulatorKeyPress(event('c', { ctrl: true }))).toEqual({ + usage: 0x06, + modifiers: [0xe0], + }); + }); +}); diff --git a/packages/presentation/ui/src/shell/simulator/index.ts b/packages/presentation/ui/src/shell/simulator/index.ts index 8a7111e4..689eedae 100644 --- a/packages/presentation/ui/src/shell/simulator/index.ts +++ b/packages/presentation/ui/src/shell/simulator/index.ts @@ -1,6 +1,8 @@ +export type { SimulatorKeyPress } from './keymap'; export type { SimulatorScreenFrame, SimulatorScreenPoint, SimulatorScreenProps, + SimulatorScreenTouchPhase, } from './simulator-screen'; export { SimulatorScreen } from './simulator-screen'; diff --git a/packages/presentation/ui/src/shell/simulator/keymap.ts b/packages/presentation/ui/src/shell/simulator/keymap.ts new file mode 100644 index 00000000..407b8129 --- /dev/null +++ b/packages/presentation/ui/src/shell/simulator/keymap.ts @@ -0,0 +1,102 @@ +/** + * Browser `KeyboardEvent` → HID keyboard usage (page 7), US layout — the shape the simulator's + * HID keyboard injection consumes. Meta/Command combos return `null` so application shortcuts + * (palette, panels) keep working; characters outside the US layout (IME output, option-composed + * accents) also return `null` and are left to the browser. + */ + +export interface SimulatorKeyPress { + usage: number; + /** Modifier usages (`0xE0..`) held around the key. */ + modifiers: number[]; +} + +const SHIFT_USAGE = 0xe1; + +const NAMED_KEYS: Readonly> = { + Enter: 0x28, + Escape: 0x29, + Backspace: 0x2a, + Tab: 0x2b, + ' ': 0x2c, + Delete: 0x4c, + ArrowRight: 0x4f, + ArrowLeft: 0x50, + ArrowDown: 0x51, + ArrowUp: 0x52, +}; + +/** Punctuation → `[usage, needsShift]`, US layout. */ +const PUNCTUATION: Readonly> = { + '-': [0x2d, false], + _: [0x2d, true], + '=': [0x2e, false], + '+': [0x2e, true], + '[': [0x2f, false], + '{': [0x2f, true], + ']': [0x30, false], + '}': [0x30, true], + '\\': [0x31, false], + '|': [0x31, true], + ';': [0x33, false], + ':': [0x33, true], + "'": [0x34, false], + '"': [0x34, true], + '`': [0x35, false], + '~': [0x35, true], + ',': [0x36, false], + '<': [0x36, true], + '.': [0x37, false], + '>': [0x37, true], + '/': [0x38, false], + '?': [0x38, true], + '!': [0x1e, true], + '@': [0x1f, true], + '#': [0x20, true], + $: [0x21, true], + '%': [0x22, true], + '^': [0x23, true], + '&': [0x24, true], + '*': [0x25, true], + '(': [0x26, true], + ')': [0x27, true], +}; + +/** Decompose a key event; `null` = not ours (let the browser/app handle it). */ +export function simulatorKeyPress(event: { + key: string; + ctrlKey: boolean; + shiftKey: boolean; + altKey: boolean; + metaKey: boolean; +}): SimulatorKeyPress | null { + // Command combos belong to the app (palette, panel toggles), option composes characters. + if (event.metaKey || event.altKey) return null; + const modifiers: number[] = []; + if (event.ctrlKey) modifiers.push(0xe0); + if (event.shiftKey) modifiers.push(SHIFT_USAGE); + + const named = NAMED_KEYS[event.key]; + if (named !== undefined) return { usage: named, modifiers }; + if (event.key.length !== 1) return null; + + const code = event.key.codePointAt(0) ?? 0; + // a-z + if (code >= 0x61 && code <= 0x7a) return { usage: 0x04 + code - 0x61, modifiers }; + // A-Z (caps lock may produce these without shiftKey; the device needs shift either way) + if (code >= 0x41 && code <= 0x5a) { + return { usage: 0x04 + code - 0x41, modifiers: withShift(modifiers) }; + } + // 1-9, then 0 + if (code >= 0x31 && code <= 0x39) return { usage: 0x1e + code - 0x31, modifiers }; + if (event.key === '0') return { usage: 0x27, modifiers }; + + const punctuation = PUNCTUATION[event.key]; + if (punctuation === undefined) return null; + const [usage, needsShift] = punctuation; + return { usage, modifiers: needsShift ? withShift(modifiers) : modifiers }; +} + +function withShift(modifiers: number[]): number[] { + return modifiers.includes(SHIFT_USAGE) ? modifiers : [...modifiers, SHIFT_USAGE]; +} diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index ed68be3a..5981d9a3 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -3,14 +3,21 @@ import { clamp } from 'foxts/clamp'; import { noop } from 'foxts/noop'; import { useRef, useState } from 'react'; import { cn } from '../../lib/cn'; +import type { SimulatorKeyPress } from './keymap'; +import { simulatorKeyPress } from './keymap'; export interface SimulatorScreenPoint { x: number; y: number; } -/** Below this pointer travel (in element px) a press counts as a tap, not a swipe. */ -const SWIPE_THRESHOLD_PX = 8; +/** One phase of a streamed touch gesture. */ +export type SimulatorScreenTouchPhase = 'down' | 'move' | 'up'; + +/** Minimum interval between forwarded `move` phases (≈60 Hz). */ +const TOUCH_MOVE_INTERVAL_MS = 16; +/** A wheel stream idle for this long ends its synthetic drag gesture. */ +const WHEEL_IDLE_MS = 120; // Chassis proportions as fractions of the native screen width, matched against Simulator.app's // iPhone chrome: a thin black display band inside a titanium rim, with side-button bumps. @@ -44,10 +51,12 @@ export interface SimulatorScreenProps { /** Feed of stream frames; returns the unsubscribe. H.264 units decode through WebCodecs * (hardware, GPU-resident output); JPEG frames decode via `createImageBitmap`. */ subscribeFrames: (onFrame: (frame: SimulatorScreenFrame) => void) => () => void; - /** Press in normalized [0,1] device coordinates. */ - onTap?: (point: SimulatorScreenPoint) => void; - /** Drag in normalized [0,1] device coordinates with its real duration. */ - onSwipe?: (from: SimulatorScreenPoint, to: SimulatorScreenPoint, durationMs: number) => void; + /** A streamed touch phase in normalized [0,1] device coordinates: exactly one `down`, any + * number of `move`s, one final `up` per gesture. Forwarded in real time, so the device's own + * gesture recognition decides tap vs drag vs long-press. */ + onTouch?: (phase: SimulatorScreenTouchPhase, point: SimulatorScreenPoint) => void; + /** A key press (typed on the focused screen) decomposed to HID usages — see {@link simulatorKeyPress}. */ + onKey?: (press: SimulatorKeyPress) => void; /** The device's real screen-outline mask (image URL, framebuffer-sized): clips the stream to * the exact screen shape, and its measured corner radius keeps the chassis curve concentric. * Absent → a generic rounding. */ @@ -89,8 +98,8 @@ interface PaintState { */ export function SimulatorScreen({ subscribeFrames, - onTap, - onSwipe, + onTouch, + onKey, maskUrl, placeholder, className, @@ -103,9 +112,11 @@ export function SimulatorScreen({ chassis: null, chassisKey: '', }); - const pressRef = useRef<{ pointerId: number; start: SimulatorScreenPoint; at: number } | null>( - null, - ); + const pressRef = useRef<{ + pointerId: number; + last: SimulatorScreenPoint; + moveSentAt: number; + } | null>(null); /** `w / h` of the whole painted device (screen + bezel); sizes the canvas box. */ const [deviceAspect, setDeviceAspect] = useState(null); @@ -229,43 +240,95 @@ export function SimulatorScreen({ const handlePointerDown = (event: React.PointerEvent): void => { if (event.button !== 0) return; event.currentTarget.setPointerCapture(event.pointerId); - pressRef.current = { - pointerId: event.pointerId, - start: normalizedPoint(event), - at: performance.now(), - }; + const point = normalizedPoint(event); + pressRef.current = { pointerId: event.pointerId, last: point, moveSentAt: performance.now() }; + onTouch?.('down', point); + }; + + const handlePointerMove = (event: React.PointerEvent): void => { + const press = pressRef.current; + if (press?.pointerId !== event.pointerId) return; + const now = performance.now(); + if (now - press.moveSentAt < TOUCH_MOVE_INTERVAL_MS) return; + const point = normalizedPoint(event); + press.last = point; + press.moveSentAt = now; + onTouch?.('move', point); }; const handlePointerUp = (event: React.PointerEvent): void => { const press = pressRef.current; if (press?.pointerId !== event.pointerId) return; pressRef.current = null; - const end = normalizedPoint(event); + onTouch?.('up', normalizedPoint(event)); + }; + + const handlePointerCancel = (event: React.PointerEvent): void => { + const press = pressRef.current; + if (press?.pointerId !== event.pointerId) return; + pressRef.current = null; + // End the gesture where it last was — a stuck touch would keep the device pressed forever. + onTouch?.('up', press.last); + }; + + const handleKeyDown = (event: React.KeyboardEvent): void => { + if (onKey === undefined || event.nativeEvent.isComposing) return; + const press = simulatorKeyPress(event); + if (press === null) return; + event.preventDefault(); + onKey(press); + }; + + // Trackpad/wheel scrolling becomes a synthetic drag: touch down where the cursor sits, move + // opposite the scroll deltas (finger-follows-content, like the real device), up after idle. + const wheelRef = useRef<{ + pos: SimulatorScreenPoint; + endTimer: ReturnType; + } | null>(null); + const handleWheel = (event: React.WheelEvent): void => { + if (pressRef.current !== null) return; const screen = screenRectOnPage(event.currentTarget); - const travelPx = Math.hypot( - (end.x - press.start.x) * screen.width, - (end.y - press.start.y) * screen.height, - ); - if (travelPx < SWIPE_THRESHOLD_PX) onTap?.(end); - else onSwipe?.(press.start, end, Math.round(performance.now() - press.at)); + const endGesture = (): void => { + const wheel = wheelRef.current; + wheelRef.current = null; + if (wheel) onTouch?.('up', wheel.pos); + }; + let wheel = wheelRef.current; + if (wheel === null) { + const start = normalizedPoint(event); + wheel = { pos: start, endTimer: setTimeout(endGesture, WHEEL_IDLE_MS) }; + wheelRef.current = wheel; + onTouch?.('down', start); + } else { + clearTimeout(wheel.endTimer); + wheel.endTimer = setTimeout(endGesture, WHEEL_IDLE_MS); + } + wheel.pos = { + x: clamp(wheel.pos.x - event.deltaX / screen.width, 0, 1), + y: clamp(wheel.pos.y - event.deltaY / screen.height, 0, 1), + }; + onTouch?.('move', wheel.pos); }; return (
+ {/* Focusable so plain typing reaches the device; app-level chords (⌘…) pass through. */} { - pressRef.current = null; - }} + onPointerCancel={handlePointerCancel} + onWheel={handleWheel} + onKeyDown={handleKeyDown} /> {deviceAspect === null && placeholder}
@@ -453,7 +516,11 @@ function screenRectOnPage(canvas: HTMLCanvasElement): { }; } -function normalizedPoint(event: React.PointerEvent): SimulatorScreenPoint { +function normalizedPoint(event: { + clientX: number; + clientY: number; + currentTarget: HTMLCanvasElement; +}): SimulatorScreenPoint { const screen = screenRectOnPage(event.currentTarget); return { x: clamp((event.clientX - screen.left) / screen.width, 0, 1), From fcb83ad1a91ec36303394493f1fd71eba9dc0c8a Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 07:16:19 +0800 Subject: [PATCH 18/26] docs(sim-sidecar): document the touch and key ops --- crates/linkcode-sim/PROTOCOL.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/linkcode-sim/PROTOCOL.md b/crates/linkcode-sim/PROTOCOL.md index ae240a4f..7b4e7d18 100644 --- a/crates/linkcode-sim/PROTOCOL.md +++ b/crates/linkcode-sim/PROTOCOL.md @@ -71,11 +71,15 @@ Off macOS, or when SimulatorKit is unavailable, every P1 op fails with `xcodeMis | `type` | Params | Result | | --- | --- | --- | | `tap` | `udid`, `x`, `y` (normalised 0..1) | `{}` | +| `touch` | `udid`, `phase` (`down`/`move`/`up`), `x`, `y` | `{}` — one phase of a caller-driven touch stream | | `swipe` | `udid`, `x0`, `y0`, `x1`, `y1`, `durationMs?` | `{}` | | `button` | `udid`, `button` (`home`/`lock`) | `{}` | +| `key` | `udid`, `usage` (HID page-7 usage), `modifiers?` (usages `0xE0..`) | `{}` — modifier-downs → key-down → key-up → modifier-ups | | `streamStart` | `udid`, `fps?` (60), `quality?` (0.6), `scale?` (1.0), `codec?` (`jpeg` default, `h264`) | `{ streaming, fps, scale, codec }` — frames then arrive on `STREAM_FRAME`s (jpeg) or `STREAM_FRAME_H264`s | | `streamStop` | `udid` | `{}` | +`touch` streams a gesture in real time (the client forwards pointer events, so the device sees the finger during a drag — long-press, rubber-banding, icon drags all come from the device's own gesture recognition). The sidecar tracks one active touch identifier per udid (`down` allocates, `up` releases; a `move`/`up` without one no-ops). `touch` and `key` requests are handled **inline on the read loop** — never on a per-request thread — so phases and keystrokes keep stdio order. + `scale` (0.1..1.0) downscales each frame before JPEG encode: at native resolution the encode bounds the frame rate near ~55 fps, so `scale` below 1.0 both lifts the achievable rate toward the display's 60 Hz and cuts bandwidth (e.g. `0.5` ≈ one third the bytes). `tap`/`swipe` stay in normalized 0..1 coordinates, so a downscaled stream needs no coordinate adjustment. `codec: h264` switches the stream to hardware H.264 (VideoToolbox): the retained framebuffer `IOSurface` is wrapped in a `CVPixelBuffer` and encoded on the media engine — pixels never enter CPU memory — at **native resolution** (`scale`/`quality` are ignored; bitrate carries the bandwidth budget, ~10-25× below the JPEG stream). Access units are Annex-B with SPS/PPS prepended on keyframes (≤2s apart) — exactly what a WebCodecs `VideoDecoder` configured without a `description` consumes. Delivery is **ordered and lossless** (deltas depend on each other); if the sidecar must drop (stalled consumer), it drops through the next keyframe. If the private worker gives up, the stream degrades to slow simctl JPEG `STREAM_FRAME`s — each wire frame declares its own encoding, so a mixed stream stays decodable. From e4c6bb4cece6bd0a0c87a810bb0965d0e71f0ead Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 07:52:41 +0800 Subject: [PATCH 19/26] feat(sim,schema,engine,client,ui): two-finger pinch + IME pasteboard input (wire 49) --- .../src/__tests__/sim-mcp-endpoint.test.ts | 2 + apps/desktop/e2e/simulator-panel.e2e.mts | 18 +++- crates/linkcode-sim/PROTOCOL.md | 2 + crates/linkcode-sim/src/interactive.rs | 61 +++++++++++-- crates/linkcode-sim/src/main.rs | 14 ++- crates/linkcode-sim/src/private/input.rs | 73 ++++++++++------ crates/linkcode-sim/src/rpc.rs | 12 +++ crates/linkcode-sim/src/simctl.rs | 85 ++++++++++++++----- packages/client/core/src/client.ts | 16 ++++ .../client/core/src/client/control-channel.ts | 30 +++++++ .../client/workbench/src/simulator/panel.tsx | 20 +++++ .../foundation/schema/src/wire/message.ts | 3 +- .../foundation/schema/src/wire/simulator.ts | 21 +++++ .../simulator-request-handler.test.ts | 2 + .../src/__tests__/simulator-service.test.ts | 2 + packages/host/engine/src/simulator/backend.ts | 9 ++ .../engine/src/simulator/request-handler.ts | 22 +++++ packages/host/engine/src/simulator/service.ts | 16 ++++ .../host/engine/src/wire/request-router.ts | 2 + packages/host/sim/src/client.ts | 15 ++++ .../src/shell/simulator/simulator-screen.tsx | 76 ++++++++++++++--- 21 files changed, 434 insertions(+), 67 deletions(-) diff --git a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts index 79998839..cfa19a75 100644 --- a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts +++ b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts @@ -36,6 +36,8 @@ function fakeBackend(): SimulatorBackend { screenMask: vi.fn(() => Promise.resolve(new Uint8Array([0x89, 0x50]))), tap: vi.fn(asyncNoop), touch: vi.fn(asyncNoop), + pinch: vi.fn(asyncNoop), + paste: vi.fn(asyncNoop), key: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), diff --git a/apps/desktop/e2e/simulator-panel.e2e.mts b/apps/desktop/e2e/simulator-panel.e2e.mts index 2a0189aa..40f2d9a1 100644 --- a/apps/desktop/e2e/simulator-panel.e2e.mts +++ b/apps/desktop/e2e/simulator-panel.e2e.mts @@ -29,7 +29,7 @@ const PORT = 43000 + (process.pid % 1000); /** Must match `WIRE_PROTOCOL_VERSION` (node can't load the raw-TS schema barrel); a mismatch is * silently discarded by the daemon, surfacing here as the session.start timeout. */ -const WIRE_VERSION = 48; +const WIRE_VERSION = 49; function fail(message: string): never { console.error(`FAIL: ${message}`); @@ -316,6 +316,22 @@ async function run(win: Page, chatRoot: string, deepPass: boolean): Promise OpError { } #[cfg(target_os = "macos")] -pub use imp::{available, button, key, stream_start, stream_stop, swipe, tap, touch}; +pub use imp::{available, button, key, pinch, stream_start, stream_stop, swipe, tap, touch}; #[cfg(not(target_os = "macos"))] mod stubs { @@ -36,6 +36,14 @@ mod stubs { pub fn touch(_udid: &str, _phase: TouchPhase, _x: f64, _y: f64) -> Result { Err(unsupported()) } + pub fn pinch( + _udid: &str, + _phase: TouchPhase, + _a: (f64, f64), + _b: (f64, f64), + ) -> Result { + Err(unsupported()) + } pub fn swipe( _udid: &str, _x0: f64, @@ -68,7 +76,7 @@ mod stubs { } #[cfg(not(target_os = "macos"))] -pub use stubs::{available, button, key, stream_start, stream_stop, swipe, tap, touch}; +pub use stubs::{available, button, key, pinch, stream_start, stream_stop, swipe, tap, touch}; #[cfg(target_os = "macos")] mod imp { @@ -97,6 +105,8 @@ mod imp { streams: HashMap, /// Active streamed-touch identifiers by udid (down allocates, up removes). touches: HashMap, + /// Active two-finger identifiers by udid (pinch streams). + pinches: HashMap, } struct StreamHandle { @@ -115,6 +125,7 @@ mod imp { inputs: HashMap::new(), streams: HashMap::new(), touches: HashMap::new(), + pinches: HashMap::new(), }) }) } @@ -123,6 +134,14 @@ mod imp { private::interactive_available() } + fn to_hid_phase(phase: TouchPhase) -> private::Phase { + match phase { + TouchPhase::Down => private::Phase::Down, + TouchPhase::Move => private::Phase::Move, + TouchPhase::Up => private::Phase::Up, + } + } + /// Resolve (and cache) a warmed HID client for `udid`. fn input_for(udid: &str) -> Result, OpError> { let mut reg = registry().lock().expect("interactive registry poisoned"); @@ -163,18 +182,44 @@ mod imp { let Some(identifier) = identifier else { return Ok(json!({})); }; - let hid_phase = match phase { - TouchPhase::Down => private::Phase::Down, - TouchPhase::Move => private::Phase::Move, - TouchPhase::Up => private::Phase::Up, - }; - if input.touch_phase(x, y, identifier, hid_phase) { + if input.touch_phase(x, y, identifier, to_hid_phase(phase)) { Ok(json!({})) } else { Err(OpError::new(ErrorCode::SimctlFailed, "touch failed")) } } + /// One phase of a streamed two-finger gesture (pinch/zoom). Identifiers are allocated on + /// `down` and released on `up`; a stray `move`/`up` no-ops like [`touch`]. + pub fn pinch( + udid: &str, + phase: TouchPhase, + a: (f64, f64), + b: (f64, f64), + ) -> Result { + let input = input_for(udid)?; + let mut reg = registry().lock().expect("interactive registry poisoned"); + let ids = match phase { + TouchPhase::Down => { + let ids = [input.allocate_touch(), input.allocate_touch()]; + reg.pinches.insert(udid.to_owned(), ids); + Some(ids) + } + TouchPhase::Move => reg.pinches.get(udid).copied(), + TouchPhase::Up => reg.pinches.remove(udid), + }; + drop(reg); + let Some([id0, id1]) = ids else { + return Ok(json!({})); + }; + let hid_phase = to_hid_phase(phase); + if input.touch_pair([(a.0, a.1, id0), (b.0, b.1, id1)], hid_phase) { + Ok(json!({})) + } else { + Err(OpError::new(ErrorCode::SimctlFailed, "pinch failed")) + } + } + pub fn swipe( udid: &str, x0: f64, diff --git a/crates/linkcode-sim/src/main.rs b/crates/linkcode-sim/src/main.rs index fe7db07a..9cf06401 100644 --- a/crates/linkcode-sim/src/main.rs +++ b/crates/linkcode-sim/src/main.rs @@ -165,7 +165,10 @@ fn main() { // Touch phases and key presses must keep stdio order — a per-request thread would // race a gesture's down/move/up or reorder typed characters. Each is a handful of // fast HID sends, so inline handling is cheap and stays off the inflight gate. - if matches!(request.op, Op::Touch { .. } | Op::Key { .. }) { + if matches!( + request.op, + Op::Touch { .. } | Op::Pinch { .. } | Op::Key { .. } + ) { serve(request, &tx); } else { gate.acquire(); @@ -256,6 +259,15 @@ fn serve(request: Request, tx: &Sender) { } Op::Tap { udid, x, y } => interactive::tap(&udid, x, y), Op::Touch { udid, phase, x, y } => interactive::touch(&udid, phase, x, y), + Op::Pinch { + udid, + phase, + x0, + y0, + x1, + y1, + } => interactive::pinch(&udid, phase, (x0, y0), (x1, y1)), + Op::Paste { udid, text } => simctl::set_pasteboard(&udid, &text), Op::Swipe { udid, x0, diff --git a/crates/linkcode-sim/src/private/input.rs b/crates/linkcode-sim/src/private/input.rs index 748d9d0d..b3d37013 100644 --- a/crates/linkcode-sim/src/private/input.rs +++ b/crates/linkcode-sim/src/private/input.rs @@ -164,6 +164,12 @@ impl Input { self.send_touch(x, y, identifier, phase) } + /// Inject one phase of a caller-driven **two-finger** stream (pinch/zoom). Both fingers share + /// the phase; the caller owns the pair of identifiers and the cadence. + pub fn touch_pair(&self, fingers: [(f64, f64, u32); 2], phase: Phase) -> bool { + self.send_fingers(&fingers, phase) + } + /// Single-finger tap at a normalised (0..1) point, holding for `hold`. pub fn tap(&self, x: f64, y: f64, hold: Duration) -> bool { let id = self.next_identifier(); @@ -256,10 +262,18 @@ impl Input { if id == 0 { 1 } else { id } } - /// Build one digitizer parent+finger event, wrap it, patch the target/edge byte slots, and send. + /// Single-finger convenience over [`send_fingers`]. fn send_touch(&self, x: f64, y: f64, identifier: u32, phase: Phase) -> bool { - let x = clamp01(x); - let y = clamp01(y); + self.send_fingers(&[(x, y, identifier)], phase) + } + + /// Build one digitizer parent with a finger child per `(x, y, id)`, wrap it, patch the + /// target/edge byte slots, and send. The parent carries the first finger's position; each + /// child gets its own index/identifier so the device tracks the fingers independently. + fn send_fingers(&self, fingers: &[(f64, f64, u32)], phase: Phase) -> bool { + let Some(&(px, py, parent_id)) = fingers.first() else { + return false; + }; let mask = phase.event_mask(); let range = phase.range(); let touch = phase.touch(); @@ -273,11 +287,11 @@ impl Input { now, TRANSDUCER_FINGER, 0, - identifier, + parent_id, mask, 0, - x, - y, + clamp01(px), + clamp01(py), 0.0, 0.0, 0.0, @@ -289,32 +303,37 @@ impl Input { if parent.is_null() { return false; } - let finger = unsafe { - (self.symbols.create_finger)( - ptr::null(), - now, - 0, - identifier, - mask, - x, - y, - 0.0, - 0.0, - 0.0, - range, - touch, - 0, - ) - }; - if !finger.is_null() { - // SAFETY: append copies the child into the parent; both stay valid here. - unsafe { (self.symbols.append_event)(parent, finger, 0) }; + let mut children = Vec::with_capacity(fingers.len()); + for (index, &(x, y, id)) in fingers.iter().enumerate() { + // SAFETY: same ABI as the digitizer creator; +1 ref appended and released below. + let finger = unsafe { + (self.symbols.create_finger)( + ptr::null(), + now, + index as u32, + id, + mask, + clamp01(x), + clamp01(y), + 0.0, + 0.0, + 0.0, + range, + touch, + 0, + ) + }; + if !finger.is_null() { + // SAFETY: append copies the child into the parent; both stay valid here. + unsafe { (self.symbols.append_event)(parent, finger, 0) }; + children.push(finger); + } } // SAFETY: wrapper reads the parent event and returns a malloc'd Indigo message (or null). let message = unsafe { (self.symbols.trackpad_wrap)(parent.cast_const()) }; // The wrapper has copied out what it needs; release the events regardless of outcome. unsafe { - if !finger.is_null() { + for finger in children { CFRelease(finger.cast_const()); } CFRelease(parent.cast_const()); diff --git a/crates/linkcode-sim/src/rpc.rs b/crates/linkcode-sim/src/rpc.rs index 482876a3..67538310 100644 --- a/crates/linkcode-sim/src/rpc.rs +++ b/crates/linkcode-sim/src/rpc.rs @@ -66,6 +66,18 @@ pub enum Op { x: f64, y: f64, }, + /// One phase of a caller-driven **two-finger** stream (pinch/zoom); private API. Both finger + /// positions are normalised. Sequencing and stdio ordering match `touch`. + Pinch { + udid: String, + phase: TouchPhase, + x0: f64, + y0: f64, + x1: f64, + y1: f64, + }, + /// Set the device pasteboard (public `simctl pbcopy`); pair with a Cmd+V key press. + Paste { udid: String, text: String }, /// Swipe between two normalised (0..1) points over `duration_ms` (private API; P1). Swipe { udid: String, diff --git a/crates/linkcode-sim/src/simctl.rs b/crates/linkcode-sim/src/simctl.rs index 8e8f6208..f682e241 100644 --- a/crates/linkcode-sim/src/simctl.rs +++ b/crates/linkcode-sim/src/simctl.rs @@ -4,7 +4,7 @@ //! [`ErrorCode::Timeout`]. A missing `xcrun` (no Xcode / no Command Line Tools) surfaces as //! [`ErrorCode::XcodeMissing`] so the daemon can gate the capability instead of retrying. -use std::io::Read; +use std::io::{Read, Write}; use std::process::{Command, Stdio}; use std::sync::atomic::{AtomicU64, Ordering}; use std::thread; @@ -160,6 +160,46 @@ pub fn device_type_bundle_path(udid: &str) -> Result Result { + let mut child = simctl(["pbcopy", udid]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| match e.kind() { + std::io::ErrorKind::NotFound => OpError::new( + ErrorCode::XcodeMissing, + format!("{XCRUN} not found; install Xcode with the iOS platform"), + ), + _ => OpError::new(ErrorCode::Io, format!("spawn {XCRUN} pbcopy: {e}")), + })?; + // Write on a thread so a full pipe buffer can never deadlock against exit polling. + let stdin = child.stdin.take().expect("stdin piped above"); + let bytes = text.as_bytes().to_vec(); + let writer = thread::spawn(move || { + let mut stdin = stdin; + let _ = stdin.write_all(&bytes); + // Drop closes the pipe so pbcopy sees EOF. + }); + let stderr = drain(child.stderr.take().expect("stderr piped above")); + let status = wait_with_timeout(&mut child, XCRUN, DEFAULT_TIMEOUT)?; + let _ = writer.join(); + if status.success() { + Ok(json!({})) + } else { + Err(OpError::new( + ErrorCode::SimctlFailed, + format!( + "{XCRUN} pbcopy exited with {status}: {}", + join_drained(stderr).trim() + ), + )) + } +} + fn simctl<'a>(args: impl IntoIterator) -> Command { let mut cmd = apple_tool(XCRUN); cmd.arg("simctl").args(args); @@ -202,24 +242,7 @@ fn run_ok(mut cmd: Command, timeout: Duration) -> Result { let stdout = drain(child.stdout.take().expect("stdout piped above")); let stderr = drain(child.stderr.take().expect("stderr piped above")); - let deadline = Instant::now() + timeout; - let status = loop { - match child.try_wait() { - Ok(Some(status)) => break status, - Ok(None) if Instant::now() >= deadline => { - let _ = child.kill(); - let _ = child.wait(); - return Err(OpError::new( - ErrorCode::Timeout, - format!("{program} timed out after {}s", timeout.as_secs()), - )); - } - Ok(None) => thread::sleep(Duration::from_millis(20)), - Err(e) => { - return Err(OpError::new(ErrorCode::Io, format!("wait {program}: {e}"))); - } - } - }; + let status = wait_with_timeout(&mut child, &program, timeout)?; let stdout = join_drained(stdout); let stderr = join_drained(stderr); @@ -238,6 +261,30 @@ fn run_ok(mut cmd: Command, timeout: Duration) -> Result { } } +/// Wait for `child` under `timeout`; kill and report `Timeout` if it overruns. +fn wait_with_timeout( + child: &mut std::process::Child, + program: &str, + timeout: Duration, +) -> Result { + let deadline = Instant::now() + timeout; + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status), + Ok(None) if Instant::now() >= deadline => { + let _ = child.kill(); + let _ = child.wait(); + return Err(OpError::new( + ErrorCode::Timeout, + format!("{program} timed out after {}s", timeout.as_secs()), + )); + } + Ok(None) => thread::sleep(Duration::from_millis(20)), + Err(e) => return Err(OpError::new(ErrorCode::Io, format!("wait {program}: {e}"))), + } + } +} + fn drain(mut pipe: impl Read + Send + 'static) -> thread::JoinHandle { thread::spawn(move || { let mut buf = Vec::new(); diff --git a/packages/client/core/src/client.ts b/packages/client/core/src/client.ts index 4f04b2df..8341de69 100644 --- a/packages/client/core/src/client.ts +++ b/packages/client/core/src/client.ts @@ -731,6 +731,22 @@ export class LinkCodeClient { return this.control.simulatorTouch(sessionId, udid, phase, x, y); } + /** One phase of a streamed two-finger gesture (pinch/zoom); both finger positions normalized. */ + simulatorPinch( + sessionId: SessionId, + udid: string, + phase: SimulatorTouchPhase, + a: { x: number; y: number }, + b: { x: number; y: number }, + ): Promise { + return this.control.simulatorPinch(sessionId, udid, phase, a, b); + } + + /** Set the device pasteboard; pair with a Cmd+V key press to inject arbitrary Unicode. */ + simulatorPaste(sessionId: SessionId, udid: string, text: string): Promise { + return this.control.simulatorPaste(sessionId, udid, text); + } + simulatorSwipe( sessionId: SessionId, udid: string, diff --git a/packages/client/core/src/client/control-channel.ts b/packages/client/core/src/client/control-channel.ts index 44b6abbb..e355e45c 100644 --- a/packages/client/core/src/client/control-channel.ts +++ b/packages/client/core/src/client/control-channel.ts @@ -714,6 +714,36 @@ export class ControlChannel { })); } + simulatorPinch( + sessionId: SessionId, + udid: string, + phase: SimulatorTouchPhase, + a: { x: number; y: number }, + b: { x: number; y: number }, + ): Promise { + return this.sendCorrelated('ack', (clientReqId) => ({ + kind: 'simulator.pinch', + clientReqId, + sessionId, + udid, + phase, + x0: a.x, + y0: a.y, + x1: b.x, + y1: b.y, + })); + } + + simulatorPaste(sessionId: SessionId, udid: string, text: string): Promise { + return this.sendCorrelated('ack', (clientReqId) => ({ + kind: 'simulator.paste', + clientReqId, + sessionId, + udid, + text, + })); + } + simulatorSwipe( sessionId: SessionId, udid: string, diff --git a/packages/client/workbench/src/simulator/panel.tsx b/packages/client/workbench/src/simulator/panel.tsx index 61fbf335..116c7f3d 100644 --- a/packages/client/workbench/src/simulator/panel.tsx +++ b/packages/client/workbench/src/simulator/panel.tsx @@ -132,10 +132,28 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): // Surface a claim conflict once per gesture, not per 60 Hz move. void request.catch(phase === 'down' ? flagBusy : noop); }; + const handlePinch = ( + phase: SimulatorScreenTouchPhase, + a: SimulatorScreenPoint, + b: SimulatorScreenPoint, + ): void => { + if (ownerSessionId === null || udid === null) return; + void client + .simulatorPinch(ownerSessionId, udid, phase, a, b) + .catch(phase === 'down' ? flagBusy : noop); + }; const handleKey = (press: SimulatorKeyPress): void => { if (ownerSessionId === null || udid === null) return; void client.simulatorKey(ownerSessionId, udid, press.usage, press.modifiers).catch(flagBusy); }; + const handleText = (text: string): void => { + if (ownerSessionId === null || udid === null) return; + // Set the pasteboard, then Cmd+V (Left GUI usage 0xE3 + V usage 0x19) so iOS pastes it. + void client + .simulatorPaste(ownerSessionId, udid, text) + .then(() => client.simulatorKey(ownerSessionId, udid, 0x19, [0xe3])) + .catch(flagBusy); + }; const pressButton = (button: 'home' | 'lock'): void => { if (ownerSessionId === null || udid === null) return; void client.simulatorButton(ownerSessionId, udid, button).catch(flagBusy); @@ -211,7 +229,9 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): key={udid} subscribeFrames={subscribeFrames} onTouch={handleTouch} + onPinch={handlePinch} onKey={handleKey} + onText={handleText} maskUrl={masks[udid] ?? null} placeholder={ {t('simulatorConnecting')} diff --git a/packages/foundation/schema/src/wire/message.ts b/packages/foundation/schema/src/wire/message.ts index d260b212..ba20693a 100644 --- a/packages/foundation/schema/src/wire/message.ts +++ b/packages/foundation/schema/src/wire/message.ts @@ -19,7 +19,8 @@ import { WirePayloadSchema } from './payload'; // 47 adds the simulator screen-mask wire (CODE-397). // 48 adds the H.264 stream codec plumbing (CODE-397). // 49 adds streamed touch, wheel scroll, and HID keyboard input (CODE-397). -export const WIRE_PROTOCOL_VERSION = 49 as const; +// 50 adds two-finger pinch and IME pasteboard input (CODE-397). +export const WIRE_PROTOCOL_VERSION = 50 as const; /** Complete wire message: version + unique id + timestamp + payload. */ export const WireMessageSchema = z.object({ diff --git a/packages/foundation/schema/src/wire/simulator.ts b/packages/foundation/schema/src/wire/simulator.ts index 2f00c912..1e6d332b 100644 --- a/packages/foundation/schema/src/wire/simulator.ts +++ b/packages/foundation/schema/src/wire/simulator.ts @@ -143,6 +143,27 @@ export const simulatorWireVariants = [ x: coord, y: coord, }), + /** One phase of a streamed two-finger gesture (pinch/zoom); both finger positions normalized. */ + z.object({ + kind: z.literal('simulator.pinch'), + clientReqId: WireRequestIdSchema, + sessionId: SessionIdSchema, + udid, + phase: SimulatorTouchPhaseSchema, + x0: coord, + y0: coord, + x1: coord, + y1: coord, + }), + /** Set the device pasteboard; clients pair it with a Cmd+V key press to inject arbitrary + * Unicode (IME output, emoji) the US-ASCII key path cannot type. */ + z.object({ + kind: z.literal('simulator.paste'), + clientReqId: WireRequestIdSchema, + sessionId: SessionIdSchema, + udid, + text: z.string(), + }), z.object({ kind: z.literal('simulator.swipe'), clientReqId: WireRequestIdSchema, diff --git a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts index 4f7df8cc..3f5642b0 100644 --- a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts +++ b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts @@ -35,6 +35,8 @@ function fakeBackend() { screenMask: vi.fn(() => Promise.resolve(new Uint8Array([0x89, 0x50]))), tap: vi.fn(asyncNoop), touch: vi.fn(asyncNoop), + pinch: vi.fn(asyncNoop), + paste: vi.fn(asyncNoop), key: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), diff --git a/packages/host/engine/src/__tests__/simulator-service.test.ts b/packages/host/engine/src/__tests__/simulator-service.test.ts index db618d42..99fe86c6 100644 --- a/packages/host/engine/src/__tests__/simulator-service.test.ts +++ b/packages/host/engine/src/__tests__/simulator-service.test.ts @@ -32,6 +32,8 @@ function fakeBackend(devices: SimulatorDeviceInfo[]) { screenMask: vi.fn(() => Promise.resolve(new Uint8Array([0x89, 0x50]))), tap: vi.fn(asyncNoop), touch: vi.fn(asyncNoop), + pinch: vi.fn(asyncNoop), + paste: vi.fn(asyncNoop), key: vi.fn(asyncNoop), swipe: vi.fn(asyncNoop), button: vi.fn(asyncNoop), diff --git a/packages/host/engine/src/simulator/backend.ts b/packages/host/engine/src/simulator/backend.ts index f005aa37..7e6baaa0 100644 --- a/packages/host/engine/src/simulator/backend.ts +++ b/packages/host/engine/src/simulator/backend.ts @@ -84,6 +84,15 @@ export interface SimulatorBackend { tap(udid: string, x: number, y: number): Promise; /** One phase of a streamed touch gesture; the caller owns the down/move/up sequencing. */ touch(udid: string, phase: SimulatorTouchPhase, x: number, y: number): Promise; + /** One phase of a streamed two-finger gesture (pinch/zoom); both finger positions normalized. */ + pinch( + udid: string, + phase: SimulatorTouchPhase, + a: SimulatorPoint, + b: SimulatorPoint, + ): Promise; + /** Set the device pasteboard; pair with a Cmd+V key press to inject arbitrary Unicode. */ + paste(udid: string, text: string): Promise; /** Swipe between two normalized (0..1) points over `durationMs` (private HID; macOS only). */ swipe(udid: string, from: SimulatorPoint, to: SimulatorPoint, durationMs?: number): Promise; /** Press a hardware button (private HID; macOS only). */ diff --git a/packages/host/engine/src/simulator/request-handler.ts b/packages/host/engine/src/simulator/request-handler.ts index 711c12c4..614f49af 100644 --- a/packages/host/engine/src/simulator/request-handler.ts +++ b/packages/host/engine/src/simulator/request-handler.ts @@ -23,6 +23,8 @@ type SimulatorRequest = Extract< | 'simulator.screen-mask' | 'simulator.tap' | 'simulator.touch' + | 'simulator.pinch' + | 'simulator.paste' | 'simulator.swipe' | 'simulator.button' | 'simulator.key' @@ -178,6 +180,26 @@ export class SimulatorRequestHandler { this.responder.sendSuccess(payload.clientReqId); }), ); + case 'simulator.pinch': + return this.withSimulators(payload.clientReqId, (simulators) => + simulatorOperation('simulator.pinch', 'Failed to pinch', async () => { + await simulators.pinch( + payload.sessionId, + payload.udid, + payload.phase, + { x: payload.x0, y: payload.y0 }, + { x: payload.x1, y: payload.y1 }, + ); + this.responder.sendSuccess(payload.clientReqId); + }), + ); + case 'simulator.paste': + return this.withSimulators(payload.clientReqId, (simulators) => + simulatorOperation('simulator.paste', 'Failed to set pasteboard', async () => { + await simulators.paste(payload.sessionId, payload.udid, payload.text); + this.responder.sendSuccess(payload.clientReqId); + }), + ); case 'simulator.swipe': return this.withSimulators(payload.clientReqId, (simulators) => simulatorOperation('simulator.swipe', 'Failed to swipe', async () => { diff --git a/packages/host/engine/src/simulator/service.ts b/packages/host/engine/src/simulator/service.ts index d9f6230e..05a2bd94 100644 --- a/packages/host/engine/src/simulator/service.ts +++ b/packages/host/engine/src/simulator/service.ts @@ -185,6 +185,22 @@ export class SimulatorService { return this.backend.touch(udid, phase, x, y); } + async pinch( + sessionId: SessionId, + udid: string, + phase: SimulatorTouchPhase, + a: SimulatorPoint, + b: SimulatorPoint, + ): Promise { + this.claim(sessionId, udid); + return this.backend.pinch(udid, phase, a, b); + } + + async paste(sessionId: SessionId, udid: string, text: string): Promise { + this.claim(sessionId, udid); + return this.backend.paste(udid, text); + } + async swipe( sessionId: SessionId, udid: string, diff --git a/packages/host/engine/src/wire/request-router.ts b/packages/host/engine/src/wire/request-router.ts index 8bccfd85..880d62c7 100644 --- a/packages/host/engine/src/wire/request-router.ts +++ b/packages/host/engine/src/wire/request-router.ts @@ -141,6 +141,8 @@ export class WireRequestRouter { case 'simulator.screen-mask': case 'simulator.tap': case 'simulator.touch': + case 'simulator.pinch': + case 'simulator.paste': case 'simulator.key': case 'simulator.swipe': case 'simulator.button': diff --git a/packages/host/sim/src/client.ts b/packages/host/sim/src/client.ts index 3ce81be8..9d874f12 100644 --- a/packages/host/sim/src/client.ts +++ b/packages/host/sim/src/client.ts @@ -157,6 +157,21 @@ export class SimSidecarClient { await this.call('touch', { udid, phase, x, y }); } + /** One phase of a streamed two-finger gesture (pinch/zoom); both finger positions normalized. */ + async pinch( + udid: string, + phase: SimTouchPhase, + a: { x: number; y: number }, + b: { x: number; y: number }, + ): Promise { + await this.call('pinch', { udid, phase, x0: a.x, y0: a.y, x1: b.x, y1: b.y }); + } + + /** Set the device pasteboard; pair with a Cmd+V key press to inject arbitrary Unicode. */ + async paste(udid: string, text: string): Promise { + await this.call('paste', { udid, text }); + } + /** Swipe between two normalized (0..1) points over `durationMs` (private HID; macOS only). */ async swipe( udid: string, diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index 5981d9a3..ad022f38 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -55,8 +55,17 @@ export interface SimulatorScreenProps { * number of `move`s, one final `up` per gesture. Forwarded in real time, so the device's own * gesture recognition decides tap vs drag vs long-press. */ onTouch?: (phase: SimulatorScreenTouchPhase, point: SimulatorScreenPoint) => void; + /** A streamed two-finger phase (pinch/zoom): both fingers in normalized coordinates. Driven by + * Option-drag, mirroring Simulator.app — the two fingers are symmetric about the drag origin. */ + onPinch?: ( + phase: SimulatorScreenTouchPhase, + a: SimulatorScreenPoint, + b: SimulatorScreenPoint, + ) => void; /** A key press (typed on the focused screen) decomposed to HID usages — see {@link simulatorKeyPress}. */ onKey?: (press: SimulatorKeyPress) => void; + /** Text committed by the OS IME (composition end / non-ASCII input): pasted onto the device. */ + onText?: (text: string) => void; /** The device's real screen-outline mask (image URL, framebuffer-sized): clips the stream to * the exact screen shape, and its measured corner radius keeps the chassis curve concentric. * Absent → a generic rounding. */ @@ -99,12 +108,15 @@ interface PaintState { export function SimulatorScreen({ subscribeFrames, onTouch, + onPinch, onKey, + onText, maskUrl, placeholder, className, }: SimulatorScreenProps): React.ReactNode { const canvasRef = useRef(null); + const inputRef = useRef(null); const paintRef = useRef({ mask: null, frame: null, @@ -114,6 +126,9 @@ export function SimulatorScreen({ }); const pressRef = useRef<{ pointerId: number; + /** True while this drag is an Option-pinch (two mirrored fingers about `origin`). */ + pinch: boolean; + origin: SimulatorScreenPoint; last: SimulatorScreenPoint; moveSentAt: number; } | null>(null); @@ -237,12 +252,34 @@ export function SimulatorScreen({ [maskUrl], ); + // Option-drag pinches (Simulator.app convention): the two fingers mirror about the press + // origin, so the cursor drives one finger and its reflection drives the other. + const mirror = ( + origin: SimulatorScreenPoint, + point: SimulatorScreenPoint, + ): SimulatorScreenPoint => ({ + x: clamp(2 * origin.x - point.x, 0, 1), + y: clamp(2 * origin.y - point.y, 0, 1), + }); + const emitGesture = (phase: SimulatorScreenTouchPhase, point: SimulatorScreenPoint): void => { + const press = pressRef.current; + if (press?.pinch) onPinch?.(phase, point, mirror(press.origin, point)); + else onTouch?.(phase, point); + }; + const handlePointerDown = (event: React.PointerEvent): void => { if (event.button !== 0) return; event.currentTarget.setPointerCapture(event.pointerId); + inputRef.current?.focus({ preventScroll: true }); const point = normalizedPoint(event); - pressRef.current = { pointerId: event.pointerId, last: point, moveSentAt: performance.now() }; - onTouch?.('down', point); + pressRef.current = { + pointerId: event.pointerId, + pinch: event.altKey && onPinch !== undefined, + origin: point, + last: point, + moveSentAt: performance.now(), + }; + emitGesture('down', point); }; const handlePointerMove = (event: React.PointerEvent): void => { @@ -253,25 +290,25 @@ export function SimulatorScreen({ const point = normalizedPoint(event); press.last = point; press.moveSentAt = now; - onTouch?.('move', point); + emitGesture('move', point); }; const handlePointerUp = (event: React.PointerEvent): void => { const press = pressRef.current; if (press?.pointerId !== event.pointerId) return; + emitGesture('up', normalizedPoint(event)); pressRef.current = null; - onTouch?.('up', normalizedPoint(event)); }; const handlePointerCancel = (event: React.PointerEvent): void => { const press = pressRef.current; if (press?.pointerId !== event.pointerId) return; - pressRef.current = null; // End the gesture where it last was — a stuck touch would keep the device pressed forever. - onTouch?.('up', press.last); + emitGesture('up', press.last); + pressRef.current = null; }; - const handleKeyDown = (event: React.KeyboardEvent): void => { + const handleKeyDown = (event: React.KeyboardEvent): void => { if (onKey === undefined || event.nativeEvent.isComposing) return; const press = simulatorKeyPress(event); if (press === null) return; @@ -279,6 +316,14 @@ export function SimulatorScreen({ onKey(press); }; + // IME / non-ASCII commits arrive here (the hidden input is where composition happens). Route + // them through the pasteboard; clear the input so it never accumulates. + const handleInput = (event: React.ChangeEvent): void => { + const value = event.currentTarget.value; + event.currentTarget.value = ''; + if (value.length > 0) onText?.(value); + }; + // Trackpad/wheel scrolling becomes a synthetic drag: touch down where the cursor sits, move // opposite the scroll deltas (finger-follows-content, like the real device), up after idle. const wheelRef = useRef<{ @@ -314,12 +359,10 @@ export function SimulatorScreen({
- {/* Focusable so plain typing reaches the device; app-level chords (⌘…) pass through. */} + {/* Off-screen editable that owns keyboard focus: ASCII keydowns become HID key presses, + IME/non-ASCII commits become pasteboard text. A tap on the canvas focuses it. App-level + chords (⌘…) fall through `simulatorKeyPress` untouched. */} + {deviceAspect === null && placeholder}
From 09152d46c4b4fa7b051548561a814d74b7eb429c Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 14:41:15 +0800 Subject: [PATCH 20/26] perf(sim): 30fps stream + fire-and-forget touch/pinch move to kill per-move round-trip stutter --- .../client/core/src/client/control-channel.ts | 33 +++++++++++++ .../src/simulator/stream-registry.ts | 7 +-- .../engine/src/simulator/request-handler.ts | 48 ++++++++++++------- 3 files changed, 69 insertions(+), 19 deletions(-) diff --git a/packages/client/core/src/client/control-channel.ts b/packages/client/core/src/client/control-channel.ts index e355e45c..ed4fc947 100644 --- a/packages/client/core/src/client/control-channel.ts +++ b/packages/client/core/src/client/control-channel.ts @@ -703,6 +703,23 @@ export class ControlChannel { x: number, y: number, ): Promise { + // `move` phases fire at up to 60 Hz; a per-move round-trip would stall the gesture, so they go + // out unacked (the engine skips the reply). `down`/`up` stay correlated so a claim conflict or + // the gesture's completion is still observable. + if (phase === 'move') { + this.transport.send( + createWireMessage({ + kind: 'simulator.touch', + clientReqId: this.pending.nextClientReqId(), + sessionId, + udid, + phase, + x, + y, + }), + ); + return Promise.resolve({ ok: true }); + } return this.sendCorrelated('ack', (clientReqId) => ({ kind: 'simulator.touch', clientReqId, @@ -721,6 +738,22 @@ export class ControlChannel { a: { x: number; y: number }, b: { x: number; y: number }, ): Promise { + if (phase === 'move') { + this.transport.send( + createWireMessage({ + kind: 'simulator.pinch', + clientReqId: this.pending.nextClientReqId(), + sessionId, + udid, + phase, + x0: a.x, + y0: a.y, + x1: b.x, + y1: b.y, + }), + ); + return Promise.resolve({ ok: true }); + } return this.sendCorrelated('ack', (clientReqId) => ({ kind: 'simulator.pinch', clientReqId, diff --git a/packages/client/workbench/src/simulator/stream-registry.ts b/packages/client/workbench/src/simulator/stream-registry.ts index 53664f9a..effe08cf 100644 --- a/packages/client/workbench/src/simulator/stream-registry.ts +++ b/packages/client/workbench/src/simulator/stream-registry.ts @@ -8,9 +8,10 @@ export type SimulatorStreamClient = Pick< 'simulatorStreamStart' | 'simulatorStreamStop' >; -/** Panel-facing stream parameters: hardware H.264 at native resolution — full 60 fps at a - * fraction of a JPEG stream's bandwidth. Hosts that cannot honor it fall back to JPEG frames. */ -const STREAM_OPTIONS = { fps: 60, codec: 'h264' } as const; +/** Panel-facing stream parameters: hardware H.264 at native resolution. 30 fps keeps the + * client-side decode + native-resolution canvas composite well within one core's budget (60 fps + * saturated it and made interaction stutter); hosts without H.264 fall back to JPEG frames. */ +const STREAM_OPTIONS = { fps: 30, codec: 'h264' } as const; /** Bridges the unmount→mount gap of the docked↔maximized handoff without stopping the stream. */ const CLOSE_DELAY_MS = 250; diff --git a/packages/host/engine/src/simulator/request-handler.ts b/packages/host/engine/src/simulator/request-handler.ts index 614f49af..71e8ae30 100644 --- a/packages/host/engine/src/simulator/request-handler.ts +++ b/packages/host/engine/src/simulator/request-handler.ts @@ -2,6 +2,7 @@ import type { SessionId, WirePayload } from '@linkcode/schema'; import type { Transport } from '@linkcode/transport'; import { createWireMessage } from '@linkcode/transport'; import { Effect } from 'effect'; +import { noop } from 'foxts/noop'; import type { EngineFailure } from '../failure'; import { RequestError, toOperationFailure } from '../failure'; import type { WireResponder } from '../wire/responder'; @@ -167,32 +168,36 @@ export class SimulatorRequestHandler { this.responder.sendSuccess(payload.clientReqId); }), ); - case 'simulator.touch': + case 'simulator.touch': { + const run = (simulators: SimulatorService): Promise => + simulators.touch(payload.sessionId, payload.udid, payload.phase, payload.x, payload.y); + // High-frequency `move` phases arrive unacked (see the client); run them fire-and-forget + // so no reply is emitted. `down`/`up` stay correlated for claim/completion feedback. + if (payload.phase === 'move') return this.fireAndForget('simulator.touch', run); return this.withSimulators(payload.clientReqId, (simulators) => simulatorOperation('simulator.touch', 'Failed to inject touch', async () => { - await simulators.touch( - payload.sessionId, - payload.udid, - payload.phase, - payload.x, - payload.y, - ); + await run(simulators); this.responder.sendSuccess(payload.clientReqId); }), ); - case 'simulator.pinch': + } + case 'simulator.pinch': { + const run = (simulators: SimulatorService): Promise => + simulators.pinch( + payload.sessionId, + payload.udid, + payload.phase, + { x: payload.x0, y: payload.y0 }, + { x: payload.x1, y: payload.y1 }, + ); + if (payload.phase === 'move') return this.fireAndForget('simulator.pinch', run); return this.withSimulators(payload.clientReqId, (simulators) => simulatorOperation('simulator.pinch', 'Failed to pinch', async () => { - await simulators.pinch( - payload.sessionId, - payload.udid, - payload.phase, - { x: payload.x0, y: payload.y0 }, - { x: payload.x1, y: payload.y1 }, - ); + await run(simulators); this.responder.sendSuccess(payload.clientReqId); }), ); + } case 'simulator.paste': return this.withSimulators(payload.clientReqId, (simulators) => simulatorOperation('simulator.paste', 'Failed to set pasteboard', async () => { @@ -300,6 +305,17 @@ export class SimulatorRequestHandler { } } + /** Run an unacked simulator op (a high-frequency `move`). No reply is emitted, and a failure is + * swallowed — a dropped move is corrected by the next one, and the stream reflects reality. */ + private fireAndForget( + _operation: string, + run: (simulators: SimulatorService) => Promise, + ): Effect.Effect { + if (!this.simulators) return Effect.void; + const simulators = this.simulators; + return Effect.promise(() => run(simulators).catch(noop)); + } + private withSimulators( replyTo: string, fn: (simulators: SimulatorService) => Effect.Effect, From 88e126b811b9473a21fff66a0cc19e48c85a138a Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 14:42:54 +0800 Subject: [PATCH 21/26] refactor(ui): split SimulatorScreen into geometry/compositor/decoder modules Decouples the God component into framework-agnostic, unit-testable pure logic: device-geometry (coordinate mapping), device-compositor (canvas assembly), frame-decoder (H.264/JPEG decode). The component drops to event wiring + JSX. --- apps/desktop/e2e/simulator-live.mts | 175 ++++++++ .../__tests__/device-geometry.test.ts | 76 ++++ .../src/shell/simulator/device-compositor.ts | 179 ++++++++ .../ui/src/shell/simulator/device-geometry.ts | 97 +++++ .../ui/src/shell/simulator/frame-decoder.ts | 119 ++++++ .../ui/src/shell/simulator/index.ts | 4 +- .../src/shell/simulator/simulator-screen.tsx | 390 ++---------------- 7 files changed, 680 insertions(+), 360 deletions(-) create mode 100644 apps/desktop/e2e/simulator-live.mts create mode 100644 packages/presentation/ui/src/shell/simulator/__tests__/device-geometry.test.ts create mode 100644 packages/presentation/ui/src/shell/simulator/device-compositor.ts create mode 100644 packages/presentation/ui/src/shell/simulator/device-geometry.ts create mode 100644 packages/presentation/ui/src/shell/simulator/frame-decoder.ts diff --git a/apps/desktop/e2e/simulator-live.mts b/apps/desktop/e2e/simulator-live.mts new file mode 100644 index 00000000..db2f5449 --- /dev/null +++ b/apps/desktop/e2e/simulator-live.mts @@ -0,0 +1,175 @@ +/** + * Live simulator panel — NOT a test. Boots an isolated daemon (with the linkcode-sim sidecar) and + * the built desktop app, seeds a claim session, summons the Simulator section, and then holds the + * window open so you can drive the real booted device by hand (tap, drag, pinch with ⌥, scroll, + * type). Ctrl-C to tear it all down. Run: `pnpm -F @linkcode/desktop exec node e2e/simulator-live.mts` + */ + +import type { ChildProcess } from 'node:child_process'; +import { spawn } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { noop } from 'foxts/noop'; +import { wait } from 'foxts/wait'; +import type { ElectronApplication } from 'playwright-core'; +import { _electron } from 'playwright-core'; +import { io } from 'socket.io-client'; + +const require = createRequire(import.meta.url); +const desktopDir = resolve(import.meta.dirname, '..'); +const daemonDir = resolve(desktopDir, '../daemon'); +const repoRoot = resolve(desktopDir, '../..'); +const simSidecar = join(repoRoot, 'target', 'release', 'linkcode-sim'); +const electronBinary = require('electron') as unknown as string; + +const PORT = 43000 + (process.pid % 1000); +const WIRE_VERSION = 49; + +async function waitForDaemon(): Promise { + const deadline = Date.now() + 30000; + while (Date.now() < deadline) { + try { + await fetch(`http://127.0.0.1:${PORT}/socket.io/?EIO=4&transport=polling`); + return; + } catch { + await wait(250); + } + } + throw new Error(`daemon did not come up on :${PORT}`); +} + +function seedPiSession(cwd: string): Promise { + const socket = io(`http://127.0.0.1:${PORT}`, { transports: ['websocket'] }); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('session.start timed out')), 60000); + socket.on('frame', (raw: unknown) => { + const reply = (raw as { payload?: Record }).payload; + if (reply?.replyTo !== 'live-session') return; + clearTimeout(timer); + if (reply.kind === 'session.started' && typeof reply.sessionId === 'string') { + resolve(reply.sessionId); + } else { + reject(new Error(`session.start failed: ${JSON.stringify(reply)}`)); + } + }); + socket.on('connect', () => { + socket.emit('frame', { + v: WIRE_VERSION, + id: `live-${Date.now().toString(36)}`, + ts: Date.now(), + payload: { kind: 'session.start', clientReqId: 'live-session', opts: { kind: 'pi', cwd } }, + }); + }); + socket.on('connect_error', (error: Error) => { + clearTimeout(timer); + reject(error); + }); + }).finally(() => socket.close()); +} + +async function main(): Promise { + if (!existsSync(join(daemonDir, 'dist/index.js'))) throw new Error('build the daemon first'); + if (!existsSync(join(desktopDir, 'out/main/index.js'))) throw new Error('build desktop first'); + if (!existsSync(simSidecar)) throw new Error('build target/release/linkcode-sim first'); + + const home = mkdtempSync(join(tmpdir(), 'linkcode-live-home-')); + const userData = mkdtempSync(join(tmpdir(), 'linkcode-live-userdata-')); + const chatRoot = join(home, 'LinkCode'); + mkdirSync(chatRoot, { recursive: true }); + const profile = `live-sim-${process.pid}`; + const appSupport = join( + process.env.HOME ?? home, + 'Library', + 'Application Support', + `LinkCode Development (${profile})`, + ); + mkdirSync(appSupport, { recursive: true }); + writeFileSync( + join(appSupport, 'settings.json'), + `${JSON.stringify({ locale: 'en', historyImportOnboardingHandled: true }, null, 2)}\n`, + ); + + let daemon: ChildProcess | null = null; + let app: ElectronApplication | null = null; + const teardown = (): void => { + void app?.close().catch(noop); + daemon?.kill('SIGTERM'); + }; + process.on('SIGINT', () => { + teardown(); + process.exit(0); + }); + process.on('SIGTERM', () => { + teardown(); + process.exit(0); + }); + + daemon = spawn(process.execPath, ['dist/index.js'], { + cwd: daemonDir, + env: { + ...process.env, + HOME: home, + LINKCODE_PORT: String(PORT), + LINKCODE_PROFILE: profile, + LINKCODE_SIM_SIDECAR_PATH: simSidecar, + }, + stdio: 'ignore', + }); + await waitForDaemon(); + console.log(`daemon up on :${PORT}`); + + const sessionId = await seedPiSession(chatRoot); + console.log(`seeded claim session ${sessionId}`); + + app = await _electron.launch({ + executablePath: electronBinary, + args: [desktopDir, `--user-data-dir=${userData}`, '--use-mock-keychain'], + env: { ...process.env, HOME: home, LINKCODE_PROFILE: profile }, + }); + const win = await app.firstWindow(); + await win + .locator('button[aria-label="Toggle side panel"]:visible') + .first() + .waitFor({ state: 'visible', timeout: 30000 }); + await win.waitForTimeout(2000); + + // Open the right panel, summon the Simulator section, activate the seeded thread. + const toggle = win.locator('button[aria-label="Toggle side panel"]:visible').first(); + if ((await toggle.getAttribute('aria-pressed')) !== 'true') { + await toggle.click(); + await win.waitForTimeout(1200); + } + const plus = win.locator('button[aria-label="Open window"]:visible'); + for (const candidate of await plus.all()) { + try { + await candidate.click({ timeout: 2000 }); + break; + } catch { + // try the next matching + trigger + } + } + await win.getByRole('menuitem', { name: 'Simulator' }).click().catch(noop); + await win.waitForTimeout(1000); + const row = win.getByText(/ in LinkCode$/).first(); + await row.waitFor({ state: 'visible', timeout: 15000 }).catch(noop); + await row.click().catch(noop); + + console.log('\n─────────────────────────────────────────────'); + console.log('Live simulator panel is up. In the window:'); + console.log(' • tap / long-press / drag — single finger'); + console.log(' • ⌥ (Option) + drag — pinch to zoom'); + console.log(' • scroll — trackpad/wheel'); + console.log(' • click the screen, then type — English keys + IME (中文/emoji via paste)'); + console.log('Ctrl-C here to shut it all down.'); + console.log('─────────────────────────────────────────────\n'); + + // Hold forever; the SIGINT/SIGTERM handlers tear everything down. + await new Promise(noop); +} + +void main().catch((error: unknown) => { + console.error(error); + process.exit(1); +}); diff --git a/packages/presentation/ui/src/shell/simulator/__tests__/device-geometry.test.ts b/packages/presentation/ui/src/shell/simulator/__tests__/device-geometry.test.ts new file mode 100644 index 00000000..170b045f --- /dev/null +++ b/packages/presentation/ui/src/shell/simulator/__tests__/device-geometry.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from 'vitest'; +import { mirrorPoint, normalizedPoint, screenRectOnPage } from '../device-geometry'; + +/** The slice of `HTMLCanvasElement` the geometry helpers read. */ +type CanvasStub = Pick; + +/** A canvas stub with a controllable native size and on-page rect. */ +function fakeCanvas(native: { w: number; h: number }, rect: DOMRect): HTMLCanvasElement { + const stub: CanvasStub = { + width: native.w, + height: native.h, + getBoundingClientRect: () => rect, + }; + return stub as HTMLCanvasElement; +} + +function rect(left: number, top: number, width: number, height: number): DOMRect { + const value: Omit = { + x: left, + y: top, + left, + top, + width, + height, + right: left + width, + bottom: top + height, + }; + return value as DOMRect; +} + +describe('mirrorPoint', () => { + it('reflects about the origin and clamps to the screen', () => { + expect(mirrorPoint({ x: 0.5, y: 0.5 }, { x: 0.3, y: 0.4 })).toEqual({ x: 0.7, y: 0.6 }); + // A reflection past an edge is clamped into [0,1]. + expect(mirrorPoint({ x: 0.2, y: 0.2 }, { x: 0.9, y: 0.9 })).toEqual({ x: 0, y: 0 }); + }); +}); + +describe('screenRectOnPage / normalizedPoint', () => { + // A device whose canvas is drawn 1:1 on the page (no letterboxing). + const nativeW = 1296; + const nativeH = 2690; + const canvas = fakeCanvas({ w: nativeW, h: nativeH }, rect(0, 0, nativeW, nativeH)); + + it('insets the screen box by the chassis band and button margin', () => { + const screen = screenRectOnPage(canvas); + // The screen sits inside the device box, narrower and shorter than the full canvas. + expect(screen.left).toBeGreaterThan(0); + expect(screen.top).toBeGreaterThan(0); + expect(screen.width).toBeLessThan(nativeW); + expect(screen.height).toBeLessThan(nativeH); + // Symmetric horizontal insets. + expect(screen.left).toBeCloseTo(nativeW - (screen.left + screen.width), 0); + }); + + it('maps the screen-box corners to the normalized unit square', () => { + const screen = screenRectOnPage(canvas); + const topLeft = normalizedPoint({ + clientX: screen.left, + clientY: screen.top, + currentTarget: canvas, + }); + const bottomRight = normalizedPoint({ + clientX: screen.left + screen.width, + clientY: screen.top + screen.height, + currentTarget: canvas, + }); + expect(topLeft).toEqual({ x: 0, y: 0 }); + expect(bottomRight).toEqual({ x: 1, y: 1 }); + }); + + it('clamps a point outside the screen box into [0,1]', () => { + const outside = normalizedPoint({ clientX: -50, clientY: -50, currentTarget: canvas }); + expect(outside).toEqual({ x: 0, y: 0 }); + }); +}); diff --git a/packages/presentation/ui/src/shell/simulator/device-compositor.ts b/packages/presentation/ui/src/shell/simulator/device-compositor.ts new file mode 100644 index 00000000..3f5ec045 --- /dev/null +++ b/packages/presentation/ui/src/shell/simulator/device-compositor.ts @@ -0,0 +1,179 @@ +/** + * The device compositor: paints one framebuffer frame into a canvas as the whole machine — + * side-button bumps, a titanium rim + black display band grown from the real screen mask (so the + * band stays even around the continuous-curvature corners a `roundRect` can't match), and the + * mask-clipped screen. Pure canvas work, no React: the component owns a {@link CompositorState} + * and calls {@link paintDevice} whenever a new frame or mask arrives. + */ + +import { + BUTTON_DEPTH_FRACTION, + FALLBACK_CORNER_FRACTION, + LEFT_BUTTONS, + PAD_FRACTION, + RIGHT_BUTTONS, + RIM_FRACTION, +} from './device-geometry'; + +/** A decoded frame the compositor can draw: a bitmap (JPEG path) or a GPU-resident VideoFrame + * (H.264 path). Both close() and drawImage() uniformly; only the size accessors differ. */ +export type DecodedFrame = ImageBitmap | VideoFrame; + +export function frameWidth(frame: DecodedFrame): number { + return 'displayWidth' in frame ? frame.displayWidth : frame.width; +} + +export function frameHeight(frame: DecodedFrame): number { + return 'displayHeight' in frame ? frame.displayHeight : frame.height; +} + +/** Everything the compositor retains between paints, in native framebuffer pixels. */ +export interface CompositorState { + mask: ImageBitmap | null; + /** Last decoded frame, retained so a late-arriving mask can recomposite it. */ + frame: DecodedFrame | null; + /** Screen-sized scratch layer the mask is composited on before drawing into the chassis. */ + screenLayer: OffscreenCanvas | null; + /** Cached chassis artwork (rim + display band), rebuilt when the geometry key changes. */ + chassis: OffscreenCanvas | null; + chassisKey: string; +} + +export function createCompositorState(): CompositorState { + return { mask: null, frame: null, screenLayer: null, chassis: null, chassisKey: '' }; +} + +const RIM_COLOR = '#3a3a3c'; +const BEZEL_COLOR = '#000000'; +/** Mask dilation samples — the offsets the screen shape is stamped along to grow the chassis. */ +const DILATION_STAMPS = 24; + +/** Paint the whole device into `canvas` (resizing it to the native device dimensions). No-op + * until a frame has been set on `state`. */ +export function paintDevice(canvas: HTMLCanvasElement, state: CompositorState): void { + const frame = state.frame; + if (frame === null) return; + const screenW = frameWidth(frame); + const screenH = frameHeight(frame); + const pad = Math.round(screenW * PAD_FRACTION); + const buttonDepth = Math.round(screenW * BUTTON_DEPTH_FRACTION); + const width = screenW + 2 * pad + 2 * buttonDepth; + const height = screenH + 2 * pad; + if (canvas.width !== width || canvas.height !== height) { + canvas.width = width; + canvas.height = height; + } + const context = canvas.getContext('2d'); + if (context === null) return; + context.clearRect(0, 0, width, height); + + // Side buttons first, so the chassis paints over their inner halves. + context.fillStyle = RIM_COLOR; + const buttonWidth = buttonDepth * 3; + for (const [top, size] of LEFT_BUTTONS) { + context.beginPath(); + context.roundRect(0, pad + top * screenH, buttonWidth, size * screenH, buttonDepth); + context.fill(); + } + for (const [top, size] of RIGHT_BUTTONS) { + context.beginPath(); + context.roundRect( + width - buttonWidth, + pad + top * screenH, + buttonWidth, + size * screenH, + buttonDepth, + ); + context.fill(); + } + + context.drawImage(buildChassis(state, screenW, screenH), buttonDepth, 0); + + // Screen: composite the frame against the mask on a screen-sized layer, then inset it. + let layer = state.screenLayer; + if (layer?.width !== screenW || layer.height !== screenH) { + layer = new OffscreenCanvas(screenW, screenH); + state.screenLayer = layer; + } + const layerContext = layer.getContext('2d'); + if (layerContext === null) return; + layerContext.clearRect(0, 0, layer.width, layer.height); + if (state.mask === null) { + layerContext.save(); + layerContext.beginPath(); + layerContext.roundRect(0, 0, layer.width, layer.height, screenW * FALLBACK_CORNER_FRACTION); + layerContext.clip(); + layerContext.drawImage(frame, 0, 0); + layerContext.restore(); + } else { + layerContext.drawImage(frame, 0, 0); + layerContext.globalCompositeOperation = 'destination-in'; + layerContext.drawImage(state.mask, 0, 0, layer.width, layer.height); + layerContext.globalCompositeOperation = 'source-over'; + } + context.drawImage(layer, buttonDepth + pad, pad); +} + +/** The chassis artwork (rim + display band), cached per frame-size + mask identity. */ +function buildChassis(state: CompositorState, screenW: number, screenH: number): OffscreenCanvas { + const pad = Math.round(screenW * PAD_FRACTION); + const rim = Math.round(screenW * RIM_FRACTION); + const width = screenW + 2 * pad; + const height = screenH + 2 * pad; + const key = `${width}x${height}:${state.mask === null ? 'fallback' : 'mask'}`; + if (state.chassis !== null && state.chassisKey === key) return state.chassis; + + const chassis = new OffscreenCanvas(width, height); + const context = chassis.getContext('2d'); + if (context === null) return chassis; + context.clearRect(0, 0, width, height); + const rimShape = growScreenShape(state.mask, screenW, screenH, pad); + const bandShape = growScreenShape(state.mask, screenW, screenH, pad - rim); + colorize(rimShape, RIM_COLOR); + colorize(bandShape, BEZEL_COLOR); + context.drawImage(rimShape, 0, 0); + context.drawImage(bandShape, rim, rim); + state.chassis = chassis; + state.chassisKey = key; + return chassis; +} + +/** The screen shape expanded outward by `grow` px: the mask stamped along a circle of offsets + * (morphological dilation); a rounded rect when no mask exists. */ +function growScreenShape( + mask: ImageBitmap | null, + screenW: number, + screenH: number, + grow: number, +): OffscreenCanvas { + const shape = new OffscreenCanvas(screenW + 2 * grow, screenH + 2 * grow); + const context = shape.getContext('2d'); + if (context === null) return shape; + if (mask === null) { + context.beginPath(); + context.roundRect(0, 0, shape.width, shape.height, screenW * FALLBACK_CORNER_FRACTION + grow); + context.fill(); + return shape; + } + for (let step = 0; step < DILATION_STAMPS; step += 1) { + const angle = (2 * Math.PI * step) / DILATION_STAMPS; + context.drawImage( + mask, + grow + grow * Math.cos(angle), + grow + grow * Math.sin(angle), + screenW, + screenH, + ); + } + return shape; +} + +/** Replace every opaque pixel of `shape` with `color` in place. */ +function colorize(shape: OffscreenCanvas, color: string): void { + const context = shape.getContext('2d'); + if (context === null) return; + context.globalCompositeOperation = 'source-in'; + context.fillStyle = color; + context.fillRect(0, 0, shape.width, shape.height); + context.globalCompositeOperation = 'source-over'; +} diff --git a/packages/presentation/ui/src/shell/simulator/device-geometry.ts b/packages/presentation/ui/src/shell/simulator/device-geometry.ts new file mode 100644 index 00000000..0c1f8b14 --- /dev/null +++ b/packages/presentation/ui/src/shell/simulator/device-geometry.ts @@ -0,0 +1,97 @@ +/** + * Device ↔ screen coordinate mapping for the simulator canvas. Pure geometry, no React or DOM + * events: given the painted canvas (chassis + screen in native pixels, drawn `object-contain`), + * it recovers where the real device screen sits on the page and maps a page point into the + * device's normalized [0,1] space. Shared by the compositor (which lays the chassis out with the + * same fractions) and the input handlers. + */ + +import { clamp } from 'foxts/clamp'; + +export interface SimulatorScreenPoint { + x: number; + y: number; +} + +export interface DeviceRect { + left: number; + top: number; + width: number; + height: number; +} + +// Chassis proportions as fractions of the native screen width, matched against Simulator.app's +// iPhone chrome: a thin black display band inside a titanium rim, with side-button bumps. +/** Screen edge → chassis outer edge (black band + rim). */ +export const PAD_FRACTION = 0.028; +/** The titanium band's thickness (outermost part of the pad). */ +export const RIM_FRACTION = 0.011; +/** How far the side buttons protrude beyond the chassis. */ +export const BUTTON_DEPTH_FRACTION = 0.009; +/** Fallback screen corner radius (fraction of width) when the host has no mask to measure. */ +export const FALLBACK_CORNER_FRACTION = 0.11; + +/** Side buttons as `[top, height]` fractions of the native screen height. */ +export const LEFT_BUTTONS: ReadonlyArray = [ + [0.245, 0.045], + [0.315, 0.1], + [0.425, 0.1], +]; +export const RIGHT_BUTTONS: ReadonlyArray = [[0.355, 0.175]]; + +/** The painted device's on-page box (the canvas is `object-contain`, so it may be letterboxed). */ +export function deviceRectOnPage(canvas: HTMLCanvasElement): DeviceRect { + const rect = canvas.getBoundingClientRect(); + if (canvas.width === 0 || canvas.height === 0) return rect; + const scale = Math.min(rect.width / canvas.width, rect.height / canvas.height); + const width = canvas.width * scale; + const height = canvas.height * scale; + return { + left: rect.left + (rect.width - width) / 2, + top: rect.top + (rect.height - height) / 2, + width, + height, + }; +} + +/** The screen's on-page box: the device box inset by the chassis band and button margin. */ +export function screenRectOnPage(canvas: HTMLCanvasElement): DeviceRect { + const device = deviceRectOnPage(canvas); + if (canvas.width === 0) return device; + // Recover the native insets the painter rounded from the screen width. + const approxScreenWidth = canvas.width / (1 + 2 * PAD_FRACTION + 2 * BUTTON_DEPTH_FRACTION); + const pad = Math.round(approxScreenWidth * PAD_FRACTION); + const buttonDepth = Math.round(approxScreenWidth * BUTTON_DEPTH_FRACTION); + const scale = device.width / canvas.width; + return { + left: device.left + (pad + buttonDepth) * scale, + top: device.top + pad * scale, + width: (canvas.width - 2 * pad - 2 * buttonDepth) * scale, + height: (canvas.height - 2 * pad) * scale, + }; +} + +/** Map a page point (a pointer/wheel event) into the device's normalized [0,1] screen space. */ +export function normalizedPoint(event: { + clientX: number; + clientY: number; + currentTarget: HTMLCanvasElement; +}): SimulatorScreenPoint { + const screen = screenRectOnPage(event.currentTarget); + return { + x: clamp((event.clientX - screen.left) / screen.width, 0, 1), + y: clamp((event.clientY - screen.top) / screen.height, 0, 1), + }; +} + +/** The reflection of `point` about `origin`, clamped to the screen — the second finger of an + * Option-drag pinch (Simulator.app convention: two fingers symmetric about the press origin). */ +export function mirrorPoint( + origin: SimulatorScreenPoint, + point: SimulatorScreenPoint, +): SimulatorScreenPoint { + return { + x: clamp(2 * origin.x - point.x, 0, 1), + y: clamp(2 * origin.y - point.y, 0, 1), + }; +} diff --git a/packages/presentation/ui/src/shell/simulator/frame-decoder.ts b/packages/presentation/ui/src/shell/simulator/frame-decoder.ts new file mode 100644 index 00000000..8180aede --- /dev/null +++ b/packages/presentation/ui/src/shell/simulator/frame-decoder.ts @@ -0,0 +1,119 @@ +/** + * Framebuffer stream decoder: turns the wire's encoded frames into {@link DecodedFrame}s for the + * compositor. H.264 access units decode through WebCodecs (hardware, GPU-resident output); JPEG + * frames decode latest-wins via `createImageBitmap`. Framework-agnostic — the consumer supplies an + * `onFrame` sink and drives it with {@link SimulatorFrameDecoder.push}. + */ + +import { noop } from 'foxts/noop'; +import type { DecodedFrame } from './device-compositor'; + +/** One stream frame: a base64 JPEG image, or one ordered base64 Annex-B H.264 access unit. */ +export interface SimulatorScreenFrame { + codec: 'jpeg' | 'h264'; + key: boolean; + data: string; +} + +/** High profile at a level comfortably above any simulator resolution; the in-band SPS/PPS on + * each keyframe governs the actual stream parameters. */ +const H264_CODEC = 'avc1.640034'; + +export class SimulatorFrameDecoder { + private closed = false; + + // JPEG path: latest-wins, one decode in flight. + private latestJpeg: string | null = null; + private decoding = false; + + // H.264 path: a lazily-configured decoder that starts from a keyframe and resets on error. + private decoder: VideoDecoder | null = null; + private awaitingKey = true; + private timestamp = 0; + + constructor(private readonly onFrame: (frame: DecodedFrame) => void) {} + + /** Feed one wire frame; decoded output arrives on `onFrame`. */ + push(frame: SimulatorScreenFrame): void { + if (this.closed) return; + if (frame.codec === 'h264') { + this.decodeH264(frame); + return; + } + // A JPEG frame amid an h264 stream means the host degraded; drop the decoder. + this.resetDecoder(); + this.latestJpeg = frame.data; + this.drawNextJpeg(); + } + + /** Tear down: closes the H.264 decoder and stops emitting. */ + close(): void { + this.closed = true; + this.resetDecoder(); + } + + private emit(frame: DecodedFrame): void { + if (this.closed) { + frame.close(); + return; + } + this.onFrame(frame); + } + + private drawNextJpeg(): void { + if (this.decoding || this.latestJpeg === null || this.closed) return; + const encoded = this.latestJpeg; + this.latestJpeg = null; + this.decoding = true; + void createImageBitmap(new Blob([base64Bytes(encoded)], { type: 'image/jpeg' })) + .then((bitmap) => this.emit(bitmap)) + // A corrupt frame is dropped; the next one repaints. + .catch(noop) + .finally(() => { + this.decoding = false; + this.drawNextJpeg(); + }); + } + + private decodeH264(frame: SimulatorScreenFrame): void { + if (this.decoder === null) { + const created = new VideoDecoder({ + output: (decoded) => this.emit(decoded), + error: () => this.resetDecoder(), + }); + created.configure({ codec: H264_CODEC, optimizeForLatency: true }); + this.decoder = created; + this.awaitingKey = true; + } + if (this.awaitingKey && !frame.key) return; + this.awaitingKey = false; + this.decoder.decode( + new EncodedVideoChunk({ + type: frame.key ? 'key' : 'delta', + // Synthetic monotonic clock; frames present as they arrive, so only order matters. + timestamp: this.timestamp++, + data: base64Bytes(frame.data), + }), + ); + } + + private resetDecoder(): void { + if (this.decoder !== null && this.decoder.state !== 'closed') this.decoder.close(); + this.decoder = null; + this.awaitingKey = true; + } +} + +/** A screen mask fetched from a URL, decoded to a bitmap. */ +export function decodeMask(response: Response): Promise { + return response.blob().then((blob) => createImageBitmap(blob)); +} + +function base64Bytes(base64: string): Uint8Array { + const raw = atob(base64); + const bytes = new Uint8Array(raw.length); + for (let index = 0; index < raw.length; index += 1) { + bytes[index] = raw.codePointAt(index) ?? 0; + } + return bytes; +} diff --git a/packages/presentation/ui/src/shell/simulator/index.ts b/packages/presentation/ui/src/shell/simulator/index.ts index 689eedae..e5aadc0c 100644 --- a/packages/presentation/ui/src/shell/simulator/index.ts +++ b/packages/presentation/ui/src/shell/simulator/index.ts @@ -1,7 +1,7 @@ +export type { SimulatorScreenPoint } from './device-geometry'; +export type { SimulatorScreenFrame } from './frame-decoder'; export type { SimulatorKeyPress } from './keymap'; export type { - SimulatorScreenFrame, - SimulatorScreenPoint, SimulatorScreenProps, SimulatorScreenTouchPhase, } from './simulator-screen'; diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index ad022f38..c0d1bd32 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -3,13 +3,16 @@ import { clamp } from 'foxts/clamp'; import { noop } from 'foxts/noop'; import { useRef, useState } from 'react'; import { cn } from '../../lib/cn'; +import { createCompositorState, paintDevice } from './device-compositor'; +import type { SimulatorScreenPoint } from './device-geometry'; +import { mirrorPoint, normalizedPoint, screenRectOnPage } from './device-geometry'; +import type { SimulatorScreenFrame } from './frame-decoder'; +import { decodeMask, SimulatorFrameDecoder } from './frame-decoder'; import type { SimulatorKeyPress } from './keymap'; import { simulatorKeyPress } from './keymap'; -export interface SimulatorScreenPoint { - x: number; - y: number; -} +export type { SimulatorScreenPoint } from './device-geometry'; +export type { SimulatorScreenFrame } from './frame-decoder'; /** One phase of a streamed touch gesture. */ export type SimulatorScreenTouchPhase = 'down' | 'move' | 'up'; @@ -19,34 +22,6 @@ const TOUCH_MOVE_INTERVAL_MS = 16; /** A wheel stream idle for this long ends its synthetic drag gesture. */ const WHEEL_IDLE_MS = 120; -// Chassis proportions as fractions of the native screen width, matched against Simulator.app's -// iPhone chrome: a thin black display band inside a titanium rim, with side-button bumps. -/** Screen edge → chassis outer edge (black band + rim). */ -const PAD_FRACTION = 0.028; -/** The titanium band's thickness (outermost part of the pad). */ -const RIM_FRACTION = 0.011; -/** How far the side buttons protrude beyond the chassis. */ -const BUTTON_DEPTH_FRACTION = 0.009; -/** Side buttons as `[top, height]` fractions of the native screen height. */ -const LEFT_BUTTONS: ReadonlyArray = [ - [0.245, 0.045], - [0.315, 0.1], - [0.425, 0.1], -]; -const RIGHT_BUTTONS: ReadonlyArray = [[0.355, 0.175]]; -/** Fallback screen corner radius (fraction of width) when the host has no mask to measure. */ -const FALLBACK_CORNER_FRACTION = 0.11; - -const RIM_COLOR = '#3a3a3c'; -const BEZEL_COLOR = '#000000'; - -/** One stream frame: a base64 JPEG image, or one ordered base64 Annex-B H.264 access unit. */ -export interface SimulatorScreenFrame { - codec: 'jpeg' | 'h264'; - key: boolean; - data: string; -} - export interface SimulatorScreenProps { /** Feed of stream frames; returns the unsubscribe. H.264 units decode through WebCodecs * (hardware, GPU-resident output); JPEG frames decode via `createImageBitmap`. */ @@ -75,35 +50,12 @@ export interface SimulatorScreenProps { className?: string; } -/** A decoded frame the compositor can draw: a bitmap (JPEG path) or a GPU-resident VideoFrame - * (H.264 path). Both close() and drawImage() uniformly; only the size accessors differ. */ -type DecodedFrame = ImageBitmap | VideoFrame; - -function frameWidth(frame: DecodedFrame): number { - return 'displayWidth' in frame ? frame.displayWidth : frame.width; -} - -function frameHeight(frame: DecodedFrame): number { - return 'displayHeight' in frame ? frame.displayHeight : frame.height; -} - -/** Everything the compositor knows about one device feed, in native framebuffer pixels. */ -interface PaintState { - mask: ImageBitmap | null; - /** Last decoded frame, retained so a late-arriving mask can recomposite it. */ - frame: DecodedFrame | null; - /** Screen-sized scratch layer the mask is composited on before drawing into the chassis. */ - screenLayer: OffscreenCanvas | null; - /** Cached chassis artwork (rim + display band), rebuilt when the geometry key changes. */ - chassis: OffscreenCanvas | null; - chassisKey: string; -} - /** - * Live device screen composited like the real machine: the chassis (bezel band + concentric - * outer corners) and the mask-clipped framebuffer are painted together in native pixel space, - * so every proportion scales with the panel exactly. Latest-wins frame decode; pointer presses - * map back to normalized tap/swipe callbacks. Key this component by device. + * Live device screen: renders the framebuffer stream as the whole machine (chassis + mask-clipped + * screen) and forwards pointer/wheel/key input as normalized gestures. The decode, compositing, + * and coordinate mapping are framework-agnostic modules ({@link SimulatorFrameDecoder}, + * {@link paintDevice}, `device-geometry`); this component only wires them to React and the DOM. + * Key it by device so a switch resets the painted frame. */ export function SimulatorScreen({ subscribeFrames, @@ -117,13 +69,7 @@ export function SimulatorScreen({ }: SimulatorScreenProps): React.ReactNode { const canvasRef = useRef(null); const inputRef = useRef(null); - const paintRef = useRef({ - mask: null, - frame: null, - screenLayer: null, - chassis: null, - chassisKey: '', - }); + const compositorRef = useRef(createCompositorState()); const pressRef = useRef<{ pointerId: number; /** True while this drag is an Option-pinch (two mirrored fingers about `origin`). */ @@ -137,9 +83,8 @@ export function SimulatorScreen({ useAbortableEffect( (signal) => { - const state = paintRef.current; - - const adopt = (frame: DecodedFrame): void => { + const state = compositorRef.current; + const decoder = new SimulatorFrameDecoder((frame) => { if (signal.aborted || canvasRef.current === null) { frame.close(); return; @@ -147,77 +92,13 @@ export function SimulatorScreen({ state.frame?.close(); state.frame = frame; paintDevice(canvasRef.current, state); - const canvas = canvasRef.current; - const aspect = `${canvas.width} / ${canvas.height}`; + const aspect = `${canvasRef.current.width} / ${canvasRef.current.height}`; setDeviceAspect((previous) => (previous === aspect ? previous : aspect)); - }; - - // JPEG path: latest-wins decode via createImageBitmap. - let latestJpeg: string | null = null; - let decoding = false; - const drawNextJpeg = (): void => { - if (decoding || latestJpeg === null || signal.aborted) return; - const encoded = latestJpeg; - latestJpeg = null; - decoding = true; - void createImageBitmap(new Blob([base64Bytes(encoded)], { type: 'image/jpeg' })) - .then(adopt) - // A corrupt frame is dropped; the next one repaints. - .catch(noop) - .finally(() => { - decoding = false; - drawNextJpeg(); - }); - }; - - // H.264 path: hardware decode via WebCodecs; output VideoFrames stay GPU-resident. The - // decoder configures lazily, consumes only from a keyframe, and resets (waiting for the - // next key, ≤2s away) on any decode error. - let decoder: VideoDecoder | null = null; - let awaitingKey = true; - let timestamp = 0; - const resetDecoder = (): void => { - if (decoder !== null && decoder.state !== 'closed') decoder.close(); - decoder = null; - awaitingKey = true; - }; - const decodeH264 = (frame: SimulatorScreenFrame): void => { - if (decoder === null) { - const created = new VideoDecoder({ - output: adopt, - error: resetDecoder, - }); - // High profile at a level comfortably above any simulator resolution; the in-band - // SPS/PPS on each keyframe governs the actual stream parameters. - created.configure({ codec: 'avc1.640034', optimizeForLatency: true }); - decoder = created; - awaitingKey = true; - } - if (awaitingKey && !frame.key) return; - awaitingKey = false; - decoder.decode( - new EncodedVideoChunk({ - type: frame.key ? 'key' : 'delta', - // Synthetic monotonic clock; frames present as they arrive, so only order matters. - timestamp: timestamp++, - data: base64Bytes(frame.data), - }), - ); - }; - - const unsubscribe = subscribeFrames((frame) => { - if (frame.codec === 'h264') { - decodeH264(frame); - return; - } - // A JPEG frame amid an h264 stream means the host degraded; drop the decoder. - resetDecoder(); - latestJpeg = frame.data; - drawNextJpeg(); }); + const unsubscribe = subscribeFrames((frame) => decoder.push(frame)); return () => { unsubscribe(); - resetDecoder(); + decoder.close(); state.frame?.close(); state.frame = null; }; @@ -228,9 +109,9 @@ export function SimulatorScreen({ useAbortableEffect( (signal) => { if (maskUrl == null) return; - const state = paintRef.current; + const state = compositorRef.current; void fetch(maskUrl, { signal }) - .then(decodeResponse) + .then(decodeMask) .then((bitmap) => { if (signal.aborted) { bitmap.close(); @@ -252,18 +133,9 @@ export function SimulatorScreen({ [maskUrl], ); - // Option-drag pinches (Simulator.app convention): the two fingers mirror about the press - // origin, so the cursor drives one finger and its reflection drives the other. - const mirror = ( - origin: SimulatorScreenPoint, - point: SimulatorScreenPoint, - ): SimulatorScreenPoint => ({ - x: clamp(2 * origin.x - point.x, 0, 1), - y: clamp(2 * origin.y - point.y, 0, 1), - }); const emitGesture = (phase: SimulatorScreenTouchPhase, point: SimulatorScreenPoint): void => { const press = pressRef.current; - if (press?.pinch) onPinch?.(phase, point, mirror(press.origin, point)); + if (press?.pinch) onPinch?.(phase, point, mirrorPoint(press.origin, point)); else onTouch?.(phase, point); }; @@ -325,9 +197,12 @@ export function SimulatorScreen({ }; // Trackpad/wheel scrolling becomes a synthetic drag: touch down where the cursor sits, move - // opposite the scroll deltas (finger-follows-content, like the real device), up after idle. + // opposite the scroll deltas (finger-follows-content), up after idle. Trackpad wheels fire far + // faster than 60 Hz, so deltas accumulate into `pos` every event but a `move` is emitted at most + // once per frame — the final `up` carries the settled position so nothing is lost. const wheelRef = useRef<{ pos: SimulatorScreenPoint; + moveSentAt: number; endTimer: ReturnType; } | null>(null); const handleWheel = (event: React.WheelEvent): void => { @@ -338,10 +213,12 @@ export function SimulatorScreen({ wheelRef.current = null; if (wheel) onTouch?.('up', wheel.pos); }; + const now = performance.now(); let wheel = wheelRef.current; if (wheel === null) { const start = normalizedPoint(event); - wheel = { pos: start, endTimer: setTimeout(endGesture, WHEEL_IDLE_MS) }; + const endTimer = setTimeout(endGesture, WHEEL_IDLE_MS); + wheel = { pos: start, moveSentAt: now, endTimer }; wheelRef.current = wheel; onTouch?.('down', start); } else { @@ -352,7 +229,10 @@ export function SimulatorScreen({ x: clamp(wheel.pos.x - event.deltaX / screen.width, 0, 1), y: clamp(wheel.pos.y - event.deltaY / screen.height, 0, 1), }; - onTouch?.('move', wheel.pos); + if (now - wheel.moveSentAt >= TOUCH_MOVE_INTERVAL_MS) { + wheel.moveSentAt = now; + onTouch?.('move', wheel.pos); + } }; return ( @@ -390,209 +270,3 @@ export function SimulatorScreen({ ); } - -/** Paint the whole device in native pixels: side-button bumps, then the cached chassis (rim + - * display band, both grown from the real mask so the band stays even around the corners), then - * the mask-clipped frame. */ -function paintDevice(canvas: HTMLCanvasElement, state: PaintState): void { - const frame = state.frame; - if (frame === null) return; - const screenW = frameWidth(frame); - const screenH = frameHeight(frame); - const pad = Math.round(screenW * PAD_FRACTION); - const buttonDepth = Math.round(screenW * BUTTON_DEPTH_FRACTION); - const width = screenW + 2 * pad + 2 * buttonDepth; - const height = screenH + 2 * pad; - if (canvas.width !== width || canvas.height !== height) { - canvas.width = width; - canvas.height = height; - } - const context = canvas.getContext('2d'); - if (context === null) return; - context.clearRect(0, 0, width, height); - - // Side buttons first, so the chassis paints over their inner halves. - context.fillStyle = RIM_COLOR; - const buttonWidth = buttonDepth * 3; - for (const [top, size] of LEFT_BUTTONS) { - context.beginPath(); - context.roundRect(0, pad + top * screenH, buttonWidth, size * screenH, buttonDepth); - context.fill(); - } - for (const [top, size] of RIGHT_BUTTONS) { - context.beginPath(); - context.roundRect( - width - buttonWidth, - pad + top * screenH, - buttonWidth, - size * screenH, - buttonDepth, - ); - context.fill(); - } - - context.drawImage(buildChassis(state, screenW, screenH), buttonDepth, 0); - - // Screen: composite the frame against the mask on a screen-sized layer, then inset it. - let layer = state.screenLayer; - if (layer?.width !== screenW || layer.height !== screenH) { - layer = new OffscreenCanvas(screenW, screenH); - state.screenLayer = layer; - } - const layerContext = layer.getContext('2d'); - if (layerContext === null) return; - layerContext.clearRect(0, 0, layer.width, layer.height); - if (state.mask === null) { - layerContext.save(); - layerContext.beginPath(); - layerContext.roundRect(0, 0, layer.width, layer.height, screenW * FALLBACK_CORNER_FRACTION); - layerContext.clip(); - layerContext.drawImage(frame, 0, 0); - layerContext.restore(); - } else { - layerContext.drawImage(frame, 0, 0); - layerContext.globalCompositeOperation = 'destination-in'; - layerContext.drawImage(state.mask, 0, 0, layer.width, layer.height); - layerContext.globalCompositeOperation = 'source-over'; - } - context.drawImage(layer, buttonDepth + pad, pad); -} - -/** - * The chassis artwork: the titanium rim and the black display band, built by morphologically - * dilating the real screen mask (stamping it along a circle of offsets). Growing the true shape - * keeps the band width even the whole way around the corner and the outer curvature in the same - * family as Apple's continuous-curvature screen corners — a `roundRect`'s circular arcs visibly - * diverge from them. Cached per frame-size + mask identity; without a mask a rounded rect stands - * in for the shape. - */ -function buildChassis(state: PaintState, screenW: number, screenH: number): OffscreenCanvas { - const pad = Math.round(screenW * PAD_FRACTION); - const rim = Math.round(screenW * RIM_FRACTION); - const width = screenW + 2 * pad; - const height = screenH + 2 * pad; - const key = `${width}x${height}:${state.mask === null ? 'fallback' : 'mask'}`; - if (state.chassis !== null && state.chassisKey === key) return state.chassis; - - const chassis = new OffscreenCanvas(width, height); - const context = chassis.getContext('2d'); - if (context === null) return chassis; - context.clearRect(0, 0, width, height); - const rimShape = growScreenShape(state.mask, screenW, screenH, pad); - const bandShape = growScreenShape(state.mask, screenW, screenH, pad - rim); - colorize(rimShape, RIM_COLOR); - colorize(bandShape, BEZEL_COLOR); - context.drawImage(rimShape, 0, 0); - context.drawImage(bandShape, rim, rim); - state.chassis = chassis; - state.chassisKey = key; - return chassis; -} - -/** The screen shape expanded outward by `grow` px: the mask stamped along a circle of offsets - * (morphological dilation); a rounded rect when no mask exists. */ -function growScreenShape( - mask: ImageBitmap | null, - screenW: number, - screenH: number, - grow: number, -): OffscreenCanvas { - const shape = new OffscreenCanvas(screenW + 2 * grow, screenH + 2 * grow); - const context = shape.getContext('2d'); - if (context === null) return shape; - if (mask === null) { - context.beginPath(); - context.roundRect(0, 0, shape.width, shape.height, screenW * FALLBACK_CORNER_FRACTION + grow); - context.fill(); - return shape; - } - const STAMPS = 24; - for (let step = 0; step < STAMPS; step += 1) { - const angle = (2 * Math.PI * step) / STAMPS; - context.drawImage( - mask, - grow + grow * Math.cos(angle), - grow + grow * Math.sin(angle), - screenW, - screenH, - ); - } - return shape; -} - -/** Replace every opaque pixel of `shape` with `color` in place. */ -function colorize(shape: OffscreenCanvas, color: string): void { - const context = shape.getContext('2d'); - if (context === null) return; - context.globalCompositeOperation = 'source-in'; - context.fillStyle = color; - context.fillRect(0, 0, shape.width, shape.height); - context.globalCompositeOperation = 'source-over'; -} - -/** The painted device's on-page box (the canvas is `object-contain`, so it may be letterboxed). */ -function deviceRectOnPage(canvas: HTMLCanvasElement): { - left: number; - top: number; - width: number; - height: number; -} { - const rect = canvas.getBoundingClientRect(); - if (canvas.width === 0 || canvas.height === 0) return rect; - const scale = Math.min(rect.width / canvas.width, rect.height / canvas.height); - const width = canvas.width * scale; - const height = canvas.height * scale; - return { - left: rect.left + (rect.width - width) / 2, - top: rect.top + (rect.height - height) / 2, - width, - height, - }; -} - -/** The screen's on-page box: the device box inset by the chassis band and button margin. */ -function screenRectOnPage(canvas: HTMLCanvasElement): { - left: number; - top: number; - width: number; - height: number; -} { - const device = deviceRectOnPage(canvas); - if (canvas.width === 0) return device; - // Recover the native insets the painter rounded from the screen width. - const approxScreenWidth = canvas.width / (1 + 2 * PAD_FRACTION + 2 * BUTTON_DEPTH_FRACTION); - const pad = Math.round(approxScreenWidth * PAD_FRACTION); - const buttonDepth = Math.round(approxScreenWidth * BUTTON_DEPTH_FRACTION); - const scale = device.width / canvas.width; - return { - left: device.left + (pad + buttonDepth) * scale, - top: device.top + pad * scale, - width: (canvas.width - 2 * pad - 2 * buttonDepth) * scale, - height: (canvas.height - 2 * pad) * scale, - }; -} - -function normalizedPoint(event: { - clientX: number; - clientY: number; - currentTarget: HTMLCanvasElement; -}): SimulatorScreenPoint { - const screen = screenRectOnPage(event.currentTarget); - return { - x: clamp((event.clientX - screen.left) / screen.width, 0, 1), - y: clamp((event.clientY - screen.top) / screen.height, 0, 1), - }; -} - -function base64Bytes(base64: string): Uint8Array { - const raw = atob(base64); - const bytes = new Uint8Array(raw.length); - for (let index = 0; index < raw.length; index += 1) { - bytes[index] = raw.codePointAt(index) ?? 0; - } - return bytes; -} - -function decodeResponse(response: Response): Promise { - return response.blob().then((blob) => createImageBitmap(blob)); -} From b4ec389df99295791fc23784c5ebabe6b8966ed1 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 15:26:32 +0800 Subject: [PATCH 22/26] perf(sim): layer static chassis + per-frame screen so 60 fps stays smooth --- .../src/simulator/stream-registry.ts | 9 +- .../__tests__/device-geometry.test.ts | 84 +++++----- .../src/shell/simulator/device-compositor.ts | 120 +++++++------ .../ui/src/shell/simulator/device-geometry.ts | 62 +++---- .../src/shell/simulator/simulator-screen.tsx | 157 ++++++++++++------ 5 files changed, 236 insertions(+), 196 deletions(-) diff --git a/packages/client/workbench/src/simulator/stream-registry.ts b/packages/client/workbench/src/simulator/stream-registry.ts index effe08cf..a7e6a871 100644 --- a/packages/client/workbench/src/simulator/stream-registry.ts +++ b/packages/client/workbench/src/simulator/stream-registry.ts @@ -8,10 +8,11 @@ export type SimulatorStreamClient = Pick< 'simulatorStreamStart' | 'simulatorStreamStop' >; -/** Panel-facing stream parameters: hardware H.264 at native resolution. 30 fps keeps the - * client-side decode + native-resolution canvas composite well within one core's budget (60 fps - * saturated it and made interaction stutter); hosts without H.264 fall back to JPEG frames. */ -const STREAM_OPTIONS = { fps: 30, codec: 'h264' } as const; +/** Panel-facing stream parameters: hardware H.264 at native resolution, 60 fps. The client draws + * the framebuffer on its own layer (one draw + one mask composite per frame, vsync-aligned), so + * 60 fps no longer saturates a core the way the old whole-device repaint did; hosts without H.264 + * fall back to JPEG frames. */ +const STREAM_OPTIONS = { fps: 60, codec: 'h264' } as const; /** Bridges the unmount→mount gap of the docked↔maximized handoff without stopping the stream. */ const CLOSE_DELAY_MS = 250; diff --git a/packages/presentation/ui/src/shell/simulator/__tests__/device-geometry.test.ts b/packages/presentation/ui/src/shell/simulator/__tests__/device-geometry.test.ts index 170b045f..57f33a1f 100644 --- a/packages/presentation/ui/src/shell/simulator/__tests__/device-geometry.test.ts +++ b/packages/presentation/ui/src/shell/simulator/__tests__/device-geometry.test.ts @@ -1,18 +1,8 @@ import { describe, expect, it } from 'vitest'; -import { mirrorPoint, normalizedPoint, screenRectOnPage } from '../device-geometry'; +import { mirrorPoint, normalizedPoint, screenInset } from '../device-geometry'; -/** The slice of `HTMLCanvasElement` the geometry helpers read. */ -type CanvasStub = Pick; - -/** A canvas stub with a controllable native size and on-page rect. */ -function fakeCanvas(native: { w: number; h: number }, rect: DOMRect): HTMLCanvasElement { - const stub: CanvasStub = { - width: native.w, - height: native.h, - getBoundingClientRect: () => rect, - }; - return stub as HTMLCanvasElement; -} +/** The slice of `HTMLCanvasElement` `normalizedPoint` reads. */ +type CanvasStub = Pick; function rect(left: number, top: number, width: number, height: number): DOMRect { const value: Omit = { @@ -28,6 +18,12 @@ function rect(left: number, top: number, width: number, height: number): DOMRect return value as DOMRect; } +/** A canvas stub whose on-page rect is fixed. */ +function fakeCanvas(bounds: DOMRect): HTMLCanvasElement { + const stub: CanvasStub = { getBoundingClientRect: () => bounds }; + return stub as HTMLCanvasElement; +} + describe('mirrorPoint', () => { it('reflects about the origin and clamps to the screen', () => { expect(mirrorPoint({ x: 0.5, y: 0.5 }, { x: 0.3, y: 0.4 })).toEqual({ x: 0.7, y: 0.6 }); @@ -36,41 +32,45 @@ describe('mirrorPoint', () => { }); }); -describe('screenRectOnPage / normalizedPoint', () => { - // A device whose canvas is drawn 1:1 on the page (no letterboxing). - const nativeW = 1296; - const nativeH = 2690; - const canvas = fakeCanvas({ w: nativeW, h: nativeH }, rect(0, 0, nativeW, nativeH)); +describe('screenInset', () => { + const inset = screenInset(1296, 2690); - it('insets the screen box by the chassis band and button margin', () => { - const screen = screenRectOnPage(canvas); - // The screen sits inside the device box, narrower and shorter than the full canvas. - expect(screen.left).toBeGreaterThan(0); - expect(screen.top).toBeGreaterThan(0); - expect(screen.width).toBeLessThan(nativeW); - expect(screen.height).toBeLessThan(nativeH); - // Symmetric horizontal insets. - expect(screen.left).toBeCloseTo(nativeW - (screen.left + screen.width), 0); + it('insets the screen box inside the device box', () => { + expect(inset.left).toBeGreaterThan(0); + expect(inset.top).toBeGreaterThan(0); + expect(inset.width).toBeLessThan(1); + expect(inset.height).toBeLessThan(1); }); - it('maps the screen-box corners to the normalized unit square', () => { - const screen = screenRectOnPage(canvas); - const topLeft = normalizedPoint({ - clientX: screen.left, - clientY: screen.top, - currentTarget: canvas, + it('leaves symmetric horizontal margins (button + pad on both sides)', () => { + const rightMargin = 1 - (inset.left + inset.width); + expect(rightMargin).toBeCloseTo(inset.left, 10); + }); +}); + +describe('normalizedPoint', () => { + // The screen canvas is drawn at (0,0), 100×200 CSS px on the page. + const canvas = fakeCanvas(rect(0, 0, 100, 200)); + + it('maps the canvas corners to the normalized unit square', () => { + expect(normalizedPoint({ clientX: 0, clientY: 0, currentTarget: canvas })).toEqual({ + x: 0, + y: 0, + }); + expect(normalizedPoint({ clientX: 100, clientY: 200, currentTarget: canvas })).toEqual({ + x: 1, + y: 1, }); - const bottomRight = normalizedPoint({ - clientX: screen.left + screen.width, - clientY: screen.top + screen.height, - currentTarget: canvas, + expect(normalizedPoint({ clientX: 50, clientY: 100, currentTarget: canvas })).toEqual({ + x: 0.5, + y: 0.5, }); - expect(topLeft).toEqual({ x: 0, y: 0 }); - expect(bottomRight).toEqual({ x: 1, y: 1 }); }); - it('clamps a point outside the screen box into [0,1]', () => { - const outside = normalizedPoint({ clientX: -50, clientY: -50, currentTarget: canvas }); - expect(outside).toEqual({ x: 0, y: 0 }); + it('clamps a point outside the canvas into [0,1]', () => { + expect(normalizedPoint({ clientX: -50, clientY: 300, currentTarget: canvas })).toEqual({ + x: 0, + y: 1, + }); }); }); diff --git a/packages/presentation/ui/src/shell/simulator/device-compositor.ts b/packages/presentation/ui/src/shell/simulator/device-compositor.ts index 3f5ec045..70ed9c0b 100644 --- a/packages/presentation/ui/src/shell/simulator/device-compositor.ts +++ b/packages/presentation/ui/src/shell/simulator/device-compositor.ts @@ -1,9 +1,11 @@ /** - * The device compositor: paints one framebuffer frame into a canvas as the whole machine — - * side-button bumps, a titanium rim + black display band grown from the real screen mask (so the - * band stays even around the continuous-curvature corners a `roundRect` can't match), and the - * mask-clipped screen. Pure canvas work, no React: the component owns a {@link CompositorState} - * and calls {@link paintDevice} whenever a new frame or mask arrives. + * The device compositor: draws the two layers of the machine. The chassis (side-button bumps + a + * titanium rim + black display band grown from the real screen mask, so the band stays even around + * the continuous-curvature corners a `roundRect` can't match) is static — painted once per device + * via {@link paintChassis}. The screen is a separate layer painted every frame by + * {@link paintScreen}: just the framebuffer clipped to the mask, so the per-frame cost is one draw + * plus one mask composite and nothing static repaints. Pure canvas work; the component owns the + * two canvases and the retained frame/mask. */ import { @@ -27,34 +29,19 @@ export function frameHeight(frame: DecodedFrame): number { return 'displayHeight' in frame ? frame.displayHeight : frame.height; } -/** Everything the compositor retains between paints, in native framebuffer pixels. */ -export interface CompositorState { - mask: ImageBitmap | null; - /** Last decoded frame, retained so a late-arriving mask can recomposite it. */ - frame: DecodedFrame | null; - /** Screen-sized scratch layer the mask is composited on before drawing into the chassis. */ - screenLayer: OffscreenCanvas | null; - /** Cached chassis artwork (rim + display band), rebuilt when the geometry key changes. */ - chassis: OffscreenCanvas | null; - chassisKey: string; -} - -export function createCompositorState(): CompositorState { - return { mask: null, frame: null, screenLayer: null, chassis: null, chassisKey: '' }; -} - const RIM_COLOR = '#3a3a3c'; const BEZEL_COLOR = '#000000'; /** Mask dilation samples — the offsets the screen shape is stamped along to grow the chassis. */ const DILATION_STAMPS = 24; -/** Paint the whole device into `canvas` (resizing it to the native device dimensions). No-op - * until a frame has been set on `state`. */ -export function paintDevice(canvas: HTMLCanvasElement, state: CompositorState): void { - const frame = state.frame; - if (frame === null) return; - const screenW = frameWidth(frame); - const screenH = frameHeight(frame); +/** Paint the static chassis (buttons + rim + band) into `canvas`, resizing it to the whole device. + * Called once per device and again only when the mask arrives — never per frame. */ +export function paintChassis( + canvas: HTMLCanvasElement, + mask: ImageBitmap | null, + screenW: number, + screenH: number, +): void { const pad = Math.round(screenW * PAD_FRACTION); const buttonDepth = Math.round(screenW * BUTTON_DEPTH_FRACTION); const width = screenW + 2 * pad + 2 * buttonDepth; @@ -87,54 +74,59 @@ export function paintDevice(canvas: HTMLCanvasElement, state: CompositorState): context.fill(); } - context.drawImage(buildChassis(state, screenW, screenH), buttonDepth, 0); + context.drawImage(buildChassis(mask, screenW, screenH), buttonDepth, 0); +} - // Screen: composite the frame against the mask on a screen-sized layer, then inset it. - let layer = state.screenLayer; - if (layer?.width !== screenW || layer.height !== screenH) { - layer = new OffscreenCanvas(screenW, screenH); - state.screenLayer = layer; +/** Paint one framebuffer frame into the screen layer, clipped to the real screen shape. This is + * the entire per-frame cost: one opaque draw of the frame, then one mask composite that re-cuts + * the rounded corners the opaque frame overwrote. The canvas is the exact screen size, so it never + * needs clearing on the mask path. */ +export function paintScreen( + canvas: HTMLCanvasElement, + frame: DecodedFrame, + mask: ImageBitmap | null, +): void { + const screenW = frameWidth(frame); + const screenH = frameHeight(frame); + if (canvas.width !== screenW || canvas.height !== screenH) { + canvas.width = screenW; + canvas.height = screenH; } - const layerContext = layer.getContext('2d'); - if (layerContext === null) return; - layerContext.clearRect(0, 0, layer.width, layer.height); - if (state.mask === null) { - layerContext.save(); - layerContext.beginPath(); - layerContext.roundRect(0, 0, layer.width, layer.height, screenW * FALLBACK_CORNER_FRACTION); - layerContext.clip(); - layerContext.drawImage(frame, 0, 0); - layerContext.restore(); + const context = canvas.getContext('2d'); + if (context === null) return; + if (mask === null) { + context.clearRect(0, 0, screenW, screenH); + context.save(); + context.beginPath(); + context.roundRect(0, 0, screenW, screenH, screenW * FALLBACK_CORNER_FRACTION); + context.clip(); + context.drawImage(frame, 0, 0); + context.restore(); } else { - layerContext.drawImage(frame, 0, 0); - layerContext.globalCompositeOperation = 'destination-in'; - layerContext.drawImage(state.mask, 0, 0, layer.width, layer.height); - layerContext.globalCompositeOperation = 'source-over'; + context.globalCompositeOperation = 'source-over'; + context.drawImage(frame, 0, 0); + context.globalCompositeOperation = 'destination-in'; + context.drawImage(mask, 0, 0, screenW, screenH); + context.globalCompositeOperation = 'source-over'; } - context.drawImage(layer, buttonDepth + pad, pad); } -/** The chassis artwork (rim + display band), cached per frame-size + mask identity. */ -function buildChassis(state: CompositorState, screenW: number, screenH: number): OffscreenCanvas { +/** The chassis artwork (rim + display band) as a screen-plus-pad-sized layer. */ +function buildChassis(mask: ImageBitmap | null, screenW: number, screenH: number): OffscreenCanvas { const pad = Math.round(screenW * PAD_FRACTION); const rim = Math.round(screenW * RIM_FRACTION); const width = screenW + 2 * pad; const height = screenH + 2 * pad; - const key = `${width}x${height}:${state.mask === null ? 'fallback' : 'mask'}`; - if (state.chassis !== null && state.chassisKey === key) return state.chassis; - const chassis = new OffscreenCanvas(width, height); const context = chassis.getContext('2d'); - if (context === null) return chassis; - context.clearRect(0, 0, width, height); - const rimShape = growScreenShape(state.mask, screenW, screenH, pad); - const bandShape = growScreenShape(state.mask, screenW, screenH, pad - rim); - colorize(rimShape, RIM_COLOR); - colorize(bandShape, BEZEL_COLOR); - context.drawImage(rimShape, 0, 0); - context.drawImage(bandShape, rim, rim); - state.chassis = chassis; - state.chassisKey = key; + if (context !== null) { + const rimShape = growScreenShape(mask, screenW, screenH, pad); + const bandShape = growScreenShape(mask, screenW, screenH, pad - rim); + colorize(rimShape, RIM_COLOR); + colorize(bandShape, BEZEL_COLOR); + context.drawImage(rimShape, 0, 0); + context.drawImage(bandShape, rim, rim); + } return chassis; } diff --git a/packages/presentation/ui/src/shell/simulator/device-geometry.ts b/packages/presentation/ui/src/shell/simulator/device-geometry.ts index 0c1f8b14..6df658e3 100644 --- a/packages/presentation/ui/src/shell/simulator/device-geometry.ts +++ b/packages/presentation/ui/src/shell/simulator/device-geometry.ts @@ -1,9 +1,9 @@ /** - * Device ↔ screen coordinate mapping for the simulator canvas. Pure geometry, no React or DOM - * events: given the painted canvas (chassis + screen in native pixels, drawn `object-contain`), - * it recovers where the real device screen sits on the page and maps a page point into the - * device's normalized [0,1] space. Shared by the compositor (which lays the chassis out with the - * same fractions) and the input handlers. + * Device ↔ screen geometry for the layered simulator view. Pure geometry, no React or DOM events: + * the chassis and screen are separate DOM layers (the screen a CSS-positioned canvas inside the + * chassis box), so mapping a pointer into normalized [0,1] space is just its offset within the + * screen canvas's own rect. The chassis compositor and this module share the same fractions, so + * {@link screenInset} places the screen layer exactly over the band's cutout. */ import { clamp } from 'foxts/clamp'; @@ -13,7 +13,8 @@ export interface SimulatorScreenPoint { y: number; } -export interface DeviceRect { +/** The screen layer's box as fractions [0,1] of the whole device box, for CSS placement. */ +export interface ScreenInset { left: number; top: number; width: number; @@ -39,48 +40,33 @@ export const LEFT_BUTTONS: ReadonlyArray = [ ]; export const RIGHT_BUTTONS: ReadonlyArray = [[0.355, 0.175]]; -/** The painted device's on-page box (the canvas is `object-contain`, so it may be letterboxed). */ -export function deviceRectOnPage(canvas: HTMLCanvasElement): DeviceRect { - const rect = canvas.getBoundingClientRect(); - if (canvas.width === 0 || canvas.height === 0) return rect; - const scale = Math.min(rect.width / canvas.width, rect.height / canvas.height); - const width = canvas.width * scale; - const height = canvas.height * scale; +/** The screen layer's placement within the device box, as fractions of it. Uses the same rounded + * native insets the chassis compositor paints, so the CSS-positioned screen sits exactly over the + * band's cutout. */ +export function screenInset(screenW: number, screenH: number): ScreenInset { + const pad = Math.round(screenW * PAD_FRACTION); + const buttonDepth = Math.round(screenW * BUTTON_DEPTH_FRACTION); + const deviceW = screenW + 2 * pad + 2 * buttonDepth; + const deviceH = screenH + 2 * pad; return { - left: rect.left + (rect.width - width) / 2, - top: rect.top + (rect.height - height) / 2, - width, - height, + left: (pad + buttonDepth) / deviceW, + top: pad / deviceH, + width: screenW / deviceW, + height: screenH / deviceH, }; } -/** The screen's on-page box: the device box inset by the chassis band and button margin. */ -export function screenRectOnPage(canvas: HTMLCanvasElement): DeviceRect { - const device = deviceRectOnPage(canvas); - if (canvas.width === 0) return device; - // Recover the native insets the painter rounded from the screen width. - const approxScreenWidth = canvas.width / (1 + 2 * PAD_FRACTION + 2 * BUTTON_DEPTH_FRACTION); - const pad = Math.round(approxScreenWidth * PAD_FRACTION); - const buttonDepth = Math.round(approxScreenWidth * BUTTON_DEPTH_FRACTION); - const scale = device.width / canvas.width; - return { - left: device.left + (pad + buttonDepth) * scale, - top: device.top + pad * scale, - width: (canvas.width - 2 * pad - 2 * buttonDepth) * scale, - height: (canvas.height - 2 * pad) * scale, - }; -} - -/** Map a page point (a pointer/wheel event) into the device's normalized [0,1] screen space. */ +/** Map a page point (a pointer/wheel event on the screen layer) into normalized [0,1] screen + * space — the offset within the screen canvas's own rect. */ export function normalizedPoint(event: { clientX: number; clientY: number; currentTarget: HTMLCanvasElement; }): SimulatorScreenPoint { - const screen = screenRectOnPage(event.currentTarget); + const rect = event.currentTarget.getBoundingClientRect(); return { - x: clamp((event.clientX - screen.left) / screen.width, 0, 1), - y: clamp((event.clientY - screen.top) / screen.height, 0, 1), + x: clamp((event.clientX - rect.left) / rect.width, 0, 1), + y: clamp((event.clientY - rect.top) / rect.height, 0, 1), }; } diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index c0d1bd32..c7acd0df 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -3,9 +3,10 @@ import { clamp } from 'foxts/clamp'; import { noop } from 'foxts/noop'; import { useRef, useState } from 'react'; import { cn } from '../../lib/cn'; -import { createCompositorState, paintDevice } from './device-compositor'; -import type { SimulatorScreenPoint } from './device-geometry'; -import { mirrorPoint, normalizedPoint, screenRectOnPage } from './device-geometry'; +import type { DecodedFrame } from './device-compositor'; +import { frameHeight, frameWidth, paintChassis, paintScreen } from './device-compositor'; +import type { ScreenInset, SimulatorScreenPoint } from './device-geometry'; +import { mirrorPoint, normalizedPoint, screenInset } from './device-geometry'; import type { SimulatorScreenFrame } from './frame-decoder'; import { decodeMask, SimulatorFrameDecoder } from './frame-decoder'; import type { SimulatorKeyPress } from './keymap'; @@ -22,6 +23,14 @@ const TOUCH_MOVE_INTERVAL_MS = 16; /** A wheel stream idle for this long ends its synthetic drag gesture. */ const WHEEL_IDLE_MS = 120; +/** The device box's size + the screen layer's placement within it, once the first frame reveals + * the framebuffer dimensions. */ +interface DeviceLayout { + deviceW: number; + deviceH: number; + inset: ScreenInset; +} + export interface SimulatorScreenProps { /** Feed of stream frames; returns the unsubscribe. H.264 units decode through WebCodecs * (hardware, GPU-resident output); JPEG frames decode via `createImageBitmap`. */ @@ -42,8 +51,8 @@ export interface SimulatorScreenProps { /** Text committed by the OS IME (composition end / non-ASCII input): pasted onto the device. */ onText?: (text: string) => void; /** The device's real screen-outline mask (image URL, framebuffer-sized): clips the stream to - * the exact screen shape, and its measured corner radius keeps the chassis curve concentric. - * Absent → a generic rounding. */ + * the exact screen shape, and its grown outline keeps the chassis band even. Absent → a generic + * rounding. */ maskUrl?: string | null; /** Shown centered until the first frame arrives. */ placeholder?: React.ReactNode; @@ -51,11 +60,12 @@ export interface SimulatorScreenProps { } /** - * Live device screen: renders the framebuffer stream as the whole machine (chassis + mask-clipped - * screen) and forwards pointer/wheel/key input as normalized gestures. The decode, compositing, - * and coordinate mapping are framework-agnostic modules ({@link SimulatorFrameDecoder}, - * {@link paintDevice}, `device-geometry`); this component only wires them to React and the DOM. - * Key it by device so a switch resets the painted frame. + * Live device screen, rendered as two DOM layers so 60 fps stays cheap: a chassis canvas painted + * once ({@link paintChassis}) and a screen canvas painted every frame ({@link paintScreen}) with + * only a frame draw + a mask composite — no static artwork repaints. Decode ({@link + * SimulatorFrameDecoder}) and coordinate mapping (`device-geometry`) are framework-agnostic; this + * component wires them to React, vsync-aligns paints with `requestAnimationFrame`, and forwards + * pointer/wheel/key input as normalized gestures. Key it by device so a switch resets the frame. */ export function SimulatorScreen({ subscribeFrames, @@ -67,9 +77,15 @@ export function SimulatorScreen({ placeholder, className, }: SimulatorScreenProps): React.ReactNode { - const canvasRef = useRef(null); + const chassisRef = useRef(null); + const screenRef = useRef(null); const inputRef = useRef(null); - const compositorRef = useRef(createCompositorState()); + /** Latest decoded frame, retained so a late mask (or the next rAF) can recomposite it. */ + const frameRef = useRef(null); + const maskRef = useRef(null); + /** Coalesce paints onto the display's refresh: the decoder can outrun it, so only the newest + * frame is drawn each tick. Set by the decoder effect; the mask effect calls it to recomposite. */ + const repaintRef = useRef<() => void>(noop); const pressRef = useRef<{ pointerId: number; /** True while this drag is an Option-pinch (two mirrored fingers about `origin`). */ @@ -78,29 +94,55 @@ export function SimulatorScreen({ last: SimulatorScreenPoint; moveSentAt: number; } | null>(null); - /** `w / h` of the whole painted device (screen + bezel); sizes the canvas box. */ - const [deviceAspect, setDeviceAspect] = useState(null); + const [layout, setLayout] = useState(null); useAbortableEffect( (signal) => { - const state = compositorRef.current; + let rafId: number | null = null; + // Rebuilt only when the framebuffer size or mask presence changes — i.e. essentially once. + let chassisKey = ''; + const paint = (): void => { + rafId = null; + const frame = frameRef.current; + const chassis = chassisRef.current; + const screen = screenRef.current; + if (frame === null || chassis === null || screen === null) return; + const screenW = frameWidth(frame); + const screenH = frameHeight(frame); + const key = `${screenW}x${screenH}:${maskRef.current === null ? 'fallback' : 'mask'}`; + if (key !== chassisKey) { + chassisKey = key; + paintChassis(chassis, maskRef.current, screenW, screenH); + setLayout({ + deviceW: chassis.width, + deviceH: chassis.height, + inset: screenInset(screenW, screenH), + }); + } + paintScreen(screen, frame, maskRef.current); + }; + const schedulePaint = (): void => { + if (rafId === null) rafId = requestAnimationFrame(paint); + }; + repaintRef.current = schedulePaint; + const decoder = new SimulatorFrameDecoder((frame) => { - if (signal.aborted || canvasRef.current === null) { + if (signal.aborted) { frame.close(); return; } - state.frame?.close(); - state.frame = frame; - paintDevice(canvasRef.current, state); - const aspect = `${canvasRef.current.width} / ${canvasRef.current.height}`; - setDeviceAspect((previous) => (previous === aspect ? previous : aspect)); + frameRef.current?.close(); + frameRef.current = frame; + schedulePaint(); }); const unsubscribe = subscribeFrames((frame) => decoder.push(frame)); return () => { unsubscribe(); decoder.close(); - state.frame?.close(); - state.frame = null; + if (rafId !== null) cancelAnimationFrame(rafId); + repaintRef.current = noop; + frameRef.current?.close(); + frameRef.current = null; }; }, [subscribeFrames], @@ -109,7 +151,6 @@ export function SimulatorScreen({ useAbortableEffect( (signal) => { if (maskUrl == null) return; - const state = compositorRef.current; void fetch(maskUrl, { signal }) .then(decodeMask) .then((bitmap) => { @@ -117,17 +158,15 @@ export function SimulatorScreen({ bitmap.close(); return; } - state.mask?.close(); - state.mask = bitmap; + maskRef.current?.close(); + maskRef.current = bitmap; // Recomposite the held frame so the mask applies without waiting for the next one. - if (canvasRef.current !== null && state.frame !== null) { - paintDevice(canvasRef.current, state); - } + repaintRef.current(); }) .catch(noop); return () => { - state.mask?.close(); - state.mask = null; + maskRef.current?.close(); + maskRef.current = null; }; }, [maskUrl], @@ -207,7 +246,7 @@ export function SimulatorScreen({ } | null>(null); const handleWheel = (event: React.WheelEvent): void => { if (pressRef.current !== null) return; - const screen = screenRectOnPage(event.currentTarget); + const rect = event.currentTarget.getBoundingClientRect(); const endGesture = (): void => { const wheel = wheelRef.current; wheelRef.current = null; @@ -226,8 +265,8 @@ export function SimulatorScreen({ wheel.endTimer = setTimeout(endGesture, WHEEL_IDLE_MS); } wheel.pos = { - x: clamp(wheel.pos.x - event.deltaX / screen.width, 0, 1), - y: clamp(wheel.pos.y - event.deltaY / screen.height, 0, 1), + x: clamp(wheel.pos.x - event.deltaX / rect.width, 0, 1), + y: clamp(wheel.pos.y - event.deltaY / rect.height, 0, 1), }; if (now - wheel.moveSentAt >= TOUCH_MOVE_INTERVAL_MS) { wheel.moveSentAt = now; @@ -238,20 +277,42 @@ export function SimulatorScreen({ return (
- +
+ + +
{/* Off-screen editable that owns keyboard focus: ASCII keydowns become HID key presses, IME/non-ASCII commits become pasteboard text. A tap on the canvas focuses it. App-level chords (⌘…) fall through `simulatorKeyPress` untouched. */} @@ -266,7 +327,7 @@ export function SimulatorScreen({ onKeyDown={handleKeyDown} onChange={handleInput} /> - {deviceAspect === null && placeholder} + {layout === null && placeholder}
); } From 777a3a7395953141a33f32fb78b285a3f2719509 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 15:35:35 +0800 Subject: [PATCH 23/26] test(sim): drive the screen layer in the panel E2E after the chassis/screen split --- apps/desktop/e2e/simulator-panel.e2e.mts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/apps/desktop/e2e/simulator-panel.e2e.mts b/apps/desktop/e2e/simulator-panel.e2e.mts index 40f2d9a1..0a7ff310 100644 --- a/apps/desktop/e2e/simulator-panel.e2e.mts +++ b/apps/desktop/e2e/simulator-panel.e2e.mts @@ -221,12 +221,13 @@ async function run(win: Page, chatRoot: string, deepPass: boolean): Promise 0) { - const size = await canvas.first().evaluate((el) => { + const size = await canvas.last().evaluate((el) => { const c = el as HTMLCanvasElement; return { width: c.width, height: c.height }; }); @@ -262,8 +263,9 @@ async function run(win: Page, chatRoot: string, deepPass: boolean): Promise => deviceCanvas.evaluate((el) => { const source = el as HTMLCanvasElement; @@ -293,12 +295,12 @@ async function run(win: Page, chatRoot: string, deepPass: boolean): Promise Date: Thu, 23 Jul 2026 15:58:37 +0800 Subject: [PATCH 24/26] fix(sim): re-plant the wheel-scroll finger at screen edges so long scrolls don't stall --- .../src/shell/simulator/simulator-screen.tsx | 51 ++++++++++++++++--- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index c7acd0df..4387b30f 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -22,6 +22,14 @@ export type SimulatorScreenTouchPhase = 'down' | 'move' | 'up'; const TOUCH_MOVE_INTERVAL_MS = 16; /** A wheel stream idle for this long ends its synthetic drag gesture. */ const WHEEL_IDLE_MS = 120; +/** When a re-planted wheel finger lands, it re-plants inside this band (a real long scroll is many + * swipes; a finger clamped at the edge just stalls). Kept clear of the top/bottom edges so an + * upward re-swipe never reads as the home gesture. */ +const WHEEL_BAND_LO = 0.15; +const WHEEL_BAND_HI = 0.85; +/** How far, in normalized units, a re-planted finger is nudged so the touch reads as a drag (not a + * stray tap if the scroll idles right after re-planting). */ +const WHEEL_NUDGE = 0.02; /** The device box's size + the screen layer's placement within it, once the first frame reveals * the framebuffer dimensions. */ @@ -238,7 +246,9 @@ export function SimulatorScreen({ // Trackpad/wheel scrolling becomes a synthetic drag: touch down where the cursor sits, move // opposite the scroll deltas (finger-follows-content), up after idle. Trackpad wheels fire far // faster than 60 Hz, so deltas accumulate into `pos` every event but a `move` is emitted at most - // once per frame — the final `up` carries the settled position so nothing is lost. + // once per frame — the final `up` carries the settled position so nothing is lost. When the + // finger would run off a screen edge, it lifts and re-plants on the far side, so one long scroll + // becomes many swipes instead of a finger pinned at the edge (which stalls scrolling mid-page). const wheelRef = useRef<{ pos: SimulatorScreenPoint; moveSentAt: number; @@ -255,7 +265,11 @@ export function SimulatorScreen({ const now = performance.now(); let wheel = wheelRef.current; if (wheel === null) { - const start = normalizedPoint(event); + const cursor = normalizedPoint(event); + const start = { + x: clamp(cursor.x, WHEEL_BAND_LO, WHEEL_BAND_HI), + y: clamp(cursor.y, WHEEL_BAND_LO, WHEEL_BAND_HI), + }; const endTimer = setTimeout(endGesture, WHEEL_IDLE_MS); wheel = { pos: start, moveSentAt: now, endTimer }; wheelRef.current = wheel; @@ -264,13 +278,34 @@ export function SimulatorScreen({ clearTimeout(wheel.endTimer); wheel.endTimer = setTimeout(endGesture, WHEEL_IDLE_MS); } - wheel.pos = { - x: clamp(wheel.pos.x - event.deltaX / rect.width, 0, 1), - y: clamp(wheel.pos.y - event.deltaY / rect.height, 0, 1), - }; - if (now - wheel.moveSentAt >= TOUCH_MOVE_INTERVAL_MS) { + // Advance the synthetic finger opposite the scroll (finger follows content). + const x = wheel.pos.x - event.deltaX / rect.width; + const y = wheel.pos.y - event.deltaY / rect.height; + if (x < 0 || x > 1 || y < 0 || y > 1) { + // Ran past a screen edge: finish this swipe there, lift, then re-plant on the far side so the + // same scroll keeps dragging. The immediate nudge makes the re-planted touch a drag, so an + // idle right after re-planting ends a swipe (up), never a stray tap in place. + const edge = { x: clamp(x, 0, 1), y: clamp(y, 0, 1) }; + onTouch?.('move', edge); + onTouch?.('up', edge); + const planted = { + x: x < 0 ? WHEEL_BAND_HI : x > 1 ? WHEEL_BAND_LO : clamp(x, WHEEL_BAND_LO, WHEEL_BAND_HI), + y: y < 0 ? WHEEL_BAND_HI : y > 1 ? WHEEL_BAND_LO : clamp(y, WHEEL_BAND_LO, WHEEL_BAND_HI), + }; + onTouch?.('down', planted); + const nudged = { + x: clamp(planted.x - Math.sign(event.deltaX) * WHEEL_NUDGE, 0, 1), + y: clamp(planted.y - Math.sign(event.deltaY) * WHEEL_NUDGE, 0, 1), + }; + onTouch?.('move', nudged); + wheel.pos = nudged; wheel.moveSentAt = now; - onTouch?.('move', wheel.pos); + } else { + wheel.pos = { x, y }; + if (now - wheel.moveSentAt >= TOUCH_MOVE_INTERVAL_MS) { + wheel.moveSentAt = now; + onTouch?.('move', wheel.pos); + } } }; From e506db4f077eb10bab80e138e76cd1cff667af94 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Thu, 23 Jul 2026 16:24:28 +0800 Subject: [PATCH 25/26] docs(sim): record the native-scroll-injection dead end in the input module --- crates/linkcode-sim/src/private/input.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/crates/linkcode-sim/src/private/input.rs b/crates/linkcode-sim/src/private/input.rs index b3d37013..df03c67d 100644 --- a/crates/linkcode-sim/src/private/input.rs +++ b/crates/linkcode-sim/src/private/input.rs @@ -7,6 +7,17 @@ //! `IndigoHIDMessageForTrackpadEventFromHIDEventRef`, then patches the two byte slots the wrapper //! leaves uninitialised (the `0x32` touch-target tag and the edge bitmask). Buttons (home/lock) use //! the legacy `IndigoHIDMessageForButton`, which SpringBoard still honors on Face ID devices. +//! +//! Native scroll injection was spiked 2026-07 and is a dead end — don't re-tread it. SimulatorKit +//! exports `IndigoHIDMessageForScrollEvent{,FromHIDEventRef}` (payload target at `+0x2c`, routing +//! tags `0x35`/`0x36`), and injected messages do reach backboardd (its AttentionAwareness counter +//! ticks per event), but neither iPhone nor iPad runtimes ever deliver them to apps: iPhones only +//! route pointer devices under AssistiveTouch, and even the iPad runtime (native pointer support) +//! dropped every variant of the matrix (both targets × wrapped/direct × with/without a preceding +//! `IndigoHIDMessageForTrackpadMoveEvent` hover). Simulator.app itself never sends scroll messages +//! for iPhone/iPad — its `simDigitizerInputView:scrollEvent:` delegate only forwards to Digital +//! Crown/Dial (watch) and returns otherwise. Scrolling on iOS is a touch gesture; the client's +//! wheel→synthetic-drag translation is the platform-canonical path, not a stopgap. use std::ffi::c_void; use std::ptr; From b575f2cf89e9e306ca095d8a3684c25c39fc7d7f Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Fri, 24 Jul 2026 05:19:38 +0800 Subject: [PATCH 26/26] fix(sim,engine,ui,schema): resolve iOS Simulator panel review findings (wire 51) - streamStop no longer reacquires a released device's claim - retarget a udid's frame fan-out when a new session takes it over, and ignore a stale stop from a non-owner - decode the device screen mask from base64 instead of a CSP-blocked data-URL fetch - probe + report an `interactive` host capability so the panel gates the live stream on hosts without SimulatorKit --- .../src/__tests__/sim-mcp-endpoint.test.ts | 4 +- apps/desktop/e2e/simulator-live.mts | 2 +- apps/desktop/e2e/simulator-panel.e2e.mts | 2 +- crates/linkcode-sim/src/capture.rs | 12 +++++- crates/linkcode-sim/src/simctl.rs | 16 ++++++++ .../client/workbench/src/simulator/panel.tsx | 21 ++++++---- .../foundation/schema/src/model/simulator.ts | 3 ++ .../foundation/schema/src/wire/message.ts | 3 +- .../simulator-request-handler.test.ts | 39 +++++++++++++++++-- .../src/__tests__/simulator-service.test.ts | 32 ++++++++++++++- packages/host/engine/src/simulator/backend.ts | 3 ++ .../engine/src/simulator/request-handler.ts | 30 ++++++++++---- packages/host/engine/src/simulator/service.ts | 12 +++++- .../host/sim/src/__tests__/client.test.ts | 2 + packages/host/sim/src/schema.ts | 3 ++ packages/presentation/i18n/src/locales/en.ts | 2 + .../presentation/i18n/src/locales/zh-cn.ts | 1 + .../ui/src/shell/simulator/frame-decoder.ts | 8 ++-- .../src/shell/simulator/simulator-screen.tsx | 23 ++++++----- 19 files changed, 176 insertions(+), 42 deletions(-) diff --git a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts index cfa19a75..98c8824e 100644 --- a/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts +++ b/apps/daemon/src/__tests__/sim-mcp-endpoint.test.ts @@ -14,7 +14,9 @@ const S2 = 'session-2' as SessionId; function fakeBackend(): SimulatorBackend { return { - probe: vi.fn(() => Promise.resolve({ simctlPath: '/usr/bin/simctl', developerDir: '/dev' })), + probe: vi.fn(() => + Promise.resolve({ simctlPath: '/usr/bin/simctl', developerDir: '/dev', interactive: true }), + ), list: vi.fn(() => Promise.resolve([ { diff --git a/apps/desktop/e2e/simulator-live.mts b/apps/desktop/e2e/simulator-live.mts index db2f5449..accdf45d 100644 --- a/apps/desktop/e2e/simulator-live.mts +++ b/apps/desktop/e2e/simulator-live.mts @@ -25,7 +25,7 @@ const simSidecar = join(repoRoot, 'target', 'release', 'linkcode-sim'); const electronBinary = require('electron') as unknown as string; const PORT = 43000 + (process.pid % 1000); -const WIRE_VERSION = 49; +const WIRE_VERSION = 51; async function waitForDaemon(): Promise { const deadline = Date.now() + 30000; diff --git a/apps/desktop/e2e/simulator-panel.e2e.mts b/apps/desktop/e2e/simulator-panel.e2e.mts index 0a7ff310..96900bb1 100644 --- a/apps/desktop/e2e/simulator-panel.e2e.mts +++ b/apps/desktop/e2e/simulator-panel.e2e.mts @@ -29,7 +29,7 @@ const PORT = 43000 + (process.pid % 1000); /** Must match `WIRE_PROTOCOL_VERSION` (node can't load the raw-TS schema barrel); a mismatch is * silently discarded by the daemon, surfacing here as the session.start timeout. */ -const WIRE_VERSION = 49; +const WIRE_VERSION = 51; function fail(message: string): never { console.error(`FAIL: ${message}`); diff --git a/crates/linkcode-sim/src/capture.rs b/crates/linkcode-sim/src/capture.rs index 35d3ba38..7827c2d2 100644 --- a/crates/linkcode-sim/src/capture.rs +++ b/crates/linkcode-sim/src/capture.rs @@ -218,7 +218,17 @@ impl CaptureStream { let stopped = Arc::clone(&stopped); let dead = Arc::clone(&dead); let worker_pid = Arc::clone(&worker_pid); - move || supervise(&udid, params, &latest, &encoded, &stopped, &dead, &worker_pid) + move || { + supervise( + &udid, + params, + &latest, + &encoded, + &stopped, + &dead, + &worker_pid, + ) + } }); CaptureStream { latest, diff --git a/crates/linkcode-sim/src/simctl.rs b/crates/linkcode-sim/src/simctl.rs index f682e241..2b933cec 100644 --- a/crates/linkcode-sim/src/simctl.rs +++ b/crates/linkcode-sim/src/simctl.rs @@ -60,9 +60,25 @@ pub fn probe() -> Result { Ok(json!({ "simctlPath": simctl_path.trim(), "developerDir": developer_dir.trim(), + // Interactive framebuffer/HID needs SimulatorKit; simctl alone can't stream a screen or + // inject touches. Report it so clients gate the live panel instead of mounting a stream that + // only ever fails. The private layer resolves exactly when interactive drive is possible. + "interactive": interactive_supported(), })) } +/// Whether this host can drive simulators interactively (framebuffer stream + HID), which requires +/// the private SimulatorKit layer beyond the public `simctl` CLI. Non-macOS builds never can. +#[cfg(target_os = "macos")] +fn interactive_supported() -> bool { + crate::private::interactive_available() +} + +#[cfg(not(target_os = "macos"))] +fn interactive_supported() -> bool { + false +} + /// List available devices with their runtime names. pub fn list() -> Result { let raw = run_ok( diff --git a/packages/client/workbench/src/simulator/panel.tsx b/packages/client/workbench/src/simulator/panel.tsx index 116c7f3d..f6641216 100644 --- a/packages/client/workbench/src/simulator/panel.tsx +++ b/packages/client/workbench/src/simulator/panel.tsx @@ -15,7 +15,7 @@ import { SelectTrigger, SelectValue, } from 'coss-ui/components/select'; -import { useEffect as useAbortableEffect } from 'foxact/use-abortable-effect'; +import { useEffect } from 'foxact/use-abortable-effect'; import { noop } from 'foxts/noop'; import { useCallback, useRef, useState, useSyncExternalStore } from 'react'; import { useTranslations } from 'use-intl'; @@ -35,13 +35,13 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): const [status, setStatus] = useState(null); const [devices, setDevices] = useState(null); const [selectedUdid, setSelectedUdid] = useState(null); - /** Screen-outline masks by udid; `null` = the host has none (fall back to generic rounding). */ + /** Screen-outline masks by udid as base64 PNGs; `null` = the host has none (generic rounding). */ const [masks, setMasks] = useState>>({}); const [busy, setBusy] = useState(false); const busyTimerRef = useRef | undefined>(undefined); const leaseRef = useRef(null); - useAbortableEffect( + useEffect( (signal) => { void client .simulatorStatus() @@ -71,18 +71,22 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): const device = pickDevice(devices, selectedUdid); const udid = device?.udid ?? null; const booted = device?.state === 'Booted'; - const canStream = sessionId !== null && udid !== null && booted; + // Optimistic until the probe resolves: assume interactive so a capable host streams immediately. + // A host with simctl but no SimulatorKit reports `interactive: false`; the live stream would only + // fail there, so we gate it out and show a hint instead of an unrecoverable Retry loop. + const interactive = status?.interactive ?? true; + const canStream = sessionId !== null && udid !== null && booted && interactive; // Fetch bookkeeping lives in a ref (not `masks`) so the effect never loops on its own writes; // the cache write itself is deliberately not abort-gated — a udid switch mid-fetch must still // land the result for the next switch back. const maskFetchedRef = useRef(new Set()); - useAbortableEffect(() => { + useEffect(() => { if (udid === null || maskFetchedRef.current.has(udid)) return; maskFetchedRef.current.add(udid); void client .simulatorScreenMask(udid) - .then((data) => setMasks((prev) => ({ ...prev, [udid]: `data:image/png;base64,${data}` }))) + .then((data) => setMasks((prev) => ({ ...prev, [udid]: data }))) .catch(() => setMasks((prev) => ({ ...prev, [udid]: null }))); }, [client, udid]); @@ -224,6 +228,9 @@ export function SimulatorPanel({ sessionId }: { sessionId: SessionId | null }): {device !== null && booted && sessionId === null && ( {t('simulatorNoSession')} )} + {device !== null && booted && sessionId !== null && !interactive && ( + {t('simulatorNonInteractive')} + )} {canStream && ( {t('simulatorConnecting')} } diff --git a/packages/foundation/schema/src/model/simulator.ts b/packages/foundation/schema/src/model/simulator.ts index 50a3f9aa..bb11102b 100644 --- a/packages/foundation/schema/src/model/simulator.ts +++ b/packages/foundation/schema/src/model/simulator.ts @@ -25,6 +25,9 @@ export const SimulatorStatusSchema = z.object({ /** Where simctl lives; present when available. */ simctlPath: z.string().optional(), developerDir: z.string().optional(), + /** Whether the host can stream a framebuffer and inject HID input (private SimulatorKit), not + * just run simctl; present when available. Clients gate the live co-driving panel on it. */ + interactive: z.boolean().optional(), /** Why unavailable (e.g. Xcode missing); present when not available. */ reason: z.string().optional(), }); diff --git a/packages/foundation/schema/src/wire/message.ts b/packages/foundation/schema/src/wire/message.ts index ba20693a..bf748cdc 100644 --- a/packages/foundation/schema/src/wire/message.ts +++ b/packages/foundation/schema/src/wire/message.ts @@ -20,7 +20,8 @@ import { WirePayloadSchema } from './payload'; // 48 adds the H.264 stream codec plumbing (CODE-397). // 49 adds streamed touch, wheel scroll, and HID keyboard input (CODE-397). // 50 adds two-finger pinch and IME pasteboard input (CODE-397). -export const WIRE_PROTOCOL_VERSION = 50 as const; +// 51 adds the simulator interactive-capability flag to the status wire (CODE-397). +export const WIRE_PROTOCOL_VERSION = 51 as const; /** Complete wire message: version + unique id + timestamp + payload. */ export const WireMessageSchema = z.object({ diff --git a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts index 3f5642b0..50d3f46d 100644 --- a/packages/host/engine/src/__tests__/simulator-request-handler.test.ts +++ b/packages/host/engine/src/__tests__/simulator-request-handler.test.ts @@ -20,7 +20,9 @@ function fakeBackend() { }, ]; return { - probe: vi.fn(() => Promise.resolve({ simctlPath: '/usr/bin/simctl', developerDir: '/dev' })), + probe: vi.fn(() => + Promise.resolve({ simctlPath: '/usr/bin/simctl', developerDir: '/dev', interactive: true }), + ), list: vi.fn(() => Promise.resolve(devices)), boot: vi.fn(() => { devices[0] = { ...devices[0], state: 'Booted' }; @@ -181,11 +183,12 @@ describe('simulator wire requests', () => { const backend = fakeBackend(); const h = harness(backend); await h.engine.start(); + const s1 = await h.startSession('s1'); await h.inject({ kind: 'simulator.stream.start', clientReqId: 'ss', - sessionId: 'session-1' as never, + sessionId: s1, udid: 'U-1', scale: 0.5, }); @@ -201,7 +204,7 @@ describe('simulator wire requests', () => { listener?.({ codec: 'jpeg', data: new Uint8Array([0xff, 0xd8, 0x01]), key: true }); expect(h.sent.find((p) => p.kind === 'simulator.stream.frame')).toMatchObject({ kind: 'simulator.stream.frame', - sessionId: 'session-1', + sessionId: s1, udid: 'U-1', codec: 'jpeg', key: true, @@ -211,11 +214,39 @@ describe('simulator wire requests', () => { await h.inject({ kind: 'simulator.stream.stop', clientReqId: 'sx', - sessionId: 'session-1' as never, + sessionId: s1, udid: 'U-1', }); expect(h.reply('sx')).toMatchObject({ kind: 'request.succeeded' }); expect(backend.streamStop).toHaveBeenCalledWith('U-1'); await h.engine.stop(); }); + + it('a stream stop from a session that does not own the device is a no-op', async () => { + const backend = fakeBackend(); + const h = harness(backend); + await h.engine.start(); + const s1 = await h.startSession('s1'); + + await h.inject({ + kind: 'simulator.stream.start', + clientReqId: 'ss1', + sessionId: s1, + udid: 'U-1', + }); + expect(h.reply('ss1')).toMatchObject({ kind: 'simulator.stream.started' }); + + // A deferred stop from a session that never held U-1 (a stale panel firing after handoff) must + // not tear down the current owner's backend stream or fan-out — it just succeeds silently. + const s2 = await h.startSession('s2'); + await h.inject({ + kind: 'simulator.stream.stop', + clientReqId: 'sx', + sessionId: s2, + udid: 'U-1', + }); + expect(h.reply('sx')).toMatchObject({ kind: 'request.succeeded' }); + expect(backend.streamStop).not.toHaveBeenCalled(); + await h.engine.stop(); + }); }); diff --git a/packages/host/engine/src/__tests__/simulator-service.test.ts b/packages/host/engine/src/__tests__/simulator-service.test.ts index 99fe86c6..a678454c 100644 --- a/packages/host/engine/src/__tests__/simulator-service.test.ts +++ b/packages/host/engine/src/__tests__/simulator-service.test.ts @@ -20,7 +20,9 @@ function device(udid: string, state: string): SimulatorDeviceInfo { function fakeBackend(devices: SimulatorDeviceInfo[]) { return { - probe: vi.fn(() => Promise.resolve({ simctlPath: '/usr/bin/simctl', developerDir: '/dev' })), + probe: vi.fn(() => + Promise.resolve({ simctlPath: '/usr/bin/simctl', developerDir: '/dev', interactive: true }), + ), list: vi.fn(() => Promise.resolve(devices)), boot: vi.fn(asyncNoop), shutdownDevice: vi.fn(asyncNoop), @@ -117,6 +119,34 @@ describe('SimulatorService', () => { expect(service.ownerOf('A')).toBe(S1); }); + it('does not reacquire a released user-booted device when its stream is stopped', async () => { + const backend = fakeBackend([device('A', 'Booted')]); + const service = new SimulatorService(backend, { idleReclaimMs: 1000 }); + + await service.boot(S1, 'A'); + service.releaseSession(S1); // user-booted → claim dropped immediately + expect(service.ownerOf('A')).toBeUndefined(); + + // A deferred stream stop for the now-released device must not pin it back to the dead session. + await service.streamStop(S1, 'A'); + expect(service.ownerOf('A')).toBeUndefined(); + }); + + it('does not cancel a service-booted reclaim when a released stream is stopped', async () => { + const backend = fakeBackend([device('A', 'Shutdown')]); + const service = new SimulatorService(backend, { idleReclaimMs: 1000 }); + + await service.boot(S1, 'A'); // service-booted (was Shutdown), so release arms a reclaim + service.releaseSession(S1); + + // A deferred stop must not refresh the claim — that would disarm the reclaim and strand the device. + await service.streamStop(S1, 'A'); + + await vi.advanceTimersByTimeAsync(1000); + expect(backend.shutdownDevice).toHaveBeenCalledWith('A'); + expect(service.ownerOf('A')).toBeUndefined(); + }); + it('never shuts down a device the user booted', async () => { const backend = fakeBackend([device('A', 'Booted')]); const service = new SimulatorService(backend, { idleReclaimMs: 1000 }); diff --git a/packages/host/engine/src/simulator/backend.ts b/packages/host/engine/src/simulator/backend.ts index 7e6baaa0..ab4d71c0 100644 --- a/packages/host/engine/src/simulator/backend.ts +++ b/packages/host/engine/src/simulator/backend.ts @@ -23,6 +23,9 @@ export interface SimulatorDeviceInfo { export interface SimulatorProbe { simctlPath: string; developerDir: string; + /** Whether the host can stream a framebuffer and inject HID input (private SimulatorKit layer), + * not just run public simctl commands. */ + interactive: boolean; } export type SimulatorImageFormat = 'jpeg' | 'png'; diff --git a/packages/host/engine/src/simulator/request-handler.ts b/packages/host/engine/src/simulator/request-handler.ts index 71e8ae30..95de8ae9 100644 --- a/packages/host/engine/src/simulator/request-handler.ts +++ b/packages/host/engine/src/simulator/request-handler.ts @@ -41,8 +41,11 @@ type SimulatorRequest = Extract< * watcher, so its own commands are the only change source it can observe). */ export class SimulatorRequestHandler { - /** Active framebuffer fan-out subscriptions, keyed by udid; the value unsubscribes it. */ - private readonly frameSubs = new Map void>(); + /** Active framebuffer fan-out subscriptions, keyed by udid. The owning `sessionId` is tracked so a + * later start by a different session replaces the fan-out: a release or shutdown outside the wire + * stop path leaves the entry, and a stale callback would otherwise route new frames to the dead + * session (the current panel then receives none). */ + private readonly frameSubs = new Map void }>(); constructor( private readonly simulators: SimulatorService | undefined, @@ -261,8 +264,14 @@ export class SimulatorRequestHandler { case 'simulator.stream.stop': return this.withSimulators(payload.clientReqId, (simulators) => simulatorOperation('simulator.stream.stop', 'Failed to stop stream', async () => { - this.unsubscribeFrames(payload.udid); - await simulators.streamStop(payload.sessionId, payload.udid); + // A deferred stop can arrive after this session released the device — possibly after + // another session took it over. Only tear down the fan-out and backend stream while this + // session still owns the device; otherwise it is a no-op that leaves the new owner's + // stream and subscription intact. + if (simulators.ownerOf(payload.udid) === payload.sessionId) { + this.unsubscribeFrames(payload.udid); + await simulators.streamStop(payload.sessionId, payload.udid); + } this.responder.sendSuccess(payload.clientReqId); }), ); @@ -272,9 +281,14 @@ export class SimulatorRequestHandler { } /** Fan the device's frames out to the transport as session-scoped `simulator.stream.frame`s. - * Idempotent: a second `streamStart` for a device already fanning out keeps the one subscription. */ + * The same session re-starting keeps its one subscription (idempotent); a different session + * taking over the device replaces the fan-out so frames follow the new owner. */ private subscribeFrames(simulators: SimulatorService, sessionId: SessionId, udid: string): void { - if (this.frameSubs.has(udid)) return; + const existing = this.frameSubs.get(udid); + if (existing) { + if (existing.sessionId === sessionId) return; + existing.unsubscribe(); + } const unsubscribe = simulators.onFrame(udid, (frame) => { this.transport.send( createWireMessage({ @@ -287,11 +301,11 @@ export class SimulatorRequestHandler { }), ); }); - this.frameSubs.set(udid, unsubscribe); + this.frameSubs.set(udid, { sessionId, unsubscribe }); } private unsubscribeFrames(udid: string): void { - this.frameSubs.get(udid)?.(); + this.frameSubs.get(udid)?.unsubscribe(); this.frameSubs.delete(udid); } diff --git a/packages/host/engine/src/simulator/service.ts b/packages/host/engine/src/simulator/service.ts index 05a2bd94..ec389f4d 100644 --- a/packages/host/engine/src/simulator/service.ts +++ b/packages/host/engine/src/simulator/service.ts @@ -20,6 +20,9 @@ export interface SimulatorHostStatus { available: boolean; simctlPath?: string; developerDir?: string; + /** Whether the host can stream a framebuffer and inject HID input, not just run simctl; present + * only when available. Clients gate the live panel on it. */ + interactive?: boolean; reason?: string; } @@ -232,9 +235,14 @@ export class SimulatorService { return this.backend.streamStart(udid, options); } - /** Stop a device's framebuffer stream. Requires the session to hold the device. */ + /** Stop a device's framebuffer stream. Unlike the interactive ops this does NOT claim: the panel + * fires stops opportunistically, so a deferred stop can arrive after the owning session already + * released the device. Reacquiring here would recreate a claim for a released user-booted device + * (pinning it to a dead session) or disarm a service-booted device's idle reclaim — either way the + * device stays stuck until the daemon restarts. Only the current owner stops the stream; for a + * stale session it is a no-op that never touches another session's claim. */ async streamStop(sessionId: SessionId, udid: string): Promise { - this.claim(sessionId, udid); + if (this.claims.get(udid)?.sessionId !== sessionId) return; return this.backend.streamStop(udid); } diff --git a/packages/host/sim/src/__tests__/client.test.ts b/packages/host/sim/src/__tests__/client.test.ts index 07677e4a..dbe73d80 100644 --- a/packages/host/sim/src/__tests__/client.test.ts +++ b/packages/host/sim/src/__tests__/client.test.ts @@ -86,6 +86,8 @@ describe('SimSidecarClient', () => { await expect(probing).resolves.toEqual({ simctlPath: '/usr/bin/simctl', developerDir: '/dev/dir', + // Omitted by this sidecar response → schema default; an older sidecar reads as non-interactive. + interactive: false, }); }); diff --git a/packages/host/sim/src/schema.ts b/packages/host/sim/src/schema.ts index d201a875..9638e1b1 100644 --- a/packages/host/sim/src/schema.ts +++ b/packages/host/sim/src/schema.ts @@ -38,6 +38,9 @@ export const SimListResultSchema = z.object({ devices: z.array(SimDeviceSchema) export const SimProbeSchema = z.object({ simctlPath: z.string(), developerDir: z.string(), + /** Whether the private SimulatorKit layer (framebuffer stream + HID) is reachable; simctl alone + * is not enough to co-drive a device. Defaulted so an older sidecar reads as non-interactive. */ + interactive: z.boolean().default(false), }); export type SimProbe = z.infer; diff --git a/packages/presentation/i18n/src/locales/en.ts b/packages/presentation/i18n/src/locales/en.ts index 029d6db4..484591f6 100644 --- a/packages/presentation/i18n/src/locales/en.ts +++ b/packages/presentation/i18n/src/locales/en.ts @@ -376,6 +376,8 @@ export const en = { simulatorSelectDevice: 'Select a device', simulatorNoDevices: 'No simulator devices found', simulatorUnavailable: 'iOS Simulator is not available on this host', + simulatorNonInteractive: + "This host can list simulators but can't stream their screen (SimulatorKit unavailable)", simulatorNoSession: 'Select a thread to drive the simulator', simulatorBoot: 'Boot', simulatorBooting: 'Booting device…', diff --git a/packages/presentation/i18n/src/locales/zh-cn.ts b/packages/presentation/i18n/src/locales/zh-cn.ts index c90a412d..bbb7b3fa 100644 --- a/packages/presentation/i18n/src/locales/zh-cn.ts +++ b/packages/presentation/i18n/src/locales/zh-cn.ts @@ -366,6 +366,7 @@ export const zhCN = { simulatorSelectDevice: '选择设备', simulatorNoDevices: '未发现模拟器设备', simulatorUnavailable: '此主机不支持 iOS 模拟器', + simulatorNonInteractive: '此主机能列出模拟器，但缺少 SimulatorKit，无法实时串流画面', simulatorNoSession: '选择一个线程后即可操控模拟器', simulatorBoot: '启动', simulatorBooting: '正在启动设备…', diff --git a/packages/presentation/ui/src/shell/simulator/frame-decoder.ts b/packages/presentation/ui/src/shell/simulator/frame-decoder.ts index 8180aede..5d0bbece 100644 --- a/packages/presentation/ui/src/shell/simulator/frame-decoder.ts +++ b/packages/presentation/ui/src/shell/simulator/frame-decoder.ts @@ -104,9 +104,11 @@ export class SimulatorFrameDecoder { } } -/** A screen mask fetched from a URL, decoded to a bitmap. */ -export function decodeMask(response: Response): Promise { - return response.blob().then((blob) => createImageBitmap(blob)); +/** Decode a base64-encoded PNG screen mask to a bitmap. Takes the raw base64, not a `data:` URL: + * the desktop renderer's CSP forbids `data:` in `connect-src`, so `fetch()`-ing a data URL is + * silently blocked — building the Blob from the decoded bytes avoids the network path entirely. */ +export function decodeMask(pngBase64: string): Promise { + return createImageBitmap(new Blob([base64Bytes(pngBase64)], { type: 'image/png' })); } function base64Bytes(base64: string): Uint8Array { diff --git a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx index 4387b30f..543d970f 100644 --- a/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx +++ b/packages/presentation/ui/src/shell/simulator/simulator-screen.tsx @@ -1,4 +1,4 @@ -import { useEffect as useAbortableEffect } from 'foxact/use-abortable-effect'; +import { useEffect } from 'foxact/use-abortable-effect'; import { clamp } from 'foxts/clamp'; import { noop } from 'foxts/noop'; import { useRef, useState } from 'react'; @@ -58,10 +58,10 @@ export interface SimulatorScreenProps { onKey?: (press: SimulatorKeyPress) => void; /** Text committed by the OS IME (composition end / non-ASCII input): pasted onto the device. */ onText?: (text: string) => void; - /** The device's real screen-outline mask (image URL, framebuffer-sized): clips the stream to - * the exact screen shape, and its grown outline keeps the chassis band even. Absent → a generic - * rounding. */ - maskUrl?: string | null; + /** The device's real screen-outline mask as a base64-encoded PNG (framebuffer-sized): clips the + * stream to the exact screen shape, and its grown outline keeps the chassis band even. Absent → a + * generic rounding. Base64, not a `data:` URL — the desktop CSP blocks `fetch`-ing data URLs. */ + maskPng?: string | null; /** Shown centered until the first frame arrives. */ placeholder?: React.ReactNode; className?: string; @@ -81,7 +81,7 @@ export function SimulatorScreen({ onPinch, onKey, onText, - maskUrl, + maskPng, placeholder, className, }: SimulatorScreenProps): React.ReactNode { @@ -104,7 +104,7 @@ export function SimulatorScreen({ } | null>(null); const [layout, setLayout] = useState(null); - useAbortableEffect( + useEffect( (signal) => { let rafId: number | null = null; // Rebuilt only when the framebuffer size or mask presence changes — i.e. essentially once. @@ -156,11 +156,10 @@ export function SimulatorScreen({ [subscribeFrames], ); - useAbortableEffect( + useEffect( (signal) => { - if (maskUrl == null) return; - void fetch(maskUrl, { signal }) - .then(decodeMask) + if (maskPng == null) return; + void decodeMask(maskPng) .then((bitmap) => { if (signal.aborted) { bitmap.close(); @@ -177,7 +176,7 @@ export function SimulatorScreen({ maskRef.current = null; }; }, - [maskUrl], + [maskPng], ); const emitGesture = (phase: SimulatorScreenTouchPhase, point: SimulatorScreenPoint): void => {