From 00815b349c17ee3c47478702225c20ff3d43d8f0 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 21:10:29 +0800 Subject: [PATCH 1/6] feat(sim): P1 private-API framebuffer streaming + HID injection, crash-isolated --- Cargo.lock | 104 ++- crates/linkcode-sim/Cargo.toml | 6 + crates/linkcode-sim/NOTICE | 15 + crates/linkcode-sim/PROTOCOL.md | 27 + crates/linkcode-sim/src/capture.rs | 210 ++++++ crates/linkcode-sim/src/interactive.rs | 275 ++++++++ crates/linkcode-sim/src/main.rs | 93 ++- crates/linkcode-sim/src/private/debug.rs | 24 + crates/linkcode-sim/src/private/device.rs | 117 ++++ crates/linkcode-sim/src/private/framework.rs | 168 +++++ crates/linkcode-sim/src/private/input.rs | 387 +++++++++++ crates/linkcode-sim/src/private/mod.rs | 21 + crates/linkcode-sim/src/private/screen.rs | 663 +++++++++++++++++++ crates/linkcode-sim/src/proto.rs | 18 + crates/linkcode-sim/src/rpc.rs | 39 ++ 15 files changed, 2163 insertions(+), 4 deletions(-) create mode 100644 crates/linkcode-sim/NOTICE create mode 100644 crates/linkcode-sim/src/capture.rs create mode 100644 crates/linkcode-sim/src/interactive.rs create mode 100644 crates/linkcode-sim/src/private/debug.rs create mode 100644 crates/linkcode-sim/src/private/device.rs create mode 100644 crates/linkcode-sim/src/private/framework.rs create mode 100644 crates/linkcode-sim/src/private/input.rs create mode 100644 crates/linkcode-sim/src/private/mod.rs create mode 100644 crates/linkcode-sim/src/private/screen.rs diff --git a/Cargo.lock b/Cargo.lock index a5df9593..0de83b79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,6 +20,25 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] +name = "cc" +version = "1.2.67" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17dd265a7d0f31ef544e1b20e03add05d3b45b491b633b10d67145d2acc1a38" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -32,6 +51,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.13.0", + "block2", + "libc", + "objc2", +] + [[package]] name = "downcast-rs" version = "1.2.1" @@ -49,6 +80,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + [[package]] name = "itoa" version = "1.0.18" @@ -63,9 +100,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.186" +version = "0.2.189" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "linkcode-pty" @@ -82,6 +119,12 @@ dependencies = [ name = "linkcode-sim" version = "0.1.0" dependencies = [ + "block2", + "dispatch2", + "libc", + "objc2", + "objc2-core-foundation", + "objc2-foundation", "serde", "serde_json", ] @@ -110,6 +153,57 @@ dependencies = [ "libc", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", + "objc2-exception-helper", +] + +[[package]] +name = "objc2-core-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" +dependencies = [ + "bitflags 2.13.0", + "block2", + "dispatch2", + "libc", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-exception-helper" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7a1c5fbb72d7735b076bb47b578523aedc40f3c439bea6dfd595c089d79d98a" +dependencies = [ + "cc", +] + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.13.0", + "block2", + "libc", + "objc2", + "objc2-core-foundation", +] + [[package]] name = "portable-pty" version = "0.9.0" @@ -219,6 +313,12 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "syn" version = "2.0.118" diff --git a/crates/linkcode-sim/Cargo.toml b/crates/linkcode-sim/Cargo.toml index a5a7f6d8..bbe08c15 100644 --- a/crates/linkcode-sim/Cargo.toml +++ b/crates/linkcode-sim/Cargo.toml @@ -14,5 +14,11 @@ name = "linkcode-sim" path = "src/main.rs" [dependencies] +block2 = "0.6.2" +dispatch2 = "0.3.1" +libc = "0.2.189" +objc2 = { version = "0.6.4", features = ["exception"] } +objc2-core-foundation = "0.3.2" +objc2-foundation = "0.3.2" serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.150" diff --git a/crates/linkcode-sim/NOTICE b/crates/linkcode-sim/NOTICE new file mode 100644 index 00000000..e652a63d --- /dev/null +++ b/crates/linkcode-sim/NOTICE @@ -0,0 +1,15 @@ +linkcode-sim + +The private-framework interactive layer (framebuffer capture and HID injection under +`src/private/`) ports recipes from baguette, an iOS Simulator control tool: + + baguette — https://github.com/tddworks/baguette + Copyright (c) tddworks + Licensed under the Apache License, Version 2.0 + +The undocumented CoreSimulator / SimulatorKit call sequences reproduced here — the IOHIDEvent +digitizer tap recipe for iOS 26, the framebuffer descriptor enumeration and screen-callback +registration, and the SimulatorKit framework path resolution — were derived from baguette's +implementation. They are reimplemented in Rust; no baguette source is included verbatim. + +A copy of the Apache License 2.0 is available at https://www.apache.org/licenses/LICENSE-2.0. diff --git a/crates/linkcode-sim/PROTOCOL.md b/crates/linkcode-sim/PROTOCOL.md index 8cc63412..4ac93ce7 100644 --- a/crates/linkcode-sim/PROTOCOL.md +++ b/crates/linkcode-sim/PROTOCOL.md @@ -28,6 +28,7 @@ Sidecar to daemon: | ---: | --- | --- | | `0x81` | `RESULT` | JSON [`Result`](#result) | | `0x82` | `SCREENSHOT` | binary [`Screenshot body`](#screenshot-body) | +| `0x83` | `STREAM_FRAME` | binary [`Stream frame body`](#stream-frame-body) (unsolicited, while a stream runs) | Unknown frame types are ignored. A malformed `REQUEST` fails only that request — the sidecar replies with an `invalidRequest` `RESULT` for its `requestId` and keeps serving. If the `requestId` itself cannot be recovered, the sidecar logs to stderr and the daemon's pending request is reclaimed by its own timeout. @@ -58,6 +59,22 @@ Unknown frame types are ignored. A malformed `REQUEST` fails only that request Per-op deadlines are enforced sidecar-side (`boot` 180s, `install` 120s, `screenshot` 30s, others 60s); a child that outlives its deadline is killed and reported as `timeout`. +`probe` additionally reports `interactive: bool` — whether this host can drive simulators through the private-API path below (macOS with SimulatorKit). The daemon gates the interactive ops on it. + +### Ops (P1 — private API: HID injection + framebuffer streaming, macOS only) + +Off macOS, or when SimulatorKit is unavailable, every P1 op fails with `xcodeMissing`. + +| `type` | Params | Result | +| --- | --- | --- | +| `tap` | `udid`, `x`, `y` (normalised 0..1) | `{}` | +| `swipe` | `udid`, `x0`, `y0`, `x1`, `y1`, `durationMs?` | `{}` | +| `button` | `udid`, `button` (`home`/`lock`) | `{}` | +| `streamStart` | `udid`, `fps?` (10), `quality?` (0.6) | `{ streaming, fps }` — JPEG frames then arrive on `STREAM_FRAME`s | +| `streamStop` | `udid` | `{}` | + +Input (`tap`/`swipe`/`button`) runs in the sidecar's main process via a per-udid warmed HID client and is stable. Framebuffer streaming runs in a **crash-isolated worker subprocess**: the sidecar spawns it, reads its frames, and respawns it on the intermittent hard crashes of the private framebuffer path. If the worker crash-loops and gives up, the stream degrades to `simctl io screenshot` frames — slower, but frames never stop and the sidecar never crashes. + ## Result Success: @@ -92,6 +109,16 @@ A successful `screenshot` responds with raw image bytes instead of JSON, so capt A failed `screenshot` responds with a normal `RESULT` error. Exactly one of the two arrives per request. +## Stream frame body + +While a `streamStart` stream runs, the sidecar pushes unsolicited `STREAM_FRAME`s (raw JPEG bytes, no base64): + +```text +[u16 little-endian udid_length][udid UTF-8 bytes][image bytes] +``` + +Frames stop after `streamStop` (or when the daemon closes the pipe). The daemon routes them by `udid`. + ## Lifecycle expectations - One sidecar process serves many concurrent requests; the daemon lazily starts it on first use. diff --git a/crates/linkcode-sim/src/capture.rs b/crates/linkcode-sim/src/capture.rs new file mode 100644 index 00000000..83c08014 --- /dev/null +++ b/crates/linkcode-sim/src/capture.rs @@ -0,0 +1,210 @@ +//! Crash-isolated framebuffer streaming. +//! +//! The private-framework framebuffer path (see [`crate::private::screen`]) intermittently aborts +//! hard inside CoreSimulator's XPC-proxy machinery ("Attempt to use unknown class") — a +//! not-in-process-catchable `SIGABRT`. Rather than let that take down the sidecar (which also serves +//! the simctl lifecycle RPCs), the capture runs in a disposable **worker subprocess**: the sidecar +//! spawns `linkcode-sim capture-worker `, reads length-prefixed JPEG frames +//! from its stdout, and respawns it with backoff if it dies. Input injection and lifecycle RPCs stay +//! in the parent and are never affected by a capture crash. +//! +//! This is the standard isolation pattern for fragile native code (GPU/plugin sandboxes): contain +//! the crash, supervise, recover. + +use std::io::{self, Read, Write}; +use std::process::{Child, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::{Duration, Instant}; + +/// One framebuffer JPEG frame. +pub type Frame = Arc>; + +/// Largest accepted worker frame (a JPEG at simulator resolution is well under this). +const MAX_WORKER_FRAME: usize = 32 * 1024 * 1024; +/// Backoff between worker respawns after a crash. +const RESPAWN_BACKOFF: Duration = Duration::from_millis(500); +/// A worker that lived at least this long resets the crash-loop counter. +const HEALTHY_AFTER: Duration = Duration::from_secs(5); +/// Consecutive fast crashes before the stream gives up and reports unavailable. +const MAX_FAST_CRASHES: u32 = 6; + +/// A supervised framebuffer stream for one device. Holds the latest delivered frame; the worker +/// subprocess is spawned, read, and respawned by a background manager thread. +pub struct CaptureStream { + latest: Arc>>, + stopped: Arc, + /// Set true once the manager gives up after a crash loop, so callers can degrade. + dead: Arc, + manager: Option>, +} + +impl CaptureStream { + /// Start streaming device `udid` at `fps` and JPEG `quality` by supervising a capture worker. + pub fn start(udid: String, quality: f64, fps: u32) -> CaptureStream { + let latest = Arc::new(Mutex::new(None)); + let stopped = Arc::new(AtomicBool::new(false)); + let dead = Arc::new(AtomicBool::new(false)); + let manager = thread::spawn({ + let latest = Arc::clone(&latest); + let stopped = Arc::clone(&stopped); + let dead = Arc::clone(&dead); + move || supervise(&udid, quality, fps, &latest, &stopped, &dead) + }); + CaptureStream { + latest, + stopped, + dead, + manager: Some(manager), + } + } + + /// The most recently delivered frame, or `None` before the first frame (or once dead). + pub fn latest(&self) -> Option { + self.latest + .lock() + .expect("capture stream mutex poisoned") + .clone() + } + + /// Whether the worker crash-looped and the stream gave up. + pub fn is_dead(&self) -> bool { + self.dead.load(Ordering::Relaxed) + } +} + +impl Drop for CaptureStream { + fn drop(&mut self) { + self.stopped.store(true, Ordering::Relaxed); + if let Some(manager) = self.manager.take() { + let _ = manager.join(); + } + } +} + +/// Manager loop: (re)spawn the worker, pump its frames, back off and retry on crash, give up after +/// too many fast crashes. +fn supervise( + udid: &str, + quality: f64, + fps: u32, + latest: &Arc>>, + stopped: &Arc, + dead: &Arc, +) { + let mut fast_crashes = 0u32; + while !stopped.load(Ordering::Relaxed) { + let started = Instant::now(); + match spawn_worker(udid, quality, fps) { + Ok(mut child) => { + pump_worker(&mut child, latest, stopped); + let _ = child.wait(); + } + Err(err) => { + eprintln!("sim capture: failed to spawn worker: {err}"); + } + } + if stopped.load(Ordering::Relaxed) { + break; + } + // A worker that ran a while then died (device shut down, transient crash) is not a loop. + fast_crashes = if started.elapsed() < HEALTHY_AFTER { + fast_crashes + 1 + } else { + 0 + }; + if fast_crashes >= MAX_FAST_CRASHES { + eprintln!("sim capture: worker keeps crashing; giving up on {udid}"); + dead.store(true, Ordering::Relaxed); + break; + } + thread::sleep(RESPAWN_BACKOFF); + } +} + +fn spawn_worker(udid: &str, quality: f64, fps: u32) -> io::Result { + let exe = std::env::current_exe()?; + Command::new(exe) + .arg("capture-worker") + .arg(udid) + .arg(format!("{quality}")) + .arg(format!("{fps}")) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .spawn() +} + +/// Read length-prefixed JPEG frames from the worker until EOF (worker exit/crash) or stop. +fn pump_worker(child: &mut Child, latest: &Arc>>, stopped: &Arc) { + let Some(stdout) = child.stdout.take() else { + return; + }; + let mut reader = io::BufReader::new(stdout); + let mut header = [0u8; 4]; + while !stopped.load(Ordering::Relaxed) { + if reader.read_exact(&mut header).is_err() { + break; // worker gone + } + let len = u32::from_le_bytes(header) as usize; + if len == 0 || len > MAX_WORKER_FRAME { + break; // corrupt stream + } + let mut frame = vec![0u8; len]; + if reader.read_exact(&mut frame).is_err() { + break; + } + *latest.lock().expect("capture stream mutex poisoned") = Some(Arc::new(frame)); + } +} + +/// The worker entry point (`linkcode-sim capture-worker `): open the device's +/// framebuffer and stream length-prefixed JPEG frames to stdout at `fps`. Exits non-zero on any +/// failure; the parent respawns. A hard `SIGABRT` from the private path also just ends this process. +#[cfg(target_os = "macos")] +pub fn run_worker() -> ! { + use crate::private::{Screen, SimDevice}; + + let mut args = std::env::args().skip(2); + let udid = args.next().unwrap_or_default(); + let quality: f64 = args.next().and_then(|a| a.parse().ok()).unwrap_or(0.6); + let fps: u32 = args + .next() + .and_then(|a| a.parse().ok()) + .unwrap_or(10) + .clamp(1, 60); + + let Some(device) = SimDevice::resolve(&udid) else { + eprintln!("sim capture-worker: device {udid} not found"); + std::process::exit(2); + }; + // Warm the device's CoreSimulator XPC connection before enumerating framebuffer ports: the HID + // client init establishes it and registers the proxy classes, which otherwise race the cold + // connection and abort framebuffer open with "unknown class". + let _warm = crate::private::Input::warm(&device); + let Some(screen) = Screen::open(&device) else { + eprintln!("sim capture-worker: no framebuffer for {udid}"); + std::process::exit(3); + }; + + let interval = Duration::from_millis(1000 / u64::from(fps)); + let mut stdout = io::stdout().lock(); + loop { + let tick = Instant::now(); + if let Some(jpeg) = screen.capture_jpeg(quality) { + let len = u32::try_from(jpeg.len()).unwrap_or(0); + if len == 0 + || stdout.write_all(&len.to_le_bytes()).is_err() + || stdout.write_all(&jpeg).is_err() + || stdout.flush().is_err() + { + break; // parent closed the pipe + } + } + if let Some(rest) = interval.checked_sub(tick.elapsed()) { + thread::sleep(rest); + } + } + std::process::exit(0); +} diff --git a/crates/linkcode-sim/src/interactive.rs b/crates/linkcode-sim/src/interactive.rs new file mode 100644 index 00000000..cb9391e2 --- /dev/null +++ b/crates/linkcode-sim/src/interactive.rs @@ -0,0 +1,275 @@ +//! The interactive (private-API) op handlers: HID injection and framebuffer streaming. +//! +//! Off macOS, and on a macOS host whose Xcode lacks SimulatorKit, every op fails with a stable +//! `unsupported` error so the daemon gates the capability. On macOS the ops resolve a per-udid +//! warmed HID client (`Input`) and drive taps/swipes/buttons, or start/stop a crash-isolated +//! [`CaptureStream`](crate::capture::CaptureStream) whose frames are pushed to the daemon as +//! `STREAM_FRAME`s. + +use std::sync::mpsc::Sender; + +use serde_json::{Value, json}; + +use crate::OutMsg; +use crate::rpc::{ButtonKind, ErrorCode, OpError}; + +fn unsupported() -> OpError { + OpError::new( + ErrorCode::XcodeMissing, + "interactive simulator control is unavailable on this host", + ) +} + +#[cfg(target_os = "macos")] +pub use imp::{available, button, stream_start, stream_stop, swipe, tap}; + +#[cfg(not(target_os = "macos"))] +mod stubs { + use super::*; + + pub fn available() -> bool { + false + } + pub fn tap(_udid: &str, _x: f64, _y: f64) -> Result { + Err(unsupported()) + } + pub fn swipe( + _udid: &str, + _x0: f64, + _y0: f64, + _x1: f64, + _y1: f64, + _duration_ms: u64, + ) -> Result { + Err(unsupported()) + } + pub fn button(_udid: &str, _button: ButtonKind) -> Result { + Err(unsupported()) + } + pub fn stream_start( + _udid: &str, + _fps: u32, + _quality: f64, + _tx: &Sender, + ) -> Result { + Err(unsupported()) + } + pub fn stream_stop(_udid: &str) -> Result { + Err(unsupported()) + } +} + +#[cfg(not(target_os = "macos"))] +pub use stubs::{available, button, stream_start, stream_stop, swipe, tap}; + +#[cfg(target_os = "macos")] +mod imp { + use std::collections::HashMap; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex, OnceLock}; + use std::thread; + use std::time::{Duration, Instant}; + + use super::*; + use crate::capture::CaptureStream; + use crate::private::{self, Button, Input, SimDevice}; + use crate::proto::{STREAM_FRAME, encode_stream_frame}; + + /// Warmed HID clients and running streams, keyed by udid. Warming a client is expensive, so it + /// is cached; a stream is one crash-isolated worker plus a pusher thread. + struct Registry { + inputs: HashMap>, + streams: HashMap, + } + + struct StreamHandle { + // Held so the CaptureStream (and its worker) stay alive until the handle is removed; the + // pusher thread holds the other Arc and stops when `stop` is set. + #[allow(dead_code, reason = "ownership hold that keeps the stream alive")] + stream: Arc, + stop: Arc, + pusher: Option>, + } + + fn registry() -> &'static Mutex { + static REGISTRY: OnceLock> = OnceLock::new(); + REGISTRY.get_or_init(|| { + Mutex::new(Registry { + inputs: HashMap::new(), + streams: HashMap::new(), + }) + }) + } + + pub fn available() -> bool { + private::interactive_available() + } + + /// Resolve (and cache) a warmed HID client for `udid`. + fn input_for(udid: &str) -> Result, OpError> { + let mut reg = registry().lock().expect("interactive registry poisoned"); + if let Some(input) = reg.inputs.get(udid) { + return Ok(Arc::clone(input)); + } + let device = SimDevice::resolve(udid).ok_or_else(|| { + OpError::new(ErrorCode::SimctlFailed, format!("device {udid} not found")) + })?; + let input = Input::warm(&device).ok_or_else(unsupported).map(Arc::new)?; + reg.inputs.insert(udid.to_owned(), Arc::clone(&input)); + Ok(input) + } + + pub fn tap(udid: &str, x: f64, y: f64) -> Result { + if input_for(udid)?.tap(x, y, Duration::from_millis(80)) { + Ok(json!({})) + } else { + Err(OpError::new(ErrorCode::SimctlFailed, "tap failed")) + } + } + + pub fn swipe( + udid: &str, + x0: f64, + y0: f64, + x1: f64, + y1: f64, + duration_ms: u64, + ) -> Result { + let duration = if duration_ms == 0 { + Duration::from_millis(250) + } else { + Duration::from_millis(duration_ms) + }; + let steps = 10u32; + let step = duration / (steps + 2); + if input_for(udid)?.swipe(x0, y0, x1, y1, steps, step) { + Ok(json!({})) + } else { + Err(OpError::new(ErrorCode::SimctlFailed, "swipe failed")) + } + } + + pub fn button(udid: &str, button: ButtonKind) -> Result { + let button = match button { + ButtonKind::Home => Button::Home, + ButtonKind::Lock => Button::Lock, + }; + if input_for(udid)?.button(button, Duration::from_millis(80)) { + Ok(json!({})) + } else { + Err(OpError::new(ErrorCode::SimctlFailed, "button press failed")) + } + } + + pub fn stream_start( + udid: &str, + fps: u32, + quality: f64, + tx: &Sender, + ) -> Result { + if !available() { + return Err(unsupported()); + } + // Warming the HID connection first stabilizes the framebuffer worker's cold open. + let _ = input_for(udid); + let fps = fps.clamp(1, 60); + let mut reg = registry().lock().expect("interactive registry poisoned"); + if reg.streams.contains_key(udid) { + return Ok(json!({ "alreadyStreaming": true })); + } + let stream = Arc::new(CaptureStream::start( + udid.to_owned(), + quality.clamp(0.1, 1.0), + fps, + )); + let stop = Arc::new(AtomicBool::new(false)); + let pusher = thread::spawn({ + let stream = Arc::clone(&stream); + let stop = Arc::clone(&stop); + let tx = tx.clone(); + let udid = udid.to_owned(); + move || push_frames(&udid, fps, &stream, &stop, &tx) + }); + reg.streams.insert( + udid.to_owned(), + StreamHandle { + stream, + stop, + pusher: Some(pusher), + }, + ); + Ok(json!({ "streaming": true, "fps": fps })) + } + + pub fn stream_stop(udid: &str) -> Result { + let handle = registry() + .lock() + .expect("interactive registry poisoned") + .streams + .remove(udid); + if let Some(mut handle) = handle { + handle.stop.store(true, Ordering::Relaxed); + if let Some(pusher) = handle.pusher.take() { + let _ = pusher.join(); + } + } + Ok(json!({})) + } + + /// Push framebuffer frames to the daemon at `fps`. Prefers the fast private capture stream; if + /// its crash-isolated worker gives up (the private API is unusable on this host/state), it + /// degrades to `simctl io screenshot` — slower, but frames never stop and the sidecar never + /// crashes. De-duplicates the private frames by identity so a static screen doesn't flood. + fn push_frames( + udid: &str, + fps: u32, + stream: &CaptureStream, + stop: &AtomicBool, + tx: &Sender, + ) { + let interval = Duration::from_millis(1000 / u64::from(fps)); + // simctl screenshots cost ~200-400ms, so poll them well below the private fps. + let fallback_interval = Duration::from_millis(500); + let mut last: Option<*const Vec> = None; + while !stop.load(Ordering::Relaxed) { + let tick = Instant::now(); + if stream.is_dead() { + // Degraded path: capture via the public simctl screenshot and push it. + if let Ok(jpeg) = crate::simctl::screenshot(udid, crate::rpc::ImageFormat::Jpeg) + && let Ok(body) = encode_stream_frame(udid, &jpeg) + && tx + .send(OutMsg::Frame { + type_byte: STREAM_FRAME, + body, + }) + .is_err() + { + break; // daemon gone + } + if let Some(rest) = fallback_interval.checked_sub(tick.elapsed()) { + thread::sleep(rest); + } + continue; + } + if let Some(frame) = stream.latest() { + let ptr = Arc::as_ptr(&frame); + if last != Some(ptr) { + last = Some(ptr); + if let Ok(body) = encode_stream_frame(udid, &frame) + && tx + .send(OutMsg::Frame { + type_byte: STREAM_FRAME, + body, + }) + .is_err() + { + break; // daemon gone + } + } + } + if let Some(rest) = interval.checked_sub(tick.elapsed()) { + thread::sleep(rest); + } + } + } +} diff --git a/crates/linkcode-sim/src/main.rs b/crates/linkcode-sim/src/main.rs index f8509998..7ecc94ef 100644 --- a/crates/linkcode-sim/src/main.rs +++ b/crates/linkcode-sim/src/main.rs @@ -4,6 +4,10 @@ //! protocol (see [`proto`]). Requests arrive on stdin; results go to stdout. Each request runs //! on its own thread so a slow boot never blocks a screenshot. +mod capture; +mod interactive; +#[cfg(target_os = "macos")] +mod private; mod proto; mod rpc; mod simctl; @@ -16,12 +20,26 @@ use crate::proto::{REQUEST, RESULT, SCREENSHOT, encode_screenshot, read_frame, w use crate::rpc::{ErrorCode, Op, OpError, Request, RequestIdOnly, error_body, success_body}; /// A frame bound for the daemon, or the sentinel that tells the writer thread to stop. -enum OutMsg { +pub(crate) enum OutMsg { Frame { type_byte: u8, body: Vec }, Stop, } fn main() { + let subcommand = std::env::args().nth(1); + // The crash-isolated framebuffer capture worker (spawned by the sidecar itself). + #[cfg(target_os = "macos")] + if subcommand.as_deref() == Some("capture-worker") { + capture::run_worker(); + } + // Hidden diagnostic path to exercise the private-framework layer against a real booted device: + // `linkcode-sim diag-interactive `. + #[cfg(target_os = "macos")] + if subcommand.as_deref() == Some("diag-interactive") { + diag_interactive(); + return; + } + let (tx, rx) = channel::(); // Sole stdout owner: serializes frames from every request thread. @@ -85,7 +103,7 @@ fn main() { fn serve(request: Request, tx: &Sender) { let request_id = request.request_id; let outcome = match request.op { - Op::Probe => simctl::probe(), + Op::Probe => probe_with_capabilities(), Op::List => simctl::list(), Op::Boot { udid } => simctl::boot(&udid), Op::Shutdown { udid } => simctl::shutdown(&udid), @@ -105,6 +123,20 @@ fn serve(request: Request, tx: &Sender) { Err(e) => Err(e), } } + Op::Tap { udid, x, y } => interactive::tap(&udid, x, y), + Op::Swipe { + udid, + x0, + y0, + x1, + y1, + duration_ms, + } => interactive::swipe(&udid, x0, y0, x1, y1, duration_ms), + Op::Button { udid, button } => interactive::button(&udid, button), + Op::StreamStart { udid, fps, quality } => { + interactive::stream_start(&udid, fps, quality, tx) + } + Op::StreamStop { udid } => interactive::stream_stop(&udid), }; match outcome { Ok(result) => send(tx, RESULT, success_body(&request_id, result)), @@ -112,6 +144,15 @@ fn serve(request: Request, tx: &Sender) { } } +/// The probe result augmented with the interactive (private-API framebuffer + HID) capability bit. +fn probe_with_capabilities() -> Result { + let mut result = simctl::probe()?; + if let Some(object) = result.as_object_mut() { + object.insert("interactive".to_owned(), interactive::available().into()); + } + Ok(result) +} + fn send(tx: &Sender, type_byte: u8, body: Vec) { let _ = tx.send(OutMsg::Frame { type_byte, body }); } @@ -119,3 +160,51 @@ fn send(tx: &Sender, type_byte: u8, body: Vec) { fn send_error(tx: &Sender, request_id: &str, error: &OpError) { send(tx, RESULT, error_body(request_id, error)); } + +/// Diagnostic entry (macOS only): drive the SUPERVISED capture stream (crash-isolated worker) plus +/// input injection against a real device. `linkcode-sim diag-interactive [x] [y]`. +/// Verifies the stream survives worker crashes and keeps delivering frames. +#[cfg(target_os = "macos")] +fn diag_interactive() { + use std::time::Duration; + let udid = std::env::args() + .nth(2) + .expect("usage: diag-interactive "); + let out = std::env::args() + .nth(3) + .unwrap_or_else(|| "diag.jpg".to_owned()); + let x: f64 = std::env::args() + .nth(4) + .and_then(|a| a.parse().ok()) + .unwrap_or(0.5); + let y: f64 = std::env::args() + .nth(5) + .and_then(|a| a.parse().ok()) + .unwrap_or(0.5); + eprintln!( + "interactive available: {}", + private::interactive_available() + ); + + let device = private::SimDevice::resolve(&udid).expect("device not found"); + let input = private::Input::warm(&device).expect("HID warm failed"); + let stream = capture::CaptureStream::start(udid.clone(), 0.6, 12); + eprintln!("supervised capture stream started; driving input for ~5s"); + + let start = std::time::Instant::now(); + let mut frames = 0u32; + let mut last_len = 0usize; + while start.elapsed() < Duration::from_secs(5) && !stream.is_dead() { + input.tap(x, y, Duration::from_millis(30)); + if let Some(frame) = stream.latest() { + frames += 1; + last_len = frame.len(); + std::fs::write(&out, &**frame).expect("write jpeg"); + } + std::thread::sleep(Duration::from_millis(80)); + } + eprintln!( + "stream dead={} frames-seen={frames} last={last_len} bytes; wrote {out}", + stream.is_dead() + ); +} diff --git a/crates/linkcode-sim/src/private/debug.rs b/crates/linkcode-sim/src/private/debug.rs new file mode 100644 index 00000000..8ae49997 --- /dev/null +++ b/crates/linkcode-sim/src/private/debug.rs @@ -0,0 +1,24 @@ +//! Env-gated diagnostic logging for the private-framework layer. +//! +//! The private paths cross undocumented CoreSimulator/SimulatorKit surfaces, so keep a way to trace +//! them without shipping noise. Set `LINKCODE_SIM_DEBUG=1` to print to stderr; off by default. + +use std::sync::OnceLock; + +pub fn enabled() -> bool { + static ENABLED: OnceLock = OnceLock::new(); + *ENABLED.get_or_init(|| { + std::env::var("LINKCODE_SIM_DEBUG").is_ok_and(|v| v != "0" && !v.is_empty()) + }) +} + +/// Print a diagnostic line to stderr when `LINKCODE_SIM_DEBUG` is set. +macro_rules! dbg_log { + ($($arg:tt)*) => { + if $crate::private::debug::enabled() { + eprintln!("[sim] {}", format!($($arg)*)); + } + }; +} + +pub(crate) use dbg_log; diff --git a/crates/linkcode-sim/src/private/device.rs b/crates/linkcode-sim/src/private/device.rs new file mode 100644 index 00000000..5c3c33b2 --- /dev/null +++ b/crates/linkcode-sim/src/private/device.rs @@ -0,0 +1,117 @@ +//! Resolving a CoreSimulator `SimDevice` object by udid. +//! +//! Recipe ported from baguette's `CoreSimulators.swift`: `SimServiceContext` → default device set → +//! `availableDevices` → match `UDID.UUIDString`. Everything goes through the ObjC runtime; there is +//! no bridging header. + +use std::ptr; + +use objc2::rc::Retained; +use objc2::runtime::{AnyClass, AnyObject, Sel}; +use objc2::{msg_send, sel}; +use objc2_foundation::NSString; + +use super::framework; + +/// A resolved `SimDevice`, retained for the lifetime of a stream/input session. +pub struct SimDevice { + pub object: Retained, +} + +// SAFETY: SimDevice objects are thread-safe for the read/HID operations we perform; the sidecar +// serializes writes per device through the mux anyway. +unsafe impl Send for SimDevice {} + +impl SimDevice { + /// Resolve the booted-or-not `SimDevice` for `udid`, or `None` if CoreSimulator can't be reached + /// or no device matches. + pub fn resolve(udid: &str) -> Option { + framework::load(); + let set = default_device_set()?; + let devices = available_devices(&set); + for device in devices { + if device_udid(&device).as_deref() == Some(udid) { + return Some(SimDevice { object: device }); + } + } + None + } + + pub fn object_ptr(&self) -> *mut AnyObject { + Retained::as_ptr(&self.object).cast_mut() + } +} + +fn default_device_set() -> Option> { + let ctx = shared_service_context()?; + let sel = sel!(defaultDeviceSetWithError:); + if !responds_to(&ctx, sel) { + return None; + } + let mut err: *mut AnyObject = ptr::null_mut(); + // SAFETY: dynamic message to a method with an `(NSError**)` out-param; we pass a valid slot and + // ignore the error object. The returned set is autoreleased — retain it to hold across the call. + let set: *mut AnyObject = unsafe { msg_send![&*ctx, defaultDeviceSetWithError: &mut err] }; + retain(set) +} + +fn shared_service_context() -> Option> { + let cls = AnyClass::get(c"SimServiceContext")?; + let dev = framework::developer_dir(); + let dir = NSString::from_str(&dev.to_string_lossy()); + let mut err: *mut AnyObject = ptr::null_mut(); + // SAFETY: class method with a developer-dir NSString and an `(NSError**)` out-param. + let ctx: *mut AnyObject = + unsafe { msg_send![cls, sharedServiceContextForDeveloperDir: &*dir, error: &mut err] }; + retain(ctx) +} + +fn available_devices(set: &Retained) -> Vec> { + let key = NSString::from_str("availableDevices"); + // SAFETY: KVC read returning an NSArray (or nil). + let array: *mut AnyObject = unsafe { msg_send![&**set, valueForKey: &*key] }; + if array.is_null() { + return Vec::new(); + } + // SAFETY: array is an NSArray*; count + objectAtIndex: are its standard accessors. + let count: usize = unsafe { msg_send![array, count] }; + let mut out = Vec::with_capacity(count); + for index in 0..count { + // SAFETY: index < count. + let device: *mut AnyObject = unsafe { msg_send![array, objectAtIndex: index] }; + if let Some(device) = retain(device) { + out.push(device); + } + } + out +} + +fn device_udid(device: &Retained) -> Option { + let key = NSString::from_str("UDID"); + // SAFETY: KVC read returning an NSUUID (or nil). + let uuid: *mut AnyObject = unsafe { msg_send![&**device, valueForKey: &*key] }; + if uuid.is_null() { + return None; + } + // SAFETY: uuid is an NSUUID*; UUIDString returns an NSString. + let string: *mut NSString = unsafe { msg_send![uuid, UUIDString] }; + if string.is_null() { + return None; + } + // SAFETY: string is a valid NSString for the duration of this read. + Some(unsafe { (*string).to_string() }) +} + +fn responds_to(object: &Retained, sel: Sel) -> bool { + // SAFETY: respondsToSelector: is defined on NSObject. + unsafe { msg_send![&**object, respondsToSelector: sel] } +} + +/// Retain a +0 (autoreleased/borrowed) object pointer into an owned `Retained`, or `None` if null. +fn retain(ptr: *mut AnyObject) -> Option> { + if ptr.is_null() { + return None; + } + // SAFETY: ptr is a live ObjC object we did not create; retaining it takes shared ownership. + unsafe { Retained::retain(ptr) } +} diff --git a/crates/linkcode-sim/src/private/framework.rs b/crates/linkcode-sim/src/private/framework.rs new file mode 100644 index 00000000..16bec1c8 --- /dev/null +++ b/crates/linkcode-sim/src/private/framework.rs @@ -0,0 +1,168 @@ +//! Locating and loading the private simulator frameworks. +//! +//! Recipe ported from baguette (Apache-2.0; see crate NOTICE): CoreSimulator lives in a stable +//! system path; SimulatorKit moved from `Contents/Developer/Library/PrivateFrameworks` (Xcode ≤26) +//! up to `Contents/SharedFrameworks` (Xcode 27), so both layouts are probed oldest-first. The IOKit +//! `IOHIDEvent*` symbols are in the dyld shared cache and reachable via `RTLD_DEFAULT` once anything +//! is loaded. + +use std::ffi::{CString, c_void}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::OnceLock; + +const CORE_SIMULATOR: &str = + "/Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimulator"; +const SIMKIT_SUFFIX: &str = "SimulatorKit.framework/SimulatorKit"; + +/// A dlopen handle to SimulatorKit. `None` means no Xcode on this machine carries SimulatorKit — +/// the caller degrades to the public simctl path. +pub struct Frameworks { + pub simulator_kit: *mut c_void, +} + +// SAFETY: the handle is a process-lifetime dlopen result, only ever read to resolve symbols. +unsafe impl Send for Frameworks {} +unsafe impl Sync for Frameworks {} + +static FRAMEWORKS: OnceLock> = OnceLock::new(); + +/// Load CoreSimulator + SimulatorKit once. Returns `None` when SimulatorKit cannot be found, which +/// the sidecar reports as a non-interactive capability rather than failing. +pub fn load() -> Option<&'static Frameworks> { + FRAMEWORKS.get_or_init(load_once).as_ref() +} + +fn load_once() -> Option { + // CoreSimulator is also loaded by simctl-adjacent paths, but load it here so its classes + // (SimServiceContext, SimDevice) are registered before we message them. Best-effort. + dlopen(CORE_SIMULATOR); + + let kit_path = simulator_kit_path(&developer_dir())?; + let simulator_kit = dlopen(&kit_path.to_string_lossy())?; + Some(Frameworks { simulator_kit }) +} + +/// Resolve a developer directory that actually contains SimulatorKit: `xcode-select -p` first, then +/// a scan of `/Applications/Xcode*.app`, else the canonical default so a later error names a path +/// the user recognizes. +pub fn developer_dir() -> PathBuf { + if let Some(dir) = xcode_select_dir() + && simulator_kit_path(&dir).is_some() + { + return dir; + } + if let Some(dir) = scan_applications() { + return dir; + } + xcode_select_dir() + .unwrap_or_else(|| PathBuf::from("/Applications/Xcode.app/Contents/Developer")) +} + +/// The candidate SimulatorKit paths for a developer dir, in probe order (Xcode ≤26 then 27). +pub fn simulator_kit_candidates(developer_dir: &Path) -> [PathBuf; 2] { + let contents = developer_dir.parent().unwrap_or(developer_dir); + [ + developer_dir.join(format!("Library/PrivateFrameworks/{SIMKIT_SUFFIX}")), + contents.join(format!("SharedFrameworks/{SIMKIT_SUFFIX}")), + ] +} + +/// The first candidate that exists, or `None` when this Xcode carries no SimulatorKit. +pub fn simulator_kit_path(developer_dir: &Path) -> Option { + simulator_kit_candidates(developer_dir) + .into_iter() + .find(|path| path.exists()) +} + +fn xcode_select_dir() -> Option { + let output = Command::new("/usr/bin/xcode-select") + .arg("-p") + .output() + .ok()?; + if !output.status.success() { + return None; + } + let dir = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + if dir.is_empty() { + None + } else { + Some(PathBuf::from(dir)) + } +} + +fn scan_applications() -> Option { + let canonical = PathBuf::from("/Applications/Xcode.app/Contents/Developer"); + if simulator_kit_path(&canonical).is_some() { + return Some(canonical); + } + let mut entries: Vec<_> = std::fs::read_dir("/Applications") + .ok()? + .filter_map(|entry| entry.ok()) + .map(|entry| entry.file_name()) + .filter_map(|name| name.into_string().ok()) + .filter(|name| name.starts_with("Xcode") && name.ends_with(".app") && name != "Xcode.app") + .collect(); + entries.sort(); + for app in entries { + let dir = PathBuf::from(format!("/Applications/{app}/Contents/Developer")); + if simulator_kit_path(&dir).is_some() { + return Some(dir); + } + } + None +} + +/// `dlopen(path, RTLD_NOW | RTLD_GLOBAL)`; `None` on failure. +fn dlopen(path: &str) -> Option<*mut c_void> { + let c_path = CString::new(path).ok()?; + // SAFETY: c_path is a valid NUL-terminated C string kept alive across the call. + let handle = unsafe { libc::dlopen(c_path.as_ptr(), libc::RTLD_NOW | libc::RTLD_GLOBAL) }; + if handle.is_null() { None } else { Some(handle) } +} + +/// Resolve a symbol from a specific dlopen handle. +/// +/// # Safety +/// The returned pointer is only valid to transmute to the symbol's true ABI; the caller asserts the +/// signature. `handle` must be a live dlopen handle. +pub unsafe fn dlsym(handle: *mut c_void, name: &str) -> Option<*mut c_void> { + let c_name = CString::new(name).ok()?; + // SAFETY: forwarded to the caller's contract; c_name outlives the call. + let sym = unsafe { libc::dlsym(handle, c_name.as_ptr()) }; + if sym.is_null() { None } else { Some(sym) } +} + +/// Resolve a symbol from the dyld shared cache (`RTLD_DEFAULT`) — used for the public `IOHIDEvent*` +/// creators that are not in SimulatorKit itself. +/// +/// # Safety +/// Same contract as [`dlsym`]. +pub unsafe fn dlsym_default(name: &str) -> Option<*mut c_void> { + let c_name = CString::new(name).ok()?; + // RTLD_DEFAULT is the special handle (void*)-2 on Darwin. + let rtld_default = (-2isize) as *mut c_void; + // SAFETY: forwarded to the caller's contract; c_name outlives the call. + let sym = unsafe { libc::dlsym(rtld_default, c_name.as_ptr()) }; + if sym.is_null() { None } else { Some(sym) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn candidates_cover_both_xcode_layouts() { + let dir = Path::new("/Applications/Xcode.app/Contents/Developer"); + let candidates = simulator_kit_candidates(dir); + assert!( + candidates[0].ends_with( + "Developer/Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit" + ) + ); + assert!( + candidates[1] + .ends_with("Contents/SharedFrameworks/SimulatorKit.framework/SimulatorKit") + ); + } +} diff --git a/crates/linkcode-sim/src/private/input.rs b/crates/linkcode-sim/src/private/input.rs new file mode 100644 index 00000000..4335a6f7 --- /dev/null +++ b/crates/linkcode-sim/src/private/input.rs @@ -0,0 +1,387 @@ +//! Touch and button injection into a booted simulator. +//! +//! Recipe ported from baguette's `IndigoHIDInput.swift` + `IOHIDDigitizerDispatch.swift` +//! (Apache-2.0; see crate NOTICE). On iOS 26 a tap CANNOT go through the 9-arg +//! `IndigoHIDMessageForMouseNSEvent` — it is misread as a Home gesture or dropped. The working path +//! builds a real `IOHIDEvent` digitizer parent+finger pair, runs it through +//! `IndigoHIDMessageForTrackpadEventFromHIDEventRef`, then patches the two byte slots the wrapper +//! leaves uninitialised (the `0x32` touch-target tag and the edge bitmask). Buttons (home/lock) use +//! the legacy `IndigoHIDMessageForButton`, which SpringBoard still honors on Face ID devices. + +use std::ffi::c_void; +use std::ptr; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::thread::sleep; +use std::time::Duration; + +use objc2::msg_send; +use objc2::rc::Retained; +use objc2::runtime::{AnyClass, AnyObject}; + +use super::device::SimDevice; +use super::framework; + +// IOHIDEvent creators — public IOKit symbols in the dyld shared cache. ABIs transcribed verbatim +// from the baguette Swift typedefs. `bool` is C `_Bool` (1 byte); the AArch64/x86-64 ABIs place the +// integer- and float-class args from the declared order, so no manual register juggling is needed. +type CreateDigitizerFn = unsafe extern "C" fn( + *const c_void, // allocator (null) + u64, // timestamp + u32, // transducer type (2 = finger) + u32, // index + u32, // identifier + u32, // event mask + u32, // button mask + f64, // x + f64, // y + f64, // z + f64, // tip pressure + f64, // barrel pressure + bool, // range + bool, // touch + u32, // options +) -> *mut c_void; // IOHIDEventRef (+1, Create rule) + +type CreateFingerFn = unsafe extern "C" fn( + *const c_void, // allocator (null) + u64, // timestamp + u32, // index + u32, // identifier + u32, // event mask + f64, // x + f64, // y + f64, // z + f64, // tip pressure + f64, // twist + bool, // range + bool, // touch + u32, // options +) -> *mut c_void; // IOHIDEventRef (+1) + +type AppendEventFn = unsafe extern "C" fn(*mut c_void, *mut c_void, u32); +type TrackpadWrapFn = unsafe extern "C" fn(*const c_void) -> *mut c_void; +type ButtonFn = unsafe extern "C" fn(u32, u32, u32) -> *mut c_void; +type ServiceFn = unsafe extern "C" fn() -> *mut c_void; + +unsafe extern "C" { + fn CFRelease(cf: *const c_void); + fn malloc_size(ptr: *const c_void) -> usize; + fn mach_absolute_time() -> u64; +} + +const TRANSDUCER_FINGER: u32 = 2; +const TOUCH_TARGET: u32 = 0x32; +/// `IndigoHIDMessageForButton` arg0 for the legacy home/lock path; 3rd arg is the digitizer routing +/// target (0x33), not a timestamp. +const HOME_ARG0: u32 = 0x0; +const LOCK_ARG0: u32 = 0x1; +const LEGACY_BUTTON_TARGET: u32 = 0x33; + +/// Which hardware button to press. +#[derive(Clone, Copy)] +pub enum Button { + Home, + Lock, +} + +/// Touch phase for a streamed gesture. +#[derive(Clone, Copy, PartialEq, Eq)] +pub enum Phase { + Down, + Move, + Up, +} + +impl Phase { + fn event_mask(self) -> u32 { + match self { + // Range | Touch | Position for a sustained touch; Touch | Position for the lift. + Self::Down | Self::Move => 0x07, + Self::Up => 0x06, + } + } + fn range(self) -> bool { + self != Self::Up + } + fn touch(self) -> bool { + self != Self::Up + } +} + +struct Symbols { + create_digitizer: CreateDigitizerFn, + create_finger: CreateFingerFn, + append_event: AppendEventFn, + trackpad_wrap: TrackpadWrapFn, + button: ButtonFn, + create_pointer_service: Option, + create_mouse_service: Option, +} + +/// A warmed HID client bound to one device, plus the resolved private symbols. Created lazily; a +/// resolution failure means this host cannot inject input (the caller degrades to view-only). +pub struct Input { + client: Retained, + symbols: Symbols, + touch_counter: AtomicU32, +} + +// SAFETY: the sidecar serializes input per device; the HID client and C fn pointers are only touched +// under that serialization. +unsafe impl Send for Input {} +unsafe impl Sync for Input {} + +impl Input { + /// Resolve symbols, create + warm the `SimDeviceLegacyHIDClient` for `device`. `None` when + /// SimulatorKit or the HID client cannot be reached. + pub fn warm(device: &SimDevice) -> Option { + let symbols = resolve_symbols()?; + let client = make_hid_client(device)?; + let input = Input { + client, + symbols, + touch_counter: AtomicU32::new(0), + }; + input.warm_services(); + Some(input) + } + + /// Single-finger tap at a normalised (0..1) point, holding for `hold`. + pub fn tap(&self, x: f64, y: f64, hold: Duration) -> bool { + let id = self.next_identifier(); + if !self.send_touch(x, y, id, Phase::Down) { + return false; + } + sleep(hold.max(Duration::from_millis(20))); + self.send_touch(x, y, id, Phase::Up) + } + + /// Continuous swipe between two normalised points over `steps` interpolated moves. + pub fn swipe(&self, x0: f64, y0: f64, x1: f64, y1: f64, steps: u32, step: Duration) -> bool { + let steps = steps.max(2); + let id = self.next_identifier(); + if !self.send_touch(x0, y0, id, Phase::Down) { + return false; + } + let mut ok = 0u32; + for i in 1..=steps { + sleep(step); + let t = f64::from(i) / f64::from(steps); + let x = x0 + (x1 - x0) * t; + let y = y0 + (y1 - y0) * t; + if self.send_touch(x, y, id, Phase::Move) { + ok += 1; + } + } + sleep(step); + self.send_touch(x1, y1, id, Phase::Up) && ok >= steps / 2 + } + + /// Press a hardware button (home/lock) for `hold`. + pub fn button(&self, button: Button, hold: Duration) -> bool { + let arg0 = match button { + Button::Home => HOME_ARG0, + Button::Lock => LOCK_ARG0, + }; + // SAFETY: button fn resolved in `resolve_symbols`; direction 1=down, 2=up (0 crashes + // backboardd). Each returns a freshly malloc'd message consumed by send (freeWhenDone). + let down = unsafe { (self.symbols.button)(arg0, 1, LEGACY_BUTTON_TARGET) }; + if down.is_null() { + return false; + } + self.send_message(down); + sleep(hold.max(Duration::from_millis(20))); + let up = unsafe { (self.symbols.button)(arg0, 2, LEGACY_BUTTON_TARGET) }; + if up.is_null() { + return false; + } + self.send_message(up); + true + } + + fn next_identifier(&self) -> u32 { + let id = self + .touch_counter + .fetch_add(1, Ordering::Relaxed) + .wrapping_add(1); + if id == 0 { 1 } else { id } + } + + /// Build one digitizer parent+finger event, wrap it, patch the target/edge byte slots, and send. + fn send_touch(&self, x: f64, y: f64, identifier: u32, phase: Phase) -> bool { + let x = clamp01(x); + let y = clamp01(y); + let mask = phase.event_mask(); + let range = phase.range(); + let touch = phase.touch(); + let now = unsafe { mach_absolute_time() }; + + // SAFETY: creators resolved in resolve_symbols; args match the transcribed ABIs. Returns are + // +1 IOHIDEventRefs we own and release below. + let parent = unsafe { + (self.symbols.create_digitizer)( + ptr::null(), + now, + TRANSDUCER_FINGER, + 0, + identifier, + mask, + 0, + x, + y, + 0.0, + 0.0, + 0.0, + range, + touch, + 0, + ) + }; + if parent.is_null() { + return false; + } + let finger = unsafe { + (self.symbols.create_finger)( + ptr::null(), + now, + 0, + identifier, + mask, + x, + y, + 0.0, + 0.0, + 0.0, + range, + touch, + 0, + ) + }; + if !finger.is_null() { + // SAFETY: append copies the child into the parent; both stay valid here. + unsafe { (self.symbols.append_event)(parent, finger, 0) }; + } + // SAFETY: wrapper reads the parent event and returns a malloc'd Indigo message (or null). + let message = unsafe { (self.symbols.trackpad_wrap)(parent.cast_const()) }; + // The wrapper has copied out what it needs; release the events regardless of outcome. + unsafe { + if !finger.is_null() { + CFRelease(finger.cast_const()); + } + CFRelease(parent.cast_const()); + } + if message.is_null() { + return false; + } + patch_message(message); + self.send_message(message); + true + } + + fn warm_services(&self) { + for create in [ + self.symbols.create_pointer_service, + self.symbols.create_mouse_service, + ] + .into_iter() + .flatten() + { + // SAFETY: service creators take no args and return a malloc'd message consumed by send. + let message = unsafe { create() }; + if !message.is_null() { + self.send_message(message); + sleep(Duration::from_millis(20)); + } + } + } + + /// Dispatch a malloc'd Indigo message; `freeWhenDone: true` hands ownership to the client. + fn send_message(&self, message: *mut c_void) { + let null_obj: *mut AnyObject = ptr::null_mut(); + // SAFETY: the client implements sendWithMessage:freeWhenDone:completionQueue:completion:; + // message is a live malloc'd buffer the client frees. + unsafe { + let _: () = msg_send![ + &*self.client, + sendWithMessage: message, + freeWhenDone: true, + completionQueue: null_obj, + completion: null_obj, + ]; + } + } +} + +/// Patch the two byte slots the trackpad wrapper leaves uninitialised: the touch-target routing tag +/// (offset 0x6c, and 0x10c on the larger layout) and the edge bitmask (0x3a/0x3b, 0xda/0xdb). We +/// only ever inject interior touches, so the edge bits stay zero. +fn patch_message(message: *mut c_void) { + // SAFETY: message is a malloc'd buffer at least 0x70 bytes; malloc_size gates the larger writes. + unsafe { + message + .byte_add(0x6c) + .cast::() + .write_unaligned(TOUCH_TARGET); + let size = malloc_size(message.cast_const()); + if size >= 0x110 { + message + .byte_add(0x10c) + .cast::() + .write_unaligned(TOUCH_TARGET); + } + message.byte_add(0x3a).cast::().write_unaligned(0); + message.byte_add(0x3b).cast::().write_unaligned(0); + if size >= 0xdc { + message.byte_add(0xda).cast::().write_unaligned(0); + message.byte_add(0xdb).cast::().write_unaligned(0); + } + } +} + +fn make_hid_client(device: &SimDevice) -> Option> { + let cls = AnyClass::get(c"_TtC12SimulatorKit24SimDeviceLegacyHIDClient")?; + let mut err: *mut AnyObject = ptr::null_mut(); + // SAFETY: alloc + initWithDevice:error: on the resolved class; the device object outlives the + // call. init returns a +1 object we own. + let client: *mut AnyObject = unsafe { + let allocated: *mut AnyObject = msg_send![cls, alloc]; + msg_send![allocated, initWithDevice: device.object_ptr(), error: &mut err] + }; + if client.is_null() { + return None; + } + // SAFETY: init already returned +1; adopt it without an extra retain. + unsafe { Retained::from_raw(client) } +} + +fn resolve_symbols() -> Option { + let frameworks = framework::load()?; + let kit = frameworks.simulator_kit; + // SAFETY: each dlsym result is transmuted to the ABI its symbol name denotes; the fn pointers + // are only invoked with matching argument types above. + unsafe { + let create_digitizer = framework::dlsym_default("IOHIDEventCreateDigitizerEvent")?; + let create_finger = framework::dlsym_default("IOHIDEventCreateDigitizerFingerEvent")?; + let append_event = framework::dlsym_default("IOHIDEventAppendEvent")?; + let trackpad_wrap = + framework::dlsym(kit, "IndigoHIDMessageForTrackpadEventFromHIDEventRef")?; + let button = framework::dlsym(kit, "IndigoHIDMessageForButton")?; + Some(Symbols { + create_digitizer: std::mem::transmute::<*mut c_void, CreateDigitizerFn>( + create_digitizer, + ), + create_finger: std::mem::transmute::<*mut c_void, CreateFingerFn>(create_finger), + append_event: std::mem::transmute::<*mut c_void, AppendEventFn>(append_event), + trackpad_wrap: std::mem::transmute::<*mut c_void, TrackpadWrapFn>(trackpad_wrap), + button: std::mem::transmute::<*mut c_void, ButtonFn>(button), + create_pointer_service: framework::dlsym(kit, "IndigoHIDMessageToCreatePointerService") + .map(|p| std::mem::transmute::<*mut c_void, ServiceFn>(p)), + create_mouse_service: framework::dlsym(kit, "IndigoHIDMessageToCreateMouseService") + .map(|p| std::mem::transmute::<*mut c_void, ServiceFn>(p)), + }) + } +} + +fn clamp01(v: f64) -> f64 { + v.clamp(0.0, 1.0) +} diff --git a/crates/linkcode-sim/src/private/mod.rs b/crates/linkcode-sim/src/private/mod.rs new file mode 100644 index 00000000..9c307caa --- /dev/null +++ b/crates/linkcode-sim/src/private/mod.rs @@ -0,0 +1,21 @@ +//! Private-framework layer: live framebuffer capture and HID injection via CoreSimulator + +//! SimulatorKit. macOS-only; recipes ported from baguette (Apache-2.0, see the crate NOTICE). +//! +//! Everything here is best-effort and degrades: [`interactive_available`] reports whether the +//! frameworks resolved, and the sidecar falls back to public `simctl` (screenshot polling, no touch) +//! when they don't — so an Xcode without SimulatorKit is view-only, not broken. + +pub(crate) mod debug; +mod device; +mod framework; +mod input; +mod screen; + +pub use device::SimDevice; +pub use input::{Button, Input}; +pub use screen::Screen; + +/// Whether this host can drive simulators interactively (private frameworks resolved). +pub fn interactive_available() -> bool { + framework::load().is_some() +} diff --git a/crates/linkcode-sim/src/private/screen.rs b/crates/linkcode-sim/src/private/screen.rs new file mode 100644 index 00000000..7c364c66 --- /dev/null +++ b/crates/linkcode-sim/src/private/screen.rs @@ -0,0 +1,663 @@ +//! Capturing the simulator framebuffer as JPEG. +//! +//! Recipe ported from baguette's `SimulatorKitScreen.swift` + `JPEGEncoder.swift` (Apache-2.0; see +//! crate NOTICE): `device.io` → `deviceIOPorts` → the `com.apple.framebuffer.display` descriptor(s), +//! whose `framebufferSurface` is a live `IOSurface`. Registering frame callbacks is what makes +//! CoreSimulator composite and deliver frames without a Simulator.app viewer; the callback copies the +//! surface pixels out (the reader thread encodes them) — surfaces are recycled buffers that cannot be +//! held past the callback, and the CoreGraphics/ImageIO encode is unstable on the callback thread. +//! +//! This path is fragile on the Xcode 26 / iOS 26 `SimStreamProcessor` era — it intermittently aborts +//! hard inside CoreSimulator's XPC-proxy machinery — so it is only ever driven from the crash-isolated +//! worker subprocess ([`crate::capture`]), never in the sidecar's main process. + +use std::ffi::{CString, c_ulong, c_void}; +use std::ptr; + +use dispatch2::DispatchQueue; +use objc2::rc::Retained; +use objc2::runtime::{AnyObject, Sel}; +use objc2::sel; +use objc2_foundation::{NSString, NSUUID}; + +use super::debug::dbg_log; +use super::device::SimDevice; + +// A hand-rolled ObjC block that captures a `*const FrameSink` context, used as the framebuffer +// frame/surfaces callbacks. This mirrors baguette's model exactly: `framebufferSurface` only returns +// a live surface once delivery is active, and delivery only activates when a real callback fires — +// so the callback must grab the surface when SimulatorKit signals a frame. block2's `RcBlock` +// crashes the SimulatorKit callback path; a hand-rolled block with a POD capture and a signature is +// what the API accepts. It is registered as a copyable global block whose captured pointer aims at a +// process-lifetime (leaked-with-registration) sink. +#[repr(C)] +struct BlockDescriptor { + reserved: c_ulong, + size: c_ulong, + /// Objective-C type encoding of the block signature; required with `BLOCK_HAS_SIGNATURE`. + /// SimulatorKit reads it, so a signature-less block makes registration throw. + signature: *const i8, +} + +// SAFETY: the descriptor is immutable; its signature points at a static NUL-terminated string. +unsafe impl Sync for BlockDescriptor {} + +#[repr(C)] +struct CaptureBlock { + isa: *const c_void, + flags: i32, + reserved: i32, + invoke: unsafe extern "C" fn(*mut CaptureBlock), + descriptor: *const BlockDescriptor, + /// Captured context: the sink the callback stashes the latest surface into. + sink: *const FrameSink, +} + +unsafe extern "C" { + static _NSConcreteGlobalBlock: c_void; +} + +const BLOCK_IS_GLOBAL: i32 = 1 << 28; +const BLOCK_HAS_SIGNATURE: i32 = 1 << 30; +/// `void (^)(void)`: void return (`v`), 8-byte frame, block self at offset 0 (`@?`). +const BLOCK_SIGNATURE: &[u8] = b"v8@?0\0"; + +static CAPTURE_DESCRIPTOR: BlockDescriptor = BlockDescriptor { + reserved: 0, + size: size_of::() as c_ulong, + signature: BLOCK_SIGNATURE.as_ptr().cast::(), +}; + +/// The callback: SimulatorKit signalled a frame, so grab the current largest surface into the sink. +unsafe extern "C" fn capture_invoke(block: *mut CaptureBlock) { + // SAFETY: block is our own CaptureBlock; sink is a live FrameSink for the registration lifetime. + let sink = unsafe { &*(*block).sink }; + sink.grab_latest(); +} + +/// A raw BGRA frame copied out of an `IOSurface`. Owned pixels, so it outlives the surface and can +/// be encoded off the callback thread. +struct RawFrame { + width: usize, + height: usize, + stride: usize, + pixels: Vec, +} + +/// Shared state between the framebuffer callbacks (SimulatorKit's queue thread) and the capture +/// reader (the sidecar's request thread). The framebuffer surfaces are live, recycled buffers that +/// must NOT be held past the callback — and the CoreGraphics/ImageIO JPEG encode intermittently +/// aborts when run on the callback thread. So the callback does only the minimal safe work (lock → +/// memcpy the pixels → unlock) and the reader thread encodes the owned copy, where it is stable. +struct FrameSink { + descriptors: Vec<*mut AnyObject>, + /// Gate: callbacks can fire for one descriptor while `register` is still wiring the next, so + /// `grab_latest` must not touch a descriptor until every registration has completed. Set true + /// only after the registration loop finishes. + active: std::sync::atomic::AtomicBool, + latest: std::sync::Mutex>>, +} + +// SAFETY: `latest` is mutex-guarded; the descriptor pointers are only messaged (thread-safe reads). +unsafe impl Send for FrameSink {} +unsafe impl Sync for FrameSink {} + +impl FrameSink { + /// Pick the largest live surface across descriptors and copy its pixels into `latest`. Runs on + /// SimulatorKit's callback thread; only a lock + memcpy touch the surface. + fn grab_latest(&self) { + // Ignore callbacks that race the registration loop — a not-yet-registered descriptor aborts + // when messaged for its surface. + if !self.active.load(std::sync::atomic::Ordering::Acquire) { + return; + } + let frame = objc2::rc::autoreleasepool(|_| { + let mut best: *mut c_void = ptr::null_mut(); + let mut best_area = 0usize; + for &descriptor in &self.descriptors { + // SAFETY: framebufferSurface returns the current IOSurface (or nil). + let surface = + unsafe { send_obj(descriptor, sel!(framebufferSurface)) }.cast::(); + if surface.is_null() { + continue; + } + // SAFETY: surface is a live IOSurface. + let area = unsafe { IOSurfaceGetWidth(surface) * IOSurfaceGetHeight(surface) }; + if area > best_area { + best = surface; + best_area = area; + } + } + if best.is_null() { + return None; + } + copy_surface(best) + }); + if let Some(frame) = frame { + *self.latest.lock().expect("frame sink mutex poisoned") = + Some(std::sync::Arc::new(frame)); + } + } + + /// The most recent raw frame, or `None` if none has arrived. + fn latest(&self) -> Option> { + self.latest + .lock() + .expect("frame sink mutex poisoned") + .clone() + } +} + +/// Copy the pixels of a locked BGRA surface into an owned [`RawFrame`]. Minimal surface access: read +/// dimensions, lock read-only, memcpy row by row (surfaces can be padded), unlock. +fn copy_surface(surface: *mut c_void) -> Option { + // SAFETY: surface is a live IOSurface; read-only lock, bounded row copies, matching unlock. + unsafe { + const READ_ONLY: u32 = 1; + if IOSurfaceLock(surface, READ_ONLY, ptr::null_mut()) != 0 { + return None; + } + let width = IOSurfaceGetWidth(surface); + let height = IOSurfaceGetHeight(surface); + let stride = IOSurfaceGetBytesPerRow(surface); + let base = IOSurfaceGetBaseAddress(surface); + let frame = if base.is_null() || width == 0 || height == 0 || stride < width * 4 { + None + } else { + let mut pixels = vec![0u8; stride * height]; + std::ptr::copy_nonoverlapping(base.cast::(), pixels.as_mut_ptr(), stride * height); + Some(RawFrame { + width, + height, + stride, + pixels, + }) + }; + IOSurfaceUnlock(surface, READ_ONLY, ptr::null_mut()); + frame + } +} + +/// Build a leaked capturing block bound to `sink` for the registration callbacks. +fn capture_block(sink: *const FrameSink) -> *mut c_void { + let block = Box::new(CaptureBlock { + isa: (&raw const _NSConcreteGlobalBlock).cast::(), + flags: BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE, + reserved: 0, + invoke: capture_invoke, + descriptor: &raw const CAPTURE_DESCRIPTOR, + sink, + }); + // Leak: SimulatorKit retains the block for the whole registration; the sink outlives it too. + (Box::into_raw(block) as *mut c_void).cast() +} + +// The framebuffer ports are CoreSimulator XPC proxies (`ROCKRemoteProxy`) that answer via +// `forwardInvocation:` — the messaged selectors are not in their class method lists, so objc2's +// `msg_send!` (which verifies against the method list) rejects them. Raw `objc_msgSend` uses the +// runtime's real dispatch, which honors forwarding — the same path baguette's ObjC `perform:` takes. +unsafe extern "C" { + fn objc_msgSend(); +} + +/// Send a no-argument, object-returning message via raw `objc_msgSend`. +/// +/// # Safety +/// `receiver` must be a live object that handles `sel` (directly or by forwarding); `sel` must name +/// a zero-argument, object-returning method. +unsafe fn send_obj(receiver: *mut AnyObject, sel: Sel) -> *mut AnyObject { + // SAFETY: transmute the runtime dispatcher to this call's concrete ABI, per the fn contract. + let f: unsafe extern "C" fn(*mut AnyObject, Sel) -> *mut AnyObject = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(receiver, sel) } +} + +/// Send `respondsToSelector:` via raw `objc_msgSend`. +/// +/// # Safety +/// `receiver` must be a live object. +unsafe fn responds_to(receiver: *mut AnyObject, target: Sel) -> bool { + // SAFETY: respondsToSelector: takes a SEL and returns BOOL; transmute to that ABI. + let f: unsafe extern "C" fn(*mut AnyObject, Sel, Sel) -> bool = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(receiver, sel!(respondsToSelector:), target) } +} + +/// Send `objectAtIndex:` via raw `objc_msgSend`. +/// +/// # Safety +/// `receiver` must be a live NSArray and `index` within bounds. +unsafe fn object_at_index(receiver: *mut AnyObject, index: usize) -> *mut AnyObject { + // SAFETY: objectAtIndex: takes NSUInteger, returns id; transmute to that ABI. + let f: unsafe extern "C" fn(*mut AnyObject, Sel, usize) -> *mut AnyObject = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(receiver, sel!(objectAtIndex:), index) } +} + +/// Send `count` via raw `objc_msgSend`. +/// +/// # Safety +/// `receiver` must be a live NSArray. +unsafe fn array_count(receiver: *mut AnyObject) -> usize { + // SAFETY: count returns NSUInteger; transmute to that ABI. + let f: unsafe extern "C" fn(*mut AnyObject, Sel) -> usize = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(receiver, sel!(count)) } +} + +/// Send `valueForKey:` via raw `objc_msgSend`. +/// +/// # Safety +/// `receiver` must be a live object and `key` a live NSString. +unsafe fn value_for_key_raw(receiver: *mut AnyObject, key: *mut AnyObject) -> *mut AnyObject { + // SAFETY: valueForKey: takes id, returns id; transmute to that ABI. + let f: unsafe extern "C" fn(*mut AnyObject, Sel, *mut AnyObject) -> *mut AnyObject = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(receiver, sel!(valueForKey:), key) } +} + +// IOSurface (public framework). +#[link(name = "IOSurface", kind = "framework")] +unsafe extern "C" { + fn IOSurfaceLock(surface: *mut c_void, options: u32, seed: *mut u32) -> i32; + fn IOSurfaceUnlock(surface: *mut c_void, options: u32, seed: *mut u32) -> i32; + fn IOSurfaceGetBaseAddress(surface: *mut c_void) -> *mut c_void; + fn IOSurfaceGetWidth(surface: *mut c_void) -> usize; + fn IOSurfaceGetHeight(surface: *mut c_void) -> usize; + fn IOSurfaceGetBytesPerRow(surface: *mut c_void) -> usize; +} + +// CoreGraphics (public framework). +#[link(name = "CoreGraphics", kind = "framework")] +unsafe extern "C" { + fn CGColorSpaceCreateDeviceRGB() -> *mut c_void; + fn CGColorSpaceRelease(space: *mut c_void); + fn CGBitmapContextCreate( + data: *mut c_void, + width: usize, + height: usize, + bits_per_component: usize, + bytes_per_row: usize, + space: *mut c_void, + bitmap_info: u32, + ) -> *mut c_void; + fn CGBitmapContextCreateImage(context: *mut c_void) -> *mut c_void; + fn CGContextRelease(context: *mut c_void); + fn CGImageRelease(image: *mut c_void); +} + +// ImageIO (public framework). +#[link(name = "ImageIO", kind = "framework")] +unsafe extern "C" { + fn CGImageDestinationCreateWithData( + data: *mut c_void, + type_: *const c_void, + count: usize, + options: *const c_void, + ) -> *mut c_void; + fn CGImageDestinationAddImage(dest: *mut c_void, image: *mut c_void, properties: *const c_void); + fn CGImageDestinationFinalize(dest: *mut c_void) -> bool; + static kCGImageDestinationLossyCompressionQuality: *const c_void; +} + +// CoreFoundation (public framework). +#[link(name = "CoreFoundation", kind = "framework")] +unsafe extern "C" { + fn CFRelease(cf: *const c_void); + fn CFDataCreateMutable(allocator: *const c_void, capacity: isize) -> *mut c_void; + fn CFDataGetLength(data: *const c_void) -> isize; + fn CFDataGetBytePtr(data: *const c_void) -> *const u8; + fn CFNumberCreate( + allocator: *const c_void, + type_: isize, + value: *const c_void, + ) -> *const c_void; + fn CFStringCreateWithCString( + allocator: *const c_void, + cstr: *const i8, + encoding: u32, + ) -> *const c_void; + fn CFDictionaryCreate( + allocator: *const c_void, + keys: *const *const c_void, + values: *const *const c_void, + count: isize, + key_callbacks: *const c_void, + value_callbacks: *const c_void, + ) -> *const c_void; +} + +const KCG_ALPHA_PREMULTIPLIED_FIRST: u32 = 2; +const KCG_BYTE_ORDER_32_LITTLE: u32 = 2 << 12; +const KCF_NUMBER_DOUBLE_TYPE: isize = 13; +const KCF_STRING_ENCODING_UTF8: u32 = 0x0800_0100; + +/// The framebuffer of one booted device, held for a streaming session. Registration (which the +/// leaked no-op block backs) makes CoreSimulator composite the framebuffer headlessly; the +/// descriptor's `framebufferSurface` is then polled on demand under a per-capture autorelease pool. +/// The registration UUID is retained to unregister on drop. +pub struct Screen { + /// Every `com.apple.framebuffer.display` descriptor, retained. A device exposes several (main + /// LCD plus secondary/overlay planes); the sink polls all and keeps the largest live surface. + descriptors: Vec>, + uuid: Retained, + /// The callbacks stash the latest surface here; capture reads it. Boxed so the callback's + /// captured raw pointer stays valid, and dropped only after callbacks are unregistered. + sink: Box, + /// Held to keep the callback dispatch queue alive for the registration's lifetime. + _queue: dispatch2::DispatchRetained, +} + +// SAFETY: the descriptors are messaged read-only and the sink is internally synchronized. +unsafe impl Send for Screen {} + +impl Drop for Screen { + fn drop(&mut self) { + // Unregister first so no further callback can fire into the sink we are about to drop. + let uuid = Retained::as_ptr(&self.uuid).cast_mut().cast::(); + for descriptor in &self.descriptors { + let descriptor = Retained::as_ptr(descriptor).cast_mut(); + if unsafe { responds_to(descriptor, sel!(unregisterScreenCallbacksWithUUID:)) } { + // SAFETY: unregister takes the UUID we registered with. + let f: unsafe extern "C" fn(*mut AnyObject, Sel, *mut AnyObject) = + unsafe { std::mem::transmute(objc_msgSend as *const c_void) }; + unsafe { f(descriptor, sel!(unregisterScreenCallbacksWithUUID:), uuid) }; + } + } + } +} + +impl Screen { + /// Resolve the largest framebuffer descriptor for a booted device. `None` when the device has no + /// framebuffer (not booted) or the IO plumbing is unavailable. + pub fn open(device: &SimDevice) -> Option { + // The framebuffer probe messages CoreSimulator XPC proxies whose forwarding autoreleases + // reply objects; hold a pool for the whole probe, and retain the descriptors (they survive + // the drain) before returning. + objc2::rc::autoreleasepool(|_| { + let io = io_client(device)?; + // Populate the port list lazily, as SimulatorKitScreen does. + // SAFETY: io is a live SimDeviceIOClient that handles updateIOPorts (returns void). + unsafe { send_obj(Retained::as_ptr(&io).cast_mut(), sel!(updateIOPorts)) }; + let ports = value_for_key(&io, "deviceIOPorts")?; + let ports_ptr = Retained::as_ptr(&ports).cast_mut(); + // SAFETY: deviceIOPorts is an NSArray. + let count = unsafe { array_count(ports_ptr) }; + dbg_log!("open: {count} io ports"); + // Collect every framebuffer.display descriptor: a device exposes several planes and only + // the active one(s) carry a live surface, so we register on all and pick the largest. + let mut descriptors = Vec::new(); + for index in 0..count { + // SAFETY: index < count. + let port = unsafe { object_at_index(ports_ptr, index) }; + if port.is_null() { + continue; + } + if let Some(descriptor) = framebuffer_descriptor(port, index) { + descriptors.push(descriptor); + } + } + if descriptors.is_empty() { + return None; + } + Some(register(descriptors)) + }) + } + + /// The latest framebuffer frame encoded as JPEG at `quality` (0..1), or `None` if none has been + /// delivered yet. The callback copies raw pixels; this reader thread does the CoreGraphics/ImageIO + /// encode (stable off the callback thread). + pub fn capture_jpeg(&self, quality: f64) -> Option> { + // The first frame arrives asynchronously after registration; poll the sink briefly so the + // first read doesn't spuriously fail. Steady-state reads find a frame immediately. + for _ in 0..60 { + if let Some(frame) = self.sink.latest() { + return encode_bgra_jpeg(&frame, quality.clamp(0.1, 1.0)); + } + std::thread::sleep(std::time::Duration::from_millis(10)); + } + None + } +} + +/// Encode an owned BGRA [`RawFrame`] to JPEG bytes via CoreGraphics + ImageIO. Runs on the reader +/// thread from owned pixels — no live surface involved. +fn encode_bgra_jpeg(frame: &RawFrame, quality: f64) -> Option> { + // SAFETY: pixels back a `width×height`, `stride`-padded BGRA buffer; every CF/CG object created + // here is released on all paths. + unsafe { + let color_space = CGColorSpaceCreateDeviceRGB(); + let context = CGBitmapContextCreate( + frame.pixels.as_ptr().cast_mut().cast::(), + frame.width, + frame.height, + 8, + frame.stride, + color_space, + KCG_ALPHA_PREMULTIPLIED_FIRST | KCG_BYTE_ORDER_32_LITTLE, + ); + CGColorSpaceRelease(color_space); + if context.is_null() { + return None; + } + let image = CGBitmapContextCreateImage(context); + CGContextRelease(context); + if image.is_null() { + return None; + } + let bytes = encode_cgimage_jpeg(image, quality); + CGImageRelease(image); + bytes + } +} + +/// Write a `CGImage` to JPEG via ImageIO at `quality`. +/// +/// # Safety +/// `image` must be a live CGImageRef. +unsafe fn encode_cgimage_jpeg(image: *mut c_void, quality: f64) -> Option> { + unsafe { + let data = CFDataCreateMutable(ptr::null(), 0); + if data.is_null() { + return None; + } + let jpeg_uti = CString::new("public.jpeg").ok()?; + let type_ = + CFStringCreateWithCString(ptr::null(), jpeg_uti.as_ptr(), KCF_STRING_ENCODING_UTF8); + let options = quality_dictionary(quality); + let dest = CGImageDestinationCreateWithData(data, type_, 1, options); + if !type_.is_null() { + CFRelease(type_); + } + if !options.is_null() { + CFRelease(options); + } + if dest.is_null() { + CFRelease(data.cast_const()); + return None; + } + CGImageDestinationAddImage(dest, image, ptr::null()); + let ok = CGImageDestinationFinalize(dest); + CFRelease(dest.cast_const()); + let out = if ok { + let len = CFDataGetLength(data.cast_const()); + let bytes = CFDataGetBytePtr(data.cast_const()); + if len > 0 && !bytes.is_null() { + Some(std::slice::from_raw_parts(bytes, len as usize).to_vec()) + } else { + None + } + } else { + None + }; + CFRelease(data.cast_const()); + out + } +} + +/// A one-entry `{ kCGImageDestinationLossyCompressionQuality: quality }` dictionary, or null on +/// failure. Uses null CF callbacks — legal for a transient dict whose keys/values we release after. +/// +/// # Safety +/// Caller releases the returned dictionary (which owns nothing under null callbacks) and must keep +/// the quality value alive only until this returns. +unsafe fn quality_dictionary(quality: f64) -> *const c_void { + unsafe { + let number = CFNumberCreate( + ptr::null(), + KCF_NUMBER_DOUBLE_TYPE, + (&raw const quality).cast(), + ); + if number.is_null() { + return ptr::null(); + } + let keys = [kCGImageDestinationLossyCompressionQuality]; + let values = [number]; + let dict = CFDictionaryCreate( + ptr::null(), + keys.as_ptr(), + values.as_ptr(), + 1, + ptr::null(), + ptr::null(), + ); + CFRelease(number); + dict + } +} + +/// Register frame/surfaces callbacks on every framebuffer descriptor so CoreSimulator composites and +/// delivers frames (no Simulator.app viewer needed); each delivery stashes the largest live surface +/// into the shared sink. Returns the assembled `Screen`. +fn register(descriptors: Vec>) -> Screen { + let uuid = NSUUID::new(); + let queue = DispatchQueue::new("ai.linkcode.sim.framebuffer", None); + // The sink is boxed and its pointer captured by the callback blocks; it must not move and must + // outlive the registration (Screen drops it only after unregistering). + let sink = Box::new(FrameSink { + descriptors: descriptors + .iter() + .map(|d| Retained::as_ptr(d).cast_mut()) + .collect(), + active: std::sync::atomic::AtomicBool::new(false), + latest: std::sync::Mutex::new(None), + }); + let sink_ptr: *const FrameSink = &*sink; + let register_sel = sel!(registerScreenCallbacksWithUUID:callbackQueue:frameCallback:surfacesChangedCallback:propertiesChangedCallback:); + let mut registered = 0; + for descriptor in &descriptors { + let descriptor_ptr = Retained::as_ptr(descriptor).cast_mut(); + if !unsafe { responds_to(descriptor_ptr, register_sel) } { + continue; + } + // A fresh capturing block per callback slot, all pointing at the one sink. + let frame = capture_block(sink_ptr); + let surfaces = capture_block(sink_ptr); + let props = capture_block(sink_ptr); + // SAFETY: the 5-object-argument registration selector. UUID stays alive in the Screen; the + // dispatch queue is retained by CoreSimulator for the registration; the blocks are leaked + // and point at the boxed sink which outlives the registration. + unsafe { + let f: unsafe extern "C" fn( + *mut AnyObject, + Sel, + *mut AnyObject, + *mut c_void, + *mut c_void, + *mut c_void, + *mut c_void, + ) = std::mem::transmute(objc_msgSend as *const c_void); + f( + descriptor_ptr, + register_sel, + Retained::as_ptr(&uuid).cast_mut().cast::(), + dispatch2::DispatchRetained::as_ptr(&queue) + .as_ptr() + .cast::(), + frame, + surfaces, + props, + ); + } + registered += 1; + } + // All descriptors registered — callbacks may now safely touch any of them. + sink.active + .store(true, std::sync::atomic::Ordering::Release); + dbg_log!("registered {registered} framebuffer descriptors"); + Screen { + descriptors, + uuid, + sink, + _queue: queue, + } +} + +/// Probe one IO port: if it is the `com.apple.framebuffer.display` port, retain and return its +/// descriptor. +fn framebuffer_descriptor(port: *mut AnyObject, index: usize) -> Option> { + if !unsafe { responds_to(port, sel!(portIdentifier)) } { + return None; + } + let pid = unsafe { send_obj(port, sel!(portIdentifier)) }; + let identifier = if pid.is_null() { + String::new() + } else { + nsstring_utf8(pid) + }; + dbg_log!("port {index}: identifier={identifier:?}"); + if identifier != "com.apple.framebuffer.display" { + return None; + } + if !unsafe { responds_to(port, sel!(descriptor)) } { + return None; + } + let descriptor = unsafe { send_obj(port, sel!(descriptor)) }; + if descriptor.is_null() || !unsafe { responds_to(descriptor, sel!(framebufferSurface)) } { + return None; + } + // SAFETY: descriptor is a live +0 object; retain to hold across the session. + unsafe { Retained::retain(descriptor) } +} + +fn io_client(device: &SimDevice) -> Option> { + // SAFETY: `io` returns the device's SimDeviceIOClient (+0) or nil. + let io = unsafe { send_obj(device.object_ptr(), sel!(io)) }; + if io.is_null() { + return None; + } + // SAFETY: io is a live +0 object; retain to hold across the session. + unsafe { Retained::retain(io) } +} + +fn value_for_key(object: &Retained, key: &str) -> Option> { + let key = NSString::from_str(key); + let key_ptr = (&*key as *const NSString).cast_mut().cast::(); + // SAFETY: KVC read via raw dispatch. + let value = unsafe { value_for_key_raw(Retained::as_ptr(object).cast_mut(), key_ptr) }; + if value.is_null() { + return None; + } + // SAFETY: value is a live +0 object. + unsafe { Retained::retain(value) } +} + +/// Read an NSString-typed object's bytes via raw `UTF8String`. The port identifier is an NSString; +/// reading it through objc2's `NSString::to_string` would re-message via the verifying macro and can +/// trip over CoreSimulator's string proxies, so use raw dispatch and guard with respondsToSelector:. +fn nsstring_utf8(object: *mut AnyObject) -> String { + if object.is_null() || !unsafe { responds_to(object, sel!(UTF8String)) } { + return String::new(); + } + // SAFETY: UTF8String returns a const char* valid until the autorelease pool drains. + let ptr = unsafe { + let f: unsafe extern "C" fn(*mut AnyObject, Sel) -> *const i8 = + std::mem::transmute(objc_msgSend as *const c_void); + f(object, sel!(UTF8String)) + }; + if ptr.is_null() { + return String::new(); + } + // SAFETY: ptr is a NUL-terminated C string from UTF8String. + unsafe { std::ffi::CStr::from_ptr(ptr).to_string_lossy().into_owned() } +} diff --git a/crates/linkcode-sim/src/proto.rs b/crates/linkcode-sim/src/proto.rs index edbdac23..5ffd5c66 100644 --- a/crates/linkcode-sim/src/proto.rs +++ b/crates/linkcode-sim/src/proto.rs @@ -18,6 +18,24 @@ pub const REQUEST: u8 = 0x01; // Sidecar → daemon. pub const RESULT: u8 = 0x81; pub const SCREENSHOT: u8 = 0x82; +/// A streamed framebuffer frame: `[u16 LE udid_len][udid][jpeg]` (unsolicited, while a stream runs). +pub const STREAM_FRAME: u8 = 0x83; + +/// Encode a `STREAM_FRAME` body: `[u16 LE udid_len][udid][jpeg bytes]`. +pub fn encode_stream_frame(udid: &str, jpeg: &[u8]) -> io::Result> { + let id = udid.as_bytes(); + if id.is_empty() || id.len() > MAX_REQUEST_ID_LEN { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid udid length", + )); + } + let mut out = Vec::with_capacity(2 + id.len() + jpeg.len()); + out.extend_from_slice(&(id.len() as u16).to_le_bytes()); + out.extend_from_slice(id); + out.extend_from_slice(jpeg); + Ok(out) +} /// Read one frame. Returns `Ok(None)` on a clean end-of-stream (the daemon closed the pipe). pub fn read_frame(reader: &mut impl Read) -> io::Result)>> { diff --git a/crates/linkcode-sim/src/rpc.rs b/crates/linkcode-sim/src/rpc.rs index 84cbf343..0ffcb5bc 100644 --- a/crates/linkcode-sim/src/rpc.rs +++ b/crates/linkcode-sim/src/rpc.rs @@ -53,6 +53,45 @@ pub enum Op { #[serde(default)] format: ImageFormat, }, + /// Single-finger tap at a normalised (0..1) point (private API; P1). + Tap { udid: String, x: f64, y: f64 }, + /// Swipe between two normalised (0..1) points over `duration_ms` (private API; P1). + Swipe { + udid: String, + x0: f64, + y0: f64, + x1: f64, + y1: f64, + #[serde(default)] + duration_ms: u64, + }, + /// Press a hardware button (private API; P1). + Button { udid: String, button: ButtonKind }, + /// Start streaming the device framebuffer as JPEG `FRAME`s at `fps` (private API; P1). + StreamStart { + udid: String, + #[serde(default = "default_fps")] + fps: u32, + #[serde(default = "default_quality")] + quality: f64, + }, + /// Stop a running framebuffer stream. + StreamStop { udid: String }, +} + +fn default_fps() -> u32 { + 10 +} +fn default_quality() -> f64 { + 0.6 +} + +/// Hardware buttons exposable over the wire (extended as the private HID layer grows). +#[derive(Clone, Copy, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ButtonKind { + Home, + Lock, } /// Screenshot encodings supported by `simctl io screenshot --type`. From 2731e5e5bd2ef0905843f4c5cf7666b095d4d126 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 22:40:49 +0800 Subject: [PATCH 2/6] fix(sim): stable headless framebuffer capture on Xcode 26 Reach the frame sink through a process global instead of a block-captured pointer (ROCK's XPC delivery zeroes the capture), consume the screen callbacks' delivered surface instead of polling framebufferSurface on the XPC proxy every frame, and build the ImageIO options dict with the standard CFType callbacks. Sustained 15fps headless with no crashes. --- Cargo.lock | 3 - crates/linkcode-sim/Cargo.toml | 1 - crates/linkcode-sim/src/capture.rs | 17 +- crates/linkcode-sim/src/private/screen.rs | 406 ++++++++++++++-------- 4 files changed, 266 insertions(+), 161 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0de83b79..ffe2ed19 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,8 +58,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ "bitflags 2.13.0", - "block2", - "libc", "objc2", ] @@ -120,7 +118,6 @@ name = "linkcode-sim" version = "0.1.0" dependencies = [ "block2", - "dispatch2", "libc", "objc2", "objc2-core-foundation", diff --git a/crates/linkcode-sim/Cargo.toml b/crates/linkcode-sim/Cargo.toml index bbe08c15..de652dd5 100644 --- a/crates/linkcode-sim/Cargo.toml +++ b/crates/linkcode-sim/Cargo.toml @@ -15,7 +15,6 @@ path = "src/main.rs" [dependencies] block2 = "0.6.2" -dispatch2 = "0.3.1" libc = "0.2.189" objc2 = { version = "0.6.4", features = ["exception"] } objc2-core-foundation = "0.3.2" diff --git a/crates/linkcode-sim/src/capture.rs b/crates/linkcode-sim/src/capture.rs index 83c08014..fdb6a6a4 100644 --- a/crates/linkcode-sim/src/capture.rs +++ b/crates/linkcode-sim/src/capture.rs @@ -1,15 +1,16 @@ //! Crash-isolated framebuffer streaming. //! -//! The private-framework framebuffer path (see [`crate::private::screen`]) intermittently aborts -//! hard inside CoreSimulator's XPC-proxy machinery ("Attempt to use unknown class") — a -//! not-in-process-catchable `SIGABRT`. Rather than let that take down the sidecar (which also serves -//! the simctl lifecycle RPCs), the capture runs in a disposable **worker subprocess**: the sidecar -//! spawns `linkcode-sim capture-worker `, reads length-prefixed JPEG frames -//! from its stdout, and respawns it with backoff if it dies. Input injection and lifecycle RPCs stay -//! in the parent and are never affected by a capture crash. +//! The steady-state framebuffer path (see [`crate::private::screen`]) is stable, but the private +//! frameworks can still abort hard in ways that are not catchable in-process — most of all a cold +//! CoreSimulator XPC connection racing class registration ("Attempt to use unknown class", a +//! `SIGABRT`). Rather than let that take down the sidecar (which also serves the simctl lifecycle +//! RPCs), the capture runs in a disposable **worker subprocess**: the sidecar spawns +//! `linkcode-sim capture-worker `, reads length-prefixed JPEG frames from its +//! stdout, and respawns it with backoff if it dies. Input injection and lifecycle RPCs stay in the +//! parent and are never affected by a capture crash. //! //! This is the standard isolation pattern for fragile native code (GPU/plugin sandboxes): contain -//! the crash, supervise, recover. +//! the crash, supervise, recover — defense in depth around a path that is now stable in steady state. use std::io::{self, Read, Write}; use std::process::{Child, Command, Stdio}; diff --git a/crates/linkcode-sim/src/private/screen.rs b/crates/linkcode-sim/src/private/screen.rs index 7c364c66..1c63afe6 100644 --- a/crates/linkcode-sim/src/private/screen.rs +++ b/crates/linkcode-sim/src/private/screen.rs @@ -1,20 +1,33 @@ //! Capturing the simulator framebuffer as JPEG. //! -//! Recipe ported from baguette's `SimulatorKitScreen.swift` + `JPEGEncoder.swift` (Apache-2.0; see -//! crate NOTICE): `device.io` → `deviceIOPorts` → the `com.apple.framebuffer.display` descriptor(s), -//! whose `framebufferSurface` is a live `IOSurface`. Registering frame callbacks is what makes -//! CoreSimulator composite and deliver frames without a Simulator.app viewer; the callback copies the -//! surface pixels out (the reader thread encodes them) — surfaces are recycled buffers that cannot be -//! held past the callback, and the CoreGraphics/ImageIO encode is unstable on the callback thread. +//! Uses CoreSimulator's `SimDisplayIOSurfaceRenderable` screen: `device.io` → `deviceIOPorts` → the +//! `com.apple.framebuffer.display` descriptor(s), then +//! `registerScreenCallbacksWithUUID:callbackQueue:frameCallback:surfacesChangedCallback:propertiesChangedCallback:`. +//! Registering those callbacks is what makes CoreSimulator composite the framebuffer headlessly (no +//! Simulator.app viewer needed) — `framebufferSurface` stays nil until delivery is active. //! -//! This path is fragile on the Xcode 26 / iOS 26 `SimStreamProcessor` era — it intermittently aborts -//! hard inside CoreSimulator's XPC-proxy machinery — so it is only ever driven from the crash-isolated -//! worker subprocess ([`crate::capture`]), never in the sidecar's main process. +//! The delivery model on the Xcode 26 `SimStreamProcessor` era is: `surfacesChangedCallback` fires +//! once at activation (and again only on a resize/realloc) with the live `IOSurface`; `frameCallback` +//! fires per composited frame; and the display surface then updates in place. So the surfaces +//! callback retains the delivered surface, and the frame callback (on the same serial queue, +//! synchronized with frame production) locks-and-copies it into an owned buffer; the reader thread +//! then does the CoreGraphics/ImageIO encode off the queue. The hot per-frame path therefore touches +//! only local IOSurface C calls — it never re-messages the CoreSimulator XPC proxy, which is what +//! made the earlier "poll `framebufferSurface` every frame" recipe abort intermittently inside the +//! proxy machinery. +//! +//! Two things this path must get right or it crashes hard: the callback blocks must be captureless +//! and reach state through a process global (`CURRENT_SINK`), because ROCK's XPC delivery does not +//! preserve a pointer captured in the block body; and the ImageIO options dictionary must use the +//! standard CFType callbacks, because ImageIO deep-copies and validates it. The path is still driven +//! only from the crash-isolated worker subprocess ([`crate::capture`]), never in the sidecar's main +//! process, so any residual hard abort (e.g. a cold-connect class race) is contained. use std::ffi::{CString, c_ulong, c_void}; use std::ptr; +use std::sync::Mutex; +use std::sync::atomic::{AtomicPtr, Ordering}; -use dispatch2::DispatchQueue; use objc2::rc::Retained; use objc2::runtime::{AnyObject, Sel}; use objc2::sel; @@ -23,34 +36,48 @@ use objc2_foundation::{NSString, NSUUID}; use super::debug::dbg_log; use super::device::SimDevice; -// A hand-rolled ObjC block that captures a `*const FrameSink` context, used as the framebuffer -// frame/surfaces callbacks. This mirrors baguette's model exactly: `framebufferSurface` only returns -// a live surface once delivery is active, and delivery only activates when a real callback fires — -// so the callback must grab the surface when SimulatorKit signals a frame. block2's `RcBlock` -// crashes the SimulatorKit callback path; a hand-rolled block with a POD capture and a signature is -// what the API accepts. It is registered as a copyable global block whose captured pointer aims at a -// process-lifetime (leaked-with-registration) sink. +// Hand-rolled ObjC blocks for the three screen callbacks. block2's `RcBlock` crashes the +// CoreSimulator callback path; a hand-rolled global block with a signature is what the API accepts. +// The blocks are captureless: ROCK's XPC delivery does not preserve a pointer captured in the block +// body (it reads back null in the callback), so the callbacks reach state through [`CURRENT_SINK`] +// instead. All three share one layout so a single builder and invoke-pointer cast serve them. #[repr(C)] struct BlockDescriptor { reserved: c_ulong, size: c_ulong, /// Objective-C type encoding of the block signature; required with `BLOCK_HAS_SIGNATURE`. - /// SimulatorKit reads it, so a signature-less block makes registration throw. + /// CoreSimulator reads it, so a signature-less block makes registration throw. signature: *const i8, } -// SAFETY: the descriptor is immutable; its signature points at a static NUL-terminated string. +// SAFETY: descriptors are immutable; each signature points at a static NUL-terminated string. unsafe impl Sync for BlockDescriptor {} #[repr(C)] -struct CaptureBlock { +struct CallbackBlock { isa: *const c_void, flags: i32, reserved: i32, - invoke: unsafe extern "C" fn(*mut CaptureBlock), + /// One of the three invoke fns below, stored type-erased; the descriptor's signature tells the + /// runtime the real ABI to call it with. + invoke: *const c_void, descriptor: *const BlockDescriptor, - /// Captured context: the sink the callback stashes the latest surface into. - sink: *const FrameSink, +} + +/// The active stream's sink, reached by the callbacks. The blocks are handed to a CoreSimulator +/// `ROCKRemoteProxy` and delivered back through `ROCKInvocation`/XPC, which does NOT preserve a +/// pointer captured in the block body — a captured `sink` field reads back null in the callback and +/// segfaults. So the sink is reached through this process-global pointer instead, read from the +/// callback's own address space. `Screen::open` runs once per (worker) process, so a single global is +/// exactly right; it is set at registration and cleared on drop. +static CURRENT_SINK: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +/// The active sink, or `None` if no stream is registered. +fn current_sink() -> Option<&'static FrameSink> { + let ptr = CURRENT_SINK.load(Ordering::Acquire); + // SAFETY: when non-null, `ptr` is the boxed sink owned by the live `Screen` that set it; it is + // cleared to null in `Screen::drop` before the box is freed, so a non-null read is valid. + unsafe { ptr.as_ref() } } unsafe extern "C" { @@ -59,24 +86,55 @@ unsafe extern "C" { const BLOCK_IS_GLOBAL: i32 = 1 << 28; const BLOCK_HAS_SIGNATURE: i32 = 1 << 30; -/// `void (^)(void)`: void return (`v`), 8-byte frame, block self at offset 0 (`@?`). -const BLOCK_SIGNATURE: &[u8] = b"v8@?0\0"; +/// `void (^)(void)` — the per-frame callback (we copy on the reader, so this is a no-op). +const FRAME_SIGNATURE: &[u8] = b"v8@?0\0"; +/// `void (^)(IOSurface *unmasked, IOSurface *masked)` — the surfaces-changed callback. +const SURFACES_SIGNATURE: &[u8] = b"v24@?0@8@16\0"; +/// `void (^)(id)` — the properties-changed callback (no-op). +const PROPS_SIGNATURE: &[u8] = b"v16@?0@8\0"; + +static FRAME_DESCRIPTOR: BlockDescriptor = block_descriptor(FRAME_SIGNATURE); +static SURFACES_DESCRIPTOR: BlockDescriptor = block_descriptor(SURFACES_SIGNATURE); +static PROPS_DESCRIPTOR: BlockDescriptor = block_descriptor(PROPS_SIGNATURE); + +const fn block_descriptor(signature: &'static [u8]) -> BlockDescriptor { + BlockDescriptor { + reserved: 0, + size: size_of::() as c_ulong, + signature: signature.as_ptr().cast::(), + } +} -static CAPTURE_DESCRIPTOR: BlockDescriptor = BlockDescriptor { - reserved: 0, - size: size_of::() as c_ulong, - signature: BLOCK_SIGNATURE.as_ptr().cast::(), -}; +/// Per-composited-frame callback: copy the current surface into an owned frame. Runs on +/// CoreSimulator's serial delivery queue, synchronized with frame production, so the surface backing +/// is valid here — it is NOT valid to retain and read later on another thread (recycled buffer). Only +/// the memcpy happens here; the reader thread does the CoreGraphics/ImageIO encode. +unsafe extern "C" fn frame_invoke(_block: *mut CallbackBlock) { + if let Some(sink) = current_sink() { + sink.grab_current(); + } +} -/// The callback: SimulatorKit signalled a frame, so grab the current largest surface into the sink. -unsafe extern "C" fn capture_invoke(block: *mut CaptureBlock) { - // SAFETY: block is our own CaptureBlock; sink is a live FrameSink for the registration lifetime. - let sink = unsafe { &*(*block).sink }; - sink.grab_latest(); +/// Surfaces-changed callback: retain the delivered unmasked (full, bezel-free) surface as the current +/// framebuffer; ignore the masked one (rounded-corner alpha we don't want). Also copies it once right +/// away — the frame callback only fires on *new* composited frames, so a static screen (no motion +/// after boot) would otherwise never yield a first frame. +unsafe extern "C" fn surfaces_invoke( + _block: *mut CallbackBlock, + unmasked: *mut c_void, + _masked: *mut c_void, +) { + if let Some(sink) = current_sink() { + sink.set_current(unmasked); + sink.grab_current(); + } } -/// A raw BGRA frame copied out of an `IOSurface`. Owned pixels, so it outlives the surface and can -/// be encoded off the callback thread. +/// Properties-changed callback: no-op. +unsafe extern "C" fn props_invoke(_block: *mut CallbackBlock, _props: *mut c_void) {} + +/// A raw BGRA frame copied out of an `IOSurface`. Owned pixels, so it outlives the surface lock and +/// can be encoded off the callback queue. struct RawFrame { width: usize, height: usize, @@ -84,62 +142,69 @@ struct RawFrame { pixels: Vec, } -/// Shared state between the framebuffer callbacks (SimulatorKit's queue thread) and the capture -/// reader (the sidecar's request thread). The framebuffer surfaces are live, recycled buffers that -/// must NOT be held past the callback — and the CoreGraphics/ImageIO JPEG encode intermittently -/// aborts when run on the callback thread. So the callback does only the minimal safe work (lock → -/// memcpy the pixels → unlock) and the reader thread encodes the owned copy, where it is stable. +/// A CFRetained `IOSurface`, released on drop. The surfaces callback delivers the surface borrowed +/// (valid only for that call), so we retain it to hold across later frame callbacks — the way +/// `SimDisplayView` retains the surface it renders each display frame. Its content updates in place. +struct Held(*mut c_void); + +// SAFETY: a retained IOSurface is safe to reference until released; it is only locked/copied on the +// serial delivery queue. +unsafe impl Send for Held {} + +impl Held { + fn new(surface: *mut c_void) -> Held { + // SAFETY: surface is a live IOSurface (CFType) for the duration of the delivering callback; + // retain to hold it past that call. + unsafe { CFRetain(surface.cast_const()) }; + Held(surface) + } +} + +impl Drop for Held { + fn drop(&mut self) { + // SAFETY: balances the CFRetain in `new`. + unsafe { CFRelease(self.0.cast_const()) }; + } +} + +/// Shared state between the callbacks (CoreSimulator's serial delivery queue) and the capture reader +/// (the sidecar's request thread). `current` is the retained live surface, swapped by the surfaces +/// callback and copied by the frame callback — both on the same serial queue. `latest` is the owned +/// copy the reader encodes. struct FrameSink { - descriptors: Vec<*mut AnyObject>, - /// Gate: callbacks can fire for one descriptor while `register` is still wiring the next, so - /// `grab_latest` must not touch a descriptor until every registration has completed. Set true - /// only after the registration loop finishes. - active: std::sync::atomic::AtomicBool, - latest: std::sync::Mutex>>, + current: Mutex>, + latest: Mutex>>, } -// SAFETY: `latest` is mutex-guarded; the descriptor pointers are only messaged (thread-safe reads). +// SAFETY: both fields are mutex-guarded; the surface is only locked/copied on the delivery queue. unsafe impl Send for FrameSink {} unsafe impl Sync for FrameSink {} impl FrameSink { - /// Pick the largest live surface across descriptors and copy its pixels into `latest`. Runs on - /// SimulatorKit's callback thread; only a lock + memcpy touch the surface. - fn grab_latest(&self) { - // Ignore callbacks that race the registration loop — a not-yet-registered descriptor aborts - // when messaged for its surface. - if !self.active.load(std::sync::atomic::Ordering::Acquire) { + /// Adopt a freshly delivered surface as current, retaining it and releasing the previous one. + fn set_current(&self, surface: *mut c_void) { + if surface.is_null() { return; } - let frame = objc2::rc::autoreleasepool(|_| { - let mut best: *mut c_void = ptr::null_mut(); - let mut best_area = 0usize; - for &descriptor in &self.descriptors { - // SAFETY: framebufferSurface returns the current IOSurface (or nil). - let surface = - unsafe { send_obj(descriptor, sel!(framebufferSurface)) }.cast::(); - if surface.is_null() { - continue; - } - // SAFETY: surface is a live IOSurface. - let area = unsafe { IOSurfaceGetWidth(surface) * IOSurfaceGetHeight(surface) }; - if area > best_area { - best = surface; - best_area = area; - } - } - if best.is_null() { - return None; - } - copy_surface(best) - }); + *self.current.lock().expect("frame sink mutex poisoned") = Some(Held::new(surface)); + } + + /// Copy the current surface into `latest`. Called from the frame callback on the delivery queue, + /// where the retained surface's backing is live and coherent with the just-composited frame. + fn grab_current(&self) { + let frame = { + let guard = self.current.lock().expect("frame sink mutex poisoned"); + let Some(held) = guard.as_ref() else { + return; + }; + copy_surface(held.0) + }; if let Some(frame) = frame { *self.latest.lock().expect("frame sink mutex poisoned") = Some(std::sync::Arc::new(frame)); } } - /// The most recent raw frame, or `None` if none has arrived. fn latest(&self) -> Option> { self.latest .lock() @@ -151,6 +216,9 @@ impl FrameSink { /// Copy the pixels of a locked BGRA surface into an owned [`RawFrame`]. Minimal surface access: read /// dimensions, lock read-only, memcpy row by row (surfaces can be padded), unlock. fn copy_surface(surface: *mut c_void) -> Option { + if surface.is_null() { + return None; + } // SAFETY: surface is a live IOSurface; read-only lock, bounded row copies, matching unlock. unsafe { const READ_ONLY: u32 = 1; @@ -178,24 +246,25 @@ fn copy_surface(surface: *mut c_void) -> Option { } } -/// Build a leaked capturing block bound to `sink` for the registration callbacks. -fn capture_block(sink: *const FrameSink) -> *mut c_void { - let block = Box::new(CaptureBlock { +/// Build a leaked, captureless callback block with `invoke`/`descriptor`. It carries no context — +/// the callbacks reach the sink through [`CURRENT_SINK`], since a captured pointer does not survive +/// ROCK's XPC delivery. +fn callback_block(invoke: *const c_void, descriptor: *const BlockDescriptor) -> *mut c_void { + let block = Box::new(CallbackBlock { isa: (&raw const _NSConcreteGlobalBlock).cast::(), flags: BLOCK_IS_GLOBAL | BLOCK_HAS_SIGNATURE, reserved: 0, - invoke: capture_invoke, - descriptor: &raw const CAPTURE_DESCRIPTOR, - sink, + invoke, + descriptor, }); - // Leak: SimulatorKit retains the block for the whole registration; the sink outlives it too. + // Leak: CoreSimulator retains the block for the whole registration. (Box::into_raw(block) as *mut c_void).cast() } // The framebuffer ports are CoreSimulator XPC proxies (`ROCKRemoteProxy`) that answer via // `forwardInvocation:` — the messaged selectors are not in their class method lists, so objc2's // `msg_send!` (which verifies against the method list) rejects them. Raw `objc_msgSend` uses the -// runtime's real dispatch, which honors forwarding — the same path baguette's ObjC `perform:` takes. +// runtime's real dispatch, which honors forwarding. unsafe extern "C" { fn objc_msgSend(); } @@ -303,6 +372,7 @@ unsafe extern "C" { // CoreFoundation (public framework). #[link(name = "CoreFoundation", kind = "framework")] unsafe extern "C" { + fn CFRetain(cf: *const c_void) -> *const c_void; fn CFRelease(cf: *const c_void); fn CFDataCreateMutable(allocator: *const c_void, capacity: isize) -> *mut c_void; fn CFDataGetLength(data: *const c_void) -> isize; @@ -325,6 +395,13 @@ unsafe extern "C" { key_callbacks: *const c_void, value_callbacks: *const c_void, ) -> *const c_void; + static kCFTypeDictionaryKeyCallBacks: c_void; + static kCFTypeDictionaryValueCallBacks: c_void; +} + +// dispatch (public): a serial queue for CoreSimulator to deliver the screen callbacks on. +unsafe extern "C" { + fn dispatch_queue_create(label: *const i8, attr: *const c_void) -> *mut c_void; } const KCG_ALPHA_PREMULTIPLIED_FIRST: u32 = 2; @@ -332,28 +409,35 @@ const KCG_BYTE_ORDER_32_LITTLE: u32 = 2 << 12; const KCF_NUMBER_DOUBLE_TYPE: isize = 13; const KCF_STRING_ENCODING_UTF8: u32 = 0x0800_0100; -/// The framebuffer of one booted device, held for a streaming session. Registration (which the -/// leaked no-op block backs) makes CoreSimulator composite the framebuffer headlessly; the -/// descriptor's `framebufferSurface` is then polled on demand under a per-capture autorelease pool. -/// The registration UUID is retained to unregister on drop. +/// How long [`Screen::open`] waits for the first surface to be delivered before giving up. +const ACTIVATION_TIMEOUT_MS: u64 = 2000; + +/// The framebuffer of one booted device, held for a streaming session. Registration makes +/// CoreSimulator composite the framebuffer headlessly and push the display surface into the sink; the +/// reader copies that held surface on demand. The registration UUID is retained to unregister on drop. pub struct Screen { - /// Every `com.apple.framebuffer.display` descriptor, retained. A device exposes several (main - /// LCD plus secondary/overlay planes); the sink polls all and keeps the largest live surface. + /// Every `com.apple.framebuffer.display` descriptor we registered on, retained. A device exposes + /// several (main LCD plus secondary/overlay planes); we register on all and hold the surface each + /// delivers. descriptors: Vec>, uuid: Retained, - /// The callbacks stash the latest surface here; capture reads it. Boxed so the callback's - /// captured raw pointer stays valid, and dropped only after callbacks are unregistered. + /// The surfaces callback stashes the current surface here; the reader copies it. Boxed so the + /// callback's captured raw pointer stays valid, and dropped only after callbacks are unregistered. sink: Box, - /// Held to keep the callback dispatch queue alive for the registration's lifetime. - _queue: dispatch2::DispatchRetained, + /// The serial dispatch queue CoreSimulator delivers callbacks on; retained for the registration. + queue: *mut c_void, } -// SAFETY: the descriptors are messaged read-only and the sink is internally synchronized. +// SAFETY: the descriptors are messaged read-only, the sink is internally synchronized, and the queue +// is only handed to CoreSimulator and released on drop. unsafe impl Send for Screen {} impl Drop for Screen { fn drop(&mut self) { - // Unregister first so no further callback can fire into the sink we are about to drop. + // Stop the callbacks from reaching the sink, then unregister. Cleared before the boxed sink is + // freed so an in-flight callback either sees the live sink or (after this) sees null. + CURRENT_SINK.store(ptr::null_mut(), Ordering::Release); + // Unregister so no further callback can fire. let uuid = Retained::as_ptr(&self.uuid).cast_mut().cast::(); for descriptor in &self.descriptors { let descriptor = Retained::as_ptr(descriptor).cast_mut(); @@ -364,19 +448,26 @@ impl Drop for Screen { unsafe { f(descriptor, sel!(unregisterScreenCallbacksWithUUID:), uuid) }; } } + if !self.queue.is_null() { + // SAFETY: the queue was created +1 by dispatch_queue_create and handed to CoreSimulator, + // which retained it for the registration we just tore down; release our reference. + unsafe { CFRelease(self.queue.cast_const()) }; + } } } impl Screen { - /// Resolve the largest framebuffer descriptor for a booted device. `None` when the device has no - /// framebuffer (not booted) or the IO plumbing is unavailable. + /// Open the framebuffer of a booted device and start surface delivery. `None` when the device has + /// no framebuffer (not booted), the IO plumbing is unavailable, no descriptor supports the screen + /// callbacks, or no surface is delivered within the activation timeout (the caller degrades to + /// simctl). pub fn open(device: &SimDevice) -> Option { // The framebuffer probe messages CoreSimulator XPC proxies whose forwarding autoreleases // reply objects; hold a pool for the whole probe, and retain the descriptors (they survive // the drain) before returning. objc2::rc::autoreleasepool(|_| { let io = io_client(device)?; - // Populate the port list lazily, as SimulatorKitScreen does. + // Populate the port list lazily. // SAFETY: io is a live SimDeviceIOClient that handles updateIOPorts (returns void). unsafe { send_obj(Retained::as_ptr(&io).cast_mut(), sel!(updateIOPorts)) }; let ports = value_for_key(&io, "deviceIOPorts")?; @@ -384,8 +475,6 @@ impl Screen { // SAFETY: deviceIOPorts is an NSArray. let count = unsafe { array_count(ports_ptr) }; dbg_log!("open: {count} io ports"); - // Collect every framebuffer.display descriptor: a device exposes several planes and only - // the active one(s) carry a live surface, so we register on all and pick the largest. let mut descriptors = Vec::new(); for index in 0..count { // SAFETY: index < count. @@ -400,23 +489,38 @@ impl Screen { if descriptors.is_empty() { return None; } - Some(register(descriptors)) + let screen = register(descriptors)?; + // Wait for delivery to activate (the surfaces callback fires ~once at activation, and the + // proxy also exposes `framebufferSurface` once live). If neither yields a surface, the + // connection is dead/degraded — report None so the caller degrades to simctl. + if screen.await_first_surface() { + Some(screen) + } else { + dbg_log!("open: no surface within activation timeout"); + None + } }) } - /// The latest framebuffer frame encoded as JPEG at `quality` (0..1), or `None` if none has been - /// delivered yet. The callback copies raw pixels; this reader thread does the CoreGraphics/ImageIO - /// encode (stable off the callback thread). - pub fn capture_jpeg(&self, quality: f64) -> Option> { - // The first frame arrives asynchronously after registration; poll the sink briefly so the - // first read doesn't spuriously fail. Steady-state reads find a frame immediately. - for _ in 0..60 { - if let Some(frame) = self.sink.latest() { - return encode_bgra_jpeg(&frame, quality.clamp(0.1, 1.0)); + /// Wait for the first frame to be copied out by the frame callback, up to + /// [`ACTIVATION_TIMEOUT_MS`]. A copied frame (not just a delivered surface pointer) proves the + /// whole delivery path is live. + fn await_first_surface(&self) -> bool { + for _ in 0..(ACTIVATION_TIMEOUT_MS / 10) { + if self.sink.latest().is_some() { + return true; } std::thread::sleep(std::time::Duration::from_millis(10)); } - None + false + } + + /// The latest copied framebuffer encoded as JPEG at `quality` (0..1), or `None` if no frame has + /// been copied yet. The frame callback did the surface copy on the delivery queue; this reader + /// thread only encodes the owned pixels — the CoreSimulator proxy is never messaged on this path. + pub fn capture_jpeg(&self, quality: f64) -> Option> { + let frame = self.sink.latest()?; + encode_bgra_jpeg(&frame, quality.clamp(0.1, 1.0)) } } @@ -496,11 +600,12 @@ unsafe fn encode_cgimage_jpeg(image: *mut c_void, quality: f64) -> Option *const c_void { unsafe { let number = CFNumberCreate( @@ -518,31 +623,30 @@ unsafe fn quality_dictionary(quality: f64) -> *const c_void { keys.as_ptr(), values.as_ptr(), 1, - ptr::null(), - ptr::null(), + (&raw const kCFTypeDictionaryKeyCallBacks).cast(), + (&raw const kCFTypeDictionaryValueCallBacks).cast(), ); + // The dict retained the number under the CFType callbacks; drop our reference. CFRelease(number); dict } } -/// Register frame/surfaces callbacks on every framebuffer descriptor so CoreSimulator composites and -/// delivers frames (no Simulator.app viewer needed); each delivery stashes the largest live surface -/// into the shared sink. Returns the assembled `Screen`. -fn register(descriptors: Vec>) -> Screen { +/// Register the three screen callbacks on every framebuffer descriptor so CoreSimulator composites +/// and delivers the display surface (no Simulator.app viewer needed). Returns the assembled `Screen`, +/// or `None` if no descriptor supports the registration. +fn register(descriptors: Vec>) -> Option { let uuid = NSUUID::new(); - let queue = DispatchQueue::new("ai.linkcode.sim.framebuffer", None); - // The sink is boxed and its pointer captured by the callback blocks; it must not move and must - // outlive the registration (Screen drops it only after unregistering). + let label = c"ai.linkcode.sim.framebuffer"; + // SAFETY: dispatch_queue_create with a static label and null (serial) attr returns a +1 queue. + let queue = unsafe { dispatch_queue_create(label.as_ptr(), ptr::null()) }; + // The sink is boxed (stable address) and owned by the Screen; the callbacks reach it through + // CURRENT_SINK. Publish it before registering so an immediately-delivered callback finds it. let sink = Box::new(FrameSink { - descriptors: descriptors - .iter() - .map(|d| Retained::as_ptr(d).cast_mut()) - .collect(), - active: std::sync::atomic::AtomicBool::new(false), - latest: std::sync::Mutex::new(None), + current: Mutex::new(None), + latest: Mutex::new(None), }); - let sink_ptr: *const FrameSink = &*sink; + CURRENT_SINK.store((&*sink as *const FrameSink).cast_mut(), Ordering::Release); let register_sel = sel!(registerScreenCallbacksWithUUID:callbackQueue:frameCallback:surfacesChangedCallback:propertiesChangedCallback:); let mut registered = 0; for descriptor in &descriptors { @@ -550,13 +654,14 @@ fn register(descriptors: Vec>) -> Screen { if !unsafe { responds_to(descriptor_ptr, register_sel) } { continue; } - // A fresh capturing block per callback slot, all pointing at the one sink. - let frame = capture_block(sink_ptr); - let surfaces = capture_block(sink_ptr); - let props = capture_block(sink_ptr); - // SAFETY: the 5-object-argument registration selector. UUID stays alive in the Screen; the - // dispatch queue is retained by CoreSimulator for the registration; the blocks are leaked - // and point at the boxed sink which outlives the registration. + let frame = callback_block(frame_invoke as *const c_void, &raw const FRAME_DESCRIPTOR); + let surfaces = callback_block( + surfaces_invoke as *const c_void, + &raw const SURFACES_DESCRIPTOR, + ); + let props = callback_block(props_invoke as *const c_void, &raw const PROPS_DESCRIPTOR); + // SAFETY: the 5-object-argument registration selector. UUID + queue stay alive in the Screen; + // the blocks are leaked captureless globals; the sink is reached via CURRENT_SINK. unsafe { let f: unsafe extern "C" fn( *mut AnyObject, @@ -571,9 +676,7 @@ fn register(descriptors: Vec>) -> Screen { descriptor_ptr, register_sel, Retained::as_ptr(&uuid).cast_mut().cast::(), - dispatch2::DispatchRetained::as_ptr(&queue) - .as_ptr() - .cast::(), + queue, frame, surfaces, props, @@ -581,16 +684,21 @@ fn register(descriptors: Vec>) -> Screen { } registered += 1; } - // All descriptors registered — callbacks may now safely touch any of them. - sink.active - .store(true, std::sync::atomic::Ordering::Release); dbg_log!("registered {registered} framebuffer descriptors"); - Screen { + if registered == 0 { + CURRENT_SINK.store(ptr::null_mut(), Ordering::Release); + if !queue.is_null() { + // SAFETY: nothing adopted the queue; release the +1 from dispatch_queue_create. + unsafe { CFRelease(queue.cast_const()) }; + } + return None; + } + Some(Screen { descriptors, uuid, sink, - _queue: queue, - } + queue, + }) } /// Probe one IO port: if it is the `com.apple.framebuffer.display` port, retain and return its From 8a1c97bed09409b8657151d64269164b03b99f4a Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 22:56:21 +0800 Subject: [PATCH 3/6] feat(sim): bench-encode subcommand for the capture encode ceiling --- crates/linkcode-sim/src/main.rs | 49 +++++++++++++ crates/linkcode-sim/src/private/mod.rs | 2 +- crates/linkcode-sim/src/private/screen.rs | 87 +++++++++++++++++++++++ 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/crates/linkcode-sim/src/main.rs b/crates/linkcode-sim/src/main.rs index 7ecc94ef..d49ae82c 100644 --- a/crates/linkcode-sim/src/main.rs +++ b/crates/linkcode-sim/src/main.rs @@ -39,6 +39,13 @@ fn main() { diag_interactive(); return; } + // Benchmark the JPEG encode ceiling (the capture stream's single-thread frame-rate bound): + // `linkcode-sim bench-encode [iters]`. No simulator needed. + #[cfg(target_os = "macos")] + if subcommand.as_deref() == Some("bench-encode") { + bench_encode(); + return; + } let (tx, rx) = channel::(); @@ -208,3 +215,45 @@ fn diag_interactive() { stream.is_dead() ); } + +/// Benchmark entry (macOS only): time the JPEG encode across a resolution/quality sweep and print the +/// implied max frame rate. `linkcode-sim bench-encode [iters]` (default 120). The encode runs on one +/// reader thread, so `1000 / avg_ms` is the sustainable stream ceiling; the capture memcpy runs on a +/// separate thread and is not the bound. +#[cfg(target_os = "macos")] +fn bench_encode() { + let iters: u32 = std::env::args() + .nth(2) + .and_then(|a| a.parse().ok()) + .unwrap_or(120); + // Native iPhone 17 Pro framebuffer is 1206×2622; sweep it and progressive downscales at the + // default stream quality, then the same full res at a lower quality to show quality is not the + // bound. + let configs = [ + (1206usize, 2622usize, 0.6f64), + (904, 1966, 0.6), + (603, 1311, 0.6), + (402, 874, 0.6), + (1206, 2622, 0.3), + ]; + println!( + "encode bench — {iters} iters/config (single reader thread = stream fps ceiling)\n{:>11} {:>5} {:>8} {:>8} {:>7} {:>9} {:>8}", + "resolution", "q", "avg ms", "p95 ms", "size", "fps(avg)", "fps(peak)" + ); + for (w, h, q) in configs { + match private::bench_encode(w, h, q, iters) { + Some(b) => println!( + "{:>4}x{:<6} {:>5.2} {:>8.2} {:>8.2} {:>6}K {:>9.1} {:>8.1}", + b.width, + b.height, + b.quality, + b.avg_ms, + b.p95_ms, + b.out_kib, + b.fps(), + b.peak_fps() + ), + None => println!("{w}x{h} q{q}: encode failed"), + } + } +} diff --git a/crates/linkcode-sim/src/private/mod.rs b/crates/linkcode-sim/src/private/mod.rs index 9c307caa..8bcea54e 100644 --- a/crates/linkcode-sim/src/private/mod.rs +++ b/crates/linkcode-sim/src/private/mod.rs @@ -13,7 +13,7 @@ mod screen; pub use device::SimDevice; pub use input::{Button, Input}; -pub use screen::Screen; +pub use screen::{Screen, bench_encode}; /// Whether this host can drive simulators interactively (private frameworks resolved). pub fn interactive_available() -> bool { diff --git a/crates/linkcode-sim/src/private/screen.rs b/crates/linkcode-sim/src/private/screen.rs index 1c63afe6..c3a599c9 100644 --- a/crates/linkcode-sim/src/private/screen.rs +++ b/crates/linkcode-sim/src/private/screen.rs @@ -555,6 +555,93 @@ fn encode_bgra_jpeg(frame: &RawFrame, quality: f64) -> Option> { } } +/// The measured cost of encoding one framebuffer at a given resolution/quality, with the frame rate +/// it implies (a single reader thread does the encode, so `1000 / avg_ms` is the sustainable ceiling). +pub struct EncodeBench { + pub width: usize, + pub height: usize, + pub quality: f64, + pub avg_ms: f64, + pub best_ms: f64, + pub p95_ms: f64, + pub out_kib: usize, +} + +impl EncodeBench { + /// Sustainable frames per second at the average encode cost. + pub fn fps(&self) -> f64 { + if self.avg_ms > 0.0 { + 1000.0 / self.avg_ms + } else { + 0.0 + } + } + + /// Best-case frames per second (fastest encode observed). + pub fn peak_fps(&self) -> f64 { + if self.best_ms > 0.0 { + 1000.0 / self.best_ms + } else { + 0.0 + } + } +} + +/// Benchmark the JPEG encode hot path (the capture pipeline's single-thread ceiling) on a synthetic +/// BGRA frame of `width`×`height`, timing `iters` encodes at `quality` after a warmup. No simulator +/// is needed — this isolates the CoreGraphics/ImageIO cost that bounds the stream's frame rate. +pub fn bench_encode(width: usize, height: usize, quality: f64, iters: u32) -> Option { + let stride = width * 4; + // A gradient plus a per-pixel wobble so the JPEG has realistic entropy (a flat frame would + // compress trivially and understate the encode cost). + let mut pixels = vec![0u8; stride * height]; + for y in 0..height { + let row = &mut pixels[y * stride..y * stride + width * 4]; + for x in 0..width { + let p = &mut row[x * 4..x * 4 + 4]; + p[0] = (x ^ y) as u8; // B + p[1] = (x.wrapping_add(y)) as u8; // G + p[2] = (x.wrapping_mul(3) ^ y) as u8; // R + p[3] = 0xFF; // A + } + } + let frame = RawFrame { + width, + height, + stride, + pixels, + }; + let q = quality.clamp(0.1, 1.0); + + // Warm up: the first encode pays one-time ImageIO/CoreGraphics setup. + let mut out_len = 0usize; + for _ in 0..3 { + out_len = encode_bgra_jpeg(&frame, q)?.len(); + } + + let mut samples = Vec::with_capacity(iters as usize); + for _ in 0..iters { + let start = std::time::Instant::now(); + let jpeg = encode_bgra_jpeg(&frame, q)?; + samples.push(start.elapsed().as_secs_f64() * 1000.0); + out_len = jpeg.len(); + } + samples.sort_by(|a, b| a.partial_cmp(b).expect("no NaN durations")); + let avg_ms = samples.iter().sum::() / samples.len() as f64; + let best_ms = samples[0]; + let p95_ms = samples[(samples.len() * 95 / 100).min(samples.len() - 1)]; + + Some(EncodeBench { + width, + height, + quality: q, + avg_ms, + best_ms, + p95_ms, + out_kib: out_len / 1024, + }) +} + /// Write a `CGImage` to JPEG via ImageIO at `quality`. /// /// # Safety From d7a75f794f51f011583c512a6d9ba880d4f1ce27 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 23:00:19 +0800 Subject: [PATCH 4/6] feat(sim): default stream to 60fps and document the encode benchmark --- crates/linkcode-sim/PROTOCOL.md | 2 +- crates/linkcode-sim/README.md | 23 ++++++++++++++++++++++- crates/linkcode-sim/src/capture.rs | 2 +- crates/linkcode-sim/src/rpc.rs | 2 +- 4 files changed, 25 insertions(+), 4 deletions(-) diff --git a/crates/linkcode-sim/PROTOCOL.md b/crates/linkcode-sim/PROTOCOL.md index 4ac93ce7..25014402 100644 --- a/crates/linkcode-sim/PROTOCOL.md +++ b/crates/linkcode-sim/PROTOCOL.md @@ -70,7 +70,7 @@ Off macOS, or when SimulatorKit is unavailable, every P1 op fails with `xcodeMis | `tap` | `udid`, `x`, `y` (normalised 0..1) | `{}` | | `swipe` | `udid`, `x0`, `y0`, `x1`, `y1`, `durationMs?` | `{}` | | `button` | `udid`, `button` (`home`/`lock`) | `{}` | -| `streamStart` | `udid`, `fps?` (10), `quality?` (0.6) | `{ streaming, fps }` — JPEG frames then arrive on `STREAM_FRAME`s | +| `streamStart` | `udid`, `fps?` (60), `quality?` (0.6) | `{ streaming, fps }` — JPEG frames then arrive on `STREAM_FRAME`s | | `streamStop` | `udid` | `{}` | Input (`tap`/`swipe`/`button`) runs in the sidecar's main process via a per-udid warmed HID client and is stable. Framebuffer streaming runs in a **crash-isolated worker subprocess**: the sidecar spawns it, reads its frames, and respawns it on the intermittent hard crashes of the private framebuffer path. If the worker crash-loops and gives up, the stream degrades to `simctl io screenshot` frames — slower, but frames never stop and the sidecar never crashes. diff --git a/crates/linkcode-sim/README.md b/crates/linkcode-sim/README.md index 4815453d..0ab4e058 100644 --- a/crates/linkcode-sim/README.md +++ b/crates/linkcode-sim/README.md @@ -1,6 +1,6 @@ # linkcode-sim -The Link Code iOS Simulator host: one long-lived process the daemon spawns to drive Apple's iOS Simulator. P0 wraps the public `xcrun simctl` CLI only (lifecycle, install/launch, screenshot); the streaming/HID phase is CODE-396. The stdio wire contract is [`PROTOCOL.md`](./PROTOCOL.md); the TypeScript client is `@linkcode/sim` (`packages/host/sim`). +The Link Code iOS Simulator host: one long-lived process the daemon spawns to drive Apple's iOS Simulator. P0 wraps the public `xcrun simctl` CLI (lifecycle, install/launch, screenshot); P1 adds live framebuffer streaming (JPEG `STREAM_FRAME`s) and touch/button injection through CoreSimulator/SimulatorKit private frameworks, in a crash-isolated capture worker. The stdio wire contract is [`PROTOCOL.md`](./PROTOCOL.md); the TypeScript client is `@linkcode/sim` (`packages/host/sim`). macOS-only at runtime (it needs Xcode's simulator tooling); it compiles and answers `probe` with a structured `xcodeMissing` error everywhere else, so workspace-wide `cargo test` stays green on any platform. @@ -12,3 +12,24 @@ cargo test -p linkcode-sim --test device_loop -- --ignored # full boot→instal ``` Release binaries are staged by `apps/desktop/scripts/stage-sidecar.mts` into `apps/desktop/sidecar/${arch}` (macOS only) and shipped via electron-builder `extraResources`. + +## Benchmark + +`bench-encode` times the JPEG encode hot path — the per-frame CoreGraphics/ImageIO cost that bounds the framebuffer stream, since a single reader thread does the encode. `1000 / avg_ms` is the sustainable frame-rate ceiling. It needs no simulator: it encodes a synthetic BGRA frame across a resolution/quality sweep. + +```sh +cargo build -p linkcode-sim --release +./target/release/linkcode-sim bench-encode [iters] # default 120 iters/config +``` + +On 2026-07-22 (macOS arm64), 150 iters/config: + +| Resolution | Quality | avg ms | p95 ms | Size | fps (avg) | fps (peak) | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| 1206×2622 (native) | 0.60 | 6.49 | 8.13 | 705K | 154 | 182 | +| 904×1966 | 0.60 | 3.45 | 3.98 | 396K | 290 | 332 | +| 603×1311 | 0.60 | 1.50 | 1.73 | 177K | 665 | 730 | +| 402×874 | 0.60 | 0.77 | 0.95 | 79K | 1294 | 1441 | +| 1206×2622 | 0.30 | 6.39 | 7.74 | 705K | 156 | 181 | + +Takeaways: the native-resolution encode ceiling (~154 fps) leaves ~2.5× headroom over 60 fps; cost is set by pixel count (the DCT), not JPEG quality (0.3 ≈ 0.6); downscaling is the real lever (half resolution ≈ 4× cheaper). The synthetic frame is high-entropy, so the numbers are conservative — a mostly-flat real screen encodes a touch faster. Use it to catch encode regressions and size the fps/resolution budget, not as an absolute. diff --git a/crates/linkcode-sim/src/capture.rs b/crates/linkcode-sim/src/capture.rs index fdb6a6a4..ac2f9c81 100644 --- a/crates/linkcode-sim/src/capture.rs +++ b/crates/linkcode-sim/src/capture.rs @@ -173,7 +173,7 @@ pub fn run_worker() -> ! { let fps: u32 = args .next() .and_then(|a| a.parse().ok()) - .unwrap_or(10) + .unwrap_or(60) .clamp(1, 60); let Some(device) = SimDevice::resolve(&udid) else { diff --git a/crates/linkcode-sim/src/rpc.rs b/crates/linkcode-sim/src/rpc.rs index 0ffcb5bc..f0452e79 100644 --- a/crates/linkcode-sim/src/rpc.rs +++ b/crates/linkcode-sim/src/rpc.rs @@ -80,7 +80,7 @@ pub enum Op { } fn default_fps() -> u32 { - 10 + 60 } fn default_quality() -> f64 { 0.6 From 1cdc6441092c3494a03b8c905c87edf21efc9e31 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 23:15:12 +0800 Subject: [PATCH 5/6] perf(sim): precise frame pacing via mach_wait_until thread::sleep coalesces timers on macOS and overshoots by several ms, so a plain deadline loop drifts below target (60fps landed ~52-56). Pace on the mach absolute clock instead: locks lower rates exactly (30->29.5) and drives 60fps to the full-res pipeline ceiling (~58.5), with no busy-spin. --- crates/linkcode-sim/src/capture.rs | 85 ++++++++++++++++++++++++-- crates/linkcode-sim/src/interactive.rs | 12 ++-- 2 files changed, 86 insertions(+), 11 deletions(-) diff --git a/crates/linkcode-sim/src/capture.rs b/crates/linkcode-sim/src/capture.rs index ac2f9c81..96f2049e 100644 --- a/crates/linkcode-sim/src/capture.rs +++ b/crates/linkcode-sim/src/capture.rs @@ -22,6 +22,84 @@ use std::time::{Duration, Instant}; /// One framebuffer JPEG frame. pub type Frame = Arc>; +/// A drift-free, precise frame pacer (macOS). It advances an absolute deadline by a fixed interval +/// each tick and waits to it with `mach_wait_until`, whose accuracy is well under a millisecond. +/// `thread::sleep` on macOS coalesces timers and overshoots by several ms — enough that even a +/// deadline loop built on it caps 60 fps near 55. `mach_wait_until` does not busy-spin, so locking to +/// the target rate costs no CPU beyond encoding the frames themselves. Times are in mach ticks. +#[cfg(target_os = "macos")] +pub(crate) struct FrameClock { + interval_ticks: u64, + next: u64, +} + +#[cfg(target_os = "macos")] +impl FrameClock { + pub(crate) fn new(fps: u32) -> FrameClock { + let interval_ns = 1_000_000_000u64 / u64::from(fps.max(1)); + FrameClock { + interval_ticks: mach_time::ns_to_ticks(interval_ns), + next: mach_time::now(), + } + } + + /// Wait until the next frame deadline. If a stall put us more than a full frame behind, resync to + /// now instead of emitting a catch-up burst of duplicate frames. + pub(crate) fn tick(&mut self) { + self.next = self.next.wrapping_add(self.interval_ticks); + let now = mach_time::now(); + if self.next > now { + mach_time::wait_until(self.next); + } else if now - self.next > self.interval_ticks { + self.next = now; + } + } +} + +/// Minimal `mach_absolute_time` timebase wrappers for precise pacing (see [`FrameClock`]). +#[cfg(target_os = "macos")] +mod mach_time { + use std::sync::OnceLock; + + #[repr(C)] + struct Timebase { + numer: u32, + denom: u32, + } + + unsafe extern "C" { + fn mach_absolute_time() -> u64; + fn mach_wait_until(deadline: u64) -> i32; + fn mach_timebase_info(info: *mut Timebase) -> i32; + } + + /// `(numer, denom)`: mach ticks → nanoseconds is `ticks * numer / denom`. Constant per boot. + fn timebase() -> (u64, u64) { + static TB: OnceLock<(u64, u64)> = OnceLock::new(); + *TB.get_or_init(|| { + let mut tb = Timebase { numer: 0, denom: 0 }; + // SAFETY: fills a valid out-param; returns KERN_SUCCESS with a nonzero denom on macOS. + unsafe { mach_timebase_info(&mut tb) }; + (u64::from(tb.numer.max(1)), u64::from(tb.denom.max(1))) + }) + } + + pub(super) fn now() -> u64 { + // SAFETY: reads the monotonic mach clock; no preconditions. + unsafe { mach_absolute_time() } + } + + pub(super) fn ns_to_ticks(ns: u64) -> u64 { + let (numer, denom) = timebase(); + (u128::from(ns) * u128::from(denom) / u128::from(numer)) as u64 + } + + pub(super) fn wait_until(deadline_ticks: u64) { + // SAFETY: blocks the calling thread until the absolute mach-time deadline. + unsafe { mach_wait_until(deadline_ticks) }; + } +} + /// Largest accepted worker frame (a JPEG at simulator resolution is well under this). const MAX_WORKER_FRAME: usize = 32 * 1024 * 1024; /// Backoff between worker respawns after a crash. @@ -189,10 +267,9 @@ pub fn run_worker() -> ! { std::process::exit(3); }; - let interval = Duration::from_millis(1000 / u64::from(fps)); + let mut clock = FrameClock::new(fps); let mut stdout = io::stdout().lock(); loop { - let tick = Instant::now(); if let Some(jpeg) = screen.capture_jpeg(quality) { let len = u32::try_from(jpeg.len()).unwrap_or(0); if len == 0 @@ -203,9 +280,7 @@ pub fn run_worker() -> ! { break; // parent closed the pipe } } - if let Some(rest) = interval.checked_sub(tick.elapsed()) { - thread::sleep(rest); - } + clock.tick(); } std::process::exit(0); } diff --git a/crates/linkcode-sim/src/interactive.rs b/crates/linkcode-sim/src/interactive.rs index cb9391e2..c305366d 100644 --- a/crates/linkcode-sim/src/interactive.rs +++ b/crates/linkcode-sim/src/interactive.rs @@ -71,7 +71,7 @@ mod imp { use std::time::{Duration, Instant}; use super::*; - use crate::capture::CaptureStream; + use crate::capture::{CaptureStream, FrameClock}; use crate::private::{self, Button, Input, SimDevice}; use crate::proto::{STREAM_FRAME, encode_stream_frame}; @@ -227,14 +227,16 @@ mod imp { stop: &AtomicBool, tx: &Sender, ) { - let interval = Duration::from_millis(1000 / u64::from(fps)); + // Poll the private stream on a drift-free clock so a locked worker fps reaches the wire + // without the per-frame sleep overshoot that would otherwise sample it below target. + let mut clock = FrameClock::new(fps); // simctl screenshots cost ~200-400ms, so poll them well below the private fps. let fallback_interval = Duration::from_millis(500); let mut last: Option<*const Vec> = None; while !stop.load(Ordering::Relaxed) { - let tick = Instant::now(); if stream.is_dead() { // Degraded path: capture via the public simctl screenshot and push it. + let tick = Instant::now(); if let Ok(jpeg) = crate::simctl::screenshot(udid, crate::rpc::ImageFormat::Jpeg) && let Ok(body) = encode_stream_frame(udid, &jpeg) && tx @@ -267,9 +269,7 @@ mod imp { } } } - if let Some(rest) = interval.checked_sub(tick.elapsed()) { - thread::sleep(rest); - } + clock.tick(); } } } From 84df64e765fe5932c12ab3f633171e0114ffd027 Mon Sep 17 00:00:00 2001 From: AprilNEA Date: Wed, 22 Jul 2026 23:23:57 +0800 Subject: [PATCH 6/6] feat(sim): configurable capture scale (default 1.0) to unlock 60fps Downscale each frame before JPEG encode via a high-quality CoreGraphics resample. Native-resolution encode caps the stream near 55fps; scale below 1.0 lifts it to the display's 60Hz and cuts bandwidth (0.5 -> ~1/3 the bytes, 603x1311 locks 60fps). Threaded from streamStart through StreamParams to the worker; tap/swipe stay normalized so no coordinate change is needed. --- crates/linkcode-sim/PROTOCOL.md | 4 +- crates/linkcode-sim/src/capture.rs | 42 ++++++--- crates/linkcode-sim/src/interactive.rs | 12 ++- crates/linkcode-sim/src/main.rs | 18 +++- crates/linkcode-sim/src/private/screen.rs | 110 +++++++++++++++++++--- crates/linkcode-sim/src/rpc.rs | 5 + 6 files changed, 156 insertions(+), 35 deletions(-) diff --git a/crates/linkcode-sim/PROTOCOL.md b/crates/linkcode-sim/PROTOCOL.md index 25014402..ac2a2b34 100644 --- a/crates/linkcode-sim/PROTOCOL.md +++ b/crates/linkcode-sim/PROTOCOL.md @@ -70,9 +70,11 @@ Off macOS, or when SimulatorKit is unavailable, every P1 op fails with `xcodeMis | `tap` | `udid`, `x`, `y` (normalised 0..1) | `{}` | | `swipe` | `udid`, `x0`, `y0`, `x1`, `y1`, `durationMs?` | `{}` | | `button` | `udid`, `button` (`home`/`lock`) | `{}` | -| `streamStart` | `udid`, `fps?` (60), `quality?` (0.6) | `{ streaming, fps }` — JPEG frames then arrive on `STREAM_FRAME`s | +| `streamStart` | `udid`, `fps?` (60), `quality?` (0.6), `scale?` (1.0) | `{ streaming, fps, scale }` — JPEG frames then arrive on `STREAM_FRAME`s | | `streamStop` | `udid` | `{}` | +`scale` (0.1..1.0) downscales each frame before JPEG encode: at native resolution the encode bounds the frame rate near ~55 fps, so `scale` below 1.0 both lifts the achievable rate toward the display's 60 Hz and cuts bandwidth (e.g. `0.5` ≈ one third the bytes). `tap`/`swipe` stay in normalized 0..1 coordinates, so a downscaled stream needs no coordinate adjustment. + Input (`tap`/`swipe`/`button`) runs in the sidecar's main process via a per-udid warmed HID client and is stable. Framebuffer streaming runs in a **crash-isolated worker subprocess**: the sidecar spawns it, reads its frames, and respawns it on the intermittent hard crashes of the private framebuffer path. If the worker crash-loops and gives up, the stream degrades to `simctl io screenshot` frames — slower, but frames never stop and the sidecar never crashes. ## Result diff --git a/crates/linkcode-sim/src/capture.rs b/crates/linkcode-sim/src/capture.rs index 96f2049e..45d59c4e 100644 --- a/crates/linkcode-sim/src/capture.rs +++ b/crates/linkcode-sim/src/capture.rs @@ -109,6 +109,16 @@ const HEALTHY_AFTER: Duration = Duration::from_secs(5); /// Consecutive fast crashes before the stream gives up and reports unavailable. const MAX_FAST_CRASHES: u32 = 6; +/// Encoder/pacing parameters for a capture stream, threaded from `streamStart` down to the worker +/// (and across the process boundary as CLI args). Clamping happens at the boundary, not here. +#[derive(Clone, Copy)] +pub struct StreamParams { + pub fps: u32, + pub quality: f64, + /// Downscale factor applied before JPEG encode (0..1; 1.0 = native resolution). + pub scale: f64, +} + /// A supervised framebuffer stream for one device. Holds the latest delivered frame; the worker /// subprocess is spawned, read, and respawned by a background manager thread. pub struct CaptureStream { @@ -120,8 +130,8 @@ pub struct CaptureStream { } impl CaptureStream { - /// Start streaming device `udid` at `fps` and JPEG `quality` by supervising a capture worker. - pub fn start(udid: String, quality: f64, fps: u32) -> CaptureStream { + /// Start streaming device `udid` with `params` by supervising a capture worker. + pub fn start(udid: String, params: StreamParams) -> CaptureStream { let latest = Arc::new(Mutex::new(None)); let stopped = Arc::new(AtomicBool::new(false)); let dead = Arc::new(AtomicBool::new(false)); @@ -129,7 +139,7 @@ impl CaptureStream { let latest = Arc::clone(&latest); let stopped = Arc::clone(&stopped); let dead = Arc::clone(&dead); - move || supervise(&udid, quality, fps, &latest, &stopped, &dead) + move || supervise(&udid, params, &latest, &stopped, &dead) }); CaptureStream { latest, @@ -166,8 +176,7 @@ impl Drop for CaptureStream { /// too many fast crashes. fn supervise( udid: &str, - quality: f64, - fps: u32, + params: StreamParams, latest: &Arc>>, stopped: &Arc, dead: &Arc, @@ -175,7 +184,7 @@ fn supervise( let mut fast_crashes = 0u32; while !stopped.load(Ordering::Relaxed) { let started = Instant::now(); - match spawn_worker(udid, quality, fps) { + match spawn_worker(udid, params) { Ok(mut child) => { pump_worker(&mut child, latest, stopped); let _ = child.wait(); @@ -202,13 +211,14 @@ fn supervise( } } -fn spawn_worker(udid: &str, quality: f64, fps: u32) -> io::Result { +fn spawn_worker(udid: &str, params: StreamParams) -> io::Result { let exe = std::env::current_exe()?; Command::new(exe) .arg("capture-worker") .arg(udid) - .arg(format!("{quality}")) - .arg(format!("{fps}")) + .arg(format!("{}", params.quality)) + .arg(format!("{}", params.fps)) + .arg(format!("{}", params.scale)) .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::inherit()) @@ -238,9 +248,10 @@ fn pump_worker(child: &mut Child, latest: &Arc>>, stopped: & } } -/// The worker entry point (`linkcode-sim capture-worker `): open the device's -/// framebuffer and stream length-prefixed JPEG frames to stdout at `fps`. Exits non-zero on any -/// failure; the parent respawns. A hard `SIGABRT` from the private path also just ends this process. +/// The worker entry point (`linkcode-sim capture-worker `): open the +/// device's framebuffer and stream length-prefixed JPEG frames to stdout at `fps`, each downscaled by +/// `scale`. Exits non-zero on any failure; the parent respawns. A hard `SIGABRT` from the private +/// path also just ends this process. #[cfg(target_os = "macos")] pub fn run_worker() -> ! { use crate::private::{Screen, SimDevice}; @@ -253,6 +264,11 @@ pub fn run_worker() -> ! { .and_then(|a| a.parse().ok()) .unwrap_or(60) .clamp(1, 60); + let scale: f64 = args + .next() + .and_then(|a| a.parse::().ok()) + .unwrap_or(1.0) + .clamp(0.1, 1.0); let Some(device) = SimDevice::resolve(&udid) else { eprintln!("sim capture-worker: device {udid} not found"); @@ -270,7 +286,7 @@ pub fn run_worker() -> ! { let mut clock = FrameClock::new(fps); let mut stdout = io::stdout().lock(); loop { - if let Some(jpeg) = screen.capture_jpeg(quality) { + if let Some(jpeg) = screen.capture_jpeg(quality, scale) { let len = u32::try_from(jpeg.len()).unwrap_or(0); if len == 0 || stdout.write_all(&len.to_le_bytes()).is_err() diff --git a/crates/linkcode-sim/src/interactive.rs b/crates/linkcode-sim/src/interactive.rs index c305366d..1a55c2d9 100644 --- a/crates/linkcode-sim/src/interactive.rs +++ b/crates/linkcode-sim/src/interactive.rs @@ -50,6 +50,7 @@ mod stubs { _udid: &str, _fps: u32, _quality: f64, + _scale: f64, _tx: &Sender, ) -> Result { Err(unsupported()) @@ -165,6 +166,7 @@ mod imp { udid: &str, fps: u32, quality: f64, + scale: f64, tx: &Sender, ) -> Result { if !available() { @@ -173,14 +175,18 @@ mod imp { // Warming the HID connection first stabilizes the framebuffer worker's cold open. let _ = input_for(udid); let fps = fps.clamp(1, 60); + let scale = scale.clamp(0.1, 1.0); let mut reg = registry().lock().expect("interactive registry poisoned"); if reg.streams.contains_key(udid) { return Ok(json!({ "alreadyStreaming": true })); } let stream = Arc::new(CaptureStream::start( udid.to_owned(), - quality.clamp(0.1, 1.0), - fps, + crate::capture::StreamParams { + fps, + quality: quality.clamp(0.1, 1.0), + scale, + }, )); let stop = Arc::new(AtomicBool::new(false)); let pusher = thread::spawn({ @@ -198,7 +204,7 @@ mod imp { pusher: Some(pusher), }, ); - Ok(json!({ "streaming": true, "fps": fps })) + Ok(json!({ "streaming": true, "fps": fps, "scale": scale })) } pub fn stream_stop(udid: &str) -> Result { diff --git a/crates/linkcode-sim/src/main.rs b/crates/linkcode-sim/src/main.rs index d49ae82c..a8f4972b 100644 --- a/crates/linkcode-sim/src/main.rs +++ b/crates/linkcode-sim/src/main.rs @@ -140,9 +140,12 @@ fn serve(request: Request, tx: &Sender) { duration_ms, } => interactive::swipe(&udid, x0, y0, x1, y1, duration_ms), Op::Button { udid, button } => interactive::button(&udid, button), - Op::StreamStart { udid, fps, quality } => { - interactive::stream_start(&udid, fps, quality, tx) - } + Op::StreamStart { + udid, + fps, + quality, + scale, + } => interactive::stream_start(&udid, fps, quality, scale, tx), Op::StreamStop { udid } => interactive::stream_stop(&udid), }; match outcome { @@ -195,7 +198,14 @@ fn diag_interactive() { let device = private::SimDevice::resolve(&udid).expect("device not found"); let input = private::Input::warm(&device).expect("HID warm failed"); - let stream = capture::CaptureStream::start(udid.clone(), 0.6, 12); + let stream = capture::CaptureStream::start( + udid.clone(), + capture::StreamParams { + fps: 12, + quality: 0.6, + scale: 1.0, + }, + ); eprintln!("supervised capture stream started; driving input for ~5s"); let start = std::time::Instant::now(); diff --git a/crates/linkcode-sim/src/private/screen.rs b/crates/linkcode-sim/src/private/screen.rs index c3a599c9..61894a4d 100644 --- a/crates/linkcode-sim/src/private/screen.rs +++ b/crates/linkcode-sim/src/private/screen.rs @@ -352,9 +352,31 @@ unsafe extern "C" { ) -> *mut c_void; fn CGBitmapContextCreateImage(context: *mut c_void) -> *mut c_void; fn CGContextRelease(context: *mut c_void); + fn CGContextDrawImage(context: *mut c_void, rect: CGRect, image: *mut c_void); + fn CGContextSetInterpolationQuality(context: *mut c_void, quality: i32); fn CGImageRelease(image: *mut c_void); } +#[repr(C)] +struct CGPoint { + x: f64, + y: f64, +} +#[repr(C)] +struct CGSize { + width: f64, + height: f64, +} +#[repr(C)] +struct CGRect { + origin: CGPoint, + size: CGSize, +} + +/// `kCGInterpolationHigh` — best downscale quality (Lanczos-ish), worth it since the encode of the +/// smaller image is what we saved time on. +const KCG_INTERPOLATION_HIGH: i32 = 3; + // ImageIO (public framework). #[link(name = "ImageIO", kind = "framework")] unsafe extern "C" { @@ -515,22 +537,26 @@ impl Screen { false } - /// The latest copied framebuffer encoded as JPEG at `quality` (0..1), or `None` if no frame has - /// been copied yet. The frame callback did the surface copy on the delivery queue; this reader - /// thread only encodes the owned pixels — the CoreSimulator proxy is never messaged on this path. - pub fn capture_jpeg(&self, quality: f64) -> Option> { + /// The latest copied framebuffer encoded as JPEG at `quality` (0..1), downscaled by `scale` + /// (0..1; 1.0 = native), or `None` if no frame has been copied yet. The frame callback did the + /// surface copy on the delivery queue; this reader thread only encodes the owned pixels — the + /// CoreSimulator proxy is never messaged on this path. + pub fn capture_jpeg(&self, quality: f64, scale: f64) -> Option> { let frame = self.sink.latest()?; - encode_bgra_jpeg(&frame, quality.clamp(0.1, 1.0)) + encode_bgra_jpeg(&frame, quality.clamp(0.1, 1.0), scale.clamp(0.1, 1.0)) } } -/// Encode an owned BGRA [`RawFrame`] to JPEG bytes via CoreGraphics + ImageIO. Runs on the reader -/// thread from owned pixels — no live surface involved. -fn encode_bgra_jpeg(frame: &RawFrame, quality: f64) -> Option> { +/// Encode an owned BGRA [`RawFrame`] to JPEG bytes via CoreGraphics + ImageIO, optionally downscaled +/// by `scale`. Runs on the reader thread from owned pixels — no live surface involved. Downscaling +/// (CoreGraphics resample into a smaller context) cuts the JPEG encode cost roughly by `scale²`, +/// which is the lever for a higher locked frame rate and lower bandwidth. +fn encode_bgra_jpeg(frame: &RawFrame, quality: f64, scale: f64) -> Option> { // SAFETY: pixels back a `width×height`, `stride`-padded BGRA buffer; every CF/CG object created // here is released on all paths. unsafe { let color_space = CGColorSpaceCreateDeviceRGB(); + let bitmap_info = KCG_ALPHA_PREMULTIPLIED_FIRST | KCG_BYTE_ORDER_32_LITTLE; let context = CGBitmapContextCreate( frame.pixels.as_ptr().cast_mut().cast::(), frame.width, @@ -538,23 +564,78 @@ fn encode_bgra_jpeg(frame: &RawFrame, quality: f64) -> Option> { 8, frame.stride, color_space, - KCG_ALPHA_PREMULTIPLIED_FIRST | KCG_BYTE_ORDER_32_LITTLE, + bitmap_info, ); - CGColorSpaceRelease(color_space); if context.is_null() { + CGColorSpaceRelease(color_space); return None; } let image = CGBitmapContextCreateImage(context); CGContextRelease(context); if image.is_null() { + CGColorSpaceRelease(color_space); return None; } - let bytes = encode_cgimage_jpeg(image, quality); + // Optionally resample into a smaller context before encoding. + let scaled = if scale < 0.999 { + downscale_image( + image, + frame.width, + frame.height, + scale, + color_space, + bitmap_info, + ) + } else { + ptr::null_mut() + }; + CGColorSpaceRelease(color_space); + let target = if scaled.is_null() { image } else { scaled }; + let bytes = encode_cgimage_jpeg(target, quality); + if !scaled.is_null() { + CGImageRelease(scaled); + } CGImageRelease(image); bytes } } +/// Draw `image` into a `scale`-sized context and return the resampled `CGImage` (caller releases), or +/// null on failure. `color_space`/`bitmap_info` must match the source so colors are preserved. +/// +/// # Safety +/// `image` must be a live CGImageRef and `color_space` a live CGColorSpaceRef. +unsafe fn downscale_image( + image: *mut c_void, + width: usize, + height: usize, + scale: f64, + color_space: *mut c_void, + bitmap_info: u32, +) -> *mut c_void { + unsafe { + let w = (((width as f64) * scale).round() as usize).max(1); + let h = (((height as f64) * scale).round() as usize).max(1); + // bytes_per_row 0 lets CoreGraphics pick an aligned stride for the allocation it owns. + let ctx = CGBitmapContextCreate(ptr::null_mut(), w, h, 8, 0, color_space, bitmap_info); + if ctx.is_null() { + return ptr::null_mut(); + } + CGContextSetInterpolationQuality(ctx, KCG_INTERPOLATION_HIGH); + let rect = CGRect { + origin: CGPoint { x: 0.0, y: 0.0 }, + size: CGSize { + width: w as f64, + height: h as f64, + }, + }; + CGContextDrawImage(ctx, rect, image); + let scaled = CGBitmapContextCreateImage(ctx); + CGContextRelease(ctx); + scaled + } +} + /// The measured cost of encoding one framebuffer at a given resolution/quality, with the frame rate /// it implies (a single reader thread does the encode, so `1000 / avg_ms` is the sustainable ceiling). pub struct EncodeBench { @@ -613,16 +694,17 @@ pub fn bench_encode(width: usize, height: usize, quality: f64, iters: u32) -> Op }; let q = quality.clamp(0.1, 1.0); - // Warm up: the first encode pays one-time ImageIO/CoreGraphics setup. + // Warm up: the first encode pays one-time ImageIO/CoreGraphics setup. The sweep sizes the frame + // directly, so encode at native scale here. let mut out_len = 0usize; for _ in 0..3 { - out_len = encode_bgra_jpeg(&frame, q)?.len(); + out_len = encode_bgra_jpeg(&frame, q, 1.0)?.len(); } let mut samples = Vec::with_capacity(iters as usize); for _ in 0..iters { let start = std::time::Instant::now(); - let jpeg = encode_bgra_jpeg(&frame, q)?; + let jpeg = encode_bgra_jpeg(&frame, q, 1.0)?; samples.push(start.elapsed().as_secs_f64() * 1000.0); out_len = jpeg.len(); } diff --git a/crates/linkcode-sim/src/rpc.rs b/crates/linkcode-sim/src/rpc.rs index f0452e79..cd148452 100644 --- a/crates/linkcode-sim/src/rpc.rs +++ b/crates/linkcode-sim/src/rpc.rs @@ -74,6 +74,8 @@ pub enum Op { fps: u32, #[serde(default = "default_quality")] quality: f64, + #[serde(default = "default_scale")] + scale: f64, }, /// Stop a running framebuffer stream. StreamStop { udid: String }, @@ -85,6 +87,9 @@ fn default_fps() -> u32 { fn default_quality() -> f64 { 0.6 } +fn default_scale() -> f64 { + 1.0 +} /// Hardware buttons exposable over the wire (extended as the private HID layer grows). #[derive(Clone, Copy, Deserialize)]