From a6cb03bac4cc5d3187b7ac07e8f3ff77085b26aa Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 00:02:23 -0700 Subject: [PATCH 01/26] feat(core): signal output demand from bus lanes, holds, and events Watch channels never announce a new subscriber, so a producer that wants to idle while nothing consumes its output has no way to learn that a consumer arrived short of polling receiver counts. The bus now owns an OutputDemandSignal: every lane subscription, every OutputDemandHold, and every published event notifies it, and output_receiver_count sums the receivers on every latest-value output stream, per-zone canvases included. Holds cover consumers that read published frames by borrowing a sender instead of holding a receiver. The device registry gains a change notification that fires whenever its generation moves, and the backend manager exposes its routed devices with their backend ids. Together these let the render loop sleep on real signals and wake the instant a device or consumer appears. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-core/src/bus/demand.rs | 78 +++++++++++++ crates/hypercolor-core/src/bus/mod.rs | 106 ++++++++++++++++-- .../src/device/manager/mapping.rs | 10 ++ crates/hypercolor-core/src/device/registry.rs | 15 ++- crates/hypercolor-core/tests/bus_tests.rs | 87 ++++++++++++++ crates/hypercolor-core/tests/device_tests.rs | 34 ++++++ 6 files changed, 320 insertions(+), 10 deletions(-) create mode 100644 crates/hypercolor-core/src/bus/demand.rs diff --git a/crates/hypercolor-core/src/bus/demand.rs b/crates/hypercolor-core/src/bus/demand.rs new file mode 100644 index 000000000..dd020270a --- /dev/null +++ b/crates/hypercolor-core/src/bus/demand.rs @@ -0,0 +1,78 @@ +//! Output-demand signalling for the latest-value bus lanes. +//! +//! A producer that publishes onto watch lanes can stop producing while +//! nothing consumes its output, but watch channels never announce a new +//! subscriber. [`OutputDemandSignal`] closes that gap: every lane +//! subscription, every [`OutputDemandHold`], and every discrete event +//! published on the bus notifies it, so an idle producer can sleep on the +//! signal and re-evaluate the moment a consumer appears or observable state +//! changes, instead of polling for either. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + +use tokio::sync::Notify; +use tokio::sync::futures::Notified; + +/// Wake signal for producers that idle while nothing consumes their output. +#[derive(Debug, Default)] +pub struct OutputDemandSignal { + epoch: AtomicU64, + holds: AtomicUsize, + notify: Notify, +} + +impl OutputDemandSignal { + /// Monotonic count of notifications, for observing that one happened. + #[must_use] + pub fn epoch(&self) -> u64 { + self.epoch.load(Ordering::Acquire) + } + + /// Number of live [`OutputDemandHold`]s. + #[must_use] + pub fn hold_count(&self) -> usize { + self.holds.load(Ordering::Acquire) + } + + /// Future that resolves on the next notification. + /// + /// Pin it and call [`Notified::enable`] before re-checking demand: a + /// notification that lands between the check and the await then still + /// wakes the waiter. + pub fn notified(&self) -> Notified<'_> { + self.notify.notified() + } + + /// Wake every current waiter. + pub fn notify(&self) { + self.epoch.fetch_add(1, Ordering::AcqRel); + self.notify.notify_waiters(); + } + + pub(super) fn acquire_hold(self: &Arc) -> OutputDemandHold { + self.holds.fetch_add(1, Ordering::AcqRel); + self.notify(); + OutputDemandHold { + signal: Arc::clone(self), + } + } +} + +/// Keeps output demand registered for consumers that read the bus without +/// holding a lane receiver, such as a borrow of a per-zone sender. +/// +/// Dropping the hold withdraws the demand. Release needs no notification: +/// a producer only loses demand on its own next evaluation. +#[derive(Debug)] +#[must_use = "dropping the hold withdraws the output demand it registers"] +pub struct OutputDemandHold { + signal: Arc, +} + +impl Drop for OutputDemandHold { + fn drop(&mut self) { + let previous = self.signal.holds.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "output demand hold underflow"); + } +} diff --git a/crates/hypercolor-core/src/bus/mod.rs b/crates/hypercolor-core/src/bus/mod.rs index 57f18f567..a04c8e0c2 100644 --- a/crates/hypercolor-core/src/bus/mod.rs +++ b/crates/hypercolor-core/src/bus/mod.rs @@ -9,12 +9,17 @@ //! - **Canvas previews** — latest render and screen-source canvases via `tokio::sync::watch`. //! - **Authoritative scene canvases** — latest full-scene display surface for non-preview consumers. //! - **Per-zone canvases** — latest render-zone canvases via per-zone `tokio::sync::watch`. +//! - **Output demand**: [`OutputDemandSignal`] wakes a producer that idles while +//! nothing consumes its output, whenever a lane gains a receiver, a demand hold is +//! taken, or an event is published. //! //! The bus is `Send + Sync` and cloneable. Channel operations are lock-free; //! low-frequency status-event deduplication uses a short critical section. +mod demand; mod filter; +pub use demand::{OutputDemandHold, OutputDemandSignal}; pub use filter::{EventFilter, FilteredEventReceiver}; use std::collections::HashMap; @@ -632,24 +637,41 @@ pub struct WatchLane { sender: watch::Sender, published: AtomicU64, revision: AtomicU64, + demand: Arc, } impl WatchLane { /// Create a lane with its initial latest value. #[must_use] pub fn new(initial: T) -> Self { + Self::with_demand_signal(initial, Arc::default()) + } + + fn with_demand_signal(initial: T, demand: Arc) -> Self { let (sender, _) = watch::channel(initial); Self { sender, published: AtomicU64::new(0), revision: AtomicU64::new(0), + demand, } } /// Subscribe to the latest value. + /// + /// A new receiver is new output demand, so subscribing notifies the + /// lane's [`OutputDemandSignal`]. #[must_use] pub fn subscribe(&self) -> watch::Receiver { - self.sender.subscribe() + let receiver = self.sender.subscribe(); + self.demand.notify(); + receiver + } + + /// Number of live receivers. + #[must_use] + pub fn receiver_count(&self) -> usize { + self.sender.receiver_count() } /// Borrow the latest published value. @@ -739,16 +761,37 @@ pub struct BusLanes { } impl BusLanes { - fn new() -> Self { + fn new(demand: &Arc) -> Self { Self { - frame: WatchLane::new(FrameData::empty()), - spectrum: WatchLane::new(SpectrumData::empty()), - previews: std::array::from_fn(|_| WatchLane::new(CanvasFrame::empty())), - screen_zones: WatchLane::new(ScreenZonesFrame::default()), - zone_preview: WatchLane::new(Arc::from([])), + frame: WatchLane::with_demand_signal(FrameData::empty(), Arc::clone(demand)), + spectrum: WatchLane::with_demand_signal(SpectrumData::empty(), Arc::clone(demand)), + previews: std::array::from_fn(|_| { + WatchLane::with_demand_signal(CanvasFrame::empty(), Arc::clone(demand)) + }), + screen_zones: WatchLane::with_demand_signal( + ScreenZonesFrame::default(), + Arc::clone(demand), + ), + zone_preview: WatchLane::with_demand_signal(Arc::from([]), Arc::clone(demand)), } } + /// Live receivers across every fixed lane. + #[must_use] + pub fn receiver_count(&self) -> usize { + self.frame + .receiver_count() + .saturating_add(self.spectrum.receiver_count()) + .saturating_add( + self.previews + .iter() + .map(WatchLane::receiver_count) + .fold(0, usize::saturating_add), + ) + .saturating_add(self.screen_zones.receiver_count()) + .saturating_add(self.zone_preview.receiver_count()) + } + #[must_use] pub const fn frame(&self) -> &WatchLane { &self.frame @@ -812,6 +855,9 @@ pub struct HypercolorBus { /// Display-output-authored final display surface metadata keyed by zone id. display_zone_output_routes: Arc>, + /// Notified when output demand appears or observable state changes. + output_demand: Arc, + /// Clock every event publication is stamped against. clock: PublicationClock, } @@ -831,15 +877,17 @@ impl HypercolorBus { #[must_use] pub fn with_clock(clock: PublicationClock) -> Self { let (events, _) = broadcast::channel(EVENT_CHANNEL_CAPACITY); + let output_demand = Arc::new(OutputDemandSignal::default()); Self { events, input_status_events: Arc::new(Mutex::new(HashMap::new())), - lanes: Arc::new(BusLanes::new()), + lanes: Arc::new(BusLanes::new(&output_demand)), zone_canvases: Arc::new(Mutex::new(HashMap::new())), display_zone_targets: Arc::new(Mutex::new(DisplayZoneTargetRegistry::default())), display_zone_output_routes: Arc::new(Mutex::new( DisplayZoneOutputRouteRegistry::default(), )), + output_demand, clock, } } @@ -868,6 +916,9 @@ impl HypercolorBus { }; // Ignore send errors -- they mean no subscribers exist. let _ = self.events.send(timestamped); + // Discrete events mark state changes an idle producer may need to + // render before anyone asks for a frame. + self.output_demand.notify(); } /// Publish a source-health transition when its operational state changed. @@ -1126,7 +1177,44 @@ impl HypercolorBus { /// Subscribe to a zone's canvas updates. #[must_use] pub fn zone_canvas_receiver(&self, id: ZoneId) -> watch::Receiver { - self.zone_canvas_sender(id).subscribe() + let receiver = self.zone_canvas_sender(id).subscribe(); + self.output_demand.notify(); + receiver + } + + /// Live receivers across every per-zone canvas stream. + #[must_use] + pub fn zone_canvas_receiver_count(&self) -> usize { + self.zone_canvases + .lock() + .expect("zone canvas registry should not be poisoned") + .values() + .map(watch::Sender::receiver_count) + .fold(0, usize::saturating_add) + } + + // ── Output demand ──────────────────────────────────────────────── + + /// The signal notified by lane subscriptions, demand holds, and event + /// publication. + #[must_use] + pub fn output_demand(&self) -> &OutputDemandSignal { + &self.output_demand + } + + /// Register output demand for a consumer that reads published frames + /// without holding a lane receiver. + pub fn hold_output_demand(&self) -> OutputDemandHold { + self.output_demand.acquire_hold() + } + + /// Live receivers on every latest-value output stream: the fixed lanes + /// and the per-zone canvas streams. + #[must_use] + pub fn output_receiver_count(&self) -> usize { + self.lanes + .receiver_count() + .saturating_add(self.zone_canvas_receiver_count()) } /// Number of tracked per-zone canvas streams. diff --git a/crates/hypercolor-core/src/device/manager/mapping.rs b/crates/hypercolor-core/src/device/manager/mapping.rs index 69b331867..978ed1ef9 100644 --- a/crates/hypercolor-core/src/device/manager/mapping.rs +++ b/crates/hypercolor-core/src/device/manager/mapping.rs @@ -148,6 +148,16 @@ impl BackendManager { self.device_map.len() } + /// Every routed physical device as `(backend_id, device_id)`. + /// + /// A device bound under several layout ids appears once per binding; + /// callers that count devices deduplicate. + pub fn mapped_device_targets(&self) -> impl Iterator + '_ { + self.device_map + .values() + .map(|mapping| (mapping.backend_id.as_str(), mapping.device_id)) + } + /// Each spatial layout `device_id` with the physical device it routes to. /// /// Anything derived from these bindings stays valid until diff --git a/crates/hypercolor-core/src/device/registry.rs b/crates/hypercolor-core/src/device/registry.rs index 7816c177a..139ad94a3 100644 --- a/crates/hypercolor-core/src/device/registry.rs +++ b/crates/hypercolor-core/src/device/registry.rs @@ -9,7 +9,8 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use hypercolor_driver_api::{DiscoveredDevice, DiscoveryConnectBehavior}; -use tokio::sync::RwLock; +use tokio::sync::futures::Notified; +use tokio::sync::{Notify, RwLock}; use tracing::{debug, info, trace, warn}; use hypercolor_types::device::{ @@ -67,6 +68,7 @@ impl TrackedDevice { pub struct DeviceRegistry { inner: Arc>, generation: Arc, + changed: Arc, } #[derive(Debug, Default)] @@ -152,6 +154,7 @@ impl DeviceRegistry { Self { inner: Arc::new(RwLock::new(RegistryInner::default())), generation: Arc::new(AtomicU64::new(0)), + changed: Arc::new(Notify::new()), } } @@ -561,6 +564,15 @@ impl DeviceRegistry { self.generation.load(Ordering::Relaxed) } + /// Future that resolves the next time [`Self::generation`] moves. + /// + /// Pin it and call [`Notified::enable`] before reading the registry, so + /// a mutation that lands between the read and the await still wakes the + /// waiter. + pub fn changed(&self) -> Notified<'_> { + self.changed.notified() + } + /// Number of devices currently tracked. pub async fn len(&self) -> usize { let inner = self.inner.read().await; @@ -805,6 +817,7 @@ impl DeviceRegistry { fn bump_generation(&self) { self.generation.fetch_add(1, Ordering::Relaxed); + self.changed.notify_waiters(); } } diff --git a/crates/hypercolor-core/tests/bus_tests.rs b/crates/hypercolor-core/tests/bus_tests.rs index 0ec84ddc2..8c8ec7e2e 100644 --- a/crates/hypercolor-core/tests/bus_tests.rs +++ b/crates/hypercolor-core/tests/bus_tests.rs @@ -1534,3 +1534,90 @@ async fn concurrent_input_status_publications_are_generation_ordered() { .all(|pair| pair[0] < pair[1]) ); } + +// ── Output demand ───────────────────────────────────────────────────────── + +/// Arm a waiter on the bus output-demand signal the way an idle producer +/// does, run `trigger`, and report whether the waiter woke. +async fn output_demand_wakes_on(bus: &HypercolorBus, trigger: impl FnOnce()) -> bool { + let notified = bus.output_demand().notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + trigger(); + timeout(Duration::from_millis(200), notified).await.is_ok() +} + +#[tokio::test] +async fn lane_subscriptions_wake_output_demand_waiters() { + let bus = HypercolorBus::new(); + let epoch = bus.output_demand().epoch(); + + let mut receivers = Vec::new(); + assert!(output_demand_wakes_on(&bus, || receivers.push(bus.canvas_receiver())).await); + assert!(output_demand_wakes_on(&bus, || receivers.push(bus.scene_canvas_receiver())).await); + assert!(output_demand_wakes_on(&bus, || receivers.push(bus.screen_canvas_receiver())).await); + let mut frames = None; + assert!(output_demand_wakes_on(&bus, || frames = Some(bus.frame_receiver())).await); + let mut zone_canvas = None; + assert!( + output_demand_wakes_on(&bus, || zone_canvas = + Some(bus.zone_canvas_receiver(ZoneId::new()))) + .await + ); + + assert!(bus.output_demand().epoch() >= epoch + 5); +} + +#[tokio::test] +async fn output_receiver_count_spans_every_output_stream() { + let bus = HypercolorBus::new(); + assert_eq!(bus.output_receiver_count(), 0); + + let frames = bus.frame_receiver(); + let spectrum = bus.spectrum_receiver(); + let canvas = bus.canvas_receiver(); + let scene = bus.scene_canvas_receiver(); + let screen = bus.screen_canvas_receiver(); + let web = bus.web_viewport_canvas_receiver(); + let zones = bus.screen_zones_receiver(); + let previews = bus.zone_preview_receiver(); + let zone_canvas = bus.zone_canvas_receiver(ZoneId::new()); + assert_eq!(bus.output_receiver_count(), 9); + assert_eq!(bus.zone_canvas_receiver_count(), 1); + + drop(( + frames, spectrum, canvas, scene, screen, web, zones, previews, + )); + assert_eq!(bus.output_receiver_count(), 1); + drop(zone_canvas); + assert_eq!(bus.output_receiver_count(), 0); +} + +#[tokio::test] +async fn output_demand_holds_count_until_dropped_and_wake_waiters() { + let bus = HypercolorBus::new(); + let mut hold = None; + + assert!(output_demand_wakes_on(&bus, || hold = Some(bus.hold_output_demand())).await); + assert_eq!(bus.output_demand().hold_count(), 1); + let second = bus.hold_output_demand(); + assert_eq!(bus.output_demand().hold_count(), 2); + + drop(hold); + drop(second); + assert_eq!(bus.output_demand().hold_count(), 0); +} + +#[tokio::test] +async fn published_events_wake_output_demand_waiters() { + let bus = HypercolorBus::new(); + + assert!(output_demand_wakes_on(&bus, || bus.publish(paused_event())).await); +} + +#[tokio::test] +async fn output_demand_waiter_that_is_not_triggered_stays_asleep() { + let bus = HypercolorBus::new(); + + assert!(!output_demand_wakes_on(&bus, || {}).await); +} diff --git a/crates/hypercolor-core/tests/device_tests.rs b/crates/hypercolor-core/tests/device_tests.rs index e7a992022..143ca3620 100644 --- a/crates/hypercolor-core/tests/device_tests.rs +++ b/crates/hypercolor-core/tests/device_tests.rs @@ -405,6 +405,40 @@ async fn registry_add_and_get() { assert_eq!(tracked.state, DeviceState::Known); } +#[tokio::test] +async fn registry_changes_wake_waiters_armed_before_they_land() { + let registry = DeviceRegistry::new(); + let device = mock_device_info("Case Fans"); + let id = device.id; + + let changed = registry.changed(); + tokio::pin!(changed); + changed.as_mut().enable(); + registry.add(device).await; + tokio::time::timeout(Duration::from_millis(200), changed) + .await + .expect("adding a device should wake a waiter armed before it"); + + let changed = registry.changed(); + tokio::pin!(changed); + changed.as_mut().enable(); + assert!(registry.set_state(&id, DeviceState::Connected).await); + tokio::time::timeout(Duration::from_millis(200), changed) + .await + .expect("a state change should wake a waiter armed before it"); + + let changed = registry.changed(); + tokio::pin!(changed); + changed.as_mut().enable(); + assert!(registry.set_state(&id, DeviceState::Connected).await); + assert!( + tokio::time::timeout(Duration::from_millis(100), changed) + .await + .is_err(), + "a no-op state write should not wake waiters" + ); +} + #[tokio::test] async fn registry_add_returns_existing_id() { let registry = DeviceRegistry::new(); From b3fc8b4c1addb8b4d7d83012ae62494b50f0ccea Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 00:02:31 -0700 Subject: [PATCH 02/26] feat(core): let effect renderers park and report warming A Servo page keeps working after the render loop stops asking for frames: its script thread schedules its own animation ticks every 30 ms while requestAnimationFrame callbacks are live, and its timers keep firing, so an unobserved WebGL effect keeps the WebGL thread and the GPU driver busy. EffectRenderer::park lets the render loop quiesce that work. Servo and web viewport renderers queue a fire-and-forget SetThrottled command, which throttles the webview (no self-scheduled ticks, timers clamped to the minimum duration); the next render command unthrottles it as it always has, so resume needs no counterpart. EffectRenderer::is_warming reports a renderer that has not yet shown its current content. Servo tracks a page from load start to its first completed frame, so the render loop can keep a freshly applied effect rendering until it is warm instead of parking on a placeholder. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/effect/builtin/web_viewport.rs | 10 +++ crates/hypercolor-core/src/effect/pool.rs | 15 +++++ .../src/effect/servo/renderer.rs | 26 ++++++++ .../src/effect/servo/renderer/frame_poll.rs | 2 + .../src/effect/servo/renderer/load.rs | 1 + .../src/effect/servo/session.rs | 12 ++++ .../src/effect/servo/worker.rs | 12 ++++ .../src/effect/servo/worker/test_support.rs | 5 ++ .../src/effect/servo/worker_client.rs | 63 +++++++++++++++++++ crates/hypercolor-core/src/effect/traits.rs | 22 +++++++ 10 files changed, 168 insertions(+) diff --git a/crates/hypercolor-core/src/effect/builtin/web_viewport.rs b/crates/hypercolor-core/src/effect/builtin/web_viewport.rs index fa93735bc..3b47f9aa4 100644 --- a/crates/hypercolor-core/src/effect/builtin/web_viewport.rs +++ b/crates/hypercolor-core/src/effect/builtin/web_viewport.rs @@ -286,6 +286,16 @@ impl EffectRenderer for WebViewportRenderer { Ok(()) } + fn park(&mut self) { + // A live web page keeps its own timers and animations running between + // renders; throttling it stops that work until the next render. + if let Some(session) = self.session.as_ref() + && let Err(error) = session.park() + { + tracing::debug!(%error, "Could not throttle the parked web viewport page"); + } + } + fn apply_controls(&mut self, batch: &ControlDeltaBatch<'_>) -> anyhow::Result<()> { for (control_id, value) in batch.changes { match control_id.as_str() { diff --git a/crates/hypercolor-core/src/effect/pool.rs b/crates/hypercolor-core/src/effect/pool.rs index dbf6222aa..efd707836 100644 --- a/crates/hypercolor-core/src/effect/pool.rs +++ b/crates/hypercolor-core/src/effect/pool.rs @@ -241,6 +241,21 @@ impl EffectPool { } } + /// Park every renderer because the render loop stopped requesting + /// frames. See [`EffectRenderer::park`]. + pub fn park(&mut self) { + for slot in self.slots.values_mut() { + slot.renderer.park(); + } + } + + /// Whether any renderer is still warming up its current content. See + /// [`EffectRenderer::is_warming`]. + #[must_use] + pub fn any_warming(&self) -> bool { + self.slots.values().any(|slot| slot.renderer.is_warming()) + } + pub fn remove_zone(&mut self, zone_id: ZoneId) { if self.slots.keys().any(|key| key.zone_id == zone_id) { let next_generation = self.next_generation(); diff --git a/crates/hypercolor-core/src/effect/servo/renderer.rs b/crates/hypercolor-core/src/effect/servo/renderer.rs index bc60f717c..948ed0d28 100644 --- a/crates/hypercolor-core/src/effect/servo/renderer.rs +++ b/crates/hypercolor-core/src/effect/servo/renderer.rs @@ -16,6 +16,7 @@ use hypercolor_types::effect::{ControlKind, EffectMetadata, EffectSource}; use std::collections::HashMap; use std::path::PathBuf; use std::time::Duration; +use tracing::debug; use super::ServoSessionHandle; #[cfg(test)] @@ -80,6 +81,10 @@ pub struct ServoRenderer { animation_cadence: AnimationCadence, host_driven_animation: bool, last_submit_time_secs: Option, + /// Set when a page starts loading and cleared by its first completed + /// frame. A reload keeps the previous page's canvas on screen, so the + /// canvas alone cannot tell whether the current page has rendered. + awaiting_first_frame: bool, producer_role: ServoProducerRole, display_descriptor: Option, } @@ -133,6 +138,7 @@ impl ServoRenderer { animation_cadence: AnimationCadence::MatchRenderLoop, host_driven_animation: false, last_submit_time_secs: None, + awaiting_first_frame: false, producer_role, display_descriptor: None, } @@ -262,6 +268,25 @@ impl EffectRenderer for ServoRenderer { self.display_descriptor = descriptor; } + fn park(&mut self) { + // Servo's script thread schedules its own animation ticks for a page + // with live requestAnimationFrame callbacks and keeps its timers + // running, so an unobserved page keeps drawing WebGL until it is + // throttled. The next render request unthrottles the webview. + if let Some(session) = self.session.as_ref() + && let Err(error) = session.park() + { + debug!(%error, "Could not throttle the parked Servo page"); + } + } + + fn is_warming(&self) -> bool { + self.initialized + && self.load_failed.is_none() + && self.hard_stall.is_none() + && (self.load_task.is_some() || (self.session.is_some() && self.awaiting_first_frame)) + } + fn destroy(&mut self) { self.load_task = None; if let Some(session) = self.session.take() { @@ -302,6 +327,7 @@ impl EffectRenderer for ServoRenderer { self.animation_cadence = AnimationCadence::MatchRenderLoop; self.host_driven_animation = false; self.last_submit_time_secs = None; + self.awaiting_first_frame = false; } } diff --git a/crates/hypercolor-core/src/effect/servo/renderer/frame_poll.rs b/crates/hypercolor-core/src/effect/servo/renderer/frame_poll.rs index ea5d4d371..b65b930cf 100644 --- a/crates/hypercolor-core/src/effect/servo/renderer/frame_poll.rs +++ b/crates/hypercolor-core/src/effect/servo/renderer/frame_poll.rs @@ -49,6 +49,7 @@ impl ServoRenderer { self.clear_stall_state(); self.last_gpu_frame = Some(frame); self.warned_fallback_frame = false; + self.awaiting_first_frame = false; } Ok(Some(EffectRenderOutput::Pending)) => {} Ok(None) => self.update_stall_state(pending, soft_stall_timeout), @@ -75,6 +76,7 @@ impl ServoRenderer { pub(super) fn accept_completed_canvas(&mut self, canvas: Canvas, clear_gpu_frame: bool) { self.clear_stall_state(); + self.awaiting_first_frame = false; self.last_canvas = Some(canvas); self.clear_cached_gpu_frame_if_needed(clear_gpu_frame); self.warned_fallback_frame = false; diff --git a/crates/hypercolor-core/src/effect/servo/renderer/load.rs b/crates/hypercolor-core/src/effect/servo/renderer/load.rs index 1ad304cea..e764ebf89 100644 --- a/crates/hypercolor-core/src/effect/servo/renderer/load.rs +++ b/crates/hypercolor-core/src/effect/servo/renderer/load.rs @@ -162,6 +162,7 @@ impl ServoRenderer { let display_descriptor = (self.producer_role == ServoProducerRole::DisplayFaceHtml) .then(|| self.display_descriptor.clone()) .flatten(); + self.awaiting_first_frame = true; self.load_task = Some(start_servo_load_task( metadata.name.clone(), path.clone(), diff --git a/crates/hypercolor-core/src/effect/servo/session.rs b/crates/hypercolor-core/src/effect/servo/session.rs index 29bc49c85..93697b592 100644 --- a/crates/hypercolor-core/src/effect/servo/session.rs +++ b/crates/hypercolor-core/src/effect/servo/session.rs @@ -273,6 +273,18 @@ impl ServoSessionHandle { } } + /// Throttle this session's page while the render loop requests no + /// frames, so its animation timers and requestAnimationFrame loop stop + /// running unobserved. The next render request unthrottles it. + /// + /// # Errors + /// + /// Returns an error when the worker's control queue cannot accept the + /// command. + pub fn park(&self) -> Result<()> { + self.worker.set_session_throttled(self.session_id, true) + } + pub fn resize(&mut self, width: u32, height: u32) { self.render_width = width.max(1); self.render_height = height.max(1); diff --git a/crates/hypercolor-core/src/effect/servo/worker.rs b/crates/hypercolor-core/src/effect/servo/worker.rs index eaaa48cb4..e8b8da192 100644 --- a/crates/hypercolor-core/src/effect/servo/worker.rs +++ b/crates/hypercolor-core/src/effect/servo/worker.rs @@ -507,6 +507,18 @@ impl ServoWorkerRuntime { record_servo_destroy_wait(destroy_started.elapsed()); let _ = response_tx.send(result); } + ScheduledServoWork::Command(WorkerCommand::SetThrottled { + session_id, + throttled, + }) => { + if let Err(error) = self.set_webview_throttled(session_id, throttled) { + debug!(?session_id, throttled, %error, "Skipped Servo webview throttle"); + } else { + // Deliver the throttle to the page's script thread now + // rather than on the next render, which may be far off. + self.servo.spin_event_loop(); + } + } ScheduledServoWork::Command(WorkerCommand::MemoryReport { response_tx }) => { let result = self.memory_report(); let _ = response_tx.send(result); diff --git a/crates/hypercolor-core/src/effect/servo/worker/test_support.rs b/crates/hypercolor-core/src/effect/servo/worker/test_support.rs index b595a8e4d..3880127b9 100644 --- a/crates/hypercolor-core/src/effect/servo/worker/test_support.rs +++ b/crates/hypercolor-core/src/effect/servo/worker/test_support.rs @@ -69,6 +69,7 @@ pub fn spawn_test_worker() -> (ServoWorker, Arc) { let _ = response_tx.send(()); break; } + WorkerCommand::SetThrottled { .. } => {} WorkerCommand::MemoryReport { response_tx } => { let _ = response_tx.send(Ok(empty_memory_report())); } @@ -170,6 +171,7 @@ pub fn spawn_render_test_worker() -> ( WorkerCommand::LoadUrl { response_tx, .. } => { let _ = response_tx.send(Ok(())); } + WorkerCommand::SetThrottled { .. } => {} WorkerCommand::MemoryReport { response_tx } => { let _ = response_tx.send(Ok(empty_memory_report())); } @@ -251,6 +253,7 @@ pub fn spawn_contended_render_test_worker() -> ContendedRenderTestWorker { let _ = response_tx.send(()); break; } + WorkerCommand::SetThrottled { .. } => {} WorkerCommand::MemoryReport { response_tx } => { let _ = response_tx.send(Ok(empty_memory_report())); } @@ -322,6 +325,7 @@ pub fn spawn_load_test_worker() -> ( let _ = response_tx.send(()); break; } + WorkerCommand::SetThrottled { .. } => {} WorkerCommand::MemoryReport { response_tx } => { let _ = response_tx.send(Ok(empty_memory_report())); } @@ -391,6 +395,7 @@ pub fn spawn_blocking_load_test_worker() -> ( let _ = response_tx.send(()); break; } + WorkerCommand::SetThrottled { .. } => {} WorkerCommand::MemoryReport { response_tx } => { let _ = response_tx.send(Ok(empty_memory_report())); } diff --git a/crates/hypercolor-core/src/effect/servo/worker_client.rs b/crates/hypercolor-core/src/effect/servo/worker_client.rs index c867bc87d..d197b038c 100644 --- a/crates/hypercolor-core/src/effect/servo/worker_client.rs +++ b/crates/hypercolor-core/src/effect/servo/worker_client.rs @@ -199,6 +199,12 @@ pub(super) enum WorkerCommand { session_id: ServoSessionId, response_tx: SyncSender>, }, + /// Throttle or unthrottle one session's webview without waiting for + /// the worker to apply it. + SetThrottled { + session_id: ServoSessionId, + throttled: bool, + }, MemoryReport { response_tx: SyncSender>, }, @@ -799,6 +805,28 @@ impl ServoWorkerClient { Ok(()) } + /// Queue a webview throttle change for one session. + /// + /// Commands are ordered barriers on the worker, so a throttle queued + /// behind an in-flight render applies after that render finishes, and + /// the next render command unthrottles the webview again. + pub(super) fn set_session_throttled( + &self, + session_id: ServoSessionId, + throttled: bool, + ) -> Result<()> { + if !self.has_session(session_id) { + return Ok(()); + } + self.try_send_command( + WorkerCommand::SetThrottled { + session_id, + throttled, + }, + "set-throttled", + ) + } + pub(super) fn memory_report(&self) -> Result { let (response_tx, response_rx) = mpsc::sync_channel(1); self.try_send_command(WorkerCommand::MemoryReport { response_tx }, "memory-report")?; @@ -921,6 +949,41 @@ mod tests { assert_eq!(queued_id, session_id); } + #[test] + fn session_throttle_queues_without_waiting_and_skips_unknown_sessions() { + let (command_tx, command_rx) = mpsc::sync_channel(4); + let client = + ServoWorkerClient::new(command_tx, Arc::new(ServoWorkerClientSharedState::new())); + let session_id = ServoSessionId(11); + + client + .set_session_throttled(session_id, true) + .expect("an unknown session has nothing to throttle"); + assert!(command_rx.try_recv().is_err()); + + client.with_sessions(|sessions| { + sessions.insert( + session_id, + ClientStateSlot::new(ServoProducerRole::SceneHtml), + ); + }); + client + .set_session_throttled(session_id, true) + .expect("throttle should queue without waiting for the worker"); + + let WorkerCommand::SetThrottled { + session_id: queued_id, + throttled, + } = command_rx + .try_recv() + .expect("throttle command should be queued") + else { + panic!("expected a throttle command"); + }; + assert_eq!(queued_id, session_id); + assert!(throttled); + } + #[test] fn render_submission_retains_work_without_blocking_when_command_queue_is_full() { let (command_tx, _command_rx) = mpsc::sync_channel(1); diff --git a/crates/hypercolor-core/src/effect/traits.rs b/crates/hypercolor-core/src/effect/traits.rs index ef65ceb8f..a6d369870 100644 --- a/crates/hypercolor-core/src/effect/traits.rs +++ b/crates/hypercolor-core/src/effect/traits.rs @@ -295,6 +295,28 @@ pub trait EffectRenderer: Send { None } + /// The render loop stopped requesting frames because nothing consumes + /// them. + /// + /// Renderers that keep working between frames on their own, such as a + /// web page whose animation timers fire without being asked, should + /// quiesce that work here. The next render call resumes the renderer, + /// so parking needs no counterpart. The default no-op covers renderers + /// that only work when called. + fn park(&mut self) {} + + /// Whether the renderer has not yet produced output for its current + /// content, for example while a page loads or before its first frame + /// lands. + /// + /// The render loop keeps rendering while any producer warms up, so a + /// renderer that is parked once warm shows its real content the moment + /// a consumer arrives. Renderers that produce output synchronously keep + /// the default. + fn is_warming(&self) -> bool { + false + } + /// Tear down the renderer and release all resources. /// /// Called when the effect transitions to `Destroying`. After this call, From e3cd0d4db6c0989b26d97e88da7d56df9f5da473 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 00:02:41 -0700 Subject: [PATCH 03/26] feat(daemon): park the render loop while nothing consumes frames With an HTML effect active the loop rendered at its full tier whether or not anything received the frames: a Mac with no devices and no client spent about 5 ms of CPU per frame, 31% of a core at 60 fps, on output no one saw. The idle throttle only engaged when no effect ran. The contended resource is CPU and GPU time spent on unobserved frames; no observable cadence changes, because the moment anything can observe a frame the loop renders at its full tier again. This is not a nerf. Before each frame the output gate counts consumers: renderable devices in the registry or routed by the backend manager (simulated displays only while display output holds demand for a preview), receivers on any bus output stream (WebSocket streams, display backdrops, and canvas captures all hold one), stream subscribers to republished host input events, and explicit demand holds. With none, and once every producer is warm for the current scene, effects, and canvas, the loop parks: it withdraws its input capture demand, parks the producers so Servo pages throttle, and sleeps on the bus demand signal, the registry, input demand revisions, and the scene transaction queue, with a one second safety tick. Rigs with a connected device never park. On wake the frame clock restarts one interval back, so effect time steps by a single frame across the park with no catch-up burst. A canvas capture skips the wake frame, whose asynchronous producers still show output requested before the park, and returns the next one. Status and diagnose report render_loop.output_demand with the park state, idle reason, consumer counts, and last wake reason; delivered and actual FPS read zero while parked and frame_liveness passes instead of failing. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/Cargo.toml | 4 + crates/hypercolor-daemon/src/api/system.rs | 9 +- .../hypercolor-daemon/src/canvas_capture.rs | 31 +- .../src/display_output/mod.rs | 32 + .../src/domain/diagnostics.rs | 33 +- crates/hypercolor-daemon/src/lib.rs | 1 + crates/hypercolor-daemon/src/output_demand.rs | 136 ++++ crates/hypercolor-daemon/src/performance.rs | 48 ++ .../hypercolor-daemon/src/preview_runtime.rs | 28 + crates/hypercolor-daemon/src/render_thread.rs | 1 + .../src/render_thread/frame_executor.rs | 54 +- .../src/render_thread/frame_policy.rs | 31 + .../src/render_thread/input_publication.rs | 21 +- .../src/render_thread/output_gate.rs | 411 ++++++++++++ .../src/render_thread/pipeline_driver.rs | 43 +- .../src/render_thread/pipeline_runtime.rs | 2 + .../src/render_thread/render_zones.rs | 10 + .../render_zones/tests/pipeline_acceptance.rs | 11 +- .../src/scene_transactions.rs | 18 +- .../tests/output_demand_tests.rs | 611 ++++++++++++++++++ .../tests/render_thread_tests.rs | 5 + .../hypercolor-tui/tests/rest_client_tests.rs | 1 + crates/hypercolor-types/src/api/diagnose.rs | 4 + crates/hypercolor-types/src/api/system.rs | 42 ++ 24 files changed, 1567 insertions(+), 20 deletions(-) create mode 100644 crates/hypercolor-daemon/src/output_demand.rs create mode 100644 crates/hypercolor-daemon/src/render_thread/output_gate.rs create mode 100644 crates/hypercolor-daemon/tests/output_demand_tests.rs diff --git a/crates/hypercolor-daemon/Cargo.toml b/crates/hypercolor-daemon/Cargo.toml index d7ab0e044..1f88628ea 100644 --- a/crates/hypercolor-daemon/Cargo.toml +++ b/crates/hypercolor-daemon/Cargo.toml @@ -43,6 +43,10 @@ required-features = ["persistence-test-hooks"] name = "render_thread_tests" required-features = ["persistence-test-hooks"] +[[test]] +name = "output_demand_tests" +required-features = ["persistence-test-hooks"] + [[test]] name = "simulator_tests" required-features = ["persistence-test-hooks"] diff --git a/crates/hypercolor-daemon/src/api/system.rs b/crates/hypercolor-daemon/src/api/system.rs index fd8166542..00b6e5013 100644 --- a/crates/hypercolor-daemon/src/api/system.rs +++ b/crates/hypercolor-daemon/src/api/system.rs @@ -158,6 +158,8 @@ async fn system_status_with_privacy( ) }; + let output_demand = performance.output_demand.status(); + // Query the live render loop for timing data. let render_loop_status = { let rl = state.render_loop.read().await; @@ -170,20 +172,23 @@ async fn system_status_with_privacy( } else { 0.0 }; + // A parked loop delivers no frames; `output_demand` says why. + let delivering = snapshot.state == RenderLoopState::Running && !output_demand.parked; RenderLoopStatus { state: snapshot.state.to_string(), fps_tier: snapshot.tier.to_string(), target_fps: snapshot.tier.fps(), ceiling_fps: snapshot.max_tier.fps(), capacity_fps, - delivered_fps: if snapshot.state == RenderLoopState::Running { + delivered_fps: if delivering { round_1(performance.delivered_fps) } else { 0.0 }, - actual_fps: capacity_fps, + actual_fps: if delivering { capacity_fps } else { 0.0 }, consecutive_misses: snapshot.consecutive_misses, total_frames: snapshot.total_frames, + output_demand, } }; let running = render_loop_is_operational(render_loop_status.state.as_str()); diff --git a/crates/hypercolor-daemon/src/canvas_capture.rs b/crates/hypercolor-daemon/src/canvas_capture.rs index 67daf9db0..e8973ce61 100644 --- a/crates/hypercolor-daemon/src/canvas_capture.rs +++ b/crates/hypercolor-daemon/src/canvas_capture.rs @@ -173,7 +173,7 @@ pub async fn capture_canvas( if index > 0 && !request.interval.is_zero() { tokio::time::sleep(request.interval).await; } - let (frame, fresh) = next_frame(&mut receiver).await; + let (frame, fresh) = next_frame(preview, &mut receiver).await; if frame.width == 0 || frame.height == 0 { return Err(DomainError::service_unavailable( "No canvas frame has been rendered yet; apply an effect and capture again", @@ -190,13 +190,28 @@ pub async fn capture_canvas( /// Take the next frame the receiver has not seen yet, waiting for one when /// necessary, and fall back to the latest published frame when none arrives /// in time. -async fn next_frame(receiver: &mut PreviewFrameReceiver) -> (CanvasFrame, bool) { - let fresh = matches!( - tokio::time::timeout(FRESH_FRAME_TIMEOUT, receiver.changed()).await, - Ok(Ok(())) - ); - let frame = receiver.borrow_and_update().clone(); - (frame, fresh) +/// +/// A capture of an idle daemon wakes a parked render loop. The first frame +/// after that wake still composes producer output requested before the +/// park, so the capture waits for the frame after it, which is rendered +/// from the capture's own request. +async fn next_frame( + preview: &PreviewRuntime, + receiver: &mut PreviewFrameReceiver, +) -> (CanvasFrame, bool) { + let deadline = tokio::time::Instant::now() + FRESH_FRAME_TIMEOUT; + let mut published = false; + loop { + let changed = tokio::time::timeout_at(deadline, receiver.changed()).await; + let frame = receiver.borrow_and_update().clone(); + if !matches!(changed, Ok(Ok(()))) { + return (frame, published); + } + published = true; + if !preview.is_render_resume_frame(frame.frame_number) { + return (frame, true); + } + } } async fn encode( diff --git a/crates/hypercolor-daemon/src/display_output/mod.rs b/crates/hypercolor-daemon/src/display_output/mod.rs index c8a0db449..848061f79 100644 --- a/crates/hypercolor-daemon/src/display_output/mod.rs +++ b/crates/hypercolor-daemon/src/display_output/mod.rs @@ -22,6 +22,7 @@ use tracing::{debug, info, warn}; use hypercolor_core::bus::{ CanvasFrame, DisplayZoneFrame, DisplayZoneOutputRoute, DisplayZoneViewport, HypercolorBus, + OutputDemandHold, }; use hypercolor_core::device::{BackendManager, DeviceRegistry}; use hypercolor_core::spatial::is_display_zone; @@ -394,6 +395,12 @@ async fn run_display_output(state: DisplayOutputState, mut shutdown_rx: oneshot: .await; let mut scene_canvas_rx = display_requires_scene_canvas(initial_targets.targets.as_ref()) .then(|| state.preview_runtime.scene_canvas_receiver()); + let mut output_demand_hold = None; + sync_output_demand_hold( + &mut output_demand_hold, + &state.event_bus, + !initial_targets.targets.is_empty(), + ); reconcile_display_workers(&state, &mut workers, initial_targets.targets.as_ref()).await; let mut last_reconciled_target_version = Some(initial_targets.version); let mut last_dispatched_sources = @@ -450,6 +457,11 @@ async fn run_display_output(state: DisplayOutputState, mut shutdown_rx: oneshot: state.preview_runtime.as_ref(), display_requires_scene_canvas(targets.targets.as_ref()), ); + sync_output_demand_hold( + &mut output_demand_hold, + &state.event_bus, + !targets.targets.is_empty(), + ); if targets.targets.is_empty() { last_dispatched_sources.clear(); continue; @@ -596,6 +608,26 @@ fn sync_display_canvas_receiver( } } +/// Hold render output demand while any display target exists. +/// +/// A face rendered straight to its zone is read from the zone's canvas +/// sender, not through a receiver, so the render loop would not count it as +/// a consumer. Physical displays are already counted as devices; this hold +/// is what keeps a previewed simulated display rendering. +fn sync_output_demand_hold( + hold: &mut Option, + event_bus: &HypercolorBus, + has_targets: bool, +) { + if has_targets { + if hold.is_none() { + *hold = Some(event_bus.hold_output_demand()); + } + } else { + let _ = hold.take(); + } +} + fn display_requires_scene_canvas(targets: &[Arc]) -> bool { targets.iter().any(|target| target.requires_scene_canvas()) } diff --git a/crates/hypercolor-daemon/src/domain/diagnostics.rs b/crates/hypercolor-daemon/src/domain/diagnostics.rs index bee1e6f4f..720ea6ec9 100644 --- a/crates/hypercolor-daemon/src/domain/diagnostics.rs +++ b/crates/hypercolor-daemon/src/domain/diagnostics.rs @@ -361,18 +361,42 @@ fn render_checks(inputs: &DiagnosticsInputs) -> Vec { let render_elapsed_ms = inputs.render_elapsed_ms(); let running = inputs.render_loop.running; let render_loop_stats = inputs.render_loop.stats; + let parked = performance.output_demand.parked_since.is_some(); let mut checks = vec![DiagnoseCheck { category: "render".to_owned(), name: "render_loop".to_owned(), status: if running { "pass" } else { "warning" }.to_owned(), - detail: format!( - "state={}, tier={}", - render_loop_stats.state, render_loop_stats.tier - ), + detail: if parked { + format!( + "state={}, tier={}, parked={}", + render_loop_stats.state, + render_loop_stats.tier, + crate::output_demand::IDLE_REASON_NO_OUTPUT_CONSUMERS + ) + } else { + format!( + "state={}, tier={}", + render_loop_stats.state, render_loop_stats.tier + ) + }, }]; if !running { return checks; } + if parked { + // A parked loop renders nothing by design, so frame age and LED + // freshness describe the last frame before the park, not a stall. + checks.push(DiagnoseCheck { + category: "render".to_owned(), + name: "frame_liveness".to_owned(), + status: "pass".to_owned(), + detail: format!( + "parked: nothing consumes rendered frames (no output devices, streams, or captures); the next consumer resumes rendering at the full tier. parks_total={}", + performance.output_demand.parks_total + ), + }); + return checks; + } let (status, detail) = render_frame_liveness_status(performance.latest_frame.as_ref(), render_elapsed_ms); @@ -738,6 +762,7 @@ fn build_render_snapshot( ) -> DiagnoseRenderSnapshot { let pacing = performance.pacing; DiagnoseRenderSnapshot { + output_demand: performance.output_demand.status(), latest_frame: performance .latest_frame .as_ref() diff --git a/crates/hypercolor-daemon/src/lib.rs b/crates/hypercolor-daemon/src/lib.rs index b670715e8..150b6006e 100644 --- a/crates/hypercolor-daemon/src/lib.rs +++ b/crates/hypercolor-daemon/src/lib.rs @@ -41,6 +41,7 @@ pub mod macos_tcc_canary; pub mod mcp; pub mod mdns; pub mod network; +pub mod output_demand; pub mod output_power; pub mod path_migration; pub mod performance; diff --git a/crates/hypercolor-daemon/src/output_demand.rs b/crates/hypercolor-daemon/src/output_demand.rs new file mode 100644 index 000000000..277d2f33b --- /dev/null +++ b/crates/hypercolor-daemon/src/output_demand.rs @@ -0,0 +1,136 @@ +//! Output demand: what consumes the frames the render loop produces. +//! +//! Rendering costs CPU and GPU every frame whether or not anything receives +//! the result. The render loop therefore asks, before each frame, whether +//! any consumer exists, and parks while none does (see +//! `render_thread::output_gate`). This module owns the definition of a +//! consumer so the render thread and the status surfaces count the same +//! things: +//! +//! - a connected device, LED or display, that receives frames: one the +//! device registry reports renderable, or one the backend manager routes +//! frames to. Simulated displays are excluded here: they consume frames +//! only while previewed, and display output holds demand for them then; +//! - a live receiver on any latest-value output stream of the bus: frame +//! data, spectrum, the canvas and scene-canvas previews (WebSocket +//! streams, display backdrops, and canvas captures all subscribe one), +//! screen canvas, screen zones, web viewport, zone previews, and the +//! per-zone canvas streams; +//! - a stream or diagnostic consumer of host input events, which the +//! render loop republishes from the interaction input it routes; +//! - an explicit [`hypercolor_core::bus::OutputDemandHold`], taken by +//! consumers that read published frames without a receiver. + +use std::time::Instant; + +use hypercolor_core::bus::HypercolorBus; +use hypercolor_core::device::DeviceRegistry; +use hypercolor_types::api::system::RenderOutputDemandStatus; +use hypercolor_types::device::DeviceInfo; + +use crate::simulators::SIMULATED_DISPLAY_BACKEND_ID; + +/// Why a parked render loop is producing no frames. +pub const IDLE_REASON_NO_OUTPUT_CONSUMERS: &str = "no_output_consumers"; + +/// Consumers of rendered output observed at one instant. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct OutputConsumers { + /// Connected devices that receive frames. + pub output_devices: u32, + /// Live receivers on the bus output streams. + pub stream_receivers: u32, + /// Stream and diagnostic consumers of republished host input events. + pub input_event_streams: u32, + /// Explicit output-demand holds. + pub demand_holds: u32, +} + +impl OutputConsumers { + /// Whether anything consumes rendered frames. + #[must_use] + pub const fn any(self) -> bool { + self.output_devices > 0 + || self.stream_receivers > 0 + || self.input_event_streams > 0 + || self.demand_holds > 0 + } + + /// Combine the bus-side consumers with counts the caller observed. + #[must_use] + pub fn from_bus(bus: &HypercolorBus, output_devices: u32, input_event_streams: u32) -> Self { + Self { + output_devices, + stream_receivers: saturating_u32(bus.output_receiver_count()), + input_event_streams, + demand_holds: saturating_u32(bus.output_demand().hold_count()), + } + } +} + +/// Whether frames routed to `backend_id` reach a consumer on their own, +/// without anyone watching them. +#[must_use] +pub fn is_output_backend(backend_id: &str) -> bool { + backend_id != SIMULATED_DISPLAY_BACKEND_ID +} + +/// Whether a registry device in `renderable` state consumes frames on its +/// own, without anyone watching it. +#[must_use] +pub fn is_output_device(info: &DeviceInfo, renderable: bool) -> bool { + renderable && is_output_backend(info.output_backend_id()) +} + +/// Count the connected devices that receive frames. +pub async fn output_device_count(registry: &DeviceRegistry) -> u32 { + let count = registry + .list() + .await + .iter() + .filter(|tracked| is_output_device(&tracked.info, tracked.state.is_renderable())) + .count(); + saturating_u32(count) +} + +/// Park bookkeeping the render thread records for status surfaces. +#[derive(Debug, Clone, Copy, Default)] +pub struct OutputDemandTelemetry { + /// When the current park began; `None` while the loop renders. + pub parked_since: Option, + /// Parks since the daemon started. + pub parks_total: u64, + /// What last woke a parked loop. + pub last_wake_reason: Option<&'static str>, + /// The consumers the render loop counted at its latest evaluation: + /// every frame while it renders, every wake signal while it is parked. + pub consumers: OutputConsumers, +} + +impl OutputDemandTelemetry { + /// Project the recorded state onto the status contract. + #[must_use] + pub fn status(self) -> RenderOutputDemandStatus { + let consumers = self.consumers; + let parked_ms = self + .parked_since + .map(|since| u64::try_from(since.elapsed().as_millis()).unwrap_or(u64::MAX)); + RenderOutputDemandStatus { + parked: self.parked_since.is_some(), + idle_reason: self + .parked_since + .map(|_| IDLE_REASON_NO_OUTPUT_CONSUMERS.to_owned()), + parked_ms, + output_devices: consumers.output_devices, + stream_receivers: consumers.stream_receivers, + input_event_streams: consumers.input_event_streams, + demand_holds: consumers.demand_holds, + parks_total: self.parks_total, + last_wake_reason: self.last_wake_reason.map(str::to_owned), + } + } +} + +fn saturating_u32(value: usize) -> u32 { + u32::try_from(value).unwrap_or(u32::MAX) +} diff --git a/crates/hypercolor-daemon/src/performance.rs b/crates/hypercolor-daemon/src/performance.rs index 02df8e2a6..7574214bc 100644 --- a/crates/hypercolor-daemon/src/performance.rs +++ b/crates/hypercolor-daemon/src/performance.rs @@ -3,6 +3,8 @@ use std::collections::VecDeque; use std::time::{Duration, Instant}; +use crate::output_demand::OutputDemandTelemetry; + const FRAME_HISTORY_CAPACITY: usize = 120; const LATENCY_BUCKET_WIDTH_US: u32 = 100; const LATENCY_BUCKET_COUNT: usize = 4096; @@ -306,6 +308,7 @@ pub(crate) struct PerformanceSnapshot { pub full_frame_copy_count_total: u64, pub full_frame_copy_frames_total: u64, pub full_frame_copy_bytes_total: u64, + pub output_demand: OutputDemandTelemetry, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -427,11 +430,55 @@ pub struct PerformanceTracker { full_frame_copy_count_total: u64, full_frame_copy_frames_total: u64, full_frame_copy_bytes_total: u64, + output_demand: OutputDemandTelemetry, } impl PerformanceTracker { + /// Record that the render loop parked because nothing consumes its + /// frames. + /// + /// A loop that wakes and parks again without rendering stays in the + /// same park, so only the first call of a park counts it. The interval + /// history is dropped because a parked loop delivers nothing: delivered + /// FPS then reads zero, and the parked gap stays out of the first + /// interval measured after the loop resumes. + /// + /// Returns whether this call began a new park. + pub(crate) fn note_output_parked(&mut self, at: Instant) -> bool { + self.frame_intervals_us.clear(); + self.last_frame_recorded_at = None; + if self.output_demand.parked_since.is_some() { + return false; + } + self.output_demand.parked_since = Some(at); + self.output_demand.parks_total = self.output_demand.parks_total.saturating_add(1); + true + } + + /// Record what woke a parked loop. The park ends with the next + /// recorded frame. + pub(crate) fn note_output_woken(&mut self, reason: &'static str) { + self.output_demand.last_wake_reason = Some(reason); + } + + /// Record the consumers the render loop counted at its latest + /// evaluation. + pub(crate) fn note_output_consumers( + &mut self, + consumers: crate::output_demand::OutputConsumers, + ) { + self.output_demand.consumers = consumers; + } + + /// Park state the render thread last recorded. + #[must_use] + pub fn output_demand_telemetry(&self) -> OutputDemandTelemetry { + self.output_demand + } + /// Record one completed frame. pub(crate) fn record_frame(&mut self, metrics: &LatestFrameMetrics) { + self.output_demand.parked_since = None; let recorded_at = Instant::now(); if let Some(previous) = self.last_frame_recorded_at.replace(recorded_at) { self.frame_intervals_us.push_back(duration_micros_u64( @@ -550,6 +597,7 @@ impl PerformanceTracker { full_frame_copy_count_total: self.full_frame_copy_count_total, full_frame_copy_frames_total: self.full_frame_copy_frames_total, full_frame_copy_bytes_total: self.full_frame_copy_bytes_total, + output_demand: self.output_demand, } } diff --git a/crates/hypercolor-daemon/src/preview_runtime.rs b/crates/hypercolor-daemon/src/preview_runtime.rs index 3e698685a..bf64f5144 100644 --- a/crates/hypercolor-daemon/src/preview_runtime.rs +++ b/crates/hypercolor-daemon/src/preview_runtime.rs @@ -84,8 +84,14 @@ struct PreviewRuntimeTelemetry { screen_canvas: PreviewObservation, web_viewport_canvas: PreviewObservation, zone_preview: PreviewObservation, + /// Frame number of the first frame after the render loop last woke + /// from a park, tagged with [`RENDER_RESUME_FRAME_SET`]; zero until the + /// loop first wakes. + render_resume_frame: AtomicU64, } +const RENDER_RESUME_FRAME_SET: u64 = 1 << 32; + impl PreviewRuntimeTelemetry { fn preview(&self, kind: PreviewKind) -> &PreviewObservation { match kind { @@ -288,6 +294,28 @@ impl PreviewRuntime { self.interactive_executor.store(None); } + /// Record the frame number the render loop renders first after waking + /// from an output-demand park. + pub fn note_render_resume(&self, frame_number: u32) { + self.telemetry.render_resume_frame.store( + RENDER_RESUME_FRAME_SET | u64::from(frame_number), + Ordering::Release, + ); + } + + /// Whether `frame_number` is the first frame after the most recent wake + /// from a park. + /// + /// Asynchronous producers deliver one tick late, so that frame composes + /// producer output requested before the park. Consumers that need output + /// rendered after they asked, such as canvas captures, wait one more + /// frame. + #[must_use] + pub fn is_render_resume_frame(&self, frame_number: u32) -> bool { + self.telemetry.render_resume_frame.load(Ordering::Acquire) + == RENDER_RESUME_FRAME_SET | u64::from(frame_number) + } + pub fn note_canvas_frame(&self, frame_number: u32, timestamp_ms: u32) { self.telemetry .preview(PreviewKind::Canvas) diff --git a/crates/hypercolor-daemon/src/render_thread.rs b/crates/hypercolor-daemon/src/render_thread.rs index b005018d6..e99c5cfe2 100644 --- a/crates/hypercolor-daemon/src/render_thread.rs +++ b/crates/hypercolor-daemon/src/render_thread.rs @@ -39,6 +39,7 @@ pub mod gpu_device; mod input_publication; mod layer_runtime; mod lighting_feed; +mod output_gate; mod pipeline_driver; mod pipeline_runtime; mod producer_queue; diff --git a/crates/hypercolor-daemon/src/render_thread/frame_executor.rs b/crates/hypercolor-daemon/src/render_thread/frame_executor.rs index 29411c281..f4c9e9305 100644 --- a/crates/hypercolor-daemon/src/render_thread/frame_executor.rs +++ b/crates/hypercolor-daemon/src/render_thread/frame_executor.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; -use tracing::{info, warn}; +use tracing::{debug, info, warn}; use hypercolor_core::bus::{CanvasFrame, DisplayZoneFrame}; use hypercolor_core::device::BackendManager; @@ -14,10 +14,11 @@ use hypercolor_types::session::OffOutputBehavior; use super::frame_composer::{ComposeRequest, compose_frame}; use super::frame_io::{FramePublicationRequest, FramePublicationSurfaces, publish_frame_updates}; use super::frame_metrics::{ActiveFrameMetricsInput, summarize_active_frame}; -use super::frame_policy::{FrameExecution, SkipDecision}; +use super::frame_policy::{FrameExecution, FramePolicy, SkipDecision}; use super::frame_reporting::{FrameCompletionReport, report_active_frame_completion}; use super::frame_sampling::{LedSamplingOutcome, resolve_led_sampling}; use super::frame_throttle::{maybe_idle_throttle, maybe_sleep_throttle}; +use super::output_gate::{GateDecision, ProducerSignature}; use super::pipeline_runtime::PreparedCanvasResize; use super::pipeline_runtime::{ FrameLoopState, OutputFrameSource, OutputReuseKey, PendingSamplingWork, PipelineRuntime, @@ -339,6 +340,11 @@ pub(crate) async fn execute_frame( .frame_policy .complete_throttled_frame(&mut render_loop, Duration::ZERO); } + if let Some(parked) = + gate_output_demand(state, frame_loop, render, &mut runtime.frame_policy).await + { + return parked; + } render.sparkleflinger.wait_for_gpu_frame_budget(); let mut scene_snapshot = build_frame_scene_snapshot( state, @@ -916,6 +922,50 @@ pub(crate) async fn execute_frame( } } +/// Park the loop instead of rendering a frame nothing consumes. +/// +/// The frame-rate inputs (audio, screen, interaction capture) feed only the +/// frames, so their capture demand is withdrawn with them and comes back +/// with the next rendered frame, and the effect producers quiesce work +/// they would otherwise keep doing between frames. +async fn gate_output_demand( + state: &RenderThreadState, + frame_loop: &mut FrameLoopState, + render: &mut RenderCaches, + frame_policy: &mut FramePolicy, +) -> Option { + let consumers = frame_loop + .output_gate + .consumers(state, &frame_loop.input_demands) + .await; + let signature = ProducerSignature::observe(state).await; + let decision = frame_loop.output_gate.decide( + consumers.any(), + signature, + render.render_zone_runtime.producers_warming(), + Instant::now(), + ); + if decision == GateDecision::Render { + return None; + } + + frame_loop.clear_input_demands(); + let _ = render.screen_queue.clear_latest(); + render.sparkleflinger.release_native_screen_caches(); + render.render_zone_runtime.park_producers(); + render.sparkleflinger.end_gpu_frame(); + let began = state + .performance + .write() + .await + .note_output_parked(Instant::now()); + if began { + debug!("Render loop parked: nothing consumes its frames"); + } + let mut render_loop = state.render_loop.write().await; + Some(frame_policy.park_execution(&mut render_loop)) +} + fn publish_input_demands( frame_loop: &mut super::pipeline_runtime::FrameLoopState, effect_demand: super::scene_snapshot::EffectDemand, diff --git a/crates/hypercolor-daemon/src/render_thread/frame_policy.rs b/crates/hypercolor-daemon/src/render_thread/frame_policy.rs index 364cb034d..7b4e8ca43 100644 --- a/crates/hypercolor-daemon/src/render_thread/frame_policy.rs +++ b/crates/hypercolor-daemon/src/render_thread/frame_policy.rs @@ -49,6 +49,11 @@ impl FrameExecution { pub(crate) fn resolve_deadline(self, scheduled_start: Instant, now: Instant) -> Instant { self.next_wake.resolve_deadline(scheduled_start, now) } + + /// Whether the loop parked because nothing consumes its frames. + pub(crate) const fn parks(&self) -> bool { + matches!(self.next_wake, NextWake::Park) + } } /// Scheduler decision for when the next render iteration should begin. @@ -57,6 +62,8 @@ pub(crate) enum NextWake { Interval(Duration), /// Hold the loop for a fixed delay before checking again. Delay(Duration), + /// Sleep until output demand appears; the pipeline driver owns the wait. + Park, } impl NextWake { @@ -64,6 +71,7 @@ impl NextWake { match self { Self::Interval(interval) => advance_deadline(scheduled_start, interval, now), Self::Delay(delay) => now.checked_add(delay).unwrap_or(now), + Self::Park => now, } } } @@ -129,6 +137,15 @@ impl FramePolicy { self.complete_throttled_frame(render_loop, SESSION_SLEEP_THROTTLE_DELAY) } + /// Close the admitted frame without rendering it and park the loop. + pub(crate) fn park_execution(&mut self, render_loop: &mut RenderLoop) -> FrameExecution { + let _ = render_loop.frame_complete(); + FrameExecution { + next_wake: NextWake::Park, + next_skip_decision: SkipDecision::None, + } + } + pub(crate) fn inactive_loop_execution( &self, loop_state: RenderLoopState, @@ -306,6 +323,20 @@ mod tests { assert_eq!(execution.next_skip_decision, SkipDecision::None); } + #[test] + fn park_execution_parks_without_a_skip_decision() { + let mut render_loop = RenderLoop::new(60); + render_loop.start(); + assert!(render_loop.tick()); + let mut policy = FramePolicy::new(FpsTier::Full); + + let execution = policy.park_execution(&mut render_loop); + + assert!(execution.parks()); + assert_eq!(execution.next_skip_decision, SkipDecision::None); + assert_eq!(render_loop.frame_number(), 1); + } + #[test] fn sleep_throttle_execution_uses_sleep_delay() { let mut render_loop = RenderLoop::new(60); diff --git a/crates/hypercolor-daemon/src/render_thread/input_publication.rs b/crates/hypercolor-daemon/src/render_thread/input_publication.rs index 69c365cae..4866b9fe0 100644 --- a/crates/hypercolor-daemon/src/render_thread/input_publication.rs +++ b/crates/hypercolor-daemon/src/render_thread/input_publication.rs @@ -729,6 +729,25 @@ impl InputPublicationDemandHandle { self.snapshot().registration_count(consumer) } + /// Count live stream and diagnostic registrations that request `source`. + /// + /// These consumers render nothing themselves; they read what the + /// authoritative renderer republishes from that source, such as host + /// input events, so they need the renderer running. + #[must_use] + pub fn stream_registrations_requesting(&self, source: SourceKind) -> usize { + self.snapshot() + .entries + .iter() + .filter(|entry| { + matches!( + entry.consumer, + InputPublicationConsumer::PassiveStream | InputPublicationConsumer::Diagnostic + ) && entry.demand.requested_hz(source) > 0 + }) + .count() + } + /// Read the current aggregate rate for one source domain. #[must_use] pub fn requested_hz(&self, source: SourceKind) -> u32 { @@ -751,7 +770,7 @@ impl InputPublicationDemandHandle { self.registry.latest.load_full() } - fn subscribe_revision(&self) -> watch::Receiver { + pub(super) fn subscribe_revision(&self) -> watch::Receiver { self.registry.revision_tx.subscribe() } diff --git a/crates/hypercolor-daemon/src/render_thread/output_gate.rs b/crates/hypercolor-daemon/src/render_thread/output_gate.rs new file mode 100644 index 000000000..ada227576 --- /dev/null +++ b/crates/hypercolor-daemon/src/render_thread/output_gate.rs @@ -0,0 +1,411 @@ +//! Output-demand gate: park the render loop while nothing consumes frames. +//! +//! Every frame costs producer, compositor, and GPU work, so a loop that +//! renders with no device, display, preview, or capture attached spends a +//! fixed per-frame cost on output no one receives. The gate removes that +//! cost without changing any observable cadence: before each frame it +//! counts consumers (see [`crate::output_demand`]), and with none it parks +//! the loop instead of rendering. A parked loop sleeps on the bus output +//! demand signal, the device registry, the input demand registry, and the +//! scene transaction queue, so the first consumer to appear wakes it at +//! once and it resumes at its full tier. A long safety tick re-evaluates in +//! case some consumer arrived without signalling. +//! +//! Parking waits for the producers to settle first. After the scene, the +//! effect catalog, or the canvas changes, the loop keeps rendering until +//! every producer is warm, so an effect applied while nobody watches is +//! loaded and showing its own content by the time a consumer arrives. + +use std::time::{Duration, Instant}; + +use hypercolor_core::input::SourceKind; +use tokio::pin; + +use super::RenderThreadState; +use super::input_publication::InputPublicationDemandHandle; +use super::pipeline_runtime::FrameLoopState; +use crate::output_demand::{OutputConsumers, is_output_backend, output_device_count}; + +/// How often a parked loop re-evaluates demand without any wake signal. +pub(crate) const PARKED_SAFETY_TICK: Duration = Duration::from_secs(1); + +/// Consecutive frame starts with no producer warming before the loop may +/// park. The first frame after a change reconciles the producers, so one +/// clean start alone does not prove a new producer is warm. +const SETTLE_WARM_FRAMES: u32 = 3; + +/// Longest the loop keeps rendering for producers that never finish +/// warming, such as a page that fails to load. Matches the Servo page-load +/// timeout. +const SETTLE_LIMIT: Duration = Duration::from_secs(10); + +/// Everything the producers render from, so a change to any of it is work +/// the loop must render before it parks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ProducerSignature { + scene_generation: u64, + zones_revision: u64, + effect_registry_generation: u64, + zone_layout_preview_generation: u64, + canvas_width: u32, + canvas_height: u32, +} + +impl ProducerSignature { + pub(crate) async fn observe(state: &RenderThreadState) -> Self { + let (scene_generation, zones_revision) = { + let plan = state.scene_plan.load(); + (plan.generation, plan.zones_revision) + }; + let effect_registry_generation = state.effect_registry.read().await.generation(); + Self { + scene_generation, + zones_revision, + effect_registry_generation, + zone_layout_preview_generation: state.zone_layout_previews.generation(), + canvas_width: state.canvas_dims.width(), + canvas_height: state.canvas_dims.height(), + } + } +} + +/// Whether the next frame renders or the loop parks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum GateDecision { + Render, + Park, +} + +/// Why a parked loop woke. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RenderWakeReason { + /// A device, stream receiver, or demand hold appeared. + OutputConsumer, + /// A queued scene transaction has a caller waiting on it. + SceneTransaction, + /// The scene, effect catalog, or canvas changed, so the producers must + /// settle on the new state. + ProducerState, + /// The loop was paused or stopped, which the main loop handles. + RenderLoopState, +} + +impl RenderWakeReason { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::OutputConsumer => "output_consumer", + Self::SceneTransaction => "scene_transaction", + Self::ProducerState => "producer_state", + Self::RenderLoopState => "render_loop_state", + } + } +} + +/// Run of frame starts under one producer signature. +#[derive(Clone, Copy, Debug)] +struct WarmStreak { + signature: ProducerSignature, + warm_frames: u32, + since: Instant, +} + +/// Render-thread state for the output-demand gate. +#[derive(Debug, Default)] +pub(crate) struct OutputGate { + /// Registry output device count keyed by the registry generation it + /// was read at. + registry_devices: Option<(u64, u32)>, + /// Consumers last recorded into render telemetry. + recorded: Option, + streak: Option, +} + +impl OutputGate { + /// Count consumers and record them for status surfaces when they + /// changed. + /// + /// The device registry is read only when its generation moved. A rig + /// with a registered device stops there; otherwise the backend + /// manager's routes are counted too, so a device mapped for output + /// counts even before the registry reports it renderable. + pub(crate) async fn consumers( + &mut self, + state: &RenderThreadState, + input_demands: &InputPublicationDemandHandle, + ) -> OutputConsumers { + let generation = state.device_registry.generation(); + let registry_devices = match self.registry_devices { + Some((counted_at, count)) if counted_at == generation => count, + _ => { + // A mutation racing the count leaves the cache keyed to the + // older generation, so the next evaluation counts again. + let count = output_device_count(&state.device_registry).await; + self.registry_devices = Some((generation, count)); + count + } + }; + let output_devices = if registry_devices > 0 { + registry_devices + } else { + routed_output_device_count(state).await + }; + let input_event_streams = + u32::try_from(input_demands.stream_registrations_requesting(SourceKind::Interaction)) + .unwrap_or(u32::MAX); + let consumers = + OutputConsumers::from_bus(&state.event_bus, output_devices, input_event_streams); + if self.recorded != Some(consumers) { + self.recorded = Some(consumers); + state + .performance + .write() + .await + .note_output_consumers(consumers); + } + consumers + } + + /// Decide whether this frame renders. + /// + /// `producers_warming` describes the producers as the previous frame + /// left them. Frames that consumers drive count toward settling too, so + /// losing the last consumer parks at once when the producers are warm. + pub(crate) fn decide( + &mut self, + consumed: bool, + signature: ProducerSignature, + producers_warming: bool, + now: Instant, + ) -> GateDecision { + let streak = match &mut self.streak { + Some(streak) if streak.signature == signature => streak, + _ => self.streak.insert(WarmStreak { + signature, + warm_frames: 0, + since: now, + }), + }; + streak.warm_frames = if producers_warming { + 0 + } else { + streak.warm_frames.saturating_add(1) + }; + if consumed { + return GateDecision::Render; + } + let settled = streak.warm_frames >= SETTLE_WARM_FRAMES + || now.saturating_duration_since(streak.since) >= SETTLE_LIMIT; + if settled { + GateDecision::Park + } else { + GateDecision::Render + } + } + + /// Why a parked loop should resume now, if it should. + pub(crate) async fn resume_reason( + &mut self, + state: &RenderThreadState, + input_demands: &InputPublicationDemandHandle, + ) -> Option { + if !state.render_loop.read().await.is_running() { + return Some(RenderWakeReason::RenderLoopState); + } + if state.scene_transactions.has_pending() { + return Some(RenderWakeReason::SceneTransaction); + } + if self.consumers(state, input_demands).await.any() { + return Some(RenderWakeReason::OutputConsumer); + } + let signature = ProducerSignature::observe(state).await; + if self + .streak + .is_none_or(|streak| streak.signature != signature) + { + return Some(RenderWakeReason::ProducerState); + } + None + } +} + +/// Count devices the backend manager routes frames to, simulated displays +/// excluded, each counted once however many layout ids bind it. +/// +/// Rigs carry a handful of routes, so the quadratic dedupe stays cheaper +/// than allocating a set on every frame. +async fn routed_output_device_count(state: &RenderThreadState) -> u32 { + let manager = state.backend_manager.lock().await; + let routed = || { + manager + .mapped_device_targets() + .filter(|(backend_id, _)| is_output_backend(backend_id)) + .map(|(_, device_id)| device_id) + }; + let count = routed() + .enumerate() + .filter(|(index, device_id)| !routed().take(*index).any(|seen| seen == *device_id)) + .count(); + u32::try_from(count).unwrap_or(u32::MAX) +} + +/// Sleep until a parked loop has a reason to run a frame. +/// +/// Each signal is armed before demand is re-evaluated, so a consumer that +/// appears between the evaluation and the sleep still wakes the loop. +pub(crate) async fn wait_for_output_demand( + state: &RenderThreadState, + frame_loop: &mut FrameLoopState, +) -> RenderWakeReason { + loop { + let output_demand = state.event_bus.output_demand().notified(); + let devices = state.device_registry.changed(); + let transactions = state.scene_transactions.queued(); + let mut input_demand_revisions = frame_loop.input_demands.subscribe_revision(); + pin!(output_demand, devices, transactions); + output_demand.as_mut().enable(); + devices.as_mut().enable(); + transactions.as_mut().enable(); + + if let Some(reason) = frame_loop + .output_gate + .resume_reason(state, &frame_loop.input_demands) + .await + { + return reason; + } + + tokio::select! { + () = &mut output_demand => {} + () = &mut devices => {} + () = &mut transactions => {} + _ = input_demand_revisions.changed() => {} + () = tokio::time::sleep(PARKED_SAFETY_TICK) => {} + } + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use super::{GateDecision, OutputGate, ProducerSignature, SETTLE_LIMIT}; + + fn signature(scene_generation: u64) -> ProducerSignature { + ProducerSignature { + scene_generation, + zones_revision: 1, + effect_registry_generation: 1, + zone_layout_preview_generation: 0, + canvas_width: 640, + canvas_height: 480, + } + } + + #[test] + fn unconsumed_loop_renders_until_producers_settle_then_parks() { + let mut gate = OutputGate::default(); + let now = Instant::now(); + + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Park + ); + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Park + ); + } + + #[test] + fn warming_producer_holds_the_loop_awake_until_it_warms() { + let mut gate = OutputGate::default(); + let now = Instant::now(); + + for _ in 0..8 { + assert_eq!( + gate.decide(false, signature(1), true, now), + GateDecision::Render + ); + } + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Park + ); + } + + #[test] + fn producer_that_never_warms_parks_after_the_settle_limit() { + let mut gate = OutputGate::default(); + let start = Instant::now(); + + assert_eq!( + gate.decide(false, signature(1), true, start), + GateDecision::Render + ); + let late = start + SETTLE_LIMIT + Duration::from_millis(1); + assert_eq!( + gate.decide(false, signature(1), true, late), + GateDecision::Park + ); + } + + #[test] + fn consumers_always_render_and_count_toward_settling() { + let mut gate = OutputGate::default(); + let now = Instant::now(); + + for _ in 0..4 { + assert_eq!( + gate.decide(true, signature(1), false, now), + GateDecision::Render + ); + } + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Park, + "producers already warm under consumers should park as soon as the last consumer leaves" + ); + } + + #[test] + fn producer_change_while_unconsumed_settles_again() { + let mut gate = OutputGate::default(); + let now = Instant::now(); + for _ in 0..3 { + let _ = gate.decide(false, signature(1), false, now); + } + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Park + ); + + assert_eq!( + gate.decide(false, signature(2), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(2), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(2), false, now), + GateDecision::Park + ); + } +} diff --git a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs index 4097308d3..2c82fab36 100644 --- a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs +++ b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs @@ -5,10 +5,11 @@ use tracing::{debug, info}; use crate::deadline::wait_until_deadline; use hypercolor_core::engine::RenderLoopState; -use super::RenderThreadState; use super::frame_executor::{execute_frame, service_scene_transactions}; use super::frame_policy::SkipDecision; +use super::output_gate::{RenderWakeReason, wait_for_output_demand}; use super::pipeline_runtime::PipelineRuntime; +use super::{RenderThreadState, u64_to_u32}; pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: PipelineRuntime) { info!( @@ -54,6 +55,13 @@ pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: Pipeline runtime.frame_loop.clock.rebase(scheduled_start); } let frame = execute_frame(&state, &mut runtime, scheduled_start, skip_decision).await; + if frame.parks() { + let reason = wait_for_output_demand(&state, &mut runtime.frame_loop).await; + resume_from_park(&state, &mut runtime, reason).await; + skip_decision = SkipDecision::None; + next_frame_at = Instant::now(); + continue; + } skip_decision = frame.next_skip_decision; next_frame_at = frame.resolve_deadline(scheduled_start, Instant::now()); } @@ -61,6 +69,39 @@ pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: Pipeline info!("render pipeline exited"); } +/// Pick the loop back up after a park without a jump in effect time. +/// +/// Effects advance by the frame clock's delta, so the clock restarts one +/// target interval back: the first frame after the wake steps effect time +/// forward by exactly one frame instead of by the whole parked span, and no +/// catch-up frames follow. Capture freshness skips the wake frame itself, +/// whose asynchronous producers still show output requested before the +/// park. +async fn resume_from_park( + state: &RenderThreadState, + runtime: &mut PipelineRuntime, + reason: RenderWakeReason, +) { + let now = Instant::now(); + let (interval, resume_frame) = { + let render_loop = state.render_loop.read().await; + (render_loop.target_interval(), render_loop.frame_number()) + }; + runtime + .frame_loop + .clock + .rebase(now.checked_sub(interval).unwrap_or(now)); + state + .preview_runtime + .note_render_resume(u64_to_u32(resume_frame)); + state + .performance + .write() + .await + .note_output_woken(reason.as_str()); + debug!(reason = reason.as_str(), "Render loop woke from a park"); +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct RenderTickSnapshot { should_render: bool, diff --git a/crates/hypercolor-daemon/src/render_thread/pipeline_runtime.rs b/crates/hypercolor-daemon/src/render_thread/pipeline_runtime.rs index 908d2e9fb..4d4ddd9d4 100644 --- a/crates/hypercolor-daemon/src/render_thread/pipeline_runtime.rs +++ b/crates/hypercolor-daemon/src/render_thread/pipeline_runtime.rs @@ -1228,6 +1228,7 @@ pub(crate) struct FrameLoopState { pub(crate) clock: FrameClockState, pub(crate) inputs: InputReuseState, pub(crate) throttle: ThrottleState, + pub(crate) output_gate: super::output_gate::OutputGate, pub(crate) publication_cadence: PublicationCadenceState, pub(crate) output_reuse: OutputReuseState, pub(crate) lighting_feed: super::lighting_feed::LightingFeedState, @@ -2355,6 +2356,7 @@ impl PipelineRuntime { clock: FrameClockState::default(), inputs: InputReuseState::with_routing(input_reader, interaction_routing), throttle: ThrottleState::default(), + output_gate: super::output_gate::OutputGate::default(), publication_cadence: PublicationCadenceState::default(), output_reuse: OutputReuseState::default(), lighting_feed: super::lighting_feed::LightingFeedState::default(), diff --git a/crates/hypercolor-daemon/src/render_thread/render_zones.rs b/crates/hypercolor-daemon/src/render_thread/render_zones.rs index 981341c3b..b69ae5e60 100644 --- a/crates/hypercolor-daemon/src/render_thread/render_zones.rs +++ b/crates/hypercolor-daemon/src/render_thread/render_zones.rs @@ -349,6 +349,16 @@ impl ZoneRuntime { Ok(runtime) } + /// Park every effect producer while the render loop requests no frames. + pub(crate) fn park_producers(&mut self) { + self.effect_pool.park(); + } + + /// Whether any effect producer has not yet shown its current content. + pub(crate) fn producers_warming(&self) -> bool { + self.effect_pool.any_warming() + } + pub(crate) fn drain_layer_runtime_events(&mut self) -> Vec { self.layer_runtime.drain_events() } diff --git a/crates/hypercolor-daemon/src/render_thread/render_zones/tests/pipeline_acceptance.rs b/crates/hypercolor-daemon/src/render_thread/render_zones/tests/pipeline_acceptance.rs index 64e37b664..c02114ce4 100644 --- a/crates/hypercolor-daemon/src/render_thread/render_zones/tests/pipeline_acceptance.rs +++ b/crates/hypercolor-daemon/src/render_thread/render_zones/tests/pipeline_acceptance.rs @@ -50,6 +50,10 @@ struct SteppedPipeline { state: RenderThreadState, /// Boxed so the futures that hold the pipeline stay small. runtime: Box, + /// The harness reads each published frame off the frame lane without a + /// receiver, so it holds output demand to keep the pipeline rendering + /// whether or not a case attaches a device. + _output_demand: hypercolor_core::bus::OutputDemandHold, } impl SteppedPipeline { @@ -101,7 +105,12 @@ impl SteppedPipeline { .await .expect("test scenes should commit"); state.render_loop.write().await.start(); - Some(Self { state, runtime }) + let output_demand = state.event_bus.hold_output_demand(); + Some(Self { + state, + runtime, + _output_demand: output_demand, + }) } async fn activate(&self, scene_id: SceneId) { diff --git a/crates/hypercolor-daemon/src/scene_transactions.rs b/crates/hypercolor-daemon/src/scene_transactions.rs index a7a3af801..17649e4c0 100644 --- a/crates/hypercolor-daemon/src/scene_transactions.rs +++ b/crates/hypercolor-daemon/src/scene_transactions.rs @@ -4,7 +4,8 @@ use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Mutex as StdMutex}; use thiserror::Error; -use tokio::sync::{Mutex, OwnedMutexGuard, oneshot}; +use tokio::sync::futures::Notified; +use tokio::sync::{Mutex, Notify, OwnedMutexGuard, oneshot}; use hypercolor_core::spatial::{SpatialEngine, SpatialPlanError}; use hypercolor_types::scene::{SceneId, UnassignedBehavior, Zone}; @@ -320,6 +321,9 @@ struct SceneTransactionQueueState { #[derive(Clone)] pub struct SceneTransactionQueue { inner: Arc>, + /// Wakes a render loop that parked because nothing consumed its frames; + /// a queued transaction has a caller waiting on it. + queued: Arc, scene_activation_lock: Arc>, layout_update_lock: Arc>, } @@ -575,6 +579,7 @@ impl SceneTransactionQueue { pub fn new() -> Self { Self { inner: Arc::default(), + queued: Arc::default(), scene_activation_lock: Arc::default(), layout_update_lock: Arc::default(), } @@ -592,9 +597,20 @@ impl SceneTransactionQueue { return Err(LayoutTransactionRejection::RendererStopped); } state.pending.push_back(transaction); + drop(state); + self.queued.notify_waiters(); Ok(()) } + /// Future that resolves the next time a transaction is queued. + /// + /// Pin it and call [`Notified::enable`] before checking + /// [`Self::has_pending`], so a push that lands between the check and the + /// await still wakes the waiter. + pub(crate) fn queued(&self) -> Notified<'_> { + self.queued.notified() + } + fn submit_layout_preparation( &self, spatial_engine: SpatialEngine, diff --git a/crates/hypercolor-daemon/tests/output_demand_tests.rs b/crates/hypercolor-daemon/tests/output_demand_tests.rs new file mode 100644 index 000000000..90c219fed --- /dev/null +++ b/crates/hypercolor-daemon/tests/output_demand_tests.rs @@ -0,0 +1,611 @@ +//! Integration tests for output-demand gating. +//! +//! The render loop parks while nothing consumes its frames and resumes at +//! its full tier the moment a consumer appears. These tests drive the real +//! render thread and prove each consumer kind keeps or wakes it, that a +//! capture after a park returns a freshly rendered frame, and that effect +//! time carries across the park without a jump. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::sync::{Mutex, RwLock, watch}; + +use hypercolor_core::asset::AssetLibrary; +use hypercolor_core::bus::HypercolorBus; +use hypercolor_core::device::mock::{MockDeviceBackend, MockDeviceConfig}; +use hypercolor_core::device::{BackendManager, DeviceRegistry}; +use hypercolor_core::effect::{EffectRegistry, builtin::register_builtin_effects}; +use hypercolor_core::engine::{FpsTier, RenderLoop}; +use hypercolor_core::input::InputManager; +use hypercolor_core::scene::SceneManager; +use hypercolor_core::spatial::SpatialEngine; +use hypercolor_daemon::SceneTransactionQueue; +use hypercolor_daemon::canvas_capture::{CanvasCaptureRequest, CanvasImageFormat, capture_canvas}; +use hypercolor_daemon::domain::scene::SceneService; +use hypercolor_daemon::domain::spatial::SpatialService; +use hypercolor_daemon::output_demand::{OutputDemandTelemetry, is_output_device}; +use hypercolor_daemon::output_power::OutputPowerState; +use hypercolor_daemon::performance::PerformanceTracker; +use hypercolor_daemon::preview_runtime::PreviewRuntime; +use hypercolor_daemon::render_thread::{CanvasDims, RenderThread, RenderThreadState}; +use hypercolor_daemon::simulators::SimulatedDisplayExt; +use hypercolor_types::api::simulators::SimulatedDisplay; +use hypercolor_types::config::RenderAccelerationMode; +use hypercolor_types::control::ControlValue; +use hypercolor_types::device::{DeviceId, DeviceInfo, DeviceState}; +use hypercolor_types::effect::EffectMetadata; +use hypercolor_types::event::FrameData; +use hypercolor_types::spatial::{ + EdgeBehavior, LedTopology, NormalizedPosition, Output, SamplingMode, SpatialLayout, +}; + +/// Deadline for liveness waits. Generous on purpose: these bound hangs, +/// they do not assert latency. +const WAIT_DEADLINE: Duration = Duration::from_secs(10); + +/// Upper bound on how long an event-driven wake may take. The parked +/// safety tick is a full second, so a wake inside this bound came from the +/// demand signal, not the tick. +const WAKE_LATENCY_BOUND: Duration = Duration::from_millis(500); + +/// Window used to prove a running loop renders at its tier rather than +/// trickling. +const RATE_WINDOW: Duration = Duration::from_millis(600); + +/// Frames a 60 fps loop must render inside [`RATE_WINDOW`]. Well below the +/// 36 an unloaded machine renders, so a busy test host does not flake, and +/// far above anything a parked or tick-driven loop produces. +const MIN_FRAMES_IN_RATE_WINDOW: u64 = 15; + +fn test_layout(zones: Vec) -> SpatialLayout { + SpatialLayout { + id: "output-demand".into(), + name: "Output Demand".into(), + description: None, + canvas_width: 320, + canvas_height: 200, + zones, + default_sampling_mode: SamplingMode::Bilinear, + default_edge_behavior: EdgeBehavior::Clamp, + version: 1, + } +} + +fn point_zone(id: &str, device_id: &str, x: f32, y: f32) -> Output { + Output { + id: id.into(), + name: id.into(), + device_id: device_id.into(), + zone_name: None, + position: NormalizedPosition { x, y }, + size: NormalizedPosition { x: 0.2, y: 0.2 }, + rotation: 0.0, + scale: 1.0, + orientation: None, + topology: LedTopology::Point, + led_positions: Vec::new(), + led_mapping: None, + sampling_mode: None, + edge_behavior: None, + shape: None, + shape_preset: None, + display_order: 0, + attachment: None, + brightness: None, + } +} + +fn builtin_effect_registry() -> EffectRegistry { + let mut registry = EffectRegistry::new(Vec::new()); + register_builtin_effects(&mut registry); + registry +} + +fn builtin_effect_metadata(stem: &str) -> EffectMetadata { + builtin_effect_registry() + .iter() + .find_map(|(_, entry)| { + (entry.metadata.source.source_stem() == Some(stem)).then_some(entry.metadata.clone()) + }) + .expect("builtin effect should exist") +} + +fn solid_color_controls(r: u8, g: u8, b: u8) -> HashMap { + HashMap::from([( + "color".into(), + ControlValue::linear_color([ + f32::from(r) / 255.0, + f32::from(g) / 255.0, + f32::from(b) / 255.0, + 1.0, + ]), + )]) +} + +fn test_asset_library() -> Arc> { + let asset_tempdir = tempfile::tempdir().expect("test asset tempdir should be created"); + let asset_dir = asset_tempdir.path().join("assets"); + Arc::new(RwLock::new( + AssetLibrary::open(asset_dir).expect("test asset library should open"), + )) +} + +/// A render state running `effect` on one point zone at the canvas center, +/// with no devices attached and nothing subscribed. +fn make_render_state(effect: &str, controls: HashMap) -> RenderThreadState { + let (_, power_state) = watch::channel(OutputPowerState::default()); + let event_bus = Arc::new(HypercolorBus::new()); + let layout = test_layout(vec![point_zone("zone_main", "mock:main", 0.5, 0.5)]); + let spatial_engine = SpatialEngine::new(layout.clone()); + let mut scene_manager = SceneManager::with_default(); + scene_manager + .upsert_primary_zone(&builtin_effect_metadata(effect), controls, None, layout) + .expect("test render state should seed a default primary zone"); + let scene_manager = SceneService::with_temporary_store(scene_manager, Arc::clone(&event_bus)) + .expect("temporary scene store should open"); + let scene_plan = scene_manager.plan_reader(); + RenderThreadState { + effect_registry: Arc::new(RwLock::new(builtin_effect_registry())), + asset_library: test_asset_library(), + spatial_engine: SpatialService::new(spatial_engine), + backend_manager: Arc::new(Mutex::new(BackendManager::new())), + device_registry: DeviceRegistry::new(), + performance: Arc::new(RwLock::new(PerformanceTracker::default())), + discovery_runtime: None, + event_bus: Arc::clone(&event_bus), + preview_runtime: Arc::new(PreviewRuntime::new(event_bus)), + zone_layout_previews: Arc::new( + hypercolor_daemon::zone_layout_preview::ZoneLayoutPreviewStore::default(), + ), + render_loop: Arc::new(RwLock::new(RenderLoop::new(60))), + scene_manager, + scene_plan, + input_manager: InputManager::new(), + interaction_routing: + hypercolor_daemon::interaction_routing::InteractionRoutingControl::default(), + power_state, + scene_transactions: SceneTransactionQueue::default(), + screen_capture_configured: false, + canvas_dims: CanvasDims::new(320, 200), + render_acceleration_mode: RenderAccelerationMode::Cpu, + #[cfg(feature = "wgpu")] + render_gpu_device: None, + configured_max_fps_tier: FpsTier::Full.into(), + face_fps_cap: 30, + } +} + +async fn start(state: &RenderThreadState) -> RenderThread { + state.render_loop.write().await.start(); + RenderThread::spawn(state.clone()) +} + +async fn stop(state: &RenderThreadState, mut render_thread: RenderThread) { + state.render_loop.write().await.stop(); + render_thread.shutdown().await.expect("shutdown"); +} + +async fn telemetry(state: &RenderThreadState) -> OutputDemandTelemetry { + state.performance.read().await.output_demand_telemetry() +} + +async fn frame_number(state: &RenderThreadState) -> u64 { + state.render_loop.read().await.frame_number() +} + +/// Wait until the loop has parked `parks` times in total and return the +/// render loop frame number it parked at. +async fn wait_until_parked(state: &RenderThreadState, parks: u64) -> u64 { + tokio::time::timeout(WAIT_DEADLINE, async { + loop { + let telemetry = telemetry(state).await; + if telemetry.parked_since.is_some() && telemetry.parks_total >= parks { + return frame_number(state).await; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("render loop should park (park #{parks}) in time")) +} + +/// Wait until the loop renders past `frame`, returning how long it took. +async fn wait_until_rendering_past(state: &RenderThreadState, frame: u64) -> Duration { + let started = Instant::now(); + tokio::time::timeout(WAIT_DEADLINE, async { + while frame_number(state).await <= frame { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("render loop should resume rendering in time"); + started.elapsed() +} + +async fn frames_rendered_in(state: &RenderThreadState, window: Duration) -> u64 { + let before = frame_number(state).await; + tokio::time::sleep(window).await; + frame_number(state).await.saturating_sub(before) +} + +async fn assert_renders_at_full_tier(state: &RenderThreadState, consumer: &str) { + let frames = frames_rendered_in(state, RATE_WINDOW).await; + assert!( + frames >= MIN_FRAMES_IN_RATE_WINDOW, + "{consumer} should keep the loop rendering at its tier, rendered {frames} frames in {RATE_WINDOW:?}" + ); + assert_eq!( + state.render_loop.read().await.stats().tier, + FpsTier::Full, + "{consumer} should not change the FPS tier" + ); +} + +fn mock_device_info(name: &str) -> DeviceInfo { + let backend = MockDeviceBackend::new().with_device(&MockDeviceConfig { + name: name.into(), + led_count: 8, + topology: LedTopology::Point, + id: Some(DeviceId::new()), + }); + backend.device_infos()[0].clone() +} + +async fn connect_device(registry: &DeviceRegistry, info: DeviceInfo) -> DeviceId { + let device_id = registry.add(info).await; + assert!( + registry.set_state(&device_id, DeviceState::Connected).await, + "test device should move to connected" + ); + device_id +} + +fn zone_main_color(frame: &FrameData) -> [u8; 3] { + frame + .zones + .iter() + .find(|zone| zone.zone_id == "zone_main") + .and_then(|zone| zone.colors.first().copied()) + .expect("frame should carry the main zone") +} + +/// HSV hue of an 8-bit color, in degrees. +fn hue_degrees([r, g, b]: [u8; 3]) -> f32 { + let (r, g, b) = (f32::from(r), f32::from(g), f32::from(b)); + let max = r.max(g).max(b); + let delta = max - r.min(g).min(b); + if delta <= f32::EPSILON { + return 0.0; + } + let sector = if (max - r).abs() <= f32::EPSILON { + ((g - b) / delta).rem_euclid(6.0) + } else if (max - g).abs() <= f32::EPSILON { + (b - r) / delta + 2.0 + } else { + (r - g) / delta + 4.0 + }; + sector * 60.0 +} + +fn hue_distance(a: f32, b: f32) -> f32 { + let difference = (a - b).rem_euclid(360.0); + difference.min(360.0 - difference) +} + +#[tokio::test] +async fn unconsumed_render_loop_parks_and_renders_nothing() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + + let parked_at = wait_until_parked(&state, 1).await; + tokio::time::sleep(Duration::from_millis(1_500)).await; + + assert_eq!( + frame_number(&state).await, + parked_at, + "a parked loop must not render, safety ticks included" + ); + let telemetry = telemetry(&state).await; + assert!(telemetry.parked_since.is_some()); + assert_eq!(telemetry.parks_total, 1); + assert_eq!( + state.render_loop.read().await.stats().tier, + FpsTier::Full, + "parking leaves the tier where it was" + ); + assert!( + !telemetry.consumers.any(), + "the gate should have counted no consumers: {:?}", + telemetry.consumers + ); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn connected_device_keeps_the_loop_rendering_without_parking() { + let state = make_render_state("rainbow", HashMap::new()); + connect_device(&state.device_registry, mock_device_info("Desk Strip")).await; + let render_thread = start(&state).await; + + assert_renders_at_full_tier(&state, "a connected device").await; + assert_renders_at_full_tier(&state, "a connected device").await; + assert_eq!( + telemetry(&state).await.parks_total, + 0, + "a rig with a connected device never parks" + ); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn device_connect_wakes_a_parked_loop_and_disconnect_parks_it_again() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + + let device_id = connect_device(&state.device_registry, mock_device_info("Fan Ring")).await; + let latency = wait_until_rendering_past(&state, parked_at).await; + assert!( + latency < WAKE_LATENCY_BOUND, + "device connect should wake the loop at once, took {latency:?}" + ); + assert_renders_at_full_tier(&state, "a newly connected device").await; + assert_eq!( + telemetry(&state).await.last_wake_reason, + Some("output_consumer") + ); + + state + .device_registry + .set_state(&device_id, DeviceState::Reconnecting) + .await; + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn simulated_display_without_a_preview_is_not_output_demand() { + let simulator = SimulatedDisplay { + id: DeviceId::new(), + name: "Virtual LCD".into(), + width: 240, + height: 240, + circular: true, + enabled: true, + } + .device_info(); + assert!(!is_output_device(&simulator, true)); + assert!(is_output_device(&mock_device_info("Strip"), true)); + assert!(!is_output_device(&mock_device_info("Strip"), false)); + + let state = make_render_state("rainbow", HashMap::new()); + connect_device(&state.device_registry, simulator).await; + let render_thread = start(&state).await; + + let parked_at = wait_until_parked(&state, 1).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(frame_number(&state).await, parked_at); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn canvas_preview_subscription_wakes_a_parked_loop_at_full_tier() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + + let subscribed_at = Instant::now(); + let mut preview = state.preview_runtime.canvas_receiver(); + tokio::time::timeout(WAIT_DEADLINE, preview.changed()) + .await + .expect("a preview subscriber should receive a frame") + .expect("canvas lane should stay open"); + let latency = subscribed_at.elapsed(); + assert!( + latency < WAKE_LATENCY_BOUND, + "a preview subscription should wake the loop at once, took {latency:?}" + ); + assert!( + u64::from(preview.borrow_and_update().frame_number) >= parked_at, + "the preview should receive a frame rendered after the wake" + ); + assert_renders_at_full_tier(&state, "a canvas preview").await; + assert_eq!(telemetry(&state).await.consumers.stream_receivers, 1); + + drop(preview); + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn scene_canvas_receiver_for_a_display_backdrop_wakes_a_parked_loop() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + + let mut backdrop = state.preview_runtime.scene_canvas_receiver(); + let latency = wait_until_rendering_past(&state, parked_at).await; + assert!( + latency < WAKE_LATENCY_BOUND, + "a display backdrop receiver should wake the loop at once, took {latency:?}" + ); + tokio::time::timeout(WAIT_DEADLINE, backdrop.changed()) + .await + .expect("the backdrop should receive a scene canvas") + .expect("scene canvas lane should stay open"); + assert_renders_at_full_tier(&state, "a display backdrop").await; + + drop(backdrop); + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn frame_and_zone_canvas_receivers_and_demand_holds_wake_a_parked_loop() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + + let parked_at = wait_until_parked(&state, 1).await; + let frames = state.event_bus.frame_receiver(); + wait_until_rendering_past(&state, parked_at).await; + assert_renders_at_full_tier(&state, "a frame stream receiver").await; + drop(frames); + + let parked_at = wait_until_parked(&state, 2).await; + let zone_canvas = state + .event_bus + .zone_canvas_receiver(hypercolor_types::scene::ZoneId::new()); + wait_until_rendering_past(&state, parked_at).await; + drop(zone_canvas); + + let parked_at = wait_until_parked(&state, 3).await; + let hold = state.event_bus.hold_output_demand(); + wait_until_rendering_past(&state, parked_at).await; + assert_renders_at_full_tier(&state, "an output demand hold").await; + drop(hold); + wait_until_parked(&state, 4).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn canvas_capture_after_a_park_returns_a_freshly_rendered_frame() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + tokio::time::sleep(Duration::from_millis(300)).await; + + let requested_at = Instant::now(); + let captured = capture_canvas( + &state.preview_runtime, + CanvasCaptureRequest::single(CanvasImageFormat::Png), + ) + .await + .expect("capture should succeed"); + let latency = requested_at.elapsed(); + + let frame = captured.first().expect("capture should return one frame"); + assert!(frame.fresh, "a capture must wake the loop for a new frame"); + // The park closed its frame without rendering, so `parked_at` is the + // number of the first frame after the wake. That frame still composes + // producer output requested before the park; the capture skips it. + assert!( + u64::from(frame.frame_number) > parked_at, + "the capture should skip the wake frame and return one rendered after the request, got frame {} after parking at {parked_at}", + frame.frame_number + ); + assert!( + !state + .preview_runtime + .is_render_resume_frame(frame.frame_number) + ); + assert!( + latency < WAKE_LATENCY_BOUND, + "a capture of a parked loop should land within a frame or two, took {latency:?}" + ); + assert_eq!(frame.source_width, 320); + assert_eq!(frame.source_height, 200); + + // The capture's demand ends with it, so the loop parks again. + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn effect_time_stays_continuous_across_park_and_resume() { + // Rainbow turns its hue 60 degrees per second, about one degree per + // frame at 60 fps. A clock that kept running through a 1.5 second park + // would jump about 90 degrees on resume. + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + wait_until_parked(&state, 1).await; + let (parked_frame, parked_hue) = { + let frame = state.event_bus.frame_lane().borrow(); + (frame.frame_number, hue_degrees(zone_main_color(&frame))) + }; + tokio::time::sleep(Duration::from_millis(1_500)).await; + + let mut frames = state.event_bus.frame_receiver(); + let resumed = tokio::time::timeout(WAIT_DEADLINE, async { + loop { + frames.changed().await.expect("frame lane should stay open"); + let frame = frames.borrow_and_update().clone(); + if frame.frame_number > parked_frame { + break frame; + } + } + }) + .await + .expect("a frame subscriber should wake the loop"); + let resumed_hue = hue_degrees(zone_main_color(&resumed)); + let resume_step = hue_distance(parked_hue, resumed_hue); + assert!( + resume_step < 12.0, + "effect time should step one frame across the park, hue moved {resume_step} degrees ({parked_hue} -> {resumed_hue})" + ); + + tokio::time::sleep(Duration::from_millis(500)).await; + let later_hue = hue_degrees(zone_main_color(&frames.borrow_and_update())); + let progress = hue_distance(resumed_hue, later_hue); + assert!( + progress > 10.0, + "effect time should keep flowing after the resume, hue moved {progress} degrees in 500 ms" + ); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn scene_change_while_parked_renders_the_new_effect_then_parks_again() { + let state = make_render_state("solid_color", solid_color_controls(255, 0, 0)); + let render_thread = start(&state).await; + wait_until_parked(&state, 1).await; + assert_eq!( + zone_main_color(&state.event_bus.frame_lane().borrow()), + [255, 0, 0] + ); + + let layout = state.spatial_engine.snapshot().layout().as_ref().clone(); + let mut mutation = state.scene_manager.begin_mutation().await; + mutation + .upsert_primary_zone( + &builtin_effect_metadata("solid_color"), + solid_color_controls(0, 0, 255), + None, + layout, + hypercolor_types::event::ChangeTrigger::System, + None, + ) + .expect("scene mutation should stage"); + state + .scene_manager + .commit_mutation(mutation) + .await + .expect("scene mutation should commit"); + + // Nothing subscribes, yet the producers settle on the new scene so a + // later consumer sees it at once. + tokio::time::timeout(WAIT_DEADLINE, async { + while zone_main_color(&state.event_bus.frame_lane().borrow()) != [0, 0, 255] { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("the parked loop should settle the new effect"); + wait_until_parked(&state, 2).await; + assert_eq!( + telemetry(&state).await.last_wake_reason, + Some("producer_state") + ); + + stop(&state, render_thread).await; +} diff --git a/crates/hypercolor-daemon/tests/render_thread_tests.rs b/crates/hypercolor-daemon/tests/render_thread_tests.rs index ee3156f6e..9bf50dbd8 100644 --- a/crates/hypercolor-daemon/tests/render_thread_tests.rs +++ b/crates/hypercolor-daemon/tests/render_thread_tests.rs @@ -5871,6 +5871,11 @@ async fn pipeline_keeps_rendering_while_async_write_failure_disconnects() { state.discovery_runtime = Some(discovery_runtime.clone()); let mut event_rx = event_bus.subscribe_all(); + // The disconnect unmaps the only device, which would leave nothing + // consuming frames and park the loop. A frame subscriber keeps output + // demand alive, so the frames below prove the loop is not blocked on + // the disconnect I/O rather than that it parked. + let _frame_consumer = event_bus.frame_receiver(); { let mut rl = state.render_loop.write().await; rl.start(); diff --git a/crates/hypercolor-tui/tests/rest_client_tests.rs b/crates/hypercolor-tui/tests/rest_client_tests.rs index 8670766fd..10b6e0519 100644 --- a/crates/hypercolor-tui/tests/rest_client_tests.rs +++ b/crates/hypercolor-tui/tests/rest_client_tests.rs @@ -208,6 +208,7 @@ async fn get_status_maps_the_system_response_without_an_active_effect_call() { actual_fps: 59.8, consecutive_misses: 0, total_frames: 12_000, + ..RenderLoopStatus::default() }, ..SystemStatus::default() }) diff --git a/crates/hypercolor-types/src/api/diagnose.rs b/crates/hypercolor-types/src/api/diagnose.rs index dccf47d78..34a64c042 100644 --- a/crates/hypercolor-types/src/api/diagnose.rs +++ b/crates/hypercolor-types/src/api/diagnose.rs @@ -59,6 +59,10 @@ pub struct DiagnoseSnapshot { pub struct DiagnoseRenderSnapshot { pub latest_frame: Option, pub recent_window: DiagnoseRenderWindowSnapshot, + /// What consumes rendered frames, and whether the loop parked because + /// nothing does. Absent from older daemons. + #[serde(default)] + pub output_demand: crate::api::system::RenderOutputDemandStatus, } #[derive(Debug, Clone, Serialize)] diff --git a/crates/hypercolor-types/src/api/system.rs b/crates/hypercolor-types/src/api/system.rs index c5c809ccd..5d5b6c7af 100644 --- a/crates/hypercolor-types/src/api/system.rs +++ b/crates/hypercolor-types/src/api/system.rs @@ -323,6 +323,48 @@ pub struct RenderLoopStatus { pub actual_fps: f64, pub consecutive_misses: u32, pub total_frames: u64, + /// Whether anything consumes rendered frames, and why the loop parked + /// when nothing does. Absent from older daemons. + #[serde(default)] + pub output_demand: RenderOutputDemandStatus, +} + +/// Output demand: what consumes the render loop's frames. +/// +/// The render loop renders only while something consumes its output: a +/// connected device, a display, a preview or canvas stream, a capture, or +/// an explicit demand hold. With none of those it parks, renders nothing, +/// and resumes at its full tier the moment a consumer appears. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +pub struct RenderOutputDemandStatus { + /// Whether the render loop is parked because nothing consumes its frames. + pub parked: bool, + /// Why the render loop is producing no frames, when it is parked. + /// `no_output_consumers` while parked. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub idle_reason: Option, + /// How long the loop has been parked, in milliseconds. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parked_ms: Option, + /// Connected devices that receive frames. Simulated displays count only + /// while previewed, through `demand_holds`. + pub output_devices: u32, + /// Live receivers on the frame, canvas, preview, spectrum, and per-zone + /// canvas streams, including in-flight canvas captures. + pub stream_receivers: u32, + /// Stream and diagnostic consumers of host input events, which the + /// render loop republishes. + pub input_event_streams: u32, + /// Explicit demand holds, such as display output for previewed + /// simulated displays. + pub demand_holds: u32, + /// How many times the loop has parked since the daemon started. + pub parks_total: u64, + /// What last woke the loop from a park: `output_consumer`, + /// `scene_transaction`, `producer_state`, or `render_loop_state`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_wake_reason: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] From 0ab6828003058eb39b098ef560c03c6f49965eb8 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 00:02:49 -0700 Subject: [PATCH 04/26] feat(cli): explain a parked render loop in hypercolor status A daemon with nothing consuming its frames now parks its render loop, so the render line reads 0 fps on an idle machine. Without a note that looks like a stall. The status table adds one line when the loop is parked saying that nothing consumes frames and that a device, preview, or capture resumes rendering. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-cli/src/commands/status.rs | 43 +++++++++++++++++++- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/crates/hypercolor-cli/src/commands/status.rs b/crates/hypercolor-cli/src/commands/status.rs index 298688eae..e77566025 100644 --- a/crates/hypercolor-cli/src/commands/status.rs +++ b/crates/hypercolor-cli/src/commands/status.rs @@ -381,6 +381,13 @@ fn status_table_lines(data: &SystemStatus, p: &Painter) -> Vec { p.muted(&format!("{} frames", format_count(total_frames))), p.muted(""), )); + if render_loop.output_demand.parked { + lines.push(format!( + " {} {}", + p.muted(&pad("", 10)), + p.muted("parked: nothing consumes frames, so nothing renders until a device, preview, or capture appears"), + )); + } // ── Frame budget ──────────────────────────────────────────────── if let Some(latest_frame) = &data.latest_frame { @@ -611,8 +618,8 @@ mod tests { use hypercolor_types::api::system::{ LatestFrameStatus, MacosCapabilityOwner, MacosDaemonHandoverPhase, MacosDaemonOwnerConflictStatus, MacosDaemonOwnerRecoveryRequiredStatus, - MacosDaemonOwnershipStatus, PreviewRuntimeStatus, RenderLoopStatus, RenderSurfaceStatus, - ServerInfo, SystemResource, SystemStatus, + MacosDaemonOwnershipStatus, PreviewRuntimeStatus, RenderLoopStatus, + RenderOutputDemandStatus, RenderSurfaceStatus, ServerInfo, SystemResource, SystemStatus, }; use serde_json::json; @@ -958,6 +965,38 @@ mod tests { assert_eq!(format_count(1_234_567), "1,234,567"); } + #[test] + fn status_lines_explain_a_parked_render_loop() { + let painter = Painter::plain(); + let parked = SystemStatus { + render_loop: RenderLoopStatus { + state: "running".to_owned(), + target_fps: 60, + output_demand: RenderOutputDemandStatus { + parked: true, + idle_reason: Some("no_output_consumers".to_owned()), + ..RenderOutputDemandStatus::default() + }, + ..RenderLoopStatus::default() + }, + ..SystemStatus::default() + }; + let rendering = SystemStatus { + render_loop: RenderLoopStatus { + state: "running".to_owned(), + target_fps: 60, + ..RenderLoopStatus::default() + }, + ..SystemStatus::default() + }; + + let parked_lines = status_table_lines(&parked, &painter).join("\n"); + let rendering_lines = status_table_lines(&rendering, &painter).join("\n"); + + assert!(parked_lines.contains("parked: nothing consumes frames")); + assert!(!rendering_lines.contains("parked")); + } + #[test] fn status_lines_include_core_fields() { let data = SystemStatus { From 87e806ab86d3727fef574b328264111ed31643bc Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 00:02:49 -0700 Subject: [PATCH 05/26] docs: describe how the render loop parks without output consumers The render pipeline page gains a section on what counts as a consumer, how a parked loop wakes, how producers settle before parking, and how effect time and capture freshness carry across a park. The SparkleFlinger design notes record the gate next to the existing sleep and idle throttles, the REST status example shows render_loop.output_demand, and the performance troubleshooting page explains a parked render_loop check so 0 fps on an idle daemon reads as expected rather than as a stall. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/content/api/rest.md | 16 +++++++++- docs/content/architecture/render-pipeline.md | 15 +++++++++ docs/content/troubleshooting/performance.md | 6 ++++ .../30-sparkleflinger-implementation.md | 31 +++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/docs/content/api/rest.md b/docs/content/api/rest.md index 9c41a0a3c..5f23b4405 100644 --- a/docs/content/api/rest.md +++ b/docs/content/api/rest.md @@ -238,7 +238,16 @@ keyed daemon receive the public identity without the status block. "target_fps": 60, "capacity_fps": 60.0, "delivered_fps": 59.8, - "actual_fps": 60.0 + "actual_fps": 60.0, + "output_demand": { + "parked": false, + "output_devices": 3, + "stream_receivers": 2, + "input_event_streams": 0, + "demand_holds": 0, + "parks_total": 1, + "last_wake_reason": "output_consumer" + } } } }, @@ -250,6 +259,11 @@ keyed daemon receive the public identity without the status block. } ``` +`render_loop.output_demand` says what consumes rendered frames. With no device, stream, +preview, or capture attached the loop parks: `parked` turns true, `idle_reason` reads +`no_output_consumers`, `parked_ms` counts up, and `delivered_fps` and `actual_fps` read +zero until a consumer appears and the loop resumes at its full tier. + `status.effect_count` reflects whatever the registry holds at request time (native built-ins plus discovered HTML effects); treat it as live, not a fixed product number. diff --git a/docs/content/architecture/render-pipeline.md b/docs/content/architecture/render-pipeline.md index d08e5cdc3..4c8fd8321 100644 --- a/docs/content/architecture/render-pipeline.md +++ b/docs/content/architecture/render-pipeline.md @@ -166,6 +166,21 @@ fps_controller.maybe_transition(); `FrameStats` surfaces `tier`, `frame_time`, `headroom`, `ewma_frame_time`, `consecutive_misses`, and `frames_since_tier_change`, all forwarded to the event bus and visible in the dashboard. +## Rendering only what someone receives + +Every frame costs producer, compositor, and GPU work whether or not anything receives it. A daemon with no devices and no open client would otherwise render a WebGL effect at 60 fps for nobody. Before each frame the loop therefore counts the consumers of its output: + +- connected devices, LED or display, that receive frames (a simulated display counts only while someone previews it); +- live receivers on the bus output streams: frame data, spectrum, the canvas and scene canvas previews, screen canvas and screen zones, web viewport, zone previews, and the per-zone canvas streams. WebSocket subscriptions, display backdrops, and canvas captures all hold one; +- WebSocket and diagnostic subscribers to host input events, which the loop republishes; +- explicit output demand holds, taken by anything that reads published frames without a receiver. + +With none of them, the loop **parks**: it renders nothing, withdraws its audio, screen, and interaction capture demand, and throttles HTML effect pages so their own timers and `requestAnimationFrame` loops stop too. A parked loop sleeps until something can observe a frame. Subscribing to a stream, taking a demand hold, connecting a device, queuing a layout change, or publishing any bus event wakes it at once, and it resumes at its full tier. A one second safety tick re-checks in case a consumer arrived without signalling. No observable cadence changes, because the moment anything can see a frame, it renders at full rate. + +Parking waits for the producers to settle. After the scene, an effect, or the canvas changes, the loop keeps rendering until every producer has shown its new content (bounded at ten seconds), so an effect applied while nobody watches is already loaded when a client or capture arrives. Effect time carries across the park: the first frame after a wake advances effect time by one frame, not by the parked span, and no catch-up frames follow. A canvas capture that wakes the loop skips the wake frame itself, whose HTML producers still show output requested before the park, and returns the next one. + +`/api/v1/status` reports the state under `render_loop.output_demand` (`parked`, `idle_reason`, `parked_ms`, and the consumer counts the loop last saw), with `delivered_fps` and `actual_fps` at zero while parked. `/api/v1/diagnose` carries the same object under `snapshot.render.output_demand`. Rigs with a connected device never park. + ## Canvas resize and layout updates Two operations trigger mid-run reconfigurations without stalling the render thread: diff --git a/docs/content/troubleshooting/performance.md b/docs/content/troubleshooting/performance.md index 32b1995b9..6de52fea7 100644 --- a/docs/content/troubleshooting/performance.md +++ b/docs/content/troubleshooting/performance.md @@ -63,6 +63,12 @@ The `tier` field shows where the FPS controller has landed. Compare it against y The controller downshifts after 2 consecutive budget misses and only upshifts after sustained headroom, so a tier stuck below your target is a real signal. +``` +render_loop state=running, tier=full, parked=no_output_consumers +``` + +`parked` means nothing consumes rendered frames: no connected device, no open preview or stream, no capture. The loop renders nothing until one appears, then resumes at its full tier, so `0 fps` here is expected rather than a stall. `frame_liveness` passes while parked, and `hypercolor status` shows the same state on its render line. `snapshot.render.output_demand` in the JSON report lists the consumer counts the loop last saw and what last woke it. + ### `frame_liveness` ``` diff --git a/docs/design/30-sparkleflinger-implementation.md b/docs/design/30-sparkleflinger-implementation.md index 0a5602fce..052b444d9 100644 --- a/docs/design/30-sparkleflinger-implementation.md +++ b/docs/design/30-sparkleflinger-implementation.md @@ -210,6 +210,37 @@ throttle. Both throttle paths bypass composition and the producer stages but still record `LatestFrameMetrics` with pacing-relevant fields populated. +### 3.4.1 Output-demand park + +Before either throttle, right after scene transactions are serviced, the +output gate (`render_thread/output_gate.rs`) asks whether anything consumes +the frame. Consumers are defined once in `output_demand.rs`: renderable +devices in the registry or devices the backend manager routes to +(simulated displays excluded), live receivers on any bus output stream, +stream and diagnostic subscribers to republished host input events, and +explicit `OutputDemandHold`s, which display output takes while it has a +target so previewed simulators count. + +With no consumer the gate parks the loop instead of rendering. It +withdraws the authoritative input capture demand, releases the screen +queue, calls `EffectRenderer::park` on every producer (which throttles +Servo webviews so the page's script thread stops scheduling its own +animation ticks and slows its timers), and returns `NextWake::Park`. The +pipeline driver then sleeps on the bus output-demand signal (notified by +lane subscriptions, demand holds, and event publication), the device +registry change signal, the input demand revision, and the scene +transaction queue, with a one second safety tick. On wake it rebases the +frame clock one target interval back, so effect time steps by one frame, +records the wake frame for capture freshness, and continues at the +current tier. + +The gate parks only once the producers are settled: three consecutive +frame starts with no producer reporting `is_warming` under an unchanged +producer signature (scene plan generation, zones revision, effect registry +generation, zone layout preview generation, canvas size), bounded at ten +seconds. Frames that consumers drive count toward settling, so losing the +last consumer parks at once when the producers are warm. + ### 3.5 Input sampling If `skip_decision == SkipDecision::None`, `sample_inputs` acquires the From add0f78d9b08c5caf02bc0e589f4463494804784 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 00:08:21 -0700 Subject: [PATCH 06/26] chore(python): regenerate the client for render output demand The daemon's status and diagnose contracts gained output_demand, which reports whether the render loop parked because nothing consumes its frames. Both fields are optional in the schema, so this client still parses responses from daemons that predate them. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../hypercolor/_generated/models/__init__.py | 2 + .../models/diagnose_render_snapshot.py | 25 +++ .../_generated/models/render_loop_status.py | 33 +++- .../models/render_output_demand_status.py | 176 ++++++++++++++++++ 4 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 python/src/hypercolor/_generated/models/render_output_demand_status.py diff --git a/python/src/hypercolor/_generated/models/__init__.py b/python/src/hypercolor/_generated/models/__init__.py index d8f852ad0..38ce1ac7e 100644 --- a/python/src/hypercolor/_generated/models/__init__.py +++ b/python/src/hypercolor/_generated/models/__init__.py @@ -562,6 +562,7 @@ from .remove_favorite_response_200 import RemoveFavoriteResponse200 from .render_acceleration_status import RenderAccelerationStatus from .render_loop_status import RenderLoopStatus +from .render_output_demand_status import RenderOutputDemandStatus from .render_surface_status import RenderSurfaceStatus from .reorder_layers_request import ReorderLayersRequest from .reorder_live_zone_layers_response_200 import ReorderLiveZoneLayersResponse200 @@ -1183,6 +1184,7 @@ "RemoveFavoriteResponse200", "RenderAccelerationStatus", "RenderLoopStatus", + "RenderOutputDemandStatus", "RenderSurfaceStatus", "ReorderLayersRequest", "ReorderLiveZoneLayersResponse200", diff --git a/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py b/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py index 01ec30f26..e984e164a 100644 --- a/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py +++ b/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: from ..models.diagnose_latest_frame_snapshot import DiagnoseLatestFrameSnapshot from ..models.diagnose_render_window_snapshot import DiagnoseRenderWindowSnapshot + from ..models.render_output_demand_status import RenderOutputDemandStatus T = TypeVar("T", bound="DiagnoseRenderSnapshot") @@ -23,10 +24,17 @@ class DiagnoseRenderSnapshot: Attributes: recent_window (DiagnoseRenderWindowSnapshot): latest_frame (DiagnoseLatestFrameSnapshot | None | Unset): + output_demand (RenderOutputDemandStatus | Unset): Output demand: what consumes the render loop's frames. + + The render loop renders only while something consumes its output: a + connected device, a display, a preview or canvas stream, a capture, or + an explicit demand hold. With none of those it parks, renders nothing, + and resumes at its full tier the moment a consumer appears. """ recent_window: DiagnoseRenderWindowSnapshot latest_frame: DiagnoseLatestFrameSnapshot | None | Unset = UNSET + output_demand: RenderOutputDemandStatus | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -44,6 +52,10 @@ def to_dict(self) -> dict[str, Any]: else: latest_frame = self.latest_frame + output_demand: dict[str, Any] | Unset = UNSET + if not isinstance(self.output_demand, Unset): + output_demand = self.output_demand.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -53,6 +65,8 @@ def to_dict(self) -> dict[str, Any]: ) if latest_frame is not UNSET: field_dict["latest_frame"] = latest_frame + if output_demand is not UNSET: + field_dict["output_demand"] = output_demand return field_dict @@ -64,6 +78,9 @@ def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: from ..models.diagnose_render_window_snapshot import ( DiagnoseRenderWindowSnapshot, ) + from ..models.render_output_demand_status import ( + RenderOutputDemandStatus, + ) d = dict(src_dict) recent_window = DiagnoseRenderWindowSnapshot.from_dict(d.pop("recent_window")) @@ -87,9 +104,17 @@ def _parse_latest_frame( latest_frame = _parse_latest_frame(d.pop("latest_frame", UNSET)) + _output_demand = d.pop("output_demand", UNSET) + output_demand: RenderOutputDemandStatus | Unset + if isinstance(_output_demand, Unset): + output_demand = UNSET + else: + output_demand = RenderOutputDemandStatus.from_dict(_output_demand) + diagnose_render_snapshot = cls( recent_window=recent_window, latest_frame=latest_frame, + output_demand=output_demand, ) diagnose_render_snapshot.additional_properties = d diff --git a/python/src/hypercolor/_generated/models/render_loop_status.py b/python/src/hypercolor/_generated/models/render_loop_status.py index 00d61a418..fa96480a0 100644 --- a/python/src/hypercolor/_generated/models/render_loop_status.py +++ b/python/src/hypercolor/_generated/models/render_loop_status.py @@ -1,12 +1,18 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from attrs import define as _attrs_define from attrs import field as _attrs_field from typing_extensions import Self +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.render_output_demand_status import RenderOutputDemandStatus + + T = TypeVar("T", bound="RenderLoopStatus") @@ -23,6 +29,12 @@ class RenderLoopStatus: state (str): target_fps (int): total_frames (int): + output_demand (RenderOutputDemandStatus | Unset): Output demand: what consumes the render loop's frames. + + The render loop renders only while something consumes its output: a + connected device, a display, a preview or canvas stream, a capture, or + an explicit demand hold. With none of those it parks, renders nothing, + and resumes at its full tier the moment a consumer appears. """ actual_fps: float @@ -34,6 +46,7 @@ class RenderLoopStatus: state: str target_fps: int total_frames: int + output_demand: RenderOutputDemandStatus | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -55,6 +68,10 @@ def to_dict(self) -> dict[str, Any]: total_frames = self.total_frames + output_demand: dict[str, Any] | Unset = UNSET + if not isinstance(self.output_demand, Unset): + output_demand = self.output_demand.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -70,11 +87,17 @@ def to_dict(self) -> dict[str, Any]: "total_frames": total_frames, } ) + if output_demand is not UNSET: + field_dict["output_demand"] = output_demand return field_dict @classmethod def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: + from ..models.render_output_demand_status import ( + RenderOutputDemandStatus, + ) + d = dict(src_dict) actual_fps = d.pop("actual_fps") @@ -94,6 +117,13 @@ def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: total_frames = d.pop("total_frames") + _output_demand = d.pop("output_demand", UNSET) + output_demand: RenderOutputDemandStatus | Unset + if isinstance(_output_demand, Unset): + output_demand = UNSET + else: + output_demand = RenderOutputDemandStatus.from_dict(_output_demand) + render_loop_status = cls( actual_fps=actual_fps, capacity_fps=capacity_fps, @@ -104,6 +134,7 @@ def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: state=state, target_fps=target_fps, total_frames=total_frames, + output_demand=output_demand, ) render_loop_status.additional_properties = d diff --git a/python/src/hypercolor/_generated/models/render_output_demand_status.py b/python/src/hypercolor/_generated/models/render_output_demand_status.py new file mode 100644 index 000000000..0d3b61ed9 --- /dev/null +++ b/python/src/hypercolor/_generated/models/render_output_demand_status.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field +from typing_extensions import Self + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="RenderOutputDemandStatus") + + +@_attrs_define +class RenderOutputDemandStatus: + """Output demand: what consumes the render loop's frames. + + The render loop renders only while something consumes its output: a + connected device, a display, a preview or canvas stream, a capture, or + an explicit demand hold. With none of those it parks, renders nothing, + and resumes at its full tier the moment a consumer appears. + + Attributes: + demand_holds (int): Explicit demand holds, such as display output for previewed + simulated displays. + input_event_streams (int): Stream and diagnostic consumers of host input events, which the + render loop republishes. + output_devices (int): Connected devices that receive frames. Simulated displays count only + while previewed, through `demand_holds`. + parked (bool): Whether the render loop is parked because nothing consumes its frames. + parks_total (int): How many times the loop has parked since the daemon started. + stream_receivers (int): Live receivers on the frame, canvas, preview, spectrum, and per-zone + canvas streams, including in-flight canvas captures. + idle_reason (None | str | Unset): Why the render loop is producing no frames, when it is parked. + `no_output_consumers` while parked. + last_wake_reason (None | str | Unset): What last woke the loop from a park: `output_consumer`, + `scene_transaction`, `producer_state`, or `render_loop_state`. + parked_ms (int | None | Unset): How long the loop has been parked, in milliseconds. + """ + + demand_holds: int + input_event_streams: int + output_devices: int + parked: bool + parks_total: int + stream_receivers: int + idle_reason: None | str | Unset = UNSET + last_wake_reason: None | str | Unset = UNSET + parked_ms: int | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + demand_holds = self.demand_holds + + input_event_streams = self.input_event_streams + + output_devices = self.output_devices + + parked = self.parked + + parks_total = self.parks_total + + stream_receivers = self.stream_receivers + + idle_reason: None | str | Unset + if isinstance(self.idle_reason, Unset): + idle_reason = UNSET + else: + idle_reason = self.idle_reason + + last_wake_reason: None | str | Unset + if isinstance(self.last_wake_reason, Unset): + last_wake_reason = UNSET + else: + last_wake_reason = self.last_wake_reason + + parked_ms: int | None | Unset + if isinstance(self.parked_ms, Unset): + parked_ms = UNSET + else: + parked_ms = self.parked_ms + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "demand_holds": demand_holds, + "input_event_streams": input_event_streams, + "output_devices": output_devices, + "parked": parked, + "parks_total": parks_total, + "stream_receivers": stream_receivers, + } + ) + if idle_reason is not UNSET: + field_dict["idle_reason"] = idle_reason + if last_wake_reason is not UNSET: + field_dict["last_wake_reason"] = last_wake_reason + if parked_ms is not UNSET: + field_dict["parked_ms"] = parked_ms + + return field_dict + + @classmethod + def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: + d = dict(src_dict) + demand_holds = d.pop("demand_holds") + + input_event_streams = d.pop("input_event_streams") + + output_devices = d.pop("output_devices") + + parked = d.pop("parked") + + parks_total = d.pop("parks_total") + + stream_receivers = d.pop("stream_receivers") + + def _parse_idle_reason(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + idle_reason = _parse_idle_reason(d.pop("idle_reason", UNSET)) + + def _parse_last_wake_reason(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + last_wake_reason = _parse_last_wake_reason(d.pop("last_wake_reason", UNSET)) + + def _parse_parked_ms(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + parked_ms = _parse_parked_ms(d.pop("parked_ms", UNSET)) + + render_output_demand_status = cls( + demand_holds=demand_holds, + input_event_streams=input_event_streams, + output_devices=output_devices, + parked=parked, + parks_total=parks_total, + stream_receivers=stream_receivers, + idle_reason=idle_reason, + last_wake_reason=last_wake_reason, + parked_ms=parked_ms, + ) + + render_output_demand_status.additional_properties = d + return render_output_demand_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties From 74524e3ab9e7bf50a31c403ccd3301f94681ee1f Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 01:21:21 -0700 Subject: [PATCH 07/26] feat(daemon): record every thread panic through a process-wide hook The default panic hook writes only to stderr and prints a backtrace only when RUST_BACKTRACE asks for one. When stderr shares a sink that stopped accepting writes, such as a log file on a full tmpfs, a panicking thread leaves no trace and the daemon keeps serving without it. The daemon now installs a hook before any thread starts. It captures the thread name, payload, location, and a backtrace taken on the spot, hands the report to an optional observer, keeps the last few reports for code that catches the unwind, chains the previous hook so stderr output stays as it was, and records the panic through tracing at error level. Servo's macOS hang monitor swaps the hook during a sample and restores whatever it found, so installing first, before Servo starts, is what keeps this hook from being dropped. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/src/lib.rs | 1 + crates/hypercolor-daemon/src/panic_hook.rs | 337 ++++++++++++++++++ crates/hypercolor-daemon/src/process.rs | 3 + .../tests/panic_hook_tests.rs | 120 +++++++ 4 files changed, 461 insertions(+) create mode 100644 crates/hypercolor-daemon/src/panic_hook.rs create mode 100644 crates/hypercolor-daemon/tests/panic_hook_tests.rs diff --git a/crates/hypercolor-daemon/src/lib.rs b/crates/hypercolor-daemon/src/lib.rs index b670715e8..722e87695 100644 --- a/crates/hypercolor-daemon/src/lib.rs +++ b/crates/hypercolor-daemon/src/lib.rs @@ -42,6 +42,7 @@ pub mod mcp; pub mod mdns; pub mod network; pub mod output_power; +pub mod panic_hook; pub mod path_migration; pub mod performance; pub mod release_notifications; diff --git a/crates/hypercolor-daemon/src/panic_hook.rs b/crates/hypercolor-daemon/src/panic_hook.rs new file mode 100644 index 000000000..d0b1a09e6 --- /dev/null +++ b/crates/hypercolor-daemon/src/panic_hook.rs @@ -0,0 +1,337 @@ +//! Process-wide panic reporting. +//! +//! The default hook writes a panic only to stderr, and only prints a +//! backtrace when `RUST_BACKTRACE` asks for one. When stderr shares a sink +//! that stopped accepting writes (a full tmpfs, a closed pipe), a panicking +//! thread leaves no trace at all and the daemon keeps serving without it. +//! +//! This hook captures every panic on every thread: the thread name, the +//! payload, the source location, and a backtrace taken on the spot. It +//! hands the report to every registered observer (each render supervisor +//! persists its render thread's panics to the state directory, which +//! matters most in `panic = "abort"` builds where nothing runs after the +//! hook), keeps a few recent reports for code that catches the unwind, +//! chains the previous hook so stderr output stays exactly as before, and +//! records the report through `tracing` at error level. + +use std::backtrace::Backtrace; +use std::collections::VecDeque; +use std::panic::PanicHookInfo; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, Once, RwLock}; +use std::thread::ThreadId; +use std::time::SystemTime; + +/// How many recent reports stay available to [`take_recent_report`]. +const RECENT_REPORT_CAPACITY: usize = 8; + +/// Frames kept in [`PanicReport::backtrace_excerpt`]. +const BACKTRACE_EXCERPT_FRAMES: usize = 48; + +/// Upper bound on [`PanicReport::backtrace_excerpt`] in bytes. +const BACKTRACE_EXCERPT_BYTES: usize = 12 * 1024; + +/// Frames near the top of a captured backtrace that may still belong to +/// the capture and the panic machinery rather than the code that panicked. +const PANIC_MACHINERY_SCAN_FRAMES: usize = 40; + +/// Everything one panic left behind, captured inside the hook. +#[derive(Debug, Clone)] +pub struct PanicReport { + /// Identity of the panicking thread. + pub thread_id: ThreadId, + /// Name of the panicking thread, or ``. + pub thread_name: String, + /// The panic message, when the payload is a string. + pub payload: String, + /// `file:line:column` of the panic, when the runtime reports one. + pub location: Option, + /// The full backtrace captured in the hook. + pub backtrace: String, + /// Wall-clock time the hook ran. + pub occurred_at: SystemTime, +} + +impl PanicReport { + fn capture(info: &PanicHookInfo<'_>) -> Self { + let thread = std::thread::current(); + Self { + thread_id: thread.id(), + thread_name: thread.name().unwrap_or("").to_owned(), + payload: info + .payload_as_str() + .unwrap_or("non-string panic payload") + .to_owned(), + location: info.location().map(|location| { + format!( + "{}:{}:{}", + location.file(), + location.line(), + location.column() + ) + }), + backtrace: Backtrace::force_capture().to_string(), + occurred_at: SystemTime::now(), + } + } + + /// The backtrace without the capture and panic machinery frames, + /// bounded in frames and bytes so it fits a persisted record. + #[must_use] + pub fn backtrace_excerpt(&self) -> String { + backtrace_excerpt(&self.backtrace) + } +} + +/// Called with every panic report before the previous hook runs. +pub type PanicObserver = Arc; + +/// Keeps an observer registered until it is dropped. +#[must_use = "the observer is removed as soon as its registration drops"] +pub struct PanicObserverRegistration { + id: u64, +} + +impl Drop for PanicObserverRegistration { + fn drop(&mut self) { + OBSERVERS + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|(id, _)| *id != self.id); + } +} + +static INSTALL: Once = Once::new(); +static RECENT_REPORTS: Mutex> = Mutex::new(VecDeque::new()); +static OBSERVERS: RwLock> = RwLock::new(Vec::new()); +static NEXT_OBSERVER_ID: AtomicU64 = AtomicU64::new(1); + +/// Install the process-wide panic hook. +/// +/// Call this before any subsystem starts threads, and in particular before +/// Servo starts: its background hang monitor briefly swaps the hook and +/// restores whatever it found, so a hook installed while it holds the swap +/// would be lost. The previous hook is chained, never replaced. Calling +/// this more than once is a no-op. +pub fn install() { + INSTALL.call_once(|| { + let previous = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + let report = PanicReport::capture(info); + notify_observers(&report); + remember(&report); + previous(info); + tracing::error!( + target: "hypercolor_daemon::panic", + thread = %report.thread_name, + location = report.location.as_deref().unwrap_or("unknown"), + payload = %report.payload, + backtrace = %report.backtrace, + "thread panicked" + ); + })); + }); +} + +/// Register `observer` to see every panic report until the returned +/// registration drops. +pub fn add_observer(observer: PanicObserver) -> PanicObserverRegistration { + let id = NEXT_OBSERVER_ID.fetch_add(1, Ordering::Relaxed); + OBSERVERS + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push((id, observer)); + PanicObserverRegistration { id } +} + +/// Take the newest report recorded on `thread`, if one is still held. +pub fn take_recent_report(thread: ThreadId) -> Option { + take_recent_report_where(|report| report.thread_id == thread) +} + +/// Take the newest held report that `matches`. +pub fn take_recent_report_where(matches: impl Fn(&PanicReport) -> bool) -> Option { + let mut reports = RECENT_REPORTS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let index = reports.iter().rposition(matches)?; + reports.remove(index) +} + +fn notify_observers(report: &PanicReport) { + // `try_read` keeps a panic raised while the observer list is being + // changed from deadlocking inside its own hook. + let observers: Vec = match OBSERVERS.try_read() { + Ok(observers) => observers + .iter() + .map(|(_, observer)| Arc::clone(observer)) + .collect(), + Err(std::sync::TryLockError::Poisoned(poisoned)) => poisoned + .into_inner() + .iter() + .map(|(_, observer)| Arc::clone(observer)) + .collect(), + Err(std::sync::TryLockError::WouldBlock) => Vec::new(), + }; + for observer in observers { + observer(report); + } +} + +fn remember(report: &PanicReport) { + let mut reports = match RECENT_REPORTS.try_lock() { + Ok(reports) => reports, + Err(std::sync::TryLockError::Poisoned(poisoned)) => poisoned.into_inner(), + Err(std::sync::TryLockError::WouldBlock) => return, + }; + if reports.len() == RECENT_REPORT_CAPACITY { + reports.pop_front(); + } + reports.push_back(report.clone()); +} + +/// Trim a captured backtrace to the frames worth persisting. +/// +/// `Backtrace::force_capture` inside a hook records the capture itself, the +/// hook, and the panic machinery above the frame that panicked. The excerpt +/// skips that contiguous run near the top and keeps a bounded number of +/// the frames after it. Machinery frames further down (a `catch_unwind` +/// that will catch this panic, say) are never reached, because the skip +/// stops at the first frame that is not machinery. +#[must_use] +pub fn backtrace_excerpt(backtrace: &str) -> String { + let frames = split_frames(backtrace); + let start = frames + .iter() + .take(PANIC_MACHINERY_SCAN_FRAMES) + .position(|frame| is_panic_machinery(frame)) + .map_or(0, |entry| { + entry + + frames + .iter() + .skip(entry) + .take_while(|frame| is_panic_machinery(frame)) + .count() + }); + let mut excerpt = String::new(); + for frame in frames.iter().skip(start).take(BACKTRACE_EXCERPT_FRAMES) { + if excerpt.len() + frame.len() > BACKTRACE_EXCERPT_BYTES { + break; + } + excerpt.push_str(frame); + } + if excerpt.is_empty() { + let mut end = backtrace.len().min(BACKTRACE_EXCERPT_BYTES); + while !backtrace.is_char_boundary(end) { + end -= 1; + } + excerpt.push_str(&backtrace[..end]); + } + excerpt +} + +/// Split backtrace text into frames, each frame being its numbered symbol +/// line plus any `at file:line` lines that follow it. +fn split_frames(backtrace: &str) -> Vec<&str> { + let mut frames = Vec::new(); + let mut frame_start: Option = None; + let mut offset = 0; + for line in backtrace.split_inclusive('\n') { + if starts_frame(line) { + if let Some(start) = frame_start { + frames.push(&backtrace[start..offset]); + } + frame_start = Some(offset); + } + offset += line.len(); + } + if let Some(start) = frame_start { + frames.push(&backtrace[start..]); + } + frames +} + +fn starts_frame(line: &str) -> bool { + let trimmed = line.trim_start(); + let digits = trimmed.bytes().take_while(u8::is_ascii_digit).count(); + digits > 0 && trimmed[digits..].starts_with(": ") +} + +fn is_panic_machinery(frame: &str) -> bool { + let symbol = frame.lines().next().unwrap_or_default(); + [ + "std::backtrace_rs::", + "std::backtrace::", + "hypercolor_daemon::panic_hook::", + " as core::ops::function::Fn>::call", + "std::panicking::", + "std::sys::backtrace::", + "rust_begin_unwind", + "core::panicking::", + "core::option::expect_failed", + "core::result::unwrap_failed", + ] + .iter() + .any(|needle| symbol.contains(needle)) +} + +#[cfg(test)] +mod tests { + use super::backtrace_excerpt; + + const CAPTURED: &str = " 0: std::backtrace::Backtrace::force_capture + at /rustc/library/std/src/backtrace.rs:312:9 + 1: hypercolor_daemon::panic_hook::PanicReport::capture + at ./src/panic_hook.rs:60:24 + 2: std::panicking::panic_with_hook + at /rustc/library/std/src/panicking.rs:842:13 + 3: core::panicking::panic_fmt + at /rustc/library/core/src/panicking.rs:75:14 + 4: hypercolor_daemon::render_thread::pipeline_driver::run_pipeline + at ./src/render_thread/pipeline_driver.rs:40:13 + 5: tokio::runtime::park::CachedParkThread::block_on +"; + + #[test] + fn excerpt_starts_at_the_frame_that_panicked() { + let excerpt = backtrace_excerpt(CAPTURED); + + assert!( + excerpt + .trim_start() + .starts_with("4: hypercolor_daemon::render_thread::pipeline_driver::run_pipeline"), + "{excerpt}" + ); + assert!(excerpt.contains("pipeline_driver.rs:40:13")); + assert!(excerpt.contains("5: tokio::runtime::park")); + assert!(!excerpt.contains("force_capture")); + } + + #[test] + fn excerpt_without_recognizable_frames_keeps_the_raw_text() { + assert_eq!( + backtrace_excerpt("disabled backtrace"), + "disabled backtrace" + ); + } + + #[test] + fn excerpt_is_bounded_in_frames_and_bytes() { + use std::fmt::Write as _; + + let mut long = String::new(); + for index in 0..500 { + writeln!( + long, + " {index}: some::deeply::nested::frame_{index}\n at ./src/lib.rs:{index}:1" + ) + .expect("writing to a String cannot fail"); + } + + let excerpt = backtrace_excerpt(&long); + + assert!(excerpt.len() <= super::BACKTRACE_EXCERPT_BYTES); + assert!(excerpt.contains("0: some::deeply::nested::frame_0")); + assert!(!excerpt.contains("frame_48\n")); + } +} diff --git a/crates/hypercolor-daemon/src/process.rs b/crates/hypercolor-daemon/src/process.rs index 25de159b8..bf49a6081 100644 --- a/crates/hypercolor-daemon/src/process.rs +++ b/crates/hypercolor-daemon/src/process.rs @@ -298,6 +298,9 @@ impl From for ServoGpuImportMode { /// Returns an error when process bootstrap, daemon startup, serving, or /// graceful shutdown fails. pub fn run(extension_installers: &'static [&'static dyn DaemonExtensionInstaller]) -> Result<()> { + // First, before any thread exists, so every thread's panic is recorded + // and no dependency that saves and restores the hook can drop it. + crate::panic_hook::install(); #[cfg(target_os = "macos")] let (mut args, macos_owner_argument) = { let matches = DaemonArgs::command().get_matches(); diff --git a/crates/hypercolor-daemon/tests/panic_hook_tests.rs b/crates/hypercolor-daemon/tests/panic_hook_tests.rs new file mode 100644 index 000000000..b5a80e522 --- /dev/null +++ b/crates/hypercolor-daemon/tests/panic_hook_tests.rs @@ -0,0 +1,120 @@ +//! The process panic hook records every thread's panic with a backtrace, +//! hands it to the registered observers, keeps it for the code that +//! catches the unwind, and still runs the hook that was installed before. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, Once}; + +use hypercolor_daemon::panic_hook::{self, PanicObserver, PanicReport}; + +/// Observers and the hook are process-wide, so tests that panic take turns. +static PANIC_SLOT: Mutex<()> = Mutex::new(()); + +/// Panics the hook that was installed before ours has seen. +static PREVIOUS_HOOK_CALLS: AtomicUsize = AtomicUsize::new(0); + +/// Install a counting hook, then the daemon hook on top of it, once. +fn install_over_a_previous_hook() { + static INSTALL: Once = Once::new(); + INSTALL.call_once(|| { + let default = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + PREVIOUS_HOOK_CALLS.fetch_add(1, Ordering::SeqCst); + default(info); + })); + panic_hook::install(); + }); +} + +fn collecting_observer() -> (PanicObserver, Arc>>) { + let observed: Arc>> = Arc::default(); + let sink = Arc::clone(&observed); + let observer: PanicObserver = Arc::new(move |report: &PanicReport| { + sink.lock() + .expect("observer sink should lock") + .push(report.clone()); + }); + (observer, observed) +} + +fn panic_on_named_thread(name: &str, payload: &'static str) -> Option { + std::thread::Builder::new() + .name(name.to_owned()) + .spawn(move || { + let caught = std::panic::catch_unwind(|| panic!("{payload}")); + assert!(caught.is_err()); + panic_hook::take_recent_report(std::thread::current().id()) + }) + .expect("probe thread should spawn") + .join() + .expect("probe thread caught its own panic") +} + +#[test] +fn hook_reports_a_named_thread_panic_to_observers_the_catcher_and_the_previous_hook() { + let _slot = PANIC_SLOT + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + install_over_a_previous_hook(); + let (observer, observed) = collecting_observer(); + let registration = panic_hook::add_observer(observer); + let previous_calls = PREVIOUS_HOOK_CALLS.load(Ordering::SeqCst); + + let recent = panic_on_named_thread("hypercolor-panic-probe", "probe panic payload") + .expect("the hook should keep the report for the catcher"); + drop(registration); + + assert_eq!(recent.thread_name, "hypercolor-panic-probe"); + assert_eq!(recent.payload, "probe panic payload"); + assert!( + recent + .location + .as_deref() + .is_some_and(|location| location.contains("panic_hook_tests.rs")), + "location should point at the panic site: {:?}", + recent.location + ); + assert!( + recent.backtrace_excerpt().contains("panic_hook_tests"), + "the excerpt should start near the panicking frame:\n{}", + recent.backtrace_excerpt() + ); + assert!( + PREVIOUS_HOOK_CALLS.load(Ordering::SeqCst) > previous_calls, + "the hook installed before the daemon hook should still run" + ); + let observed = observed.lock().expect("observer sink should lock"); + assert!( + observed + .iter() + .any(|report| report.thread_name == "hypercolor-panic-probe" + && report.payload == "probe panic payload"), + "the observer should see the panic before the unwind is caught" + ); +} + +#[test] +fn dropping_a_registration_removes_only_that_observer() { + let _slot = PANIC_SLOT + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + install_over_a_previous_hook(); + let (first, first_seen) = collecting_observer(); + let (second, second_seen) = collecting_observer(); + let first_registration = panic_hook::add_observer(first); + let second_registration = panic_hook::add_observer(second); + + drop(first_registration); + let _ = panic_on_named_thread("hypercolor-observer-probe", "second observer payload"); + drop(second_registration); + + let seen = |reports: &Arc>>| { + reports + .lock() + .expect("observer sink should lock") + .iter() + .any(|report| report.payload == "second observer payload") + }; + assert!(!seen(&first_seen)); + assert!(seen(&second_seen)); +} From 3d64f8060a8bcc7526a2f996cb782b3762fb2104 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 01:24:30 -0700 Subject: [PATCH 08/26] feat(daemon): count and log uncaptured wgpu errors instead of panicking wgpu's default uncaptured-error handler panics the calling thread, and no device the daemon opened installed a handler or a device-lost callback, so a single validation error on the render thread took the pipeline down. Every device GpuRenderDevice opens now routes uncaptured errors into process-wide accounting: counted by kind, the latest kept for diagnostics, and logged at error level with the device label, thread, and wgpu's description. Logging is limited to ten lines a second, with a count of the suppressed ones, so a cascade of follow-on errors from one invalid resource cannot flood the log while every error is still counted. Device loss is counted and logged with the device's role: the compositor device or an auxiliary one such as the interactive preview executor's. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/src/render_thread.rs | 1 + .../src/render_thread/gpu_device.rs | 60 ++++ .../src/render_thread/gpu_errors.rs | 259 ++++++++++++++++++ 3 files changed, 320 insertions(+) create mode 100644 crates/hypercolor-daemon/src/render_thread/gpu_errors.rs diff --git a/crates/hypercolor-daemon/src/render_thread.rs b/crates/hypercolor-daemon/src/render_thread.rs index b005018d6..95c865b6e 100644 --- a/crates/hypercolor-daemon/src/render_thread.rs +++ b/crates/hypercolor-daemon/src/render_thread.rs @@ -36,6 +36,7 @@ mod frame_throttle; #[cfg(feature = "wgpu")] #[doc(hidden)] pub mod gpu_device; +pub mod gpu_errors; mod input_publication; mod layer_runtime; mod lighting_feed; diff --git a/crates/hypercolor-daemon/src/render_thread/gpu_device.rs b/crates/hypercolor-daemon/src/render_thread/gpu_device.rs index 1db097624..28fccb5a4 100644 --- a/crates/hypercolor-daemon/src/render_thread/gpu_device.rs +++ b/crates/hypercolor-daemon/src/render_thread/gpu_device.rs @@ -2,6 +2,8 @@ use std::sync::Arc; use anyhow::{Context, Result}; +use super::gpu_errors::{GpuDeviceRole, GpuErrorKind, record_device_lost, record_uncaptured_error}; + const GPU_RESOURCE_CREATION_BUDGET_PERCENT: u8 = 80; const GPU_DEVICE_LOSS_BUDGET_PERCENT: u8 = 95; @@ -125,6 +127,7 @@ impl GpuRenderDevice { trace: wgpu::Trace::Off, })) .with_context(|| format!("failed to create a {label} wgpu device"))?; + install_error_handlers(&device, label, GpuDeviceRole::Compositor); let limits = device.limits(); let info = GpuRenderDeviceInfo { @@ -193,6 +196,7 @@ impl GpuRenderDevice { trace: wgpu::Trace::Off, })) .with_context(|| format!("failed to create an independent {label} wgpu device"))?; + install_error_handlers(&device, label, GpuDeviceRole::Auxiliary); let adapter_info = adapter.get_info(); let limits = device.limits(); @@ -257,6 +261,46 @@ impl GpuRenderDevice { } } +/// Route every error and loss `device` reports into the daemon's GPU +/// error accounting. +/// +/// Without a handler wgpu panics the calling thread on the first +/// uncaptured validation error, which on the render thread takes the whole +/// pipeline down. With it the error is counted, logged at error level with +/// the device label and wgpu's description, and kept for diagnostics. +fn install_error_handlers(device: &wgpu::Device, label: &'static str, role: GpuDeviceRole) { + device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| { + let kind = match &error { + wgpu::Error::Validation { .. } => GpuErrorKind::Validation, + wgpu::Error::OutOfMemory { .. } => GpuErrorKind::OutOfMemory, + wgpu::Error::Internal { .. } => GpuErrorKind::Internal, + }; + record_uncaptured_error(label, kind, &error.to_string()); + })); + device.set_device_lost_callback(move |reason, message| { + let reason = match reason { + wgpu::DeviceLostReason::Unknown => "unknown", + wgpu::DeviceLostReason::Destroyed => "destroyed", + }; + record_device_lost(label, role, reason, &message); + }); +} + +/// Raise one uncaptured validation error on `device` on purpose. +/// +/// A buffer that is both read- and write-mappable is invalid without +/// `MAPPABLE_PRIMARY_BUFFERS`, which no daemon device requests, so wgpu +/// reports it synchronously through the uncaptured-error handler. +#[cfg(test)] +pub(crate) fn raise_uncaptured_validation_error(device: &wgpu::Device) { + let _invalid = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("injected validation error"), + size: 4, + usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::MAP_WRITE, + mapped_at_creation: false, + }); +} + /// Make test devices compile shaders the way shipped builds do on users' /// machines. /// @@ -657,6 +701,22 @@ mod tests { assert!(llvmpipe.software_adapter_reason().is_some()); } + #[test] + fn uncaptured_validation_errors_are_counted_instead_of_panicking() { + let Ok(device) = GpuRenderDevice::new_with_backend_preference( + "validation handler test device", + GpuBackendPreference::Default, + ) else { + return; + }; + let before = crate::render_thread::gpu_errors::gpu_error_totals(); + + raise_uncaptured_validation_error(device.device()); + + let after = crate::render_thread::gpu_errors::gpu_error_totals(); + assert!(after.validation > before.validation); + } + #[test] fn independent_device_reuses_adapter_without_sharing_device_or_queue() { let Ok(authoritative) = GpuRenderDevice::new_with_backend_preference( diff --git a/crates/hypercolor-daemon/src/render_thread/gpu_errors.rs b/crates/hypercolor-daemon/src/render_thread/gpu_errors.rs new file mode 100644 index 000000000..d7767d7d2 --- /dev/null +++ b/crates/hypercolor-daemon/src/render_thread/gpu_errors.rs @@ -0,0 +1,259 @@ +//! Process-wide accounting for wgpu errors no error scope captured. +//! +//! wgpu's default uncaptured-error handler panics the calling thread, which +//! on the render thread takes the whole pipeline down for a validation +//! mistake. Every device the daemon opens routes those errors here instead: +//! each one is counted, the latest is kept for diagnostics, and they are +//! logged at error level under a burst limit so a cascade of follow-on +//! errors from one invalid resource cannot flood the log. Device loss is +//! counted and logged too. +//! +//! The accounting has no wgpu types in it, so builds without the `wgpu` +//! feature still report (all zero) totals. +#![cfg_attr( + not(feature = "wgpu"), + allow( + dead_code, + reason = "only wgpu devices report errors; builds without wgpu still report zero totals" + ) +)] + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Mutex, PoisonError}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +/// Errors logged per [`LOG_WINDOW`] before the rest are only counted. +const LOG_BURST: u32 = 10; + +/// Window the [`LOG_BURST`] limit applies to. +const LOG_WINDOW: Duration = Duration::from_secs(1); + +/// Which kind of uncaptured error wgpu raised. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum GpuErrorKind { + Validation, + OutOfMemory, + Internal, +} + +impl GpuErrorKind { + const fn as_str(self) -> &'static str { + match self { + Self::Validation => "validation", + Self::OutOfMemory => "out_of_memory", + Self::Internal => "internal", + } + } +} + +/// What a device does for the daemon, reported with its loss. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum GpuDeviceRole { + /// A compositor device a render pipeline draws on. + Compositor, + /// Any other logical device, such as the interactive preview executor's. + Auxiliary, +} + +/// The most recent uncaptured error or device loss. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GpuErrorSample { + /// Label of the device that reported it. + pub device: String, + /// `validation`, `out_of_memory`, `internal`, or `device_lost`. + pub kind: String, + /// wgpu's description of the error. + pub message: String, + /// Wall-clock time it was recorded, in Unix milliseconds. + pub at_unix_ms: u64, +} + +/// Counts of every uncaptured error and device loss since process start. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct GpuErrorTotals { + pub validation: u64, + pub out_of_memory: u64, + pub internal: u64, + pub device_lost: u64, + pub last: Option, +} + +impl GpuErrorTotals { + /// Uncaptured errors of every kind, device loss excluded. + #[must_use] + pub const fn uncaptured(&self) -> u64 { + self.validation + .saturating_add(self.out_of_memory) + .saturating_add(self.internal) + } +} + +#[derive(Debug, Default)] +struct LogWindow { + started_at: Option, + logged: u32, + suppressed: u64, +} + +struct GpuErrorTelemetry { + validation: AtomicU64, + out_of_memory: AtomicU64, + internal: AtomicU64, + device_lost: AtomicU64, + last: Mutex>, + log_window: Mutex, +} + +static TELEMETRY: GpuErrorTelemetry = GpuErrorTelemetry { + validation: AtomicU64::new(0), + out_of_memory: AtomicU64::new(0), + internal: AtomicU64::new(0), + device_lost: AtomicU64::new(0), + last: Mutex::new(None), + log_window: Mutex::new(LogWindow { + started_at: None, + logged: 0, + suppressed: 0, + }), +}; + +/// Record one uncaptured error from `device`. +pub(crate) fn record_uncaptured_error(device: &str, kind: GpuErrorKind, message: &str) { + let counter = match kind { + GpuErrorKind::Validation => &TELEMETRY.validation, + GpuErrorKind::OutOfMemory => &TELEMETRY.out_of_memory, + GpuErrorKind::Internal => &TELEMETRY.internal, + }; + let total = counter.fetch_add(1, Ordering::Relaxed).saturating_add(1); + store_last(device, kind.as_str(), message); + + if let Some(suppressed) = admit_log() { + tracing::error!( + device, + kind = kind.as_str(), + thread = std::thread::current().name().unwrap_or(""), + total, + suppressed_since_last_log = suppressed, + error = message, + "uncaptured wgpu error" + ); + } +} + +/// Count and log that `device` was lost. +pub(crate) fn record_device_lost(device: &str, role: GpuDeviceRole, reason: &str, message: &str) { + let total = TELEMETRY + .device_lost + .fetch_add(1, Ordering::Relaxed) + .saturating_add(1); + let detail = if message.is_empty() { + reason.to_owned() + } else { + format!("{reason}: {message}") + }; + store_last(device, "device_lost", &detail); + tracing::error!( + device, + ?role, + reason, + total, + error = message, + "wgpu device lost" + ); +} + +/// Snapshot the totals since process start. +#[must_use] +pub fn gpu_error_totals() -> GpuErrorTotals { + GpuErrorTotals { + validation: TELEMETRY.validation.load(Ordering::Relaxed), + out_of_memory: TELEMETRY.out_of_memory.load(Ordering::Relaxed), + internal: TELEMETRY.internal.load(Ordering::Relaxed), + device_lost: TELEMETRY.device_lost.load(Ordering::Relaxed), + last: TELEMETRY + .last + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(), + } +} + +fn store_last(device: &str, kind: &str, message: &str) { + let at_unix_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }); + let sample = GpuErrorSample { + device: device.to_owned(), + kind: kind.to_owned(), + message: message.to_owned(), + at_unix_ms, + }; + *TELEMETRY + .last + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(sample); +} + +/// Admit one log line under the burst limit, returning how many errors +/// went unlogged since the last admitted one. +fn admit_log() -> Option { + let mut window = TELEMETRY + .log_window + .lock() + .unwrap_or_else(PoisonError::into_inner); + let now = Instant::now(); + if window + .started_at + .is_none_or(|started| now.saturating_duration_since(started) >= LOG_WINDOW) + { + window.started_at = Some(now); + window.logged = 0; + } + if window.logged >= LOG_BURST { + window.suppressed = window.suppressed.saturating_add(1); + return None; + } + window.logged += 1; + Some(std::mem::take(&mut window.suppressed)) +} + +#[cfg(test)] +mod tests { + use super::{ + GpuDeviceRole, GpuErrorKind, gpu_error_totals, record_device_lost, record_uncaptured_error, + }; + + #[test] + fn uncaptured_errors_count_by_kind_and_keep_the_latest() { + let before = gpu_error_totals(); + + record_uncaptured_error("unit test device", GpuErrorKind::Validation, "bad usage"); + record_uncaptured_error( + "unit test device", + GpuErrorKind::OutOfMemory, + "Out of Memory", + ); + + let after = gpu_error_totals(); + assert!(after.validation > before.validation); + assert!(after.out_of_memory > before.out_of_memory); + assert!(after.uncaptured() >= before.uncaptured() + 2); + assert!(after.last.is_some(), "the latest error should be kept"); + } + + #[test] + fn device_loss_is_counted() { + let before = gpu_error_totals(); + + record_device_lost( + "preview device", + GpuDeviceRole::Auxiliary, + "unknown", + "reset", + ); + + assert!(gpu_error_totals().device_lost > before.device_lost); + } +} From 62598c1cb8b175444f2a5afa055f7c8b52c7169a Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 01:28:52 -0700 Subject: [PATCH 09/26] feat(daemon): supervise the render pipeline and restart it in-process The render thread ran its pipeline inside one block_on whose JoinHandle was joined only at shutdown. A panic or an error exit ended rendering with no log line, no diagnose failure, and no restart, while the API kept serving and every LED froze on its last frame. The thread body now lives in a supervisor that runs each pipeline attempt under catch_unwind and records how it ended. A panic, a terminated input publication pump, or a failed rebuild is logged at error level with the payload and the backtrace the panic hook captured, persisted to render-failure.json in the daemon state directory, and followed by a rebuild: fresh compositors and GPU resources, a fresh zone runtime with new Servo sessions, fresh frame state, and a fresh input pump when the pump was what failed. The compositor GPU device is kept, since a panic or a validation error leaves it valid. A panic anywhere else on the thread, in startup or in the supervisor itself, is caught at the top of the thread and requests termination, so the thread can no longer end without the daemon knowing. Restarts back off from 250 ms, doubling to 8 s, and five are allowed in five minutes; a pipeline that ran a minute before failing starts a fresh streak. The failure after the budget, or any loss of the compositor device, stops rendering and asks the daemon to exit instead. A lost device cannot be replaced in-process: Servo's GPU import device is a process-wide once-only binding to the startup device, and the native screen bridges and the interactive preview device came from the same adapter. Each GpuRenderDevice now raises its own sticky loss signal from its device-lost callback, so a loss during a backoff or a rebuild still ends in termination. RenderSupervision carries the phase, a heartbeat and frame clock the render loop stamps every iteration, restart counters, the last failure, and the termination request. The daemon threads one through the domain graph from the state directory, and the next run reports the failure an earlier run persisted, told apart by a per-process instance id because a container restarts the daemon as pid 1 every time. Each supervision also registers a panic hook observer that persists its own render thread's panics, and in a panic=abort build any render thread panic, so a release build that never returns from the hook still leaves a record. The screen parity mailbox is now shared across restarts so its diagnostics handle stays connected to whichever pipeline is live. Test hooks behind persistence-test-hooks inject a pipeline panic, a stall, a GPU validation error, and a lost compositor device. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/src/app_state.rs | 2 + crates/hypercolor-daemon/src/domain/output.rs | 9 + crates/hypercolor-daemon/src/render_thread.rs | 196 ++- .../src/render_thread/gpu_device.rs | 53 +- .../src/render_thread/gpu_errors.rs | 85 +- .../src/render_thread/input_publication.rs | 2 +- .../src/render_thread/pipeline_driver.rs | 41 +- .../screen_parity_diagnostics.rs | 22 +- .../src/render_thread/sparkleflinger/gpu.rs | 2 +- .../sparkleflinger/gpu/runtime.rs | 12 +- .../src/render_thread/sparkleflinger/mod.rs | 26 + .../src/render_thread/supervision.rs | 1154 +++++++++++++++++ .../src/render_thread/supervisor.rs | 475 +++++++ .../src/startup/lifecycle.rs | 8 +- .../hypercolor-daemon/src/startup/services.rs | 6 + .../tests/render_thread_tests.rs | 295 ++++- 16 files changed, 2255 insertions(+), 133 deletions(-) create mode 100644 crates/hypercolor-daemon/src/render_thread/supervision.rs create mode 100644 crates/hypercolor-daemon/src/render_thread/supervisor.rs diff --git a/crates/hypercolor-daemon/src/app_state.rs b/crates/hypercolor-daemon/src/app_state.rs index aab5e5563..edc2515b8 100644 --- a/crates/hypercolor-daemon/src/app_state.rs +++ b/crates/hypercolor-daemon/src/app_state.rs @@ -590,6 +590,8 @@ impl AppState { display_frames: Arc::clone(&display_frames), device_metrics: Arc::clone(&device_metrics), input_manager: input_manager.clone(), + // An isolated AppState never spawns a render thread. + render_supervision: crate::render_thread::RenderSupervision::detached(), }, |layout| { let discovery_runtime = crate::discovery::DiscoveryRuntime { diff --git a/crates/hypercolor-daemon/src/domain/output.rs b/crates/hypercolor-daemon/src/domain/output.rs index 5c3f1c70e..4c9b02862 100644 --- a/crates/hypercolor-daemon/src/domain/output.rs +++ b/crates/hypercolor-daemon/src/domain/output.rs @@ -54,6 +54,7 @@ pub struct OutputContext { preview_runtime: Arc, devices: DeviceContext, start_time: Instant, + render_supervision: crate::render_thread::RenderSupervision, } impl OutputContext { @@ -104,6 +105,7 @@ impl OutputContext { preview_runtime: Arc, devices: DeviceContext, start_time: Instant, + render_supervision: crate::render_thread::RenderSupervision, ) -> Self { Self { output_power, @@ -116,9 +118,16 @@ impl OutputContext { preview_runtime, devices, start_time, + render_supervision, } } + /// Health and recovery state of the supervised render thread. + #[must_use] + pub const fn render_supervision(&self) -> &crate::render_thread::RenderSupervision { + &self.render_supervision + } + /// How long the daemon's subsystems have been up. #[must_use] pub fn uptime(&self) -> std::time::Duration { diff --git a/crates/hypercolor-daemon/src/render_thread.rs b/crates/hypercolor-daemon/src/render_thread.rs index 95c865b6e..275f8b498 100644 --- a/crates/hypercolor-daemon/src/render_thread.rs +++ b/crates/hypercolor-daemon/src/render_thread.rs @@ -52,6 +52,8 @@ mod screen_canvas; mod screen_parity_diagnostics; #[doc(hidden)] pub mod sparkleflinger; +pub mod supervision; +mod supervisor; mod unassigned_output; mod zone_previews; @@ -62,17 +64,17 @@ use std::sync::mpsc; use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow}; +use arc_swap::ArcSwapOption; use tokio::sync::{Mutex, RwLock, watch}; use tokio_util::sync::CancellationToken; use tracing::info; +use self::input_publication::InputPublicationMonitor; pub(crate) use self::input_publication::InputScreenBranchRequest; pub use self::input_publication::{ InputPublicationConsumer, InputPublicationDemand, InputPublicationDemandHandle, InputPublicationDemandRegistration, InputPublicationStatus, InputScreenBranchDemand, }; -use self::input_publication::{InputPublicationMonitor, InputPublicationPump}; -use self::pipeline_driver::run_pipeline; pub(crate) use self::producer_queue::ProducerFrame; pub(crate) use self::render_zones::{RenderSceneContext, ZoneFrameInputs}; pub(crate) use self::scene_dependency::SceneDependencyKey; @@ -84,6 +86,10 @@ pub(crate) use self::scene_dependency::SceneDependencyKey; pub(crate) use self::screen_parity_diagnostics::{ ScreenParityDiagnosticHandle, ScreenParityLiveSnapshot, ScreenParitySnapshotError, }; +pub use self::supervision::{ + RenderFailureKind, RenderFailureRecord, RenderRecoveryAction, RenderRestartPolicy, + RenderSupervision, RenderSupervisionSnapshot, RenderThreadPhase, +}; use crate::discovery::DiscoveryRuntime; use crate::domain::scene::{ScenePlanReader, SceneService}; use crate::domain::spatial::SpatialService; @@ -92,7 +98,7 @@ use crate::output_power::OutputPowerState; use crate::performance::PerformanceTracker; use crate::preview_runtime::PreviewRuntime; use crate::scene_transactions::SceneTransactionQueue; -use crate::startup::{StartupProgress, startup_step}; +use crate::startup::StartupProgress; use crate::zone_layout_preview::ZoneLayoutPreviewStore; use hypercolor_core::asset::AssetLibrary; use hypercolor_core::bus::HypercolorBus; @@ -247,11 +253,20 @@ const fn u8_to_fps_tier(value: u8) -> FpsTier { /// Call [`shutdown`](Self::shutdown) to stop the thread gracefully. /// The render loop must be stopped first (via `RenderLoop::stop()`) — the /// thread will exit on the next `tick()` returning `false`. +/// +/// The thread supervises its own pipeline: a panic or error exit is logged, +/// recorded in the [`RenderSupervision`] it was spawned with, and followed +/// by an in-process restart, until the restart budget runs out and the +/// supervision asks the daemon to exit. pub struct RenderThread { join_handle: Option>>, cancel: CancellationToken, input_publication_demands: InputPublicationDemandHandle, - input_publication_monitor: InputPublicationMonitor, + /// The current input pump's monitor, replaced when a failed pump is + /// restarted. + input_publication_monitor: Arc>, + supervision: RenderSupervision, + _panic_observer: Option, #[cfg(all(target_os = "macos", feature = "wgpu", feature = "screen-capture"))] screen_parity_diagnostics: ScreenParityDiagnosticHandle, } @@ -359,7 +374,16 @@ impl RenderThread { } pub fn try_spawn(state: RenderThreadState) -> Result { - Self::try_spawn_with_runtime_builder(state, build_render_runtime) + Self::try_spawn_supervised(state, RenderSupervision::detached()) + } + + /// Spawn the render thread under `supervision`, which carries the + /// restart policy and receives its health and failures. + pub fn try_spawn_supervised( + state: RenderThreadState, + supervision: RenderSupervision, + ) -> Result { + Self::spawn_inner(state, build_render_runtime, None, supervision) } /// Spawn the render thread while reporting each completed startup step @@ -376,8 +400,9 @@ impl RenderThread { pub fn try_spawn_reporting( state: RenderThreadState, progress: StartupProgress, + supervision: RenderSupervision, ) -> Result { - Self::spawn_inner(state, build_render_runtime, Some(progress)) + Self::spawn_inner(state, build_render_runtime, Some(progress), supervision) } #[doc(hidden)] @@ -388,24 +413,23 @@ impl RenderThread { where F: FnOnce() -> Result + Send + 'static, { - Self::spawn_inner(state, build_runtime, None) + Self::spawn_inner(state, build_runtime, None, RenderSupervision::detached()) } fn spawn_inner( state: RenderThreadState, build_runtime: F, progress: Option, + supervision: RenderSupervision, ) -> Result where F: FnOnce() -> Result + Send + 'static, { let input_publication_demands = InputPublicationDemandHandle::new(state.input_manager.screen_native_execution_policy()); - let pump_demands = input_publication_demands.clone(); - let pipeline_demands = input_publication_demands.clone(); let cancel = CancellationToken::new(); - let worker_cancel = cancel.clone(); - let (ready_tx, ready_rx) = mpsc::sync_channel::>(1); + let input_publication_monitor = Arc::new(ArcSwapOption::empty()); + let (ready_tx, ready_rx) = mpsc::sync_channel::>(1); #[cfg(feature = "wgpu")] let (screen_parity_diagnostics, screen_parity_mailbox) = screen_parity_diagnostics::screen_parity_diagnostic_channel(); @@ -414,118 +438,67 @@ impl RenderThread { not(all(target_os = "macos", feature = "screen-capture")) ))] let _ = screen_parity_diagnostics; - let join_handle = std::thread::Builder::new() - .name("hypercolor-render".to_owned()) - .spawn(move || -> Result<()> { - let _scene_transaction_consumer = state.scene_transactions.consumer(); - crate::process::configure_render_thread_priority(); - let runtime = - match startup_step(progress.as_ref(), "render runtime", build_runtime) { - Ok(runtime) => runtime, - Err(error) => { - let _ = ready_tx.send(Err(error)); - return Ok(()); - } - }; - let mut input_pump = match startup_step( - progress.as_ref(), - "input publication pump", - || { - runtime.block_on(InputPublicationPump::start( - state.input_manager.clone(), - pump_demands, - Some(state.interaction_routing.device_input().clone()), - )) - }, - ) { - Ok(pump) => pump, - Err(error) => { - let _ = ready_tx.send(Err(error)); - return Ok(()); - } - }; - #[cfg(all( - target_os = "macos", - feature = "wgpu", - feature = "screen-capture" - ))] - let pipeline = pipeline_runtime::PipelineRuntime::from_state( - &state, - input_pump.reader(), - pipeline_demands, - #[cfg(feature = "wgpu")] - screen_parity_mailbox, - progress.as_ref(), - ); - #[cfg(not(all( - target_os = "macos", - feature = "wgpu", - feature = "screen-capture" - )))] - let pipeline = pipeline_runtime::PipelineRuntime::from_state( - &state, - input_pump.reader(), - pipeline_demands, - #[cfg(feature = "wgpu")] - screen_parity_mailbox, - progress.as_ref(), - ); - // Startup reporting ends with the pipeline build. - drop(progress); - match pipeline { - Ok(runtime_state) => { - let monitor = input_pump.monitor(); - if ready_tx.send(Ok(monitor.clone())).is_err() { - return runtime.block_on(input_pump.shutdown()); - } - let pipeline_result = runtime.block_on(async { - tokio::select! { - () = run_pipeline(state.clone(), runtime_state) => Ok(()), - status = monitor.wait_for_terminal() => { - tracing::error!(?status, "input publication pump terminated while rendering"); - state.render_loop.write().await.stop(); - Err(anyhow!("input publication pump terminated: {status:?}")) - } - () = worker_cancel.cancelled() => { - state.render_loop.write().await.stop(); - Ok(()) - } - } - }); - let shutdown_result = runtime.block_on(input_pump.shutdown()); - pipeline_result.and(shutdown_result) - } - Err(error) => { - if let Err(shutdown_error) = runtime.block_on(input_pump.shutdown()) { - tracing::warn!(%shutdown_error, "input publication cleanup failed after render startup error"); - } - let _ = ready_tx.send(Err(error)); - Ok(()) - } - } - }) - .context("failed to spawn render thread")?; - let ready = ready_rx - .recv() - .context("render thread exited before startup completed")?; - let input_publication_monitor = match ready { - Ok(monitor) => monitor, + let panic_observer = supervision + .panic_observer() + .map(crate::panic_hook::add_observer); + supervision.set_phase(RenderThreadPhase::Starting); + let launch = supervisor::RenderThreadLaunch { + state, + build_runtime, + progress, + ready: ready_tx, + cancel: cancel.clone(), + input_demands: input_publication_demands.clone(), + input_monitor: Arc::clone(&input_publication_monitor), + supervision: supervision.clone(), + #[cfg(feature = "wgpu")] + screen_parity_mailbox, + }; + let spawned = std::thread::Builder::new() + .name(supervision::RENDER_THREAD_NAME.to_owned()) + .spawn(move || supervisor::run(launch)) + .context("failed to spawn render thread"); + let join_handle = match spawned { + Ok(join_handle) => join_handle, Err(error) => { - let _ = join_handle.join(); + supervision.set_phase(RenderThreadPhase::Stopped); return Err(error); } }; + let startup = ready_rx + .recv() + .context("render thread exited before startup completed") + .and_then(|ready| ready); + if let Err(error) = startup { + let _ = join_handle.join(); + return Err(match supervision.termination() { + Some(record) => error.context(format!( + "render thread failed during startup ({}): {}", + record.kind.as_str(), + record.message + )), + None => error, + }); + } info!("render thread spawned"); Ok(Self { join_handle: Some(join_handle), cancel, input_publication_demands, input_publication_monitor, + supervision, + _panic_observer: panic_observer, #[cfg(all(target_os = "macos", feature = "wgpu", feature = "screen-capture"))] screen_parity_diagnostics, }) } + /// The supervision this thread reports its health and failures to. + #[must_use] + pub fn supervision(&self) -> RenderSupervision { + self.supervision.clone() + } + /// Clone the lock-free demand publication used by input consumers. pub fn input_publication_demands(&self) -> InputPublicationDemandHandle { self.input_publication_demands.clone() @@ -533,7 +506,10 @@ impl RenderThread { /// Read the input-publication worker lifecycle state without blocking. pub fn input_publication_status(&self) -> InputPublicationStatus { - self.input_publication_monitor.status() + self.input_publication_monitor + .load() + .as_ref() + .map_or(InputPublicationStatus::Starting, |monitor| monitor.status()) } #[cfg(all(target_os = "macos", feature = "wgpu", feature = "screen-capture"))] @@ -584,7 +560,7 @@ fn build_render_runtime() -> Result { .worker_threads(RENDER_RUNTIME_WORKERS) .max_blocking_threads(RENDER_RUNTIME_MAX_BLOCKING_THREADS) .thread_keep_alive(RENDER_RUNTIME_THREAD_KEEP_ALIVE) - .thread_name("hypercolor-render-rt") + .thread_name(supervision::RENDER_RUNTIME_THREAD_NAME) .enable_all() .build() .context("failed to initialize render thread runtime") diff --git a/crates/hypercolor-daemon/src/render_thread/gpu_device.rs b/crates/hypercolor-daemon/src/render_thread/gpu_device.rs index 28fccb5a4..47de62be9 100644 --- a/crates/hypercolor-daemon/src/render_thread/gpu_device.rs +++ b/crates/hypercolor-daemon/src/render_thread/gpu_device.rs @@ -2,7 +2,9 @@ use std::sync::Arc; use anyhow::{Context, Result}; -use super::gpu_errors::{GpuDeviceRole, GpuErrorKind, record_device_lost, record_uncaptured_error}; +use super::gpu_errors::{ + DeviceLossSignal, GpuDeviceRole, GpuErrorKind, record_device_lost, record_uncaptured_error, +}; const GPU_RESOURCE_CREATION_BUDGET_PERCENT: u8 = 80; const GPU_DEVICE_LOSS_BUDGET_PERCENT: u8 = 95; @@ -19,6 +21,7 @@ struct GpuRenderDeviceInner { device: wgpu::Device, queue: wgpu::Queue, info: GpuRenderDeviceInfo, + loss: DeviceLossSignal, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -127,7 +130,7 @@ impl GpuRenderDevice { trace: wgpu::Trace::Off, })) .with_context(|| format!("failed to create a {label} wgpu device"))?; - install_error_handlers(&device, label, GpuDeviceRole::Compositor); + let loss = install_error_handlers(&device, label, GpuDeviceRole::Compositor); let limits = device.limits(); let info = GpuRenderDeviceInfo { @@ -150,6 +153,7 @@ impl GpuRenderDevice { device, queue, info, + loss, }), }) } @@ -196,7 +200,7 @@ impl GpuRenderDevice { trace: wgpu::Trace::Off, })) .with_context(|| format!("failed to create an independent {label} wgpu device"))?; - install_error_handlers(&device, label, GpuDeviceRole::Auxiliary); + let loss = install_error_handlers(&device, label, GpuDeviceRole::Auxiliary); let adapter_info = adapter.get_info(); let limits = device.limits(); @@ -220,6 +224,7 @@ impl GpuRenderDevice { device, queue, info, + loss, }), }) } @@ -244,6 +249,11 @@ impl GpuRenderDevice { self.inner.info.clone() } + /// Raised once wgpu reports this device lost. + pub(crate) fn loss_signal(&self) -> DeviceLossSignal { + self.inner.loss.clone() + } + pub(crate) fn require_texture_usage( &self, format: wgpu::TextureFormat, @@ -262,13 +272,17 @@ impl GpuRenderDevice { } /// Route every error and loss `device` reports into the daemon's GPU -/// error accounting. +/// error accounting, and return the signal its loss raises. /// /// Without a handler wgpu panics the calling thread on the first /// uncaptured validation error, which on the render thread takes the whole /// pipeline down. With it the error is counted, logged at error level with /// the device label and wgpu's description, and kept for diagnostics. -fn install_error_handlers(device: &wgpu::Device, label: &'static str, role: GpuDeviceRole) { +fn install_error_handlers( + device: &wgpu::Device, + label: &'static str, + role: GpuDeviceRole, +) -> DeviceLossSignal { device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| { let kind = match &error { wgpu::Error::Validation { .. } => GpuErrorKind::Validation, @@ -277,13 +291,17 @@ fn install_error_handlers(device: &wgpu::Device, label: &'static str, role: GpuD }; record_uncaptured_error(label, kind, &error.to_string()); })); + let loss = DeviceLossSignal::default(); + let raised = loss.clone(); device.set_device_lost_callback(move |reason, message| { let reason = match reason { wgpu::DeviceLostReason::Unknown => "unknown", wgpu::DeviceLostReason::Destroyed => "destroyed", }; record_device_lost(label, role, reason, &message); + raised.raise(format!("{label} lost ({reason}: {message})")); }); + loss } /// Raise one uncaptured validation error on `device` on purpose. @@ -291,7 +309,7 @@ fn install_error_handlers(device: &wgpu::Device, label: &'static str, role: GpuD /// A buffer that is both read- and write-mappable is invalid without /// `MAPPABLE_PRIMARY_BUFFERS`, which no daemon device requests, so wgpu /// reports it synchronously through the uncaptured-error handler. -#[cfg(test)] +#[cfg(any(test, feature = "persistence-test-hooks"))] pub(crate) fn raise_uncaptured_validation_error(device: &wgpu::Device) { let _invalid = device.create_buffer(&wgpu::BufferDescriptor { label: Some("injected validation error"), @@ -717,6 +735,29 @@ mod tests { assert!(after.validation > before.validation); } + #[test] + fn a_destroyed_device_raises_only_its_own_loss_signal() { + let Ok(device) = GpuRenderDevice::new_with_backend_preference( + "loss signal test device", + GpuBackendPreference::Default, + ) else { + return; + }; + let sibling = device + .independent_device("loss signal sibling device") + .expect("selected adapter should support a second logical device"); + + device.device().destroy(); + let _ = device.device().poll(wgpu::PollType::Poll); + + let lost = device + .loss_signal() + .lost() + .expect("destroying the device should run its device-lost callback"); + assert!(lost.contains("loss signal test device"), "{lost}"); + assert!(sibling.loss_signal().lost().is_none()); + } + #[test] fn independent_device_reuses_adapter_without_sharing_device_or_queue() { let Ok(authoritative) = GpuRenderDevice::new_with_backend_preference( diff --git a/crates/hypercolor-daemon/src/render_thread/gpu_errors.rs b/crates/hypercolor-daemon/src/render_thread/gpu_errors.rs index d7767d7d2..071326751 100644 --- a/crates/hypercolor-daemon/src/render_thread/gpu_errors.rs +++ b/crates/hypercolor-daemon/src/render_thread/gpu_errors.rs @@ -6,7 +6,9 @@ //! each one is counted, the latest is kept for diagnostics, and they are //! logged at error level under a burst limit so a cascade of follow-on //! errors from one invalid resource cannot flood the log. Device loss is -//! counted and logged too. +//! counted and logged too, and each device raises its own +//! [`DeviceLossSignal`], which the render supervisor waits on for the +//! device its pipeline draws with. //! //! The accounting has no wgpu types in it, so builds without the `wgpu` //! feature still report (all zero) totals. @@ -19,9 +21,11 @@ )] use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Mutex, PoisonError}; +use std::sync::{Arc, Mutex, PoisonError}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use tokio::sync::watch; + /// Errors logged per [`LOG_WINDOW`] before the rest are only counted. const LOG_BURST: u32 = 10; @@ -117,6 +121,57 @@ static TELEMETRY: GpuErrorTelemetry = GpuErrorTelemetry { }), }; +/// Sticky "this device was lost" flag for one wgpu device. +/// +/// Clones share the flag. It is set once, by the device-lost callback, and +/// stays set: a lost device never comes back, so anything that checks it +/// later, such as a pipeline rebuilt on the same device, sees the loss too. +#[derive(Debug, Clone)] +pub(crate) struct DeviceLossSignal { + sender: Arc>>>, +} + +impl Default for DeviceLossSignal { + fn default() -> Self { + Self { + sender: Arc::new(watch::Sender::new(None)), + } + } +} + +impl DeviceLossSignal { + /// Mark the device lost, keeping the first description. + pub(crate) fn raise(&self, description: String) { + self.sender.send_if_modified(|lost| { + if lost.is_some() { + return false; + } + *lost = Some(Arc::from(description)); + true + }); + } + + /// The loss description, once the device is lost. + pub(crate) fn lost(&self) -> Option> { + self.sender.borrow().clone() + } + + /// Wait until the device is lost, and return the description. + pub(crate) async fn wait(&self) -> String { + let mut receiver = self.sender.subscribe(); + let lost = receiver + .wait_for(Option::is_some) + .await + .map(|lost| lost.as_deref().unwrap_or("device lost").to_owned()); + match lost { + Ok(lost) => lost, + // This signal holds its own sender, so the channel cannot close + // while the wait borrows it. + Err(_) => std::future::pending().await, + } + } +} + /// Record one uncaptured error from `device`. pub(crate) fn record_uncaptured_error(device: &str, kind: GpuErrorKind, message: &str) { let counter = match kind { @@ -222,7 +277,8 @@ fn admit_log() -> Option { #[cfg(test)] mod tests { use super::{ - GpuDeviceRole, GpuErrorKind, gpu_error_totals, record_device_lost, record_uncaptured_error, + DeviceLossSignal, GpuDeviceRole, GpuErrorKind, gpu_error_totals, record_device_lost, + record_uncaptured_error, }; #[test] @@ -256,4 +312,27 @@ mod tests { assert!(gpu_error_totals().device_lost > before.device_lost); } + + #[tokio::test] + async fn a_raised_loss_wakes_a_waiter_and_stays_raised() { + let signal = DeviceLossSignal::default(); + assert!(signal.lost().is_none()); + let waiter = tokio::spawn({ + let signal = signal.clone(); + async move { signal.wait().await } + }); + + signal.raise("compositor lost (unknown: reset)".to_owned()); + signal.raise("a later loss report".to_owned()); + + assert_eq!( + waiter.await.expect("waiter should finish"), + "compositor lost (unknown: reset)" + ); + assert_eq!( + signal.lost().as_deref(), + Some("compositor lost (unknown: reset)") + ); + assert_eq!(signal.wait().await, "compositor lost (unknown: reset)"); + } } diff --git a/crates/hypercolor-daemon/src/render_thread/input_publication.rs b/crates/hypercolor-daemon/src/render_thread/input_publication.rs index 69c365cae..e171865c3 100644 --- a/crates/hypercolor-daemon/src/render_thread/input_publication.rs +++ b/crates/hypercolor-daemon/src/render_thread/input_publication.rs @@ -1054,7 +1054,7 @@ pub enum InputPublicationStatus { } impl InputPublicationStatus { - const fn is_terminal(&self) -> bool { + pub(crate) const fn is_terminal(&self) -> bool { matches!(self, Self::Stopped | Self::Failed(_)) } } diff --git a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs index 4097308d3..83406add7 100644 --- a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs +++ b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs @@ -9,8 +9,13 @@ use super::RenderThreadState; use super::frame_executor::{execute_frame, service_scene_transactions}; use super::frame_policy::SkipDecision; use super::pipeline_runtime::PipelineRuntime; +use super::supervision::RenderSupervision; -pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: PipelineRuntime) { +pub(crate) async fn run_pipeline( + state: RenderThreadState, + mut runtime: PipelineRuntime, + supervision: RenderSupervision, +) { info!( mode = ?state.render_acceleration_mode, "render pipeline started" @@ -23,6 +28,8 @@ pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: Pipeline loop { let scheduled_start = next_frame_at; wait_until_deadline(scheduled_start).await; + #[cfg(feature = "persistence-test-hooks")] + raise_injected_fault(&supervision, &mut runtime); let tick = { let mut render_loop = state.render_loop.write().await; @@ -42,7 +49,9 @@ pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: Pipeline if let Some(execution) = handle_inactive_render_loop(&state, &mut runtime, tick.state).await { - next_frame_at = execution.resolve_deadline(scheduled_start, Instant::now()); + let now = Instant::now(); + supervision.heartbeat(now); + next_frame_at = execution.resolve_deadline(scheduled_start, now); continue; } @@ -54,13 +63,39 @@ pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: Pipeline runtime.frame_loop.clock.rebase(scheduled_start); } let frame = execute_frame(&state, &mut runtime, scheduled_start, skip_decision).await; + let now = Instant::now(); + supervision.frame_completed(now); skip_decision = frame.next_skip_decision; - next_frame_at = frame.resolve_deadline(scheduled_start, Instant::now()); + next_frame_at = frame.resolve_deadline(scheduled_start, now); } info!("render pipeline exited"); } +/// Raise a fault a test queued, on the render thread, exactly where a real +/// one would surface: between frames, with the pipeline's state live. +#[cfg(feature = "persistence-test-hooks")] +fn raise_injected_fault(supervision: &RenderSupervision, runtime: &mut PipelineRuntime) { + use super::supervision::InjectedRenderFault; + + match supervision.take_injected_fault() { + None => {} + Some(InjectedRenderFault::Panic(message)) => panic!("{message}"), + Some(InjectedRenderFault::Stall(duration)) => std::thread::sleep(duration), + Some(InjectedRenderFault::GpuValidationError) => { + let raised = runtime + .render + .sparkleflinger + .raise_uncaptured_validation_error(); + if !raised { + tracing::warn!( + "injected GPU validation error skipped: the compositor runs on the CPU" + ); + } + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct RenderTickSnapshot { should_render: bool, diff --git a/crates/hypercolor-daemon/src/render_thread/screen_parity_diagnostics.rs b/crates/hypercolor-daemon/src/render_thread/screen_parity_diagnostics.rs index 11765d2c8..e89fb4b5b 100644 --- a/crates/hypercolor-daemon/src/render_thread/screen_parity_diagnostics.rs +++ b/crates/hypercolor-daemon/src/render_thread/screen_parity_diagnostics.rs @@ -13,7 +13,7 @@ reason = "consumed by platform diagnostic endpoints; targets without one carry the seam unused" )] -use std::sync::{Arc, mpsc}; +use std::sync::{Arc, Mutex, PoisonError, mpsc}; use anyhow::{Context, Result, anyhow}; use hypercolor_core::input::screen::implementer::{ @@ -37,8 +37,14 @@ pub(crate) struct ScreenParityDiagnosticHandle { sender: tokio_mpsc::Sender, } +/// The render thread's end of the parity mailbox. +/// +/// Clones share one receiver, so the supervisor keeps the mailbox across a +/// pipeline restart and the diagnostics handle stays connected to whichever +/// pipeline is live. +#[derive(Clone)] pub(crate) struct ScreenParityDiagnosticMailbox { - receiver: tokio_mpsc::Receiver, + receiver: Arc>>, } struct ScreenParityRequest { @@ -81,7 +87,9 @@ pub(crate) fn screen_parity_diagnostic_channel() let (sender, receiver) = tokio_mpsc::channel(REQUEST_CAPACITY); ( ScreenParityDiagnosticHandle { sender }, - ScreenParityDiagnosticMailbox { receiver }, + ScreenParityDiagnosticMailbox { + receiver: Arc::new(Mutex::new(receiver)), + }, ) } @@ -109,8 +117,12 @@ impl ScreenParityDiagnosticMailbox { descriptor: Option<&ResolvedScreenPublicationDescriptor>, spatial_engine: &SpatialEngine, ) { - let Ok(request) = self.receiver.try_recv() else { - return; + let request = { + let mut receiver = self.receiver.lock().unwrap_or_else(PoisonError::into_inner); + let Ok(request) = receiver.try_recv() else { + return; + }; + request }; let result = capture_active_snapshot( render_device, diff --git a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu.rs b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu.rs index 4acc2639d..1118c4052 100644 --- a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu.rs +++ b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu.rs @@ -191,7 +191,7 @@ fn reject_injected_gpu_preparation(fail_after_prepare: bool, resource: &'static } pub(crate) struct GpuSparkleFlinger { - _render_device: GpuRenderDevice, + render_device: GpuRenderDevice, device: wgpu::Device, queue: wgpu::Queue, /// The queue position each recent render frame ended at, oldest first, diff --git a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu/runtime.rs b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu/runtime.rs index 76b3ab071..4cb3aaff3 100644 --- a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu/runtime.rs +++ b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu/runtime.rs @@ -117,7 +117,7 @@ impl GpuSparkleFlinger { })?; Ok(Self { - _render_device: render_device, + render_device, device, queue, probe, @@ -233,6 +233,16 @@ impl GpuSparkleFlinger { self.fail_next_screen_upload_pool_saturation = true; } + /// Raised once the device this compositor draws with is lost. + pub(crate) fn device_loss_signal(&self) -> crate::render_thread::gpu_errors::DeviceLossSignal { + self.render_device.loss_signal() + } + + #[cfg(feature = "persistence-test-hooks")] + pub(crate) fn raise_uncaptured_validation_error(&self) { + crate::render_thread::gpu_device::raise_uncaptured_validation_error(&self.device); + } + pub(crate) fn supports_plan(&self, plan: &CompositionPlan) -> bool { self.canvas_gpu_admitted && matches!( diff --git a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/mod.rs b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/mod.rs index cdde0ac8d..38c3fe2a5 100644 --- a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/mod.rs +++ b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/mod.rs @@ -1142,6 +1142,32 @@ impl SparkleFlinger { } } + /// The loss signal of the GPU device this compositor draws with, or + /// `None` on the CPU backend. + pub(crate) fn device_loss_signal( + &self, + ) -> Option { + match &self.backend { + #[cfg(feature = "wgpu")] + SparkleFlingerBackend::Gpu { gpu, .. } => Some(gpu.device_loss_signal()), + SparkleFlingerBackend::Cpu(_) => None, + } + } + + /// Raise one uncaptured validation error on the compositor's GPU + /// device, returning `false` on the CPU backend, which has none. + #[cfg(feature = "persistence-test-hooks")] + pub(crate) fn raise_uncaptured_validation_error(&self) -> bool { + match &self.backend { + #[cfg(feature = "wgpu")] + SparkleFlingerBackend::Gpu { gpu, .. } => { + gpu.raise_uncaptured_validation_error(); + true + } + SparkleFlingerBackend::Cpu(_) => false, + } + } + /// Wait for the GPU to finish an earlier frame before this one queues /// more work, so GPU latency stays bounded on a device slower than the /// frame cadence. A no-op on the CPU backend. diff --git a/crates/hypercolor-daemon/src/render_thread/supervision.rs b/crates/hypercolor-daemon/src/render_thread/supervision.rs new file mode 100644 index 000000000..a6c5755be --- /dev/null +++ b/crates/hypercolor-daemon/src/render_thread/supervision.rs @@ -0,0 +1,1154 @@ +//! Shared health and recovery state for the supervised render thread. +//! +//! The render thread restarts its pipeline in-process after a panic or an +//! error exit, within a bounded budget, and asks the daemon to exit once the +//! budget runs out or the compositor GPU device is lost. Everything the rest +//! of the daemon needs to see about that lives here: the lifecycle phase, a +//! heartbeat and frame-completion clock the render loop stamps every +//! iteration, restart counters, the last failure, and the termination +//! request the daemon waits on. +//! +//! Failures are also persisted to `render-failure.json` in the daemon state +//! directory, so the record survives a log that was lost (a full tmpfs, a +//! closed pipe) and a `panic = "abort"` build that never returns from the +//! panic hook. The next daemon run loads the previous record and reports it. + +use std::collections::VecDeque; +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU8, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock, Mutex, PoisonError, Weak}; +use std::thread::ThreadId; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use tokio::sync::watch; + +#[cfg(feature = "persistence-test-hooks")] +use super::gpu_errors::DeviceLossSignal; +use super::gpu_errors::{GpuErrorTotals, gpu_error_totals}; +use crate::panic_hook::{PanicObserver, PanicReport}; + +/// Name of the OS thread that runs the render pipeline. +pub const RENDER_THREAD_NAME: &str = "hypercolor-render"; + +/// Name of the render runtime's Tokio worker threads. +pub const RENDER_RUNTIME_THREAD_NAME: &str = "hypercolor-render-rt"; + +/// File in the daemon state directory that holds the last render failure. +pub const RENDER_FAILURE_FILE: &str = "render-failure.json"; + +/// Identity of this daemon process, written into every failure record. +/// +/// A pid cannot tell runs apart: a container restarts the daemon as pid 1 +/// every time, and pids are reused across boots. This id is unique per +/// process, so a record from an earlier run is always recognized as one. +static PROCESS_INSTANCE: LazyLock = LazyLock::new(|| uuid::Uuid::now_v7().to_string()); + +/// How long a running render loop may go without completing an iteration +/// before the daemon reports itself unhealthy to its service manager's +/// watchdog. A healthy loop iterates at least every quarter second, so this +/// only trips on a thread that is wedged inside a frame, which nothing can +/// interrupt in-process. +pub const RENDER_WEDGED_AFTER: Duration = Duration::from_secs(20); + +/// Whether `name` belongs to the render thread or its runtime workers. +#[must_use] +pub fn is_render_thread_name(name: &str) -> bool { + name == RENDER_THREAD_NAME || name.starts_with(RENDER_RUNTIME_THREAD_NAME) +} + +// ── Policy ────────────────────────────────────────────────────────────────── + +/// How often and how fast a failed render pipeline restarts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RenderRestartPolicy { + /// Restarts allowed within [`window`](Self::window). The failure after + /// the last allowed restart terminates the daemon instead. + pub max_restarts: u32, + /// Rolling window the restart budget is counted over. + pub window: Duration, + /// Delay before the first restart in a streak. + pub initial_backoff: Duration, + /// Upper bound on the doubling restart delay. + pub max_backoff: Duration, + /// A pipeline that runs this long before failing starts a fresh streak, + /// so its restart waits the initial backoff again. + pub stable_uptime: Duration, +} + +impl Default for RenderRestartPolicy { + fn default() -> Self { + Self { + max_restarts: 5, + window: Duration::from_secs(300), + initial_backoff: Duration::from_millis(250), + max_backoff: Duration::from_secs(8), + stable_uptime: Duration::from_secs(60), + } + } +} + +impl RenderRestartPolicy { + fn backoff(&self, consecutive_failures: u32) -> Duration { + let doublings = consecutive_failures.saturating_sub(1).min(31); + self.initial_backoff + .saturating_mul(1_u32 << doublings) + .min(self.max_backoff) + } +} + +/// What the supervisor does after a failure. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RecoveryPlan { + /// Rebuild the pipeline after `backoff`; `attempt` counts restarts in + /// the current window. + Restart { attempt: u32, backoff: Duration }, + /// Stop rendering and ask the daemon to exit. + Terminate(TerminationCause), + /// The daemon is already stopping, so nothing restarts. + DaemonStopping, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TerminationCause { + RestartBudgetExhausted { + failures: u32, + }, + GpuDeviceLost, + /// The supervisor itself failed outside a pipeline attempt. + SupervisorFailed, +} + +/// Rolling restart budget for one render thread. +#[derive(Debug)] +pub(crate) struct RestartBudget { + policy: RenderRestartPolicy, + failures: VecDeque, + consecutive: u32, +} + +impl RestartBudget { + pub(crate) const fn new(policy: RenderRestartPolicy) -> Self { + Self { + policy, + failures: VecDeque::new(), + consecutive: 0, + } + } + + /// Account for a failure at `now` of an attempt that had run for + /// `uptime`, and decide whether to restart. + pub(crate) fn record(&mut self, now: Instant, uptime: Duration) -> RecoveryPlan { + if uptime >= self.policy.stable_uptime { + self.consecutive = 0; + } + self.consecutive = self.consecutive.saturating_add(1); + self.failures.push_back(now); + while self + .failures + .front() + .is_some_and(|failed| now.saturating_duration_since(*failed) > self.policy.window) + { + self.failures.pop_front(); + } + let failures = u32::try_from(self.failures.len()).unwrap_or(u32::MAX); + if failures > self.policy.max_restarts { + return RecoveryPlan::Terminate(TerminationCause::RestartBudgetExhausted { failures }); + } + RecoveryPlan::Restart { + attempt: failures, + backoff: self.policy.backoff(self.consecutive), + } + } + + pub(crate) const fn consecutive(&self) -> u32 { + self.consecutive + } +} + +// ── Records ───────────────────────────────────────────────────────────────── + +/// Where the render thread is in its lifecycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum RenderThreadPhase { + /// No render thread has been spawned for this supervision. + NotStarted = 0, + /// The thread is building its runtime, input pump, and first pipeline. + Starting = 1, + /// A pipeline is running. + Running = 2, + /// A pipeline failed and the supervisor is rebuilding it. + Restarting = 3, + /// The supervisor gave up; the daemon is exiting so its service manager + /// restarts it. + Terminating = 4, + /// The thread exited cleanly (daemon shutdown or a loop never started). + Stopped = 5, +} + +impl RenderThreadPhase { + const fn from_u8(value: u8) -> Self { + match value { + 1 => Self::Starting, + 2 => Self::Running, + 3 => Self::Restarting, + 4 => Self::Terminating, + 5 => Self::Stopped, + _ => Self::NotStarted, + } + } + + /// Stable snake-case name used by diagnostics and status. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::NotStarted => "not_started", + Self::Starting => "starting", + Self::Running => "running", + Self::Restarting => "restarting", + Self::Terminating => "terminating", + Self::Stopped => "stopped", + } + } +} + +/// What ended a render pipeline run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RenderFailureKind { + /// The pipeline panicked. + Panic, + /// The input publication pump the pipeline reads from stopped. + InputPumpTerminated, + /// A replacement input publication pump failed to start. + InputPumpRestartFailed, + /// A replacement pipeline failed to build. + PipelineRebuildFailed, + /// The compositor GPU device was lost. + GpuDeviceLost, +} + +impl RenderFailureKind { + /// Stable snake-case name used by diagnostics and status. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Panic => "panic", + Self::InputPumpTerminated => "input_pump_terminated", + Self::InputPumpRestartFailed => "input_pump_restart_failed", + Self::PipelineRebuildFailed => "pipeline_rebuild_failed", + Self::GpuDeviceLost => "gpu_device_lost", + } + } +} + +/// What the daemon did about a render failure. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RenderRecoveryAction { + /// The pipeline was rebuilt in-process with fresh GPU and render state. + Restart, + /// The daemon exits with an error so its service manager restarts it. + TerminateProcess, + /// The daemon was already stopping, so nothing was restarted. + DaemonStopping, + /// Written by the panic hook in a `panic = "abort"` build, which aborts + /// the process right after. + ProcessAbort, + /// Written by the panic hook before the supervisor caught the unwind. + /// The supervisor replaces it with its decision; a record still in + /// this state means the supervisor never got control back. + AwaitingSupervisor, +} + +impl RenderRecoveryAction { + /// Stable snake-case name used by diagnostics and status. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Restart => "restart", + Self::TerminateProcess => "terminate_process", + Self::DaemonStopping => "daemon_stopping", + Self::ProcessAbort => "process_abort", + Self::AwaitingSupervisor => "awaiting_supervisor", + } + } +} + +/// One render failure, as logged and persisted. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RenderFailureRecord { + pub kind: RenderFailureKind, + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thread: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub location: Option, + /// RFC 3339 UTC time of the failure. + pub occurred_at: String, + pub occurred_at_unix_ms: u64, + /// Process that recorded the failure. + pub pid: u32, + /// Unique id of the daemon process that recorded the failure. Absent + /// from records written before it existed. + #[serde(default)] + pub process_instance: String, + pub daemon_version: String, + /// Restarts in the budget window including the one this failure + /// triggered, or the failures counted when the budget ran out. + pub restart_attempt: u32, + /// Restarts since the process started, after this failure. + pub restarts_total: u64, + pub action: RenderRecoveryAction, + /// Backtrace frames from the panic site, when the failure was a panic. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backtrace: Option, +} + +/// A failure before the supervisor decides what to do with it. +#[derive(Debug, Clone)] +pub(crate) struct RenderFailure { + pub(crate) kind: RenderFailureKind, + pub(crate) message: String, + pub(crate) thread: Option, + pub(crate) location: Option, + pub(crate) backtrace: Option, +} + +impl RenderFailure { + pub(crate) fn panic(payload: String, report: Option) -> Self { + let thread = std::thread::current().name().map(str::to_owned); + match report { + Some(report) => Self { + kind: RenderFailureKind::Panic, + message: report.payload.clone(), + thread: Some(report.thread_name.clone()), + location: report.location.clone(), + backtrace: Some(report.backtrace_excerpt()), + }, + None => Self { + kind: RenderFailureKind::Panic, + message: payload, + thread, + location: None, + backtrace: None, + }, + } + } + + /// Attach a panic report's thread, location, and backtrace, keeping + /// this failure's kind and message. + pub(crate) fn with_panic_report(mut self, report: Option) -> Self { + if let Some(report) = report { + self.message = format!( + "{} (after a panic on {}: {})", + self.message, report.thread_name, report.payload + ); + self.thread = Some(report.thread_name.clone()); + self.location.clone_from(&report.location); + self.backtrace = Some(report.backtrace_excerpt()); + } + self + } + + pub(crate) fn from_error(kind: RenderFailureKind, error: &anyhow::Error) -> Self { + Self::new(kind, format!("{error:#}")) + } + + pub(crate) fn new(kind: RenderFailureKind, message: String) -> Self { + Self { + kind, + message, + thread: std::thread::current().name().map(str::to_owned), + location: None, + backtrace: None, + } + } + + /// Reclassify a failure that happened after the compositor device was + /// lost: a lost device is why it failed, and why it cannot restart. + pub(crate) fn after_device_loss(mut self, lost: &str) -> Self { + self.message = format!( + "{lost}; the pipeline then failed with {}: {}", + self.kind.as_str(), + self.message + ); + self.kind = RenderFailureKind::GpuDeviceLost; + self + } +} + +/// Everything diagnostics and status read about the render thread. +#[derive(Debug, Clone)] +pub struct RenderSupervisionSnapshot { + pub phase: RenderThreadPhase, + pub restarts_total: u64, + /// Failures since the pipeline last ran past the stable uptime. + pub consecutive_failures: u32, + pub max_restarts: u32, + /// Time since the render loop last completed an iteration of any kind, + /// paused polls included. + pub last_heartbeat_age: Option, + /// Time since the render loop last completed a frame. + pub last_frame_age: Option, + /// How long the current pipeline has been running. + pub current_run: Option, + /// The latest failure in this process, or else the failure the previous + /// daemon run persisted. + pub last_failure: Option, + /// Whether [`last_failure`](Self::last_failure) happened in this process. + pub last_failure_in_this_process: bool, + pub gpu_errors: GpuErrorTotals, +} + +fn duration_millis(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + +// ── Persistence ───────────────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +struct RenderFailureStore { + path: PathBuf, +} + +/// Prefix and suffix of a record staged before its durable replace. +const STAGING_PREFIX: &str = ".render-failure."; +const STAGING_SUFFIX: &str = ".tmp"; + +impl RenderFailureStore { + /// Remove records a killed process staged and never replaced. + fn remove_stale_staging(&self) { + let Some(directory) = self.path.parent() else { + return; + }; + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + for entry in entries.filter_map(std::result::Result::ok) { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with(STAGING_PREFIX) && name.ends_with(STAGING_SUFFIX) { + let _ = std::fs::remove_file(entry.path()); + } + } + } + + fn load(&self) -> Option { + let bytes = match std::fs::read(&self.path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, + Err(error) => { + tracing::warn!(path = %self.path.display(), %error, "could not read the previous render failure record"); + return None; + } + }; + match serde_json::from_slice(&bytes) { + Ok(record) => Some(record), + Err(error) => { + tracing::warn!(path = %self.path.display(), %error, "ignoring an unreadable render failure record"); + None + } + } + } + + /// Replace the record durably. Safe to call from the panic hook: it + /// takes no lock the render thread could be holding. + fn persist(&self, record: &RenderFailureRecord) -> Result<()> { + let directory = self + .path + .parent() + .context("render failure record path has no parent directory")?; + std::fs::create_dir_all(directory).with_context(|| { + format!( + "failed to create the render failure directory {}", + directory.display() + ) + })?; + let payload = serde_json::to_vec_pretty(record) + .context("failed to serialize the render failure record")?; + let mut staged = tempfile::Builder::new() + .prefix(STAGING_PREFIX) + .suffix(STAGING_SUFFIX) + .tempfile_in(directory) + .context("failed to stage the render failure record")?; + staged + .write_all(&payload) + .and_then(|()| staged.as_file().sync_all()) + .context("failed to write the staged render failure record")?; + let (_file, staged_path) = staged + .keep() + .context("failed to keep the staged render failure record")?; + if let Err(error) = hypercolor_platform_fs::durable_replace(&staged_path, &self.path) { + let _ = std::fs::remove_file(&staged_path); + return Err(error).with_context(|| { + format!( + "failed to replace the render failure record {}", + self.path.display() + ) + }); + } + Ok(()) + } +} + +// ── Supervision handle ────────────────────────────────────────────────────── + +/// A fault the render thread raises on itself, for tests of the recovery +/// paths. +#[cfg(feature = "persistence-test-hooks")] +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InjectedRenderFault { + /// Panic with this message at the top of the next render loop + /// iteration. + Panic(&'static str), + /// Block the render thread for the given time, like a wedged frame. + Stall(Duration), + /// Raise one uncaptured validation error on the compositor GPU device. + GpuValidationError, +} + +struct SupervisionInner { + policy: RenderRestartPolicy, + store: Option, + epoch: Instant, + phase: AtomicU8, + restarts_total: AtomicU64, + consecutive_failures: AtomicU32, + attempt_started_us: AtomicU64, + last_heartbeat_us: AtomicU64, + last_frame_us: AtomicU64, + last_failure: Mutex>, + previous_failure: Option, + termination: watch::Sender>>, + /// The `hypercolor-render` thread this supervision watches, so its + /// panic observer persists only that thread's panics. + render_thread: Mutex>, + #[cfg(feature = "persistence-test-hooks")] + injected_device_loss: DeviceLossSignal, + #[cfg(feature = "persistence-test-hooks")] + injected_faults: Mutex>, + #[cfg(feature = "persistence-test-hooks")] + injected_fault_pending: std::sync::atomic::AtomicBool, +} + +/// Shared handle to one render thread's health and recovery state. +/// +/// Cloning shares the same state. The daemon creates one at startup and +/// hands it to the render thread, diagnostics, and status. +#[derive(Clone)] +pub struct RenderSupervision { + inner: Arc, +} + +impl std::fmt::Debug for RenderSupervision { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RenderSupervision") + .field("phase", &self.phase()) + .field("policy", &self.inner.policy) + .finish_non_exhaustive() + } +} + +impl Default for RenderSupervision { + fn default() -> Self { + Self::detached() + } +} + +impl RenderSupervision { + /// Supervision with `policy` that persists failures to + /// `failure_record_path` when one is given. + /// + /// A record left by an earlier process at that path is loaded and + /// reported until this process records its own failure. + #[must_use] + pub fn new(policy: RenderRestartPolicy, failure_record_path: Option) -> Self { + let store = failure_record_path.map(|path| RenderFailureStore { path }); + if let Some(store) = store.as_ref() { + store.remove_stale_staging(); + } + let previous_failure = store + .as_ref() + .and_then(RenderFailureStore::load) + .filter(|record| record.process_instance != *PROCESS_INSTANCE); + if let Some(record) = previous_failure.as_ref() { + tracing::warn!( + kind = record.kind.as_str(), + message = %record.message, + occurred_at = %record.occurred_at, + pid = record.pid, + action = record.action.as_str(), + "an earlier daemon run recorded a render failure" + ); + } + Self { + inner: Arc::new(SupervisionInner { + policy, + store, + epoch: Instant::now(), + phase: AtomicU8::new(RenderThreadPhase::NotStarted as u8), + restarts_total: AtomicU64::new(0), + consecutive_failures: AtomicU32::new(0), + attempt_started_us: AtomicU64::new(0), + last_heartbeat_us: AtomicU64::new(0), + last_frame_us: AtomicU64::new(0), + last_failure: Mutex::new(None), + previous_failure, + termination: watch::Sender::new(None), + render_thread: Mutex::new(None), + #[cfg(feature = "persistence-test-hooks")] + injected_device_loss: DeviceLossSignal::default(), + #[cfg(feature = "persistence-test-hooks")] + injected_faults: Mutex::new(VecDeque::new()), + #[cfg(feature = "persistence-test-hooks")] + injected_fault_pending: std::sync::atomic::AtomicBool::new(false), + }), + } + } + + /// Default policy, persisting to [`RENDER_FAILURE_FILE`] in + /// `state_dir`. + #[must_use] + pub fn for_state_dir(state_dir: &Path) -> Self { + Self::new( + RenderRestartPolicy::default(), + Some(state_dir.join(RENDER_FAILURE_FILE)), + ) + } + + /// Default policy without persistence. + #[must_use] + pub fn detached() -> Self { + Self::new(RenderRestartPolicy::default(), None) + } + + #[must_use] + pub fn policy(&self) -> RenderRestartPolicy { + self.inner.policy + } + + /// Where failures are persisted, if anywhere. + #[must_use] + pub fn failure_record_path(&self) -> Option<&Path> { + self.inner.store.as_ref().map(|store| store.path.as_path()) + } + + #[must_use] + pub fn phase(&self) -> RenderThreadPhase { + RenderThreadPhase::from_u8(self.inner.phase.load(Ordering::Acquire)) + } + + /// Read every health field at once. + #[must_use] + pub fn snapshot(&self) -> RenderSupervisionSnapshot { + let now = Instant::now(); + let current = self + .inner + .last_failure + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(); + let last_failure_in_this_process = current.is_some(); + let phase = self.phase(); + RenderSupervisionSnapshot { + phase, + restarts_total: self.inner.restarts_total.load(Ordering::Relaxed), + consecutive_failures: self.inner.consecutive_failures.load(Ordering::Relaxed), + max_restarts: self.inner.policy.max_restarts, + last_heartbeat_age: self.age(&self.inner.last_heartbeat_us, now), + last_frame_age: self.age(&self.inner.last_frame_us, now), + current_run: (phase == RenderThreadPhase::Running) + .then(|| self.age(&self.inner.attempt_started_us, now)) + .flatten(), + last_failure: current.or_else(|| self.inner.previous_failure.clone()), + last_failure_in_this_process, + gpu_errors: gpu_error_totals(), + } + } + + /// Whether the render thread is healthy enough for a service manager + /// watchdog: `false` only while a running pipeline has gone + /// [`RENDER_WEDGED_AFTER`] without completing a loop iteration. + #[must_use] + pub fn watchdog_healthy(&self) -> bool { + self.watchdog_healthy_at(Instant::now()) + } + + fn watchdog_healthy_at(&self, now: Instant) -> bool { + self.phase() != RenderThreadPhase::Running + || self + .age(&self.inner.last_heartbeat_us, now) + .is_none_or(|age| age < RENDER_WEDGED_AFTER) + } + + /// The failure the render thread terminated on, once it gave up. + #[must_use] + pub fn termination(&self) -> Option> { + self.inner.termination.borrow().clone() + } + + /// Wait until the render thread gives up, and return why. + pub async fn terminated(&self) -> Arc { + let mut receiver = self.inner.termination.subscribe(); + let record = receiver + .wait_for(Option::is_some) + .await + .map(|record| record.clone()); + match record { + Ok(Some(record)) => record, + // The sender lives as long as this handle, so the wait cannot + // end any other way. + Ok(None) | Err(_) => std::future::pending().await, + } + } + + /// Bind this supervision to the render thread calling it. + pub(crate) fn bind_render_thread(&self) { + *self + .inner + .render_thread + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(std::thread::current().id()); + } + + /// The loss a test injected for this supervision's compositor device. + #[cfg(feature = "persistence-test-hooks")] + pub(crate) fn injected_device_loss(&self) -> DeviceLossSignal { + self.inner.injected_device_loss.clone() + } + + pub(crate) fn set_phase(&self, phase: RenderThreadPhase) { + self.inner.phase.store(phase as u8, Ordering::Release); + } + + /// A pipeline attempt starts running at `now`. + pub(crate) fn attempt_started(&self, now: Instant) { + let stamp = self.stamp(now); + self.inner + .attempt_started_us + .store(stamp, Ordering::Relaxed); + // A fresh pipeline gets the same grace a fresh frame would, so the + // stall check measures this attempt rather than the dead one. + self.inner.last_heartbeat_us.store(stamp, Ordering::Relaxed); + self.inner.last_frame_us.store(stamp, Ordering::Relaxed); + self.set_phase(RenderThreadPhase::Running); + } + + /// The render loop completed an idle or paused iteration at `now`. + pub(crate) fn heartbeat(&self, now: Instant) { + self.inner + .last_heartbeat_us + .store(self.stamp(now), Ordering::Relaxed); + } + + /// The render loop completed a frame at `now`. + pub(crate) fn frame_completed(&self, now: Instant) { + let stamp = self.stamp(now); + self.inner.last_heartbeat_us.store(stamp, Ordering::Relaxed); + self.inner.last_frame_us.store(stamp, Ordering::Relaxed); + if self.inner.consecutive_failures.load(Ordering::Relaxed) != 0 { + let started = self.inner.attempt_started_us.load(Ordering::Relaxed); + let stable_us = + u64::try_from(self.inner.policy.stable_uptime.as_micros()).unwrap_or(u64::MAX); + if stamp.saturating_sub(started) >= stable_us { + self.inner.consecutive_failures.store(0, Ordering::Relaxed); + } + } + } + + /// Record `failure` and the plan for it: log it, persist it, update the + /// counters and the phase. + pub(crate) fn record_failure( + &self, + failure: RenderFailure, + plan: RecoveryPlan, + consecutive_failures: u32, + ) -> RenderFailureRecord { + let (action, restart_attempt, phase) = match plan { + RecoveryPlan::Restart { attempt, .. } => ( + RenderRecoveryAction::Restart, + attempt, + RenderThreadPhase::Restarting, + ), + RecoveryPlan::Terminate(TerminationCause::RestartBudgetExhausted { failures }) => ( + RenderRecoveryAction::TerminateProcess, + failures, + RenderThreadPhase::Terminating, + ), + RecoveryPlan::Terminate( + TerminationCause::GpuDeviceLost | TerminationCause::SupervisorFailed, + ) => ( + RenderRecoveryAction::TerminateProcess, + consecutive_failures, + RenderThreadPhase::Terminating, + ), + RecoveryPlan::DaemonStopping => ( + RenderRecoveryAction::DaemonStopping, + consecutive_failures, + RenderThreadPhase::Stopped, + ), + }; + let restarts_total = if action == RenderRecoveryAction::Restart { + self.inner + .restarts_total + .fetch_add(1, Ordering::Relaxed) + .saturating_add(1) + } else { + self.inner.restarts_total.load(Ordering::Relaxed) + }; + self.inner + .consecutive_failures + .store(consecutive_failures, Ordering::Relaxed); + let record = failure_record(failure, action, restart_attempt, restarts_total); + log_failure(&record, plan, self.inner.policy.max_restarts); + if let Some(store) = self.inner.store.as_ref() + && let Err(error) = store.persist(&record) + { + tracing::warn!(path = %store.path.display(), error = %format!("{error:#}"), "could not persist the render failure record"); + } + *self + .inner + .last_failure + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(record.clone()); + self.set_phase(phase); + record + } + + /// The record for a terminating failure that could not be logged or + /// persisted, built without touching either. + pub(crate) fn unrecorded_termination(&self, failure: RenderFailure) -> RenderFailureRecord { + let restarts_total = self.inner.restarts_total.load(Ordering::Relaxed); + failure_record( + failure, + RenderRecoveryAction::TerminateProcess, + 0, + restarts_total, + ) + } + + /// Ask the daemon to exit because the render thread cannot recover. + pub(crate) fn request_termination(&self, record: Arc) { + self.set_phase(RenderThreadPhase::Terminating); + self.inner.termination.send_replace(Some(record)); + } + + /// An observer for the process panic hook that persists render thread + /// panics, or `None` when this supervision does not persist. + /// + /// In a `panic = "abort"` build this is the only code that runs between + /// a render panic and the abort, so it persists a panic on the render + /// thread or any of its runtime workers. An unwinding build persists + /// only this supervision's own render thread: the supervisor records + /// that unwind itself right after, while Tokio catches a worker panic + /// and the supervisor only learns of it if the pipeline then fails. + pub(crate) fn panic_observer(&self) -> Option { + let store = self.inner.store.clone()?; + let inner: Weak = Arc::downgrade(&self.inner); + Some(Arc::new(move |report: &PanicReport| { + let Some(inner) = inner.upgrade() else { + return; + }; + let ours = if cfg!(panic = "abort") { + is_render_thread_name(&report.thread_name) + } else { + inner + .render_thread + .try_lock() + .ok() + .and_then(|thread| *thread) + == Some(report.thread_id) + }; + if !ours { + return; + } + let action = if cfg!(panic = "abort") { + RenderRecoveryAction::ProcessAbort + } else { + RenderRecoveryAction::AwaitingSupervisor + }; + let failure = RenderFailure { + kind: RenderFailureKind::Panic, + message: report.payload.clone(), + thread: Some(report.thread_name.clone()), + location: report.location.clone(), + backtrace: Some(report.backtrace_excerpt()), + }; + let restarts_total = inner.restarts_total.load(Ordering::Relaxed); + let record = failure_record(failure, action, 0, restarts_total); + // Nothing useful can be done with an error from inside the + // panic hook; the tracing record and stderr still carry it. + let _ = store.persist(&record); + })) + } + + /// Queue a fault for the render thread to raise on itself. + #[cfg(feature = "persistence-test-hooks")] + #[doc(hidden)] + pub fn inject_fault(&self, fault: InjectedRenderFault) { + self.inner + .injected_faults + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push_back(fault); + self.inner + .injected_fault_pending + .store(true, Ordering::Release); + } + + /// Report this render thread's compositor device lost, the way the + /// device-lost callback raises a real device's loss signal. + #[cfg(feature = "persistence-test-hooks")] + #[doc(hidden)] + pub fn inject_compositor_device_lost(&self, message: &str) { + self.inner.injected_device_loss.raise(format!( + "injected compositor device lost (unknown: {message})" + )); + } + + #[cfg(feature = "persistence-test-hooks")] + pub(crate) fn take_injected_fault(&self) -> Option { + if !self.inner.injected_fault_pending.load(Ordering::Acquire) { + return None; + } + let mut faults = self + .inner + .injected_faults + .lock() + .unwrap_or_else(PoisonError::into_inner); + let fault = faults.pop_front(); + if faults.is_empty() { + self.inner + .injected_fault_pending + .store(false, Ordering::Release); + } + fault + } + + /// Microseconds since this supervision's epoch, offset by one so zero + /// can mean "never". + fn stamp(&self, now: Instant) -> u64 { + u64::try_from(now.saturating_duration_since(self.inner.epoch).as_micros()) + .unwrap_or(u64::MAX - 1) + .saturating_add(1) + } + + fn age(&self, stamp: &AtomicU64, now: Instant) -> Option { + let stamp = stamp.load(Ordering::Relaxed); + (stamp != 0).then(|| Duration::from_micros(self.stamp(now).saturating_sub(stamp))) + } +} + +fn failure_record( + failure: RenderFailure, + action: RenderRecoveryAction, + restart_attempt: u32, + restarts_total: u64, +) -> RenderFailureRecord { + let now = SystemTime::now(); + RenderFailureRecord { + kind: failure.kind, + message: failure.message, + thread: failure.thread, + location: failure.location, + occurred_at: chrono::DateTime::::from(now) + .to_rfc3339_opts(chrono::SecondsFormat::Millis, true), + occurred_at_unix_ms: now.duration_since(UNIX_EPOCH).map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }), + pid: std::process::id(), + process_instance: PROCESS_INSTANCE.clone(), + daemon_version: env!("CARGO_PKG_VERSION").to_owned(), + restart_attempt, + restarts_total, + action, + backtrace: failure.backtrace, + } +} + +fn log_failure(record: &RenderFailureRecord, plan: RecoveryPlan, max_restarts: u32) { + let backtrace = record.backtrace.as_deref().unwrap_or("unavailable"); + let thread = record.thread.as_deref().unwrap_or("unknown"); + let location = record.location.as_deref().unwrap_or("unknown"); + match plan { + RecoveryPlan::Restart { attempt, backoff } => tracing::error!( + kind = record.kind.as_str(), + message = %record.message, + thread, + location, + restart = attempt, + max_restarts, + backoff_ms = duration_millis(backoff), + backtrace, + "render pipeline failed; restarting it with fresh GPU and render state" + ), + RecoveryPlan::Terminate(TerminationCause::RestartBudgetExhausted { failures }) => { + tracing::error!( + kind = record.kind.as_str(), + message = %record.message, + thread, + location, + failures, + max_restarts, + backtrace, + "render pipeline keeps failing; shutting the daemon down so its service manager restarts it" + ); + } + RecoveryPlan::Terminate(TerminationCause::SupervisorFailed) => tracing::error!( + kind = record.kind.as_str(), + message = %record.message, + thread, + location, + backtrace, + "render supervisor failed; shutting the daemon down so its service manager restarts it" + ), + RecoveryPlan::Terminate(TerminationCause::GpuDeviceLost) => tracing::error!( + kind = record.kind.as_str(), + message = %record.message, + thread, + location, + backtrace, + "compositor GPU device lost; shutting the daemon down so it reopens the GPU in a fresh process" + ), + RecoveryPlan::DaemonStopping => tracing::error!( + kind = record.kind.as_str(), + message = %record.message, + thread, + location, + backtrace, + "render pipeline failed while the daemon was stopping" + ), + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use super::{ + RENDER_FAILURE_FILE, RENDER_WEDGED_AFTER, RecoveryPlan, RenderFailure, RenderFailureKind, + RenderFailureStore, RenderRecoveryAction, RenderRestartPolicy, RenderSupervision, + RestartBudget, TerminationCause, failure_record, + }; + + #[test] + fn a_record_from_this_process_is_not_reported_as_an_earlier_run() { + let state_dir = tempfile::tempdir().expect("state tempdir should be created"); + let path = state_dir.path().join(RENDER_FAILURE_FILE); + let store = RenderFailureStore { path: path.clone() }; + let record = failure_record( + RenderFailure::new(RenderFailureKind::Panic, "this process".to_owned()), + RenderRecoveryAction::Restart, + 1, + 1, + ); + + store.persist(&record).expect("record should persist"); + let supervision = RenderSupervision::new(RenderRestartPolicy::default(), Some(path)); + + assert!(supervision.snapshot().last_failure.is_none()); + let leftovers: Vec<_> = std::fs::read_dir(state_dir.path()) + .expect("state dir should list") + .filter_map(Result::ok) + .map(|entry| entry.file_name()) + .collect(); + assert_eq!( + leftovers, + vec![std::ffi::OsString::from(RENDER_FAILURE_FILE)] + ); + } + + #[test] + fn only_a_running_pipeline_that_stopped_iterating_fails_the_watchdog() { + let supervision = RenderSupervision::detached(); + let start = Instant::now(); + let later = start + RENDER_WEDGED_AFTER + Duration::from_secs(1); + assert!(supervision.watchdog_healthy_at(later), "not started yet"); + + supervision.attempt_started(start); + assert!(supervision.watchdog_healthy_at(start + Duration::from_secs(1))); + assert!(!supervision.watchdog_healthy_at(later)); + + supervision.heartbeat(later); + assert!(supervision.watchdog_healthy_at(later + Duration::from_secs(1))); + supervision.set_phase(super::RenderThreadPhase::Restarting); + assert!(supervision.watchdog_healthy_at(later + RENDER_WEDGED_AFTER * 2)); + } + + fn policy() -> RenderRestartPolicy { + RenderRestartPolicy { + max_restarts: 3, + window: Duration::from_secs(60), + initial_backoff: Duration::from_millis(100), + max_backoff: Duration::from_millis(350), + stable_uptime: Duration::from_secs(10), + } + } + + #[test] + fn restarts_back_off_until_the_budget_runs_out() { + let mut budget = RestartBudget::new(policy()); + let start = Instant::now(); + + let plans: Vec<_> = (0..4) + .map(|index| budget.record(start + Duration::from_secs(index), Duration::ZERO)) + .collect(); + + assert_eq!( + plans, + vec![ + RecoveryPlan::Restart { + attempt: 1, + backoff: Duration::from_millis(100) + }, + RecoveryPlan::Restart { + attempt: 2, + backoff: Duration::from_millis(200) + }, + RecoveryPlan::Restart { + attempt: 3, + backoff: Duration::from_millis(350) + }, + RecoveryPlan::Terminate(TerminationCause::RestartBudgetExhausted { failures: 4 }), + ] + ); + } + + #[test] + fn failures_outside_the_window_leave_the_budget() { + let mut budget = RestartBudget::new(policy()); + let start = Instant::now(); + for index in 0..3 { + budget.record(start + Duration::from_secs(index), Duration::ZERO); + } + + let plan = budget.record(start + Duration::from_secs(120), Duration::ZERO); + + assert!(matches!(plan, RecoveryPlan::Restart { attempt: 1, .. })); + } + + #[test] + fn a_stable_run_resets_the_backoff_streak() { + let mut budget = RestartBudget::new(policy()); + let start = Instant::now(); + budget.record(start, Duration::ZERO); + budget.record(start + Duration::from_secs(1), Duration::ZERO); + + let plan = budget.record(start + Duration::from_secs(20), Duration::from_secs(15)); + + assert_eq!( + plan, + RecoveryPlan::Restart { + attempt: 3, + backoff: Duration::from_millis(100) + } + ); + assert_eq!(budget.consecutive(), 1); + } +} diff --git a/crates/hypercolor-daemon/src/render_thread/supervisor.rs b/crates/hypercolor-daemon/src/render_thread/supervisor.rs new file mode 100644 index 000000000..bedeb377b --- /dev/null +++ b/crates/hypercolor-daemon/src/render_thread/supervisor.rs @@ -0,0 +1,475 @@ +//! Body of the `hypercolor-render` thread: start the render runtime and the +//! input publication pump, build the pipeline, and keep a pipeline running. +//! +//! A pipeline that panics, loses its input pump, or loses its compositor +//! GPU device ends its attempt. The supervisor records the failure, and +//! either rebuilds the pipeline in-process with fresh GPU and render state +//! (within the restart budget, after a backoff) or asks the daemon to exit +//! so its service manager starts a fresh process. A panic anywhere else on +//! this thread, in startup or in the supervisor itself, is caught at the +//! top of the thread and also ends in that exit, so the thread never ends +//! without the daemon knowing. +//! +//! What a restart rebuilds: both compositors and every GPU resource on +//! them, the zone runtime with its effect renderers and Servo sessions, the +//! scene snapshot and frame state, and the input pump when it is the part +//! that failed. What it keeps: the compositor GPU device, which stays valid +//! after a panic or a validation error, and the shared daemon state the +//! pipeline reads. A lost device is not rebuilt in-process: Servo's GPU +//! import device is a process-wide once-only binding to the device opened +//! at startup, and the native screen bridges and the interactive preview +//! device came from the same adapter, so only a fresh process can rebind +//! all of them. + +use std::panic::AssertUnwindSafe; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use anyhow::{Result, anyhow}; +use arc_swap::ArcSwapOption; +use tokio_util::sync::CancellationToken; +use tracing::{debug, info, warn}; + +use super::gpu_errors::DeviceLossSignal; +use super::input_publication::{InputPublicationMonitor, InputPublicationPump}; +use super::pipeline_driver::run_pipeline; +use super::pipeline_runtime::PipelineRuntime; +#[cfg(feature = "wgpu")] +use super::screen_parity_diagnostics::ScreenParityDiagnosticMailbox; +use super::supervision::{ + RENDER_RUNTIME_THREAD_NAME, RecoveryPlan, RenderFailure, RenderFailureKind, RenderSupervision, + RenderThreadPhase, RestartBudget, TerminationCause, +}; +use super::{InputPublicationDemandHandle, RenderThreadState, panic_payload_message}; +use crate::startup::{StartupProgress, startup_step}; + +/// How recent a runtime worker panic must be to explain a pump failure. +const RECENT_WORKER_PANIC: Duration = Duration::from_secs(10); + +/// Everything the render thread takes ownership of when it starts. +pub(super) struct RenderThreadLaunch { + pub(super) state: RenderThreadState, + pub(super) build_runtime: F, + pub(super) progress: Option, + pub(super) ready: mpsc::SyncSender>, + pub(super) cancel: CancellationToken, + pub(super) input_demands: InputPublicationDemandHandle, + pub(super) input_monitor: Arc>, + pub(super) supervision: RenderSupervision, + #[cfg(feature = "wgpu")] + pub(super) screen_parity_mailbox: ScreenParityDiagnosticMailbox, +} + +/// Run the render thread until shutdown, a clean loop exit, or a failure +/// the supervisor gives up on. +/// +/// Startup failures are reported through `ready` and end the thread with +/// `Ok`, so the spawner sees the error instead of the join result. A panic +/// that escapes everything below is recorded and requests termination. +pub(super) fn run(launch: RenderThreadLaunch) -> Result<()> +where + F: FnOnce() -> Result, +{ + let supervision = launch.supervision.clone(); + supervision.bind_render_thread(); + std::panic::catch_unwind(AssertUnwindSafe(|| run_supervised(launch))).unwrap_or_else( + |payload| { + let failure = RenderFailure::panic( + panic_payload_message(payload.as_ref()), + crate::panic_hook::take_recent_report(std::thread::current().id()), + ); + // Recording logs and writes a file; if that panics too, the + // termination request below must still go out. + let record = std::panic::catch_unwind(AssertUnwindSafe(|| { + supervision.record_failure( + failure.clone(), + RecoveryPlan::Terminate(TerminationCause::SupervisorFailed), + 0, + ) + })) + .unwrap_or_else(|_| supervision.unrecorded_termination(failure)); + let error = anyhow!( + "render thread panicked outside a pipeline attempt: {}", + record.message + ); + supervision.request_termination(Arc::new(record)); + Err(error) + }, + ) +} + +fn run_supervised(launch: RenderThreadLaunch) -> Result<()> +where + F: FnOnce() -> Result, +{ + let RenderThreadLaunch { + state, + build_runtime, + progress, + ready, + cancel, + input_demands, + input_monitor, + supervision, + #[cfg(feature = "wgpu")] + screen_parity_mailbox, + } = launch; + let _scene_transaction_consumer = state.scene_transactions.consumer(); + crate::process::configure_render_thread_priority(); + let runtime = match startup_step(progress.as_ref(), "render runtime", build_runtime) { + Ok(runtime) => runtime, + Err(error) => { + supervision.set_phase(RenderThreadPhase::Stopped); + let _ = ready.send(Err(error)); + return Ok(()); + } + }; + let mut input_pump = match startup_step(progress.as_ref(), "input publication pump", || { + runtime.block_on(start_input_pump(&state, input_demands.clone())) + }) { + Ok(pump) => pump, + Err(error) => { + supervision.set_phase(RenderThreadPhase::Stopped); + let _ = ready.send(Err(error)); + return Ok(()); + } + }; + let pipeline = build_pipeline( + &state, + &input_pump, + &input_demands, + #[cfg(feature = "wgpu")] + &screen_parity_mailbox, + progress.as_ref(), + ); + // Startup reporting ends with the pipeline build. + drop(progress); + let pipeline = match pipeline { + Ok(pipeline) => pipeline, + Err(error) => { + if let Err(shutdown_error) = runtime.block_on(input_pump.shutdown()) { + warn!(%shutdown_error, "input publication cleanup failed after render startup error"); + } + supervision.set_phase(RenderThreadPhase::Stopped); + let _ = ready.send(Err(error)); + return Ok(()); + } + }; + input_monitor.store(Some(Arc::new(input_pump.monitor()))); + if ready.send(Ok(())).is_err() { + supervision.set_phase(RenderThreadPhase::Stopped); + return runtime.block_on(input_pump.shutdown()); + } + + let mut supervisor = Supervisor { + runtime: &runtime, + state: &state, + cancel: &cancel, + input_demands: &input_demands, + input_monitor: &input_monitor, + budget: RestartBudget::new(supervision.policy()), + supervision: &supervision, + device_loss: None, + input_pump, + #[cfg(feature = "wgpu")] + screen_parity_mailbox: &screen_parity_mailbox, + }; + let result = supervisor.supervise(pipeline); + let shutdown_result = runtime.block_on(supervisor.input_pump.shutdown()); + result.and(shutdown_result) +} + +/// How one pipeline attempt ended. +enum AttemptOutcome { + /// The render loop is no longer running or paused. + Exited, + /// The render thread was asked to stop. + Cancelled, + Failed(RenderFailure), +} + +/// What the supervisor does next after a failure. +enum Flow { + RetryAfter(Duration), + Stop(Result<()>), +} + +struct Supervisor<'a> { + runtime: &'a tokio::runtime::Runtime, + state: &'a RenderThreadState, + cancel: &'a CancellationToken, + input_demands: &'a InputPublicationDemandHandle, + input_monitor: &'a Arc>, + supervision: &'a RenderSupervision, + budget: RestartBudget, + /// Loss signal of the GPU device the pipelines draw with. Every rebuild + /// keeps the same device, so a loss stays visible to later attempts. + device_loss: Option, + input_pump: InputPublicationPump, + #[cfg(feature = "wgpu")] + screen_parity_mailbox: &'a ScreenParityDiagnosticMailbox, +} + +impl Supervisor<'_> { + fn supervise(&mut self, first: PipelineRuntime) -> Result<()> { + let mut next = Some(first); + let mut restarted = false; + loop { + let built = match next.take() { + Some(pipeline) => Ok(pipeline), + None => self.rebuild(), + }; + let (failure, uptime) = match built { + Ok(pipeline) => { + self.device_loss = pipeline.render.sparkleflinger.device_loss_signal(); + if restarted { + info!( + restarts_total = self.supervision.snapshot().restarts_total, + "render pipeline restarted with fresh GPU and render state" + ); + } + let started = Instant::now(); + self.supervision.attempt_started(started); + match self.run_attempt(pipeline) { + AttemptOutcome::Exited => { + self.supervision.set_phase(RenderThreadPhase::Stopped); + return Ok(()); + } + AttemptOutcome::Cancelled => { + self.runtime + .block_on(async { self.state.render_loop.write().await.stop() }); + self.supervision.set_phase(RenderThreadPhase::Stopped); + return Ok(()); + } + AttemptOutcome::Failed(failure) => (failure, started.elapsed()), + } + } + Err(failure) => (failure, Duration::ZERO), + }; + match self.handle_failure(failure, uptime) { + Flow::RetryAfter(backoff) => { + if !self.wait_backoff(backoff) { + return Ok(()); + } + restarted = true; + } + Flow::Stop(result) => return result, + } + } + } + + /// Run one pipeline until it exits, fails, or the thread is cancelled. + /// + /// The pipeline future runs on this thread inside `block_on`, so a + /// panic anywhere in a frame unwinds to the `catch_unwind` here. The + /// unwind drops the pipeline and everything it owned. Shared daemon + /// state behind Tokio locks survives intact, since those do not + /// poison. A std lock the panic poisoned can make the rebuilt pipeline + /// (or its rebuild) fail again, because some of its users refuse a + /// poisoned lock; the restart budget then runs out and ends in a + /// process exit, which is the recovery such a failure needs. + fn run_attempt(&self, pipeline: PipelineRuntime) -> AttemptOutcome { + let monitor = self.input_pump.monitor(); + let state = self.state.clone(); + let supervision = self.supervision.clone(); + let cancel = self.cancel.clone(); + let device_loss = self.device_loss_wait(); + let outcome = std::panic::catch_unwind(AssertUnwindSafe(|| { + self.runtime.block_on(async move { + tokio::select! { + () = run_pipeline(state, pipeline, supervision) => AttemptOutcome::Exited, + status = monitor.wait_for_terminal() => AttemptOutcome::Failed( + RenderFailure::new( + RenderFailureKind::InputPumpTerminated, + format!("input publication pump terminated while rendering: {status:?}"), + ) + .with_panic_report(crate::panic_hook::take_recent_report_where(|report| { + report.thread_name.starts_with(RENDER_RUNTIME_THREAD_NAME) + && report + .occurred_at + .elapsed() + .is_ok_and(|age| age < RECENT_WORKER_PANIC) + })), + ), + message = device_loss => AttemptOutcome::Failed( + RenderFailure::new(RenderFailureKind::GpuDeviceLost, message), + ), + () = cancel.cancelled() => AttemptOutcome::Cancelled, + } + }) + })); + outcome.unwrap_or_else(|payload| { + let message = panic_payload_message(payload.as_ref()); + let report = crate::panic_hook::take_recent_report(std::thread::current().id()); + AttemptOutcome::Failed(RenderFailure::panic(message, report)) + }) + } + + /// The compositor device's loss description, once it is lost. + fn device_lost(&self) -> Option { + let lost = self + .device_loss + .as_ref() + .and_then(DeviceLossSignal::lost) + .map(|lost| lost.to_string()); + #[cfg(feature = "persistence-test-hooks")] + let lost = lost.or_else(|| { + self.supervision + .injected_device_loss() + .lost() + .map(|lost| lost.to_string()) + }); + lost + } + + /// Resolves when the compositor device is lost, immediately when it + /// already was. + fn device_loss_wait(&self) -> impl Future + use<> { + let device = self.device_loss.clone(); + #[cfg(feature = "persistence-test-hooks")] + let injected = self.supervision.injected_device_loss(); + async move { + let device = async move { + match device { + Some(device) => device.wait().await, + None => std::future::pending().await, + } + }; + #[cfg(feature = "persistence-test-hooks")] + let device = async move { + tokio::select! { + message = device => message, + message = injected.wait() => message, + } + }; + device.await + } + } + + fn handle_failure(&mut self, failure: RenderFailure, uptime: Duration) -> Flow { + let failure = if failure.kind != RenderFailureKind::GpuDeviceLost + && let Some(lost) = self.device_lost() + { + failure.after_device_loss(&lost) + } else { + failure + }; + let plan = if self.cancel.is_cancelled() { + RecoveryPlan::DaemonStopping + } else if failure.kind == RenderFailureKind::GpuDeviceLost { + RecoveryPlan::Terminate(TerminationCause::GpuDeviceLost) + } else { + self.budget.record(Instant::now(), uptime) + }; + let record = self + .supervision + .record_failure(failure, plan, self.budget.consecutive()); + match plan { + RecoveryPlan::Restart { backoff, .. } => Flow::RetryAfter(backoff), + RecoveryPlan::DaemonStopping => Flow::Stop(Ok(())), + RecoveryPlan::Terminate(_) => { + let error = anyhow!( + "render pipeline cannot recover ({}): {}", + record.kind.as_str(), + record.message + ); + self.supervision.request_termination(Arc::new(record)); + Flow::Stop(Err(error)) + } + } + } + + /// Sleep out a restart backoff. Returns `false` when the thread was + /// cancelled meanwhile. + fn wait_backoff(&self, backoff: Duration) -> bool { + let cancel = self.cancel.clone(); + let restart = self.runtime.block_on(async move { + tokio::select! { + () = tokio::time::sleep(backoff) => true, + () = cancel.cancelled() => false, + } + }); + if !restart { + self.supervision.set_phase(RenderThreadPhase::Stopped); + } + restart + } + + /// Build a replacement pipeline, restarting the input pump first when + /// it was the part that failed. A panic during the rebuild is a failure + /// like any other and counts against the restart budget. + fn rebuild(&mut self) -> Result { + std::panic::catch_unwind(AssertUnwindSafe(|| self.try_rebuild())).unwrap_or_else( + |payload| { + let message = panic_payload_message(payload.as_ref()); + let report = crate::panic_hook::take_recent_report(std::thread::current().id()); + let mut failure = RenderFailure::panic(message, report); + failure.message = format!("while rebuilding the pipeline: {}", failure.message); + Err(failure) + }, + ) + } + + fn try_rebuild(&mut self) -> Result { + if self.input_pump.monitor().status().is_terminal() { + self.restart_input_pump()?; + } + build_pipeline( + self.state, + &self.input_pump, + self.input_demands, + #[cfg(feature = "wgpu")] + self.screen_parity_mailbox, + None, + ) + .map_err(|error| { + RenderFailure::from_error(RenderFailureKind::PipelineRebuildFailed, &error) + }) + } + + fn restart_input_pump(&mut self) -> Result<(), RenderFailure> { + if let Err(error) = self.runtime.block_on(self.input_pump.shutdown()) { + debug!(%error, "failed input publication pump reported its failure again during cleanup"); + } + let pump = self + .runtime + .block_on(start_input_pump(self.state, self.input_demands.clone())) + .map_err(|error| { + RenderFailure::from_error(RenderFailureKind::InputPumpRestartFailed, &error) + })?; + self.input_pump = pump; + self.input_monitor + .store(Some(Arc::new(self.input_pump.monitor()))); + info!("input publication pump restarted"); + Ok(()) + } +} + +async fn start_input_pump( + state: &RenderThreadState, + demands: InputPublicationDemandHandle, +) -> Result { + InputPublicationPump::start( + state.input_manager.clone(), + demands, + Some(state.interaction_routing.device_input().clone()), + ) + .await +} + +fn build_pipeline( + state: &RenderThreadState, + input_pump: &InputPublicationPump, + input_demands: &InputPublicationDemandHandle, + #[cfg(feature = "wgpu")] screen_parity_mailbox: &ScreenParityDiagnosticMailbox, + progress: Option<&StartupProgress>, +) -> Result { + PipelineRuntime::from_state( + state, + input_pump.reader(), + input_demands.clone(), + #[cfg(feature = "wgpu")] + screen_parity_mailbox.clone(), + progress, + ) +} diff --git a/crates/hypercolor-daemon/src/startup/lifecycle.rs b/crates/hypercolor-daemon/src/startup/lifecycle.rs index 7eba9770f..9714e1e76 100644 --- a/crates/hypercolor-daemon/src/startup/lifecycle.rs +++ b/crates/hypercolor-daemon/src/startup/lifecycle.rs @@ -157,8 +157,12 @@ impl DaemonState { face_fps_cap: config.display.effective_face_fps_cap(), }; self.render_thread = Some( - RenderThread::try_spawn_reporting(rt_state, self.startup_progress.clone()) - .context("failed to spawn render thread with resolved compositor mode")?, + RenderThread::try_spawn_reporting( + rt_state, + self.startup_progress.clone(), + self.domains.output.render_supervision().clone(), + ) + .context("failed to spawn render thread with resolved compositor mode")?, ); self.startup_progress .enter(DaemonStartupPhase::StartingServices); diff --git a/crates/hypercolor-daemon/src/startup/services.rs b/crates/hypercolor-daemon/src/startup/services.rs index 504a2e2f1..d4ac2782b 100644 --- a/crates/hypercolor-daemon/src/startup/services.rs +++ b/crates/hypercolor-daemon/src/startup/services.rs @@ -801,6 +801,9 @@ impl DaemonState { display_frames: Arc::clone(&display_frames), device_metrics: Arc::clone(&device_metrics), input_manager: input_manager.clone(), + render_supervision: crate::render_thread::RenderSupervision::for_state_dir( + &state_dir, + ), }, |layout| { let discovery_runtime = crate::discovery::DiscoveryRuntime { @@ -1636,6 +1639,7 @@ pub(crate) struct DomainAssemblyResources { pub display_frames: Arc>, pub device_metrics: DeviceMetricsSnapshotStore, pub input_manager: InputManager, + pub render_supervision: crate::render_thread::RenderSupervision, } /// The assembled domain graph and the driver host it was wired around. @@ -1676,6 +1680,7 @@ pub(crate) fn assemble_domains( display_frames, device_metrics, input_manager, + render_supervision, } = resources; let runtime_projection = RuntimeSessionProjection::new( @@ -1719,6 +1724,7 @@ pub(crate) fn assemble_domains( preview_runtime, devices.clone(), start_time, + render_supervision, ); let domains = DomainContexts::assemble( runtime_session, diff --git a/crates/hypercolor-daemon/tests/render_thread_tests.rs b/crates/hypercolor-daemon/tests/render_thread_tests.rs index ee3156f6e..ce7cb286e 100644 --- a/crates/hypercolor-daemon/tests/render_thread_tests.rs +++ b/crates/hypercolor-daemon/tests/render_thread_tests.rs @@ -105,9 +105,14 @@ use hypercolor_daemon::performance::PerformanceTracker; #[cfg(feature = "wgpu")] use hypercolor_daemon::preview_runtime::ZonePreviewFrameReceiver; use hypercolor_daemon::preview_runtime::{PreviewPixelFormat, PreviewRuntime, PreviewStreamDemand}; +#[cfg(feature = "wgpu")] +use hypercolor_daemon::render_thread::gpu_errors::gpu_error_totals; +use hypercolor_daemon::render_thread::supervision::{InjectedRenderFault, RENDER_FAILURE_FILE}; use hypercolor_daemon::render_thread::{ CanvasDims, InputPublicationConsumer, InputPublicationDemand, - InputPublicationDemandRegistration, RenderThread, RenderThreadState, + InputPublicationDemandRegistration, RenderFailureKind, RenderFailureRecord, + RenderRecoveryAction, RenderRestartPolicy, RenderSupervision, RenderSupervisionSnapshot, + RenderThread, RenderThreadPhase, RenderThreadState, }; // ── Helpers ───────────────────────────────────────────────────────────────── @@ -7766,3 +7771,291 @@ async fn release_sleep_clears_published_frame_and_canvas_once() { "release sleep should publish a blank canvas instead of the stale preview" ); } + +// ── Render Thread Supervision Tests ───────────────────────────────────────── + +/// Error and warning logs from every thread, captured once per test binary +/// so assertions can prove a failure was logged. +fn captured_logs() -> Arc>> { + static LOGS: std::sync::OnceLock>>> = std::sync::OnceLock::new(); + Arc::clone(LOGS.get_or_init(|| { + let logs: Arc>> = Arc::default(); + let sink = Arc::clone(&logs); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_max_level(tracing::Level::WARN) + .with_writer(move || CapturedLogWriter(Arc::clone(&sink))) + .finish(); + tracing::subscriber::set_global_default(subscriber) + .expect("only the supervision tests install a global subscriber"); + logs + })) +} + +struct CapturedLogWriter(Arc>>); + +impl std::io::Write for CapturedLogWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Whether one captured log line contains every needle. +fn logs_contain_line(logs: &Arc>>, needles: &[&str]) -> bool { + let logs = logs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + String::from_utf8_lossy(&logs) + .lines() + .any(|line| needles.iter().all(|needle| line.contains(needle))) +} + +fn fast_restart_policy(max_restarts: u32) -> RenderRestartPolicy { + RenderRestartPolicy { + max_restarts, + window: Duration::from_secs(60), + initial_backoff: Duration::from_millis(20), + max_backoff: Duration::from_millis(80), + stable_uptime: Duration::from_secs(60), + } +} + +fn solid_strip_state() -> RenderThreadState { + make_render_state( + active_builtin_effect("solid_color", solid_color_controls(0, 255, 0)), + SpatialEngine::new(test_layout(vec![strip_zone("zone_0", "mock:strip", 8)])), + BackendManager::new(), + ) +} + +async fn wait_for_render_frames(state: &RenderThreadState, minimum_frame_number: u64) { + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + loop { + let frame_number = state.render_loop.read().await.frame_number(); + if frame_number >= minimum_frame_number { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "expected render frame {minimum_frame_number}, still at {frame_number}" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + +async fn wait_for_supervision( + supervision: &RenderSupervision, + what: &str, + predicate: F, +) -> RenderSupervisionSnapshot +where + F: Fn(&RenderSupervisionSnapshot) -> bool, +{ + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + loop { + let snapshot = supervision.snapshot(); + if predicate(&snapshot) { + return snapshot; + } + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}: phase={:?}, restarts_total={}, last_failure={:?}", + snapshot.phase, + snapshot.restarts_total, + snapshot.last_failure + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + +fn read_failure_record(path: &std::path::Path) -> RenderFailureRecord { + serde_json::from_slice(&std::fs::read(path).expect("render failure record should exist")) + .expect("render failure record should parse") +} + +#[tokio::test] +async fn injected_render_panic_is_logged_recorded_and_the_pipeline_resumes() { + hypercolor_daemon::panic_hook::install(); + let logs = captured_logs(); + let state_dir = tempfile::tempdir().expect("state tempdir should be created"); + let record_path = state_dir.path().join(RENDER_FAILURE_FILE); + let supervision = RenderSupervision::new(fast_restart_policy(3), Some(record_path.clone())); + let state = solid_strip_state(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 3).await; + + supervision.inject_fault(InjectedRenderFault::Panic( + "injected panic for the logged and recorded restart", + )); + + let snapshot = wait_for_supervision(&supervision, "the restarted pipeline", |snapshot| { + snapshot.restarts_total == 1 && snapshot.phase == RenderThreadPhase::Running + }) + .await; + let resumed_from = state.render_loop.read().await.frame_number(); + wait_for_render_frames(&state, resumed_from + 5).await; + + let failure = snapshot + .last_failure + .expect("the restart should keep the failure that caused it"); + assert!(snapshot.last_failure_in_this_process); + assert_eq!(failure.kind, RenderFailureKind::Panic); + assert_eq!( + failure.message, + "injected panic for the logged and recorded restart" + ); + assert_eq!(failure.action, RenderRecoveryAction::Restart); + assert_eq!(failure.thread.as_deref(), Some("hypercolor-render")); + assert!( + failure + .location + .as_deref() + .is_some_and(|location| location.contains("pipeline_driver.rs")), + "{failure:?}" + ); + assert!( + failure + .backtrace + .as_deref() + .is_some_and(|backtrace| backtrace.contains("pipeline_driver")), + "the record should carry the backtrace captured at the panic: {failure:?}" + ); + assert_eq!( + rt.supervision().failure_record_path(), + Some(record_path.as_path()) + ); + let persisted = read_failure_record(&record_path); + assert_eq!(persisted.kind, RenderFailureKind::Panic); + assert_eq!(persisted.action, RenderRecoveryAction::Restart); + assert_eq!(persisted.restarts_total, 1); + assert_eq!(persisted.pid, std::process::id()); + assert!(persisted.backtrace.is_some()); + assert!(logs_contain_line( + &logs, + &[ + "render pipeline failed; restarting it with fresh GPU and render state", + "injected panic for the logged and recorded restart", + ] + )); + assert!(logs_contain_line( + &logs, + &[ + "thread panicked", + "injected panic for the logged and recorded restart" + ] + )); + + state.render_loop.write().await.stop(); + rt.shutdown() + .await + .expect("a recovered render thread should shut down cleanly"); + assert_eq!(supervision.phase(), RenderThreadPhase::Stopped); +} + +#[tokio::test] +async fn repeated_render_panics_exhaust_the_restart_budget_and_request_termination() { + hypercolor_daemon::panic_hook::install(); + let state_dir = tempfile::tempdir().expect("state tempdir should be created"); + let record_path = state_dir.path().join(RENDER_FAILURE_FILE); + let supervision = RenderSupervision::new(fast_restart_policy(2), Some(record_path.clone())); + let state = solid_strip_state(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 1).await; + + for _ in 0..3 { + supervision.inject_fault(InjectedRenderFault::Panic("injected crash loop panic")); + } + + let record = tokio::time::timeout(WAIT_DEADLINE, supervision.terminated()) + .await + .expect("the third panic should exhaust a two-restart budget"); + assert_eq!(record.kind, RenderFailureKind::Panic); + assert_eq!(record.action, RenderRecoveryAction::TerminateProcess); + assert_eq!(record.restart_attempt, 3); + assert_eq!(record.restarts_total, 2); + assert_eq!(supervision.phase(), RenderThreadPhase::Terminating); + let persisted = read_failure_record(&record_path); + assert_eq!(persisted.action, RenderRecoveryAction::TerminateProcess); + let error = rt + .shutdown() + .await + .expect_err("a render thread that gave up should end with an error"); + assert!(format!("{error:#}").contains("cannot recover"), "{error:#}"); +} + +#[tokio::test] +async fn lost_compositor_device_terminates_without_an_in_process_restart() { + let supervision = RenderSupervision::new(fast_restart_policy(5), None); + let state = solid_strip_state(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 1).await; + + supervision.inject_compositor_device_lost("injected driver reset"); + + let record = tokio::time::timeout(WAIT_DEADLINE, supervision.terminated()) + .await + .expect("a lost compositor device should end the render thread"); + assert_eq!(record.kind, RenderFailureKind::GpuDeviceLost); + assert_eq!(record.action, RenderRecoveryAction::TerminateProcess); + assert!( + record.message.contains("injected driver reset"), + "{record:?}" + ); + assert_eq!(record.restarts_total, 0); + assert!(rt.shutdown().await.is_err()); +} + +#[cfg(feature = "wgpu")] +#[tokio::test] +async fn injected_uncaptured_wgpu_error_is_logged_and_counted_without_killing_the_render_thread() { + let logs = captured_logs(); + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let mut state = solid_strip_state(); + state.render_acceleration_mode = RenderAccelerationMode::Gpu; + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised GPU render thread should spawn"); + wait_for_render_frames(&state, 3).await; + let before = gpu_error_totals(); + + supervision.inject_fault(InjectedRenderFault::GpuValidationError); + + let snapshot = wait_for_supervision(&supervision, "the counted validation error", |snapshot| { + snapshot.gpu_errors.validation > before.validation + }) + .await; + let counted_at = state.render_loop.read().await.frame_number(); + wait_for_render_frames(&state, counted_at + 5).await; + + assert_eq!(snapshot.phase, RenderThreadPhase::Running); + assert_eq!(supervision.snapshot().restarts_total, 0); + let last = snapshot + .gpu_errors + .last + .expect("the latest uncaptured error should be kept"); + assert_eq!(last.kind, "validation"); + assert_eq!(last.device, "SparkleFlinger GPU compositor"); + assert!(logs_contain_line( + &logs, + &["uncaptured wgpu error", "injected validation error"] + )); + + state.render_loop.write().await.stop(); + rt.shutdown() + .await + .expect("a render thread that only logged a GPU error should shut down cleanly"); +} From 1c4249bd112a410a1e64688f532d43e77ce509e9 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 01:33:32 -0700 Subject: [PATCH 10/26] feat(daemon): exit nonzero when the render thread cannot recover A render thread that gave up must not leave the daemon serving the API with nothing rendering. The daemon now watches the render supervision: when it requests termination, the API is told to stop, the normal graceful shutdown runs, and the run returns a typed render termination error so the process exits nonzero. That error wins over any teardown error. A backstop exits with code 70 if the shutdown has not finished within 20 seconds, since a failure bad enough to exhaust the restart budget can also wedge the teardown. The API now listens to a shutdown channel the daemon owns, fed by the caller's signal (a dropped caller sender still counts as shutdown) and by the render watcher. Every launcher already restarts a daemon that exits nonzero: the systemd units with Restart=on-failure, launchd and Homebrew with KeepAlive SuccessfulExit false, and the desktop app's supervisor. The Windows service did not: its recovery actions only ran for a crash. The service host now exits without reporting a stop on a render termination, which the service manager treats as a crash, and the installer sets the failure flag so any other error stop restarts the service too. A render thread wedged inside one frame cannot be interrupted. Under systemd the daemon now withholds its watchdog keepalive while a running pipeline has gone 20 seconds without a loop iteration, so systemd kills and restarts it once WatchdogSec passes. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../hypercolor-app/tests/packaging_tests.rs | 31 +++ crates/hypercolor-daemon/src/daemon.rs | 242 +++++++++++++++++- .../hypercolor-daemon/src/windows_service.rs | 14 + crates/hypercolor-linux-session/src/lib.rs | 2 +- .../hypercolor-linux-session/src/systemd.rs | 36 ++- .../tests/watchdog_tests.rs | 18 ++ scripts/install-windows-service.ps1 | 4 + 7 files changed, 341 insertions(+), 6 deletions(-) create mode 100644 crates/hypercolor-linux-session/tests/watchdog_tests.rs diff --git a/crates/hypercolor-app/tests/packaging_tests.rs b/crates/hypercolor-app/tests/packaging_tests.rs index 0d489861d..942b4a374 100644 --- a/crates/hypercolor-app/tests/packaging_tests.rs +++ b/crates/hypercolor-app/tests/packaging_tests.rs @@ -1804,6 +1804,37 @@ fn pawnio_lookups_see_the_64_bit_install_from_any_powershell() { } } +/// The daemon exits nonzero when its render thread cannot recover in +/// process. Every launcher that runs it as a service must restart it on +/// that exit, not only on a crash: systemd on failure, and the Windows +/// service manager only once the failure flag extends its recovery actions +/// to non-crash failures. +#[test] +fn daemon_service_launchers_restart_a_daemon_that_exits_with_an_error() { + for (name, unit) in [ + ("hypercolor.service", SYSTEMD_USER_UNIT), + ("hypercolor.service.system", SYSTEMD_PACKAGED_USER_UNIT), + ] { + assert!( + unit.lines().any(|line| line.trim() == "Restart=on-failure"), + "{name} must restart the daemon after an error exit" + ); + } + let code: Vec<&str> = INSTALL_WINDOWS_SERVICE_PS1 + .lines() + .filter(|line| !line.trim_start().starts_with('#')) + .collect(); + for step in [ + "sc.exe failure $ServiceName reset= 86400 actions= restart/", + "sc.exe failureflag $ServiceName 1", + ] { + assert!( + code.iter().any(|line| line.contains(step)), + "the daemon service installer is missing {step}" + ); + } +} + /// `Start-Service` and `Stop-Service` wait without a bound while a service /// is pending, which would hang the installer. The broker installer bounds /// every wait, checks the broker stayed up, and on failure prints the diff --git a/crates/hypercolor-daemon/src/daemon.rs b/crates/hypercolor-daemon/src/daemon.rs index b02b71a26..fa38e4b72 100644 --- a/crates/hypercolor-daemon/src/daemon.rs +++ b/crates/hypercolor-daemon/src/daemon.rs @@ -18,7 +18,7 @@ use tokio::net::TcpListener; use tokio::sync::watch; use tokio::task::JoinSet; use tokio::time::{Duration, sleep}; -use tracing::{info, warn}; +use tracing::{error, info, warn}; use tracing_subscriber::EnvFilter; use crate::api; @@ -26,6 +26,7 @@ use crate::api::security::{CredentialAuthority, CredentialTier, RemoteClientFami use crate::app_state::AppState; use crate::macos_owner::{MacosDaemonOwner, MacosDaemonSessionAttestation, MacosOwnerSnapshot}; use crate::mdns::MdnsPublisher; +use crate::render_thread::{RenderFailureRecord, RenderSupervision}; use crate::startup::{DaemonState, StartupProgress, config_sources}; const MAIN_RUNTIME_WORKERS: usize = 4; @@ -34,6 +35,18 @@ const MAIN_RUNTIME_THREAD_KEEP_ALIVE: std::time::Duration = std::time::Duration: const API_LISTEN_BACKLOG: i32 = 1024; const API_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(3); +/// How long the graceful shutdown after an unrecoverable render failure may +/// take before the process exits regardless. +/// +/// The daemon must not keep serving without a renderer, and a failure bad +/// enough to exhaust the restart budget can also wedge the teardown, so +/// this backstop bounds the gap a service manager sees. +const RENDER_FAILURE_EXIT_DEADLINE: Duration = Duration::from_secs(20); + +/// Exit code of a process exit forced by an unrecoverable render failure. +/// `EX_SOFTWARE`: an internal failure. +pub(crate) const RENDER_FAILURE_EXIT_CODE: i32 = 70; + /// Runtime options for one daemon process. #[derive(Default)] pub struct DaemonRunOptions { @@ -167,10 +180,16 @@ impl PreparedDaemon { let progress = StartupProgress::default(); let handoff = api::startup::ApiHandoff::starting(progress.clone(), env!("CARGO_PKG_VERSION")); + // The API stops on the caller's shutdown signal and also when the + // render thread gives up, so the server listens to a channel this + // function owns rather than the caller's. + let (api_shutdown_tx, api_shutdown_rx) = watch::channel(*shutdown_rx.borrow()); + let _shutdown_relay = + BackgroundTask::spawn(relay_shutdown_signal(shutdown_rx, api_shutdown_tx.clone())); let server = ApiServerTask::spawn(serve_api_handoff_with_shutdown_timeout( listeners, handoff.clone(), - shutdown_rx, + api_shutdown_rx, API_GRACEFUL_SHUTDOWN_TIMEOUT, )); info!(binds = %self.listen_addr, "API listeners answering startup probes"); @@ -185,6 +204,12 @@ impl PreparedDaemon { self.options.service_status.take(), progress.clone(), )?; + let render_supervision = daemon_state.domains.output.render_supervision().clone(); + let _render_failure_watch = BackgroundTask::spawn(shut_down_after_render_failure( + render_supervision.clone(), + api_shutdown_tx.clone(), + Some(RENDER_FAILURE_EXIT_DEADLINE), + )); let ui_dir = resolve_ui_dir(self.options.ui_dir.clone()); daemon_state.session_monitors = self.options.session_monitors.take(); record_api_binding( @@ -255,9 +280,30 @@ impl PreparedDaemon { ); hypercolor_linux_session::notify_ready(); - hypercolor_linux_session::spawn_watchdog(); + // A render thread wedged inside a frame cannot be interrupted, so it + // stops the keepalives and systemd restarts the daemon instead. + let watchdog_supervision = render_supervision.clone(); + hypercolor_linux_session::spawn_watchdog_while(move || { + watchdog_supervision.watchdog_healthy() + }); - server.join().await?; + let server_result = server.join().await; + if let Some(record) = render_supervision.termination() { + // The termination error must win over any teardown error: it is + // what tells a process host, such as the Windows service, to + // exit the way its service manager restarts. + if let Err(error) = server_result { + warn!(error = %format!("{error:#}"), "API server failed while stopping after the render failure"); + } + if let Some(publisher) = mdns_publisher { + publisher.shutdown().await; + } + if let Err(error) = daemon_state.shutdown().await { + warn!(error = %format!("{error:#}"), "daemon shutdown after the render failure did not finish cleanly"); + } + return Err(render_failure_exit_error(&record)); + } + server_result?; if let Some(publisher) = mdns_publisher { publisher.shutdown().await; @@ -265,11 +311,199 @@ impl PreparedDaemon { daemon_state.shutdown().await?; + if let Some(record) = render_supervision.termination() { + return Err(render_failure_exit_error(&record)); + } info!("Hypercolor daemon exited cleanly"); Ok(()) } } +/// A spawned task that is aborted when its owner goes away. +struct BackgroundTask(tokio::task::JoinHandle<()>); + +impl BackgroundTask { + fn spawn(task: impl Future + Send + 'static) -> Self { + Self(tokio::spawn(task)) + } +} + +impl Drop for BackgroundTask { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// Forward the caller's shutdown signal into the API shutdown channel. +/// +/// A dropped caller sender counts as shutdown, which is how the API server +/// treated the caller's channel directly. +async fn relay_shutdown_signal( + mut external: watch::Receiver, + api_shutdown: watch::Sender, +) { + let _ = external.wait_for(|shutdown| *shutdown).await; + api_shutdown.send_replace(true); +} + +/// Shut the API down once the render thread gives up. +/// +/// The daemon then runs its normal graceful shutdown and returns an error, +/// so the process exits nonzero and its service manager (the systemd unit, +/// launchd, the Windows service recovery actions, or the desktop app's +/// supervisor) starts a fresh one. `exit_deadline` arms a backstop that +/// exits the process if that shutdown does not finish in time. +async fn shut_down_after_render_failure( + supervision: RenderSupervision, + api_shutdown: watch::Sender, + exit_deadline: Option, +) { + let record = supervision.terminated().await; + error!( + kind = record.kind.as_str(), + message = %record.message, + restarts_total = record.restarts_total, + failure_record = %supervision + .failure_record_path() + .map_or_else(|| "not persisted".to_owned(), |path| path.display().to_string()), + "render thread cannot recover; stopping the daemon so its service manager restarts it" + ); + if let Some(deadline) = exit_deadline { + arm_render_failure_exit_backstop(deadline); + } + api_shutdown.send_replace(true); +} + +fn arm_render_failure_exit_backstop(deadline: Duration) { + let spawned = std::thread::Builder::new() + .name("hypercolor-exit-backstop".to_owned()) + .spawn(move || { + std::thread::sleep(deadline); + error!( + deadline_ms = u64::try_from(deadline.as_millis()).unwrap_or(u64::MAX), + exit_code = RENDER_FAILURE_EXIT_CODE, + "graceful shutdown after the render failure did not finish; exiting now" + ); + std::process::exit(RENDER_FAILURE_EXIT_CODE); + }); + if let Err(spawn_error) = spawned { + error!(%spawn_error, "could not arm the exit backstop for the render failure shutdown"); + } +} + +/// A daemon run that ended because the render thread could not recover. +#[derive(Debug, thiserror::Error)] +#[error( + "render thread could not recover ({kind}: {message}); exiting so the service manager restarts the daemon" +)] +pub(crate) struct RenderTerminationError { + kind: &'static str, + message: String, +} + +fn render_failure_exit_error(record: &RenderFailureRecord) -> anyhow::Error { + anyhow::Error::new(RenderTerminationError { + kind: record.kind.as_str(), + message: record.message.clone(), + }) +} + +/// Whether a daemon run ended because the render thread could not recover. +/// +/// A process host that reports its own exit, such as the Windows service, +/// uses this to exit in a way its service manager restarts. +#[must_use] +pub fn is_render_termination(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() +} + +#[cfg(test)] +mod render_failure_shutdown_tests { + use std::sync::Arc; + use std::time::Duration; + + use tokio::sync::watch; + + use super::{ + is_render_termination, relay_shutdown_signal, render_failure_exit_error, + shut_down_after_render_failure, + }; + use crate::render_thread::{ + RenderFailureKind, RenderFailureRecord, RenderRecoveryAction, RenderSupervision, + RenderThreadPhase, + }; + + fn exhausted_record() -> RenderFailureRecord { + RenderFailureRecord { + kind: RenderFailureKind::Panic, + message: "frame panicked again".to_owned(), + thread: Some("hypercolor-render".to_owned()), + location: None, + occurred_at: "2026-10-19T22:30:00.000Z".to_owned(), + occurred_at_unix_ms: 0, + pid: std::process::id(), + process_instance: String::new(), + daemon_version: env!("CARGO_PKG_VERSION").to_owned(), + restart_attempt: 6, + restarts_total: 5, + action: RenderRecoveryAction::TerminateProcess, + backtrace: None, + } + } + + #[tokio::test] + async fn an_unrecoverable_render_failure_shuts_the_api_down_and_fails_the_run() { + let supervision = RenderSupervision::detached(); + let (api_shutdown_tx, mut api_shutdown_rx) = watch::channel(false); + let watcher = tokio::spawn(shut_down_after_render_failure( + supervision.clone(), + api_shutdown_tx, + None, + )); + tokio::task::yield_now().await; + assert!(!*api_shutdown_rx.borrow(), "nothing failed yet"); + + supervision.request_termination(Arc::new(exhausted_record())); + + tokio::time::timeout( + Duration::from_secs(5), + api_shutdown_rx.wait_for(|stop| *stop), + ) + .await + .expect("the API should be told to stop") + .expect("the watcher holds the sender until it fires"); + watcher.await.expect("watcher should finish"); + assert_eq!(supervision.phase(), RenderThreadPhase::Terminating); + let record = supervision + .termination() + .expect("termination should stay recorded for the exit path"); + let error = render_failure_exit_error(&record); + assert!(is_render_termination(&error)); + assert!(!is_render_termination(&anyhow::anyhow!( + "an unrelated failure" + ))); + let error = format!("{error:#}"); + assert!(error.contains("frame panicked again"), "{error}"); + } + + #[tokio::test] + async fn the_caller_shutdown_signal_and_its_disappearance_both_stop_the_api() { + let (external_tx, external_rx) = watch::channel(false); + let (api_tx, mut api_rx) = watch::channel(false); + let relay = tokio::spawn(relay_shutdown_signal(external_rx, api_tx)); + external_tx.send_replace(true); + relay.await.expect("relay should finish"); + assert!(*api_rx.borrow_and_update()); + + let (external_tx, external_rx) = watch::channel(false); + let (api_tx, api_rx) = watch::channel(false); + let relay = tokio::spawn(relay_shutdown_signal(external_rx, api_tx)); + drop(external_tx); + relay.await.expect("relay should finish"); + assert!(*api_rx.borrow()); + } +} + /// Publish where the API listens, before installers read it. fn record_api_binding( daemon: &mut DaemonState, diff --git a/crates/hypercolor-daemon/src/windows_service.rs b/crates/hypercolor-daemon/src/windows_service.rs index 0fff761e4..b203c618e 100644 --- a/crates/hypercolor-daemon/src/windows_service.rs +++ b/crates/hypercolor-daemon/src/windows_service.rs @@ -104,6 +104,20 @@ fn run_service() -> Result<()> { shutdown_rx, configuration.extension_installers, )); + if let Err(error) = &run_result + && daemon::is_render_termination(error) + { + // A reported stop, even with an error code, runs the recovery + // actions only when the service's failure flag is set, and + // services installed before the installer set it do not have it. + // Exiting without reporting a stop is a crash to the service + // manager, which restarts the service either way. + tracing::error!( + error = %format!("{error:#}"), + "exiting the service without reporting a stop so the service manager restarts it" + ); + std::process::exit(daemon::RENDER_FAILURE_EXIT_CODE); + } let exit_code = u32::from(run_result.is_err()); report_status( &status_handle, diff --git a/crates/hypercolor-linux-session/src/lib.rs b/crates/hypercolor-linux-session/src/lib.rs index 2729baf50..35b2ea6b5 100644 --- a/crates/hypercolor-linux-session/src/lib.rs +++ b/crates/hypercolor-linux-session/src/lib.rs @@ -19,7 +19,7 @@ pub use self::parent_death::arm_parent_death; pub use self::procfs::{process_resident_memory_mb, running_processes}; #[cfg(target_os = "linux")] pub use self::screensaver::ScreensaverMonitor; -pub use self::systemd::{notify_ready, spawn_watchdog}; +pub use self::systemd::{notify_ready, spawn_watchdog, spawn_watchdog_while}; /// Build the Linux session monitors for daemon composition. #[must_use] diff --git a/crates/hypercolor-linux-session/src/systemd.rs b/crates/hypercolor-linux-session/src/systemd.rs index 623c6c49e..5f0cbf287 100644 --- a/crates/hypercolor-linux-session/src/systemd.rs +++ b/crates/hypercolor-linux-session/src/systemd.rs @@ -25,16 +25,50 @@ pub fn notify_ready() { /// /// Off Linux this does nothing. On Linux it must run inside a Tokio runtime. pub fn spawn_watchdog() { + spawn_watchdog_while(|| true); +} + +/// Spawn the periodic watchdog keepalive, sending each keepalive only while +/// `healthy` returns `true`. +/// +/// systemd kills and restarts a service that goes `WatchdogSec` without a +/// keepalive, so withholding them is how a daemon with a wedged subsystem +/// it cannot recover in-process gets a fresh process. `healthy` runs once +/// per keepalive interval on the runtime, so it must not block. +/// +/// Off Linux this does nothing. On Linux it must run inside a Tokio runtime. +pub fn spawn_watchdog_while(healthy: F) +where + F: Fn() -> bool + Send + 'static, +{ #[cfg(target_os = "linux")] { - tokio::spawn(async { + tokio::spawn(async move { let mut interval = tokio::time::interval(WATCHDOG_INTERVAL); + let mut withheld = false; loop { interval.tick().await; + if !healthy() { + if !withheld { + tracing::error!( + "withholding the systemd watchdog keepalive because the daemon \ + reports itself unhealthy; systemd restarts it once the watchdog \ + timeout passes" + ); + withheld = true; + } + continue; + } + if withheld { + tracing::warn!("daemon is healthy again; resuming systemd watchdog keepalives"); + withheld = false; + } if let Err(error) = sd_notify::notify(&[sd_notify::NotifyState::Watchdog]) { tracing::debug!("failed to notify systemd watchdog: {error}"); } } }); } + #[cfg(not(target_os = "linux"))] + let _ = healthy; } diff --git a/crates/hypercolor-linux-session/tests/watchdog_tests.rs b/crates/hypercolor-linux-session/tests/watchdog_tests.rs new file mode 100644 index 000000000..307aa86bf --- /dev/null +++ b/crates/hypercolor-linux-session/tests/watchdog_tests.rs @@ -0,0 +1,18 @@ +//! The systemd watchdog keepalive consults the daemon's health on every +//! interval, starting with the first. + +#[cfg(target_os = "linux")] +#[tokio::test] +async fn watchdog_consults_the_health_check_on_its_first_interval() { + let (checked_tx, mut checked_rx) = tokio::sync::mpsc::unbounded_channel(); + + hypercolor_linux_session::spawn_watchdog_while(move || { + let _ = checked_tx.send(()); + false + }); + + tokio::time::timeout(std::time::Duration::from_secs(5), checked_rx.recv()) + .await + .expect("the first keepalive interval should run the health check") + .expect("the watchdog task should still hold the health check"); +} diff --git a/scripts/install-windows-service.ps1 b/scripts/install-windows-service.ps1 index d266ed7ff..8de20b6fd 100644 --- a/scripts/install-windows-service.ps1 +++ b/scripts/install-windows-service.ps1 @@ -359,6 +359,10 @@ if ($serviceEnvironment.Count -gt 0) { } sc.exe failure $ServiceName reset= 86400 actions= restart/5000/restart/15000/""/60000 | Out-Null +# Without the failure flag the service manager runs those actions only when +# the process crashes. The daemon stops itself with a nonzero exit code when +# its render thread cannot recover, and that stop must restart it too. +sc.exe failureflag $ServiceName 1 | Out-Null Write-Host "Installed $ServiceName" Write-Host " Source: $daemonSourcePath" From febfb7932ec79eb7d4c2dd230f33dd99881cdb78 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 01:33:51 -0700 Subject: [PATCH 11/26] feat(api): report render thread health in diagnose and status Nothing in diagnose noticed a dead render thread: the render checks read the render loop's own state, which stays running when the thread that drives it is gone. Diagnose gains two render checks. render_thread fails while the pipeline is restarting or the daemon is exiting after an unrecoverable failure, when the thread is gone while the loop still runs, and when a running loop has completed no frame, or a paused one has not polled, for three seconds. gpu_errors keeps the uncaptured wgpu error counts since startup in its detail, warns when the latest is under ten minutes old, and fails on a lost device, so errors that no longer panic stay visible. RenderSupervisionStatus carries the phase, restart counts, the last frame age, the last failure, and the GPU error totals, on both the diagnose render snapshot and the system status. Both fields default when absent so clients keep reading older daemons. The failure message and GPU error text appear only for callers allowed private diagnostics. The Python client is regenerated and the troubleshooting guide documents both checks. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/src/api/system.rs | 6 + .../src/domain/diagnostics.rs | 363 +++++++++++++++++- .../src/render_thread/supervision.rs | 55 +++ .../tests/render_supervision_tests.rs | 104 +++++ .../tests/render_thread_tests.rs | 115 ++++++ crates/hypercolor-types/src/api/diagnose.rs | 6 +- crates/hypercolor-types/src/api/system.rs | 73 ++++ docs/content/troubleshooting/performance.md | 20 +- .../hypercolor/_generated/models/__init__.py | 6 + .../models/diagnose_render_snapshot.py | 25 ++ .../_generated/models/gpu_error_status.py | 110 ++++++ .../models/render_failure_status.py | 149 +++++++ .../models/render_supervision_status.py | 171 +++++++++ .../_generated/models/system_status.py | 25 ++ 14 files changed, 1217 insertions(+), 11 deletions(-) create mode 100644 crates/hypercolor-daemon/tests/render_supervision_tests.rs create mode 100644 python/src/hypercolor/_generated/models/gpu_error_status.py create mode 100644 python/src/hypercolor/_generated/models/render_failure_status.py create mode 100644 python/src/hypercolor/_generated/models/render_supervision_status.py diff --git a/crates/hypercolor-daemon/src/api/system.rs b/crates/hypercolor-daemon/src/api/system.rs index fd8166542..23012b135 100644 --- a/crates/hypercolor-daemon/src/api/system.rs +++ b/crates/hypercolor-daemon/src/api/system.rs @@ -300,6 +300,12 @@ async fn system_status_with_privacy( macos_daemon_ownership, compositor_acceleration: render_acceleration_status(&state.render_acceleration), render_loop: render_loop_status, + render_supervision: state + .domains + .output + .render_supervision() + .snapshot() + .status(include_private_diagnostics), session_performance, latest_frame, effect_health, diff --git a/crates/hypercolor-daemon/src/domain/diagnostics.rs b/crates/hypercolor-daemon/src/domain/diagnostics.rs index bee1e6f4f..ff3806427 100644 --- a/crates/hypercolor-daemon/src/domain/diagnostics.rs +++ b/crates/hypercolor-daemon/src/domain/diagnostics.rs @@ -24,9 +24,17 @@ use crate::domain::input_status::{actionable_input_diagnostics, input_status_sna use crate::domain::output::{OutputContext, RenderLoopStatus}; use crate::domain::spatial::SpatialService; use crate::performance::{LatestFrameMetrics, PerformanceSnapshot}; +use crate::render_thread::gpu_errors::GpuErrorTotals; +use crate::render_thread::{RenderSupervisionSnapshot, RenderThreadPhase}; +use hypercolor_core::engine::RenderLoopState; const RENDER_FRAME_STALE_WARNING_MS: f64 = 2_000.0; const RENDER_FRAME_STALE_FAIL_MS: f64 = 10_000.0; +/// How long the render thread may go without completing a frame (or, when +/// paused, without polling) before the `render_thread` check fails. A +/// healthy loop completes a throttled idle frame at least every quarter +/// second, so a few seconds of silence means it is wedged or gone. +pub const RENDER_THREAD_STALL_FAIL: Duration = Duration::from_secs(3); const DEFAULT_SAFE_CHECKS: [&str; 8] = [ "daemon", "render", @@ -149,6 +157,7 @@ struct DiagnosticsInputs { tracked_devices: usize, config_available: bool, config_persistence: Option<(PathBuf, ConfigPersistence)>, + render_supervision: RenderSupervisionSnapshot, } impl DiagnosticsInputs { @@ -339,6 +348,7 @@ impl DiagnosticsContext { tracked_devices: self.authorities.devices.device_registry().len().await, config_available: self.authorities.platform.config_available(), config_persistence: self.authorities.platform.config_persistence(), + render_supervision: self.authorities.output.render_supervision().snapshot(), } } } @@ -361,15 +371,23 @@ fn render_checks(inputs: &DiagnosticsInputs) -> Vec { let render_elapsed_ms = inputs.render_elapsed_ms(); let running = inputs.render_loop.running; let render_loop_stats = inputs.render_loop.stats; - let mut checks = vec![DiagnoseCheck { - category: "render".to_owned(), - name: "render_loop".to_owned(), - status: if running { "pass" } else { "warning" }.to_owned(), - detail: format!( - "state={}, tier={}", - render_loop_stats.state, render_loop_stats.tier + let mut checks = vec![ + DiagnoseCheck { + category: "render".to_owned(), + name: "render_loop".to_owned(), + status: if running { "pass" } else { "warning" }.to_owned(), + detail: format!( + "state={}, tier={}", + render_loop_stats.state, render_loop_stats.tier + ), + }, + render_thread_check( + &inputs.render_supervision, + render_loop_stats.state, + RENDER_THREAD_STALL_FAIL, ), - }]; + gpu_errors_check(&inputs.render_supervision.gpu_errors), + ]; if !running { return checks; } @@ -415,6 +433,178 @@ fn render_checks(inputs: &DiagnosticsInputs) -> Vec { checks } +/// Whether the supervised render thread is alive and producing frames. +/// +/// Fails while a pipeline is restarting or the daemon is terminating after +/// an unrecoverable failure, when the render thread is gone while the loop +/// still runs, and when a running loop has completed no frame (or a paused +/// loop has not polled) for `stall_after`. +#[must_use] +pub fn render_thread_check( + supervision: &RenderSupervisionSnapshot, + loop_state: RenderLoopState, + stall_after: Duration, +) -> DiagnoseCheck { + let (status, detail) = render_thread_status(supervision, loop_state, stall_after); + DiagnoseCheck { + category: "render".to_owned(), + name: "render_thread".to_owned(), + status: status.to_owned(), + detail, + } +} + +fn render_thread_status( + supervision: &RenderSupervisionSnapshot, + loop_state: RenderLoopState, + stall_after: Duration, +) -> (&'static str, String) { + let failure = supervision.last_failure.as_ref().map_or_else( + || "none".to_owned(), + |record| { + format!( + "{}{} at {}: {}", + record.kind.as_str(), + if supervision.last_failure_in_this_process { + "" + } else { + " (earlier daemon run)" + }, + record.occurred_at, + record.message + ) + }, + ); + let loop_active = matches!( + loop_state, + RenderLoopState::Running | RenderLoopState::Paused + ); + match supervision.phase { + RenderThreadPhase::Terminating => ( + "fail", + format!( + "the render thread could not recover after {} restart(s); the daemon is \ + exiting so its service manager restarts it. Last failure: {failure}", + supervision.restarts_total + ), + ), + RenderThreadPhase::Restarting => ( + "fail", + format!( + "the render pipeline is restarting after {} failure(s) in a row ({} restart(s) \ + so far, {} allowed per window). Last failure: {failure}", + supervision.consecutive_failures, + supervision.restarts_total, + supervision.max_restarts + ), + ), + RenderThreadPhase::Running => { + let (age, activity) = if loop_state == RenderLoopState::Paused { + (supervision.last_heartbeat_age, "polled") + } else { + (supervision.last_frame_age, "completed a frame") + }; + match age { + Some(age) if loop_active && age >= stall_after => ( + "fail", + format!( + "the render thread has not {activity} for {} ms while the render loop \ + is {loop_state}; restarts={}, last_failure={failure}", + age.as_millis(), + supervision.restarts_total + ), + ), + _ => ( + "pass", + format!( + "phase=running, last_frame_age_ms={}, restarts={}, last_failure={failure}", + supervision + .last_frame_age + .map_or_else(|| "none".to_owned(), |age| age.as_millis().to_string()), + supervision.restarts_total + ), + ), + } + } + phase @ (RenderThreadPhase::NotStarted + | RenderThreadPhase::Starting + | RenderThreadPhase::Stopped) => { + if loop_active { + ( + "fail", + format!( + "the render thread is {} while the render loop is {loop_state}; nothing \ + is rendering. Last failure: {failure}", + phase.as_str() + ), + ) + } else { + ( + "pass", + format!( + "phase={}, render loop {loop_state}, last_failure={failure}", + phase.as_str() + ), + ) + } + } + } +} + +/// How recent the latest uncaptured GPU error must be for the +/// `gpu_errors` check to warn. Older errors stay in the counts. +pub const GPU_ERROR_WARNING_WINDOW: Duration = Duration::from_mins(10); + +/// Whether a GPU error went uncaptured recently. +/// +/// Uncaptured validation, out-of-memory, and internal errors no longer +/// panic the thread that raised them, so this check is where they stay +/// visible: the counts since startup are always in the detail, an error +/// within [`GPU_ERROR_WARNING_WINDOW`] warns, and a lost device fails. +#[must_use] +pub fn gpu_errors_check(totals: &GpuErrorTotals) -> DiagnoseCheck { + let now_unix_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }); + gpu_errors_check_at(totals, now_unix_ms) +} + +fn gpu_errors_check_at(totals: &GpuErrorTotals, now_unix_ms: u64) -> DiagnoseCheck { + let last_age_ms = totals + .last + .as_ref() + .map(|last| now_unix_ms.saturating_sub(last.at_unix_ms)); + let last = totals.last.as_ref().map_or_else(String::new, |last| { + format!( + "; last ({} ms ago): {}: {}: {}", + last_age_ms.unwrap_or_default(), + last.device, + last.kind, + last.message + ) + }); + let counts = format!( + "validation={}, out_of_memory={}, internal={}, device_lost={}", + totals.validation, totals.out_of_memory, totals.internal, totals.device_lost + ); + let window_ms = u64::try_from(GPU_ERROR_WARNING_WINDOW.as_millis()).unwrap_or(u64::MAX); + let status = if totals.device_lost > 0 { + "fail" + } else if totals.uncaptured() > 0 && last_age_ms.is_none_or(|age| age < window_ms) { + "warning" + } else { + "pass" + }; + DiagnoseCheck { + category: "render".to_owned(), + name: "gpu_errors".to_owned(), + status: status.to_owned(), + detail: format!("{counts}{last}"), + } +} + fn device_checks(inputs: &DiagnosticsInputs, snapshot: &DiagnoseSnapshot) -> Vec { let count = inputs.tracked_devices; let output_status = if snapshot.device_output.worker_finished_queues > 0 @@ -724,7 +914,11 @@ fn render_multi_zone_sampling_status( fn build_diagnose_snapshot(inputs: &DiagnosticsInputs) -> DiagnoseSnapshot { DiagnoseSnapshot { input: inputs.input.clone(), - render: build_render_snapshot(&inputs.performance, inputs.render_elapsed_ms()), + render: build_render_snapshot( + &inputs.performance, + inputs.render_elapsed_ms(), + &inputs.render_supervision, + ), usb: build_usb_actor_snapshot(inputs.usb_actor), display_output: build_display_output_snapshot(inputs.display_output.clone()), device_output: build_device_output_snapshot(inputs.device_metrics.as_ref()), @@ -735,6 +929,7 @@ fn build_diagnose_snapshot(inputs: &DiagnosticsInputs) -> DiagnoseSnapshot { fn build_render_snapshot( performance: &PerformanceSnapshot, render_elapsed_ms: f64, + supervision: &RenderSupervisionSnapshot, ) -> DiagnoseRenderSnapshot { let pacing = performance.pacing; DiagnoseRenderSnapshot { @@ -797,6 +992,7 @@ fn build_render_snapshot( publish_avg_ms: round_2(pacing.publish_avg_ms), publish_p95_ms: round_2(pacing.publish_p95_ms), }, + supervision: supervision.status(true), } } @@ -1096,6 +1292,7 @@ mod tests { ..PerformanceSnapshot::default() }, 0.0, + &crate::render_thread::RenderSupervision::detached().snapshot(), ); let frame = snapshot @@ -1108,6 +1305,154 @@ mod tests { assert_eq!(frame.zone_composition_skips, 2); } + fn supervision( + phase: crate::render_thread::RenderThreadPhase, + frame_age_ms: u64, + heartbeat_age_ms: u64, + ) -> crate::render_thread::RenderSupervisionSnapshot { + crate::render_thread::RenderSupervisionSnapshot { + phase, + restarts_total: 0, + consecutive_failures: 0, + max_restarts: 5, + last_heartbeat_age: Some(std::time::Duration::from_millis(heartbeat_age_ms)), + last_frame_age: Some(std::time::Duration::from_millis(frame_age_ms)), + current_run: None, + last_failure: None, + last_failure_in_this_process: false, + gpu_errors: crate::render_thread::gpu_errors::GpuErrorTotals::default(), + } + } + + #[test] + fn render_thread_check_fails_on_stalls_restarts_and_a_missing_thread() { + use crate::render_thread::RenderThreadPhase; + use hypercolor_core::engine::RenderLoopState; + + let stall = std::time::Duration::from_secs(3); + let cases = [ + ( + RenderThreadPhase::Running, + RenderLoopState::Running, + 20, + 20, + "pass", + ), + ( + RenderThreadPhase::Running, + RenderLoopState::Running, + 5_000, + 20, + "fail", + ), + ( + RenderThreadPhase::Running, + RenderLoopState::Paused, + 60_000, + 40, + "pass", + ), + ( + RenderThreadPhase::Running, + RenderLoopState::Paused, + 60_000, + 4_000, + "fail", + ), + ( + RenderThreadPhase::Restarting, + RenderLoopState::Running, + 20, + 20, + "fail", + ), + ( + RenderThreadPhase::Terminating, + RenderLoopState::Running, + 20, + 20, + "fail", + ), + ( + RenderThreadPhase::Stopped, + RenderLoopState::Running, + 20, + 20, + "fail", + ), + ( + RenderThreadPhase::Stopped, + RenderLoopState::Stopped, + 9_000, + 9_000, + "pass", + ), + ( + RenderThreadPhase::NotStarted, + RenderLoopState::Created, + 0, + 0, + "pass", + ), + ]; + for (phase, loop_state, frame_age, heartbeat_age, expected) in cases { + let check = super::render_thread_check( + &supervision(phase, frame_age, heartbeat_age), + loop_state, + stall, + ); + assert_eq!( + check.status, expected, + "{phase:?} with the loop {loop_state} and a {frame_age} ms frame age: {}", + check.detail + ); + assert_eq!(check.name, "render_thread"); + } + } + + #[test] + fn gpu_errors_check_warns_on_recent_errors_and_fails_on_device_loss() { + use crate::render_thread::gpu_errors::{GpuErrorSample, GpuErrorTotals}; + + let now_ms = 50_000_000; + assert_eq!( + super::gpu_errors_check_at(&GpuErrorTotals::default(), now_ms).status, + "pass" + ); + let validation = GpuErrorTotals { + validation: 2, + last: Some(GpuErrorSample { + device: "SparkleFlinger GPU compositor".to_owned(), + kind: "validation".to_owned(), + message: "Usage flags are invalid".to_owned(), + at_unix_ms: now_ms - 1_000, + }), + ..GpuErrorTotals::default() + }; + let check = super::gpu_errors_check_at(&validation, now_ms); + assert_eq!(check.status, "warning"); + let window_ms = u64::try_from(super::GPU_ERROR_WARNING_WINDOW.as_millis()) + .expect("the window fits in milliseconds"); + let settled = super::gpu_errors_check_at(&validation, now_ms + window_ms); + assert_eq!(settled.status, "pass"); + assert!( + settled.detail.contains("validation=2"), + "{}", + settled.detail + ); + assert!(check.detail.contains("validation=2"), "{}", check.detail); + assert!( + check.detail.contains("Usage flags are invalid"), + "{}", + check.detail + ); + let lost = GpuErrorTotals { + device_lost: 1, + ..validation + }; + assert_eq!(super::gpu_errors_check_at(&lost, now_ms).status, "fail"); + } + #[test] fn multi_zone_sampling_passes_when_every_zone_samples_on_the_gpu() { let (status, detail) = render_multi_zone_sampling_status(Some(&LatestFrameMetrics { diff --git a/crates/hypercolor-daemon/src/render_thread/supervision.rs b/crates/hypercolor-daemon/src/render_thread/supervision.rs index a6c5755be..aa8a0504a 100644 --- a/crates/hypercolor-daemon/src/render_thread/supervision.rs +++ b/crates/hypercolor-daemon/src/render_thread/supervision.rs @@ -404,6 +404,61 @@ pub struct RenderSupervisionSnapshot { pub gpu_errors: GpuErrorTotals, } +impl RenderSupervisionSnapshot { + /// The API projection of this snapshot. + /// + /// `include_private` gates the failure message and the last GPU error + /// text, which can carry source paths and resource labels. + #[must_use] + pub fn status( + &self, + include_private: bool, + ) -> hypercolor_types::api::system::RenderSupervisionStatus { + use hypercolor_types::api::system::{ + GpuErrorStatus, RenderFailureStatus, RenderSupervisionStatus, + }; + + RenderSupervisionStatus { + phase: self.phase.as_str().to_owned(), + restarts_total: self.restarts_total, + consecutive_failures: self.consecutive_failures, + max_restarts: self.max_restarts, + last_frame_age_ms: self.last_frame_age.map(duration_millis), + last_failure: self + .last_failure + .as_ref() + .map(|record| RenderFailureStatus { + kind: record.kind.as_str().to_owned(), + message: if include_private { + record.message.clone() + } else { + String::new() + }, + thread: record.thread.clone(), + location: include_private.then(|| record.location.clone()).flatten(), + occurred_at: record.occurred_at.clone(), + action: record.action.as_str().to_owned(), + restart_attempt: record.restart_attempt, + current_process: self.last_failure_in_this_process, + }), + gpu_errors: GpuErrorStatus { + validation_errors: self.gpu_errors.validation, + out_of_memory_errors: self.gpu_errors.out_of_memory, + internal_errors: self.gpu_errors.internal, + device_lost: self.gpu_errors.device_lost, + last_error: include_private + .then(|| { + self.gpu_errors + .last + .as_ref() + .map(|last| format!("{}: {}: {}", last.device, last.kind, last.message)) + }) + .flatten(), + }, + } + } +} + fn duration_millis(duration: Duration) -> u64 { u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) } diff --git a/crates/hypercolor-daemon/tests/render_supervision_tests.rs b/crates/hypercolor-daemon/tests/render_supervision_tests.rs new file mode 100644 index 000000000..5ac926e52 --- /dev/null +++ b/crates/hypercolor-daemon/tests/render_supervision_tests.rs @@ -0,0 +1,104 @@ +//! Render supervision reports the failure an earlier daemon run left in the +//! state directory, and its status projection gates private detail. + +use hypercolor_daemon::render_thread::supervision::{ + RENDER_FAILURE_FILE, RENDER_RUNTIME_THREAD_NAME, RENDER_THREAD_NAME, is_render_thread_name, +}; +use hypercolor_daemon::render_thread::{RenderSupervision, RenderThreadPhase}; + +/// A record as an earlier run wrote it, before records carried a process +/// instance id. +fn earlier_run_record() -> serde_json::Value { + serde_json::json!({ + "kind": "panic", + "message": "frame panicked in the earlier run", + "thread": "hypercolor-render", + "location": "crates/hypercolor-daemon/src/render_thread/frame_executor.rs:120:9", + "occurred_at": "2026-10-19T22:30:00.000Z", + "occurred_at_unix_ms": 1_792_449_000_000_u64, + "pid": 1, + "daemon_version": "0.7.1", + "restart_attempt": 0, + "restarts_total": 0, + "action": "process_abort", + "backtrace": " 4: hypercolor_daemon::render_thread::frame_executor::execute_frame\n", + }) +} + +#[test] +fn a_failure_persisted_by_an_earlier_run_is_reported_with_private_detail_gated() { + let state_dir = tempfile::tempdir().expect("state tempdir should be created"); + let record_path = state_dir.path().join(RENDER_FAILURE_FILE); + std::fs::write( + &record_path, + serde_json::to_vec(&earlier_run_record()).expect("record should serialize"), + ) + .expect("record should be written"); + + let supervision = RenderSupervision::for_state_dir(state_dir.path()); + + assert_eq!( + supervision.failure_record_path(), + Some(record_path.as_path()) + ); + let snapshot = supervision.snapshot(); + assert_eq!(snapshot.phase, RenderThreadPhase::NotStarted); + assert!(!snapshot.last_failure_in_this_process); + let failure = snapshot + .last_failure + .clone() + .expect("the earlier run's failure should be reported"); + assert_eq!( + failure.pid, 1, + "a container's pid 1 must not hide the record" + ); + assert!(failure.process_instance.is_empty()); + + let private = snapshot.status(true); + assert_eq!(private.phase, "not_started"); + assert_eq!(private.restarts_total, 0); + let reported = private + .last_failure + .expect("status should carry the earlier failure"); + assert_eq!(reported.kind, "panic"); + assert_eq!(reported.action, "process_abort"); + assert_eq!(reported.message, "frame panicked in the earlier run"); + assert!(reported.location.is_some()); + assert!(!reported.current_process); + + let public = snapshot + .status(false) + .last_failure + .expect("kind stays public"); + assert_eq!(public.kind, "panic"); + assert!(public.message.is_empty()); + assert!(public.location.is_none()); + assert!(snapshot.status(false).gpu_errors.last_error.is_none()); +} + +#[test] +fn an_unreadable_failure_record_is_ignored() { + let state_dir = tempfile::tempdir().expect("state tempdir should be created"); + std::fs::write(state_dir.path().join(RENDER_FAILURE_FILE), b"not a record") + .expect("record should be written"); + + let staged = state_dir + .path() + .join(".render-failure.left-by-a-killed-run.tmp"); + std::fs::write(&staged, b"half written").expect("staged record should be written"); + + let supervision = RenderSupervision::for_state_dir(state_dir.path()); + + assert!(supervision.snapshot().last_failure.is_none()); + assert!(supervision.termination().is_none()); + assert!(supervision.watchdog_healthy()); + assert!(!staged.exists(), "a stale staged record should be removed"); +} + +#[test] +fn render_thread_names_cover_the_thread_and_its_runtime_workers() { + assert!(is_render_thread_name(RENDER_THREAD_NAME)); + assert!(is_render_thread_name(RENDER_RUNTIME_THREAD_NAME)); + assert!(!is_render_thread_name("hypercolor-daemon-runtime")); + assert!(!is_render_thread_name("servo-worker")); +} diff --git a/crates/hypercolor-daemon/tests/render_thread_tests.rs b/crates/hypercolor-daemon/tests/render_thread_tests.rs index ce7cb286e..68c2699a5 100644 --- a/crates/hypercolor-daemon/tests/render_thread_tests.rs +++ b/crates/hypercolor-daemon/tests/render_thread_tests.rs @@ -99,6 +99,7 @@ use hypercolor_daemon::SceneTransactionQueue; #[cfg(feature = "wgpu")] use hypercolor_daemon::canvas_capture::{CanvasCaptureRequest, CanvasImageFormat, capture_canvas}; use hypercolor_daemon::discovery::DiscoveryRuntime; +use hypercolor_daemon::domain::diagnostics::render_thread_check; use hypercolor_daemon::logical_devices::LogicalDevice; use hypercolor_daemon::output_power::{OutputPower, OutputPowerState}; use hypercolor_daemon::performance::PerformanceTracker; @@ -8059,3 +8060,117 @@ async fn injected_uncaptured_wgpu_error_is_logged_and_counted_without_killing_th .await .expect("a render thread that only logged a GPU error should shut down cleanly"); } + +async fn render_thread_check_now( + state: &RenderThreadState, + supervision: &RenderSupervision, + stall_after: Duration, +) -> hypercolor_types::api::diagnose::DiagnoseCheck { + let loop_state = state.render_loop.read().await.state(); + render_thread_check(&supervision.snapshot(), loop_state, stall_after) +} + +async fn wait_for_render_thread_check( + state: &RenderThreadState, + supervision: &RenderSupervision, + stall_after: Duration, + expected: &str, +) -> hypercolor_types::api::diagnose::DiagnoseCheck { + wait_for_render_thread_check_where(state, supervision, stall_after, |check| { + check.status == expected + }) + .await +} + +async fn wait_for_render_thread_check_where( + state: &RenderThreadState, + supervision: &RenderSupervision, + stall_after: Duration, + accept: F, +) -> hypercolor_types::api::diagnose::DiagnoseCheck +where + F: Fn(&hypercolor_types::api::diagnose::DiagnoseCheck) -> bool, +{ + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + loop { + let check = render_thread_check_now(state, supervision, stall_after).await; + if accept(&check) { + return check; + } + assert!( + tokio::time::Instant::now() < deadline, + "render_thread check never reached the expected state: {} ({})", + check.status, + check.detail + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +#[tokio::test] +async fn render_thread_check_fails_while_frames_stall_and_passes_after_recovery() { + let stall_after = Duration::from_millis(300); + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let state = solid_strip_state(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 3).await; + wait_for_render_thread_check(&state, &supervision, stall_after, "pass").await; + + supervision.inject_fault(InjectedRenderFault::Stall(Duration::from_millis(1_500))); + + let stalled = wait_for_render_thread_check(&state, &supervision, stall_after, "fail").await; + assert!( + stalled.detail.contains("has not completed a frame"), + "{}", + stalled.detail + ); + wait_for_render_thread_check(&state, &supervision, stall_after, "pass").await; + assert_eq!(supervision.snapshot().restarts_total, 0); + + state.render_loop.write().await.stop(); + rt.shutdown().await.expect("shutdown after a stall"); +} + +#[tokio::test] +async fn render_thread_check_fails_while_the_pipeline_restarts_and_passes_after() { + let stall_after = Duration::from_millis(300); + let supervision = RenderSupervision::new( + RenderRestartPolicy { + initial_backoff: Duration::from_millis(600), + max_backoff: Duration::from_millis(600), + ..fast_restart_policy(3) + }, + None, + ); + let state = solid_strip_state(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 3).await; + + supervision.inject_fault(InjectedRenderFault::Panic( + "injected panic for the restarting check", + )); + + // The panic hook symbolizes a backtrace on the render thread, which can + // outlast the stall threshold, so a stall failure may come first. + wait_for_render_thread_check_where(&state, &supervision, stall_after, |check| { + check.status == "fail" && check.detail.contains("restarting") + }) + .await; + let recovered = wait_for_render_thread_check(&state, &supervision, stall_after, "pass").await; + assert!( + recovered.detail.contains("restarts=1"), + "{}", + recovered.detail + ); + let resumed_from = state.render_loop.read().await.frame_number(); + wait_for_render_frames(&state, resumed_from + 3).await; + + state.render_loop.write().await.stop(); + rt.shutdown() + .await + .expect("shutdown after a recovered restart"); +} diff --git a/crates/hypercolor-types/src/api/diagnose.rs b/crates/hypercolor-types/src/api/diagnose.rs index dccf47d78..aac913738 100644 --- a/crates/hypercolor-types/src/api/diagnose.rs +++ b/crates/hypercolor-types/src/api/diagnose.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; -use crate::api::system::InputStatus; +use crate::api::system::{InputStatus, RenderSupervisionStatus}; /// Optional body for `POST /api/v1/diagnose`. /// @@ -59,6 +59,10 @@ pub struct DiagnoseSnapshot { pub struct DiagnoseRenderSnapshot { pub latest_frame: Option, pub recent_window: DiagnoseRenderWindowSnapshot, + /// Supervised render thread health, restart counts, and the last + /// failure. Absent from older daemons. + #[serde(default)] + pub supervision: RenderSupervisionStatus, } #[derive(Debug, Clone, Serialize)] diff --git a/crates/hypercolor-types/src/api/system.rs b/crates/hypercolor-types/src/api/system.rs index c5c809ccd..480bb0fd8 100644 --- a/crates/hypercolor-types/src/api/system.rs +++ b/crates/hypercolor-types/src/api/system.rs @@ -33,6 +33,9 @@ pub struct SystemStatus { pub macos_daemon_ownership: Option, pub compositor_acceleration: RenderAccelerationStatus, pub render_loop: RenderLoopStatus, + /// Supervised render thread health. Absent from older daemons. + #[serde(default)] + pub render_supervision: RenderSupervisionStatus, pub session_performance: SessionPerformanceStatus, pub latest_frame: Option, pub effect_health: EffectHealthStatus, @@ -325,6 +328,76 @@ pub struct RenderLoopStatus { pub total_frames: u64, } +/// Health and recovery state of the supervised render thread. +/// +/// The render thread restarts its pipeline in-process after a panic or an +/// error exit, within a bounded budget, and the daemon exits so its +/// service manager restarts it once that budget runs out or the compositor +/// GPU device is lost. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +pub struct RenderSupervisionStatus { + /// `not_started`, `starting`, `running`, `restarting`, `terminating`, + /// or `stopped`. + pub phase: String, + /// In-process pipeline restarts since the daemon started. + pub restarts_total: u64, + /// Failures since the pipeline last ran past its stable uptime. + pub consecutive_failures: u32, + /// Restarts the budget allows within its window. + pub max_restarts: u32, + /// Milliseconds since the render loop last completed a frame. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_frame_age_ms: Option, + /// The latest render failure in this daemon run, or else the one the + /// previous run recorded in the state directory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_failure: Option, + /// Uncaptured wgpu errors and device losses since the daemon started. + #[serde(default)] + pub gpu_errors: GpuErrorStatus, +} + +/// One render failure, as the supervisor recorded it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +pub struct RenderFailureStatus { + /// `panic`, `input_pump_terminated`, `input_pump_restart_failed`, + /// `pipeline_rebuild_failed`, or `gpu_device_lost`. + pub kind: String, + /// The panic message or error chain. Empty when the caller may not + /// read private diagnostics. + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thread: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub location: Option, + /// RFC 3339 UTC time of the failure. + pub occurred_at: String, + /// `restart`, `terminate_process`, `daemon_stopping`, `process_abort`, + /// or `awaiting_supervisor`. + pub action: String, + /// Restarts counted in the budget window by this failure. + pub restart_attempt: u32, + /// Whether this daemon run recorded the failure, as opposed to an + /// earlier run whose record survived in the state directory. + pub current_process: bool, +} + +/// Uncaptured wgpu errors and device losses since the daemon started. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +pub struct GpuErrorStatus { + pub validation_errors: u64, + pub out_of_memory_errors: u64, + pub internal_errors: u64, + pub device_lost: u64, + /// `device: kind: message` of the most recent one. Absent when there + /// was none, or when the caller may not read private diagnostics. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error: Option, +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] #[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] pub struct RenderAccelerationStatus { diff --git a/docs/content/troubleshooting/performance.md b/docs/content/troubleshooting/performance.md index 32b1995b9..b051bf876 100644 --- a/docs/content/troubleshooting/performance.md +++ b/docs/content/troubleshooting/performance.md @@ -41,7 +41,7 @@ Output is a styled table by default. Use `--format plain` for a one-line text su ## Step 2: read the render checks -The `render` category has five checks for FPS and LED output problems. +The `render` category has seven checks for FPS and LED output problems. ### `render_loop` @@ -63,6 +63,24 @@ The `tier` field shows where the FPS controller has landed. Compare it against y The controller downshifts after 2 consecutive budget misses and only upshifts after sustained headroom, so a tier stuck below your target is a real signal. +### `render_thread` + +``` +render_thread phase=running, last_frame_age_ms=12, restarts=0, last_failure=none +``` + +The render thread supervises its own pipeline. When a frame panics or the pipeline loses its input feed, the daemon logs the failure with a backtrace, rebuilds the pipeline with fresh GPU and render state, and keeps going. This check fails while that restart is in progress, when the thread has gone without completing a frame for 3 seconds while the loop runs, and when the thread is gone while the loop still says it is running. + +`restarts` counts in-process restarts since the daemon started. After five restarts within five minutes, or when the GPU device is lost, the daemon stops recovering in-process and exits with an error so its service manager starts a fresh process. Under systemd, a render thread stuck inside one frame for 20 seconds also stops the service watchdog keepalive, so systemd restarts the daemon. Every failure is also written to `render-failure.json` in the daemon state directory (`~/.local/state/hypercolor/` on Linux), so it survives a log that was lost. The next run reports that record here as `(earlier daemon run)`. + +### `gpu_errors` + +``` +gpu_errors validation=0, out_of_memory=0, internal=0, device_lost=0 +``` + +GPU validation, out-of-memory, and internal errors that nothing handled are counted and logged instead of crashing the thread that raised them. One in the last 10 minutes turns this check into a warning, with the device and the latest message in the detail. The counts since startup always stay in the detail. A lost GPU device fails the check. + ### `frame_liveness` ``` diff --git a/python/src/hypercolor/_generated/models/__init__.py b/python/src/hypercolor/_generated/models/__init__.py index d8f852ad0..1b0ef8b68 100644 --- a/python/src/hypercolor/_generated/models/__init__.py +++ b/python/src/hypercolor/_generated/models/__init__.py @@ -418,6 +418,7 @@ from .get_software_conflicts_response_200 import GetSoftwareConflictsResponse200 from .get_system_response_200 import GetSystemResponse200 from .gpu_compositor_probe_status import GpuCompositorProbeStatus +from .gpu_error_status import GpuErrorStatus from .gradient_stop import GradientStop from .health_checks import HealthChecks from .health_response import HealthResponse @@ -561,7 +562,9 @@ from .rejected_control_change_attempted_value import RejectedControlChangeAttemptedValue from .remove_favorite_response_200 import RemoveFavoriteResponse200 from .render_acceleration_status import RenderAccelerationStatus +from .render_failure_status import RenderFailureStatus from .render_loop_status import RenderLoopStatus +from .render_supervision_status import RenderSupervisionStatus from .render_surface_status import RenderSurfaceStatus from .reorder_layers_request import ReorderLayersRequest from .reorder_live_zone_layers_response_200 import ReorderLiveZoneLayersResponse200 @@ -1047,6 +1050,7 @@ "GetSoftwareConflictsResponse200", "GetSystemResponse200", "GpuCompositorProbeStatus", + "GpuErrorStatus", "GradientStop", "HealthChecks", "HealthResponse", @@ -1182,7 +1186,9 @@ "RejectedControlChangeAttemptedValue", "RemoveFavoriteResponse200", "RenderAccelerationStatus", + "RenderFailureStatus", "RenderLoopStatus", + "RenderSupervisionStatus", "RenderSurfaceStatus", "ReorderLayersRequest", "ReorderLiveZoneLayersResponse200", diff --git a/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py b/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py index 01ec30f26..1cce932c8 100644 --- a/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py +++ b/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py @@ -12,6 +12,7 @@ if TYPE_CHECKING: from ..models.diagnose_latest_frame_snapshot import DiagnoseLatestFrameSnapshot from ..models.diagnose_render_window_snapshot import DiagnoseRenderWindowSnapshot + from ..models.render_supervision_status import RenderSupervisionStatus T = TypeVar("T", bound="DiagnoseRenderSnapshot") @@ -23,10 +24,17 @@ class DiagnoseRenderSnapshot: Attributes: recent_window (DiagnoseRenderWindowSnapshot): latest_frame (DiagnoseLatestFrameSnapshot | None | Unset): + supervision (RenderSupervisionStatus | Unset): Health and recovery state of the supervised render thread. + + The render thread restarts its pipeline in-process after a panic or an + error exit, within a bounded budget, and the daemon exits so its + service manager restarts it once that budget runs out or the compositor + GPU device is lost. """ recent_window: DiagnoseRenderWindowSnapshot latest_frame: DiagnoseLatestFrameSnapshot | None | Unset = UNSET + supervision: RenderSupervisionStatus | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -44,6 +52,10 @@ def to_dict(self) -> dict[str, Any]: else: latest_frame = self.latest_frame + supervision: dict[str, Any] | Unset = UNSET + if not isinstance(self.supervision, Unset): + supervision = self.supervision.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -53,6 +65,8 @@ def to_dict(self) -> dict[str, Any]: ) if latest_frame is not UNSET: field_dict["latest_frame"] = latest_frame + if supervision is not UNSET: + field_dict["supervision"] = supervision return field_dict @@ -64,6 +78,9 @@ def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: from ..models.diagnose_render_window_snapshot import ( DiagnoseRenderWindowSnapshot, ) + from ..models.render_supervision_status import ( + RenderSupervisionStatus, + ) d = dict(src_dict) recent_window = DiagnoseRenderWindowSnapshot.from_dict(d.pop("recent_window")) @@ -87,9 +104,17 @@ def _parse_latest_frame( latest_frame = _parse_latest_frame(d.pop("latest_frame", UNSET)) + _supervision = d.pop("supervision", UNSET) + supervision: RenderSupervisionStatus | Unset + if isinstance(_supervision, Unset): + supervision = UNSET + else: + supervision = RenderSupervisionStatus.from_dict(_supervision) + diagnose_render_snapshot = cls( recent_window=recent_window, latest_frame=latest_frame, + supervision=supervision, ) diagnose_render_snapshot.additional_properties = d diff --git a/python/src/hypercolor/_generated/models/gpu_error_status.py b/python/src/hypercolor/_generated/models/gpu_error_status.py new file mode 100644 index 000000000..e87572418 --- /dev/null +++ b/python/src/hypercolor/_generated/models/gpu_error_status.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field +from typing_extensions import Self + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GpuErrorStatus") + + +@_attrs_define +class GpuErrorStatus: + """Uncaptured wgpu errors and device losses since the daemon started. + + Attributes: + device_lost (int): + internal_errors (int): + out_of_memory_errors (int): + validation_errors (int): + last_error (None | str | Unset): `device: kind: message` of the most recent one. Absent when there + was none, or when the caller may not read private diagnostics. + """ + + device_lost: int + internal_errors: int + out_of_memory_errors: int + validation_errors: int + last_error: None | str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + device_lost = self.device_lost + + internal_errors = self.internal_errors + + out_of_memory_errors = self.out_of_memory_errors + + validation_errors = self.validation_errors + + last_error: None | str | Unset + if isinstance(self.last_error, Unset): + last_error = UNSET + else: + last_error = self.last_error + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "device_lost": device_lost, + "internal_errors": internal_errors, + "out_of_memory_errors": out_of_memory_errors, + "validation_errors": validation_errors, + } + ) + if last_error is not UNSET: + field_dict["last_error"] = last_error + + return field_dict + + @classmethod + def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: + d = dict(src_dict) + device_lost = d.pop("device_lost") + + internal_errors = d.pop("internal_errors") + + out_of_memory_errors = d.pop("out_of_memory_errors") + + validation_errors = d.pop("validation_errors") + + def _parse_last_error(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + last_error = _parse_last_error(d.pop("last_error", UNSET)) + + gpu_error_status = cls( + device_lost=device_lost, + internal_errors=internal_errors, + out_of_memory_errors=out_of_memory_errors, + validation_errors=validation_errors, + last_error=last_error, + ) + + gpu_error_status.additional_properties = d + return gpu_error_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/python/src/hypercolor/_generated/models/render_failure_status.py b/python/src/hypercolor/_generated/models/render_failure_status.py new file mode 100644 index 000000000..0df346890 --- /dev/null +++ b/python/src/hypercolor/_generated/models/render_failure_status.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field +from typing_extensions import Self + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="RenderFailureStatus") + + +@_attrs_define +class RenderFailureStatus: + """One render failure, as the supervisor recorded it. + + Attributes: + action (str): `restart`, `terminate_process`, `daemon_stopping`, `process_abort`, + or `awaiting_supervisor`. + current_process (bool): Whether this daemon run recorded the failure, as opposed to an + earlier run whose record survived in the state directory. + kind (str): `panic`, `input_pump_terminated`, `input_pump_restart_failed`, + `pipeline_rebuild_failed`, or `gpu_device_lost`. + message (str): The panic message or error chain. Empty when the caller may not + read private diagnostics. + occurred_at (str): RFC 3339 UTC time of the failure. + restart_attempt (int): Restarts counted in the budget window by this failure. + location (None | str | Unset): + thread (None | str | Unset): + """ + + action: str + current_process: bool + kind: str + message: str + occurred_at: str + restart_attempt: int + location: None | str | Unset = UNSET + thread: None | str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + action = self.action + + current_process = self.current_process + + kind = self.kind + + message = self.message + + occurred_at = self.occurred_at + + restart_attempt = self.restart_attempt + + location: None | str | Unset + if isinstance(self.location, Unset): + location = UNSET + else: + location = self.location + + thread: None | str | Unset + if isinstance(self.thread, Unset): + thread = UNSET + else: + thread = self.thread + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "action": action, + "current_process": current_process, + "kind": kind, + "message": message, + "occurred_at": occurred_at, + "restart_attempt": restart_attempt, + } + ) + if location is not UNSET: + field_dict["location"] = location + if thread is not UNSET: + field_dict["thread"] = thread + + return field_dict + + @classmethod + def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: + d = dict(src_dict) + action = d.pop("action") + + current_process = d.pop("current_process") + + kind = d.pop("kind") + + message = d.pop("message") + + occurred_at = d.pop("occurred_at") + + restart_attempt = d.pop("restart_attempt") + + def _parse_location(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + location = _parse_location(d.pop("location", UNSET)) + + def _parse_thread(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + thread = _parse_thread(d.pop("thread", UNSET)) + + render_failure_status = cls( + action=action, + current_process=current_process, + kind=kind, + message=message, + occurred_at=occurred_at, + restart_attempt=restart_attempt, + location=location, + thread=thread, + ) + + render_failure_status.additional_properties = d + return render_failure_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/python/src/hypercolor/_generated/models/render_supervision_status.py b/python/src/hypercolor/_generated/models/render_supervision_status.py new file mode 100644 index 000000000..ae87e5b8e --- /dev/null +++ b/python/src/hypercolor/_generated/models/render_supervision_status.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field +from typing_extensions import Self + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.gpu_error_status import GpuErrorStatus + from ..models.render_failure_status import RenderFailureStatus + + +T = TypeVar("T", bound="RenderSupervisionStatus") + + +@_attrs_define +class RenderSupervisionStatus: + """Health and recovery state of the supervised render thread. + + The render thread restarts its pipeline in-process after a panic or an + error exit, within a bounded budget, and the daemon exits so its + service manager restarts it once that budget runs out or the compositor + GPU device is lost. + + Attributes: + consecutive_failures (int): Failures since the pipeline last ran past its stable uptime. + max_restarts (int): Restarts the budget allows within its window. + phase (str): `not_started`, `starting`, `running`, `restarting`, `terminating`, + or `stopped`. + restarts_total (int): In-process pipeline restarts since the daemon started. + gpu_errors (GpuErrorStatus | Unset): Uncaptured wgpu errors and device losses since the daemon started. + last_failure (None | RenderFailureStatus | Unset): + last_frame_age_ms (int | None | Unset): Milliseconds since the render loop last completed a frame. + """ + + consecutive_failures: int + max_restarts: int + phase: str + restarts_total: int + gpu_errors: GpuErrorStatus | Unset = UNSET + last_failure: None | RenderFailureStatus | Unset = UNSET + last_frame_age_ms: int | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + from ..models.render_failure_status import RenderFailureStatus + + consecutive_failures = self.consecutive_failures + + max_restarts = self.max_restarts + + phase = self.phase + + restarts_total = self.restarts_total + + gpu_errors: dict[str, Any] | Unset = UNSET + if not isinstance(self.gpu_errors, Unset): + gpu_errors = self.gpu_errors.to_dict() + + last_failure: dict[str, Any] | None | Unset + if isinstance(self.last_failure, Unset): + last_failure = UNSET + elif isinstance(self.last_failure, RenderFailureStatus): + last_failure = self.last_failure.to_dict() + else: + last_failure = self.last_failure + + last_frame_age_ms: int | None | Unset + if isinstance(self.last_frame_age_ms, Unset): + last_frame_age_ms = UNSET + else: + last_frame_age_ms = self.last_frame_age_ms + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "consecutive_failures": consecutive_failures, + "max_restarts": max_restarts, + "phase": phase, + "restarts_total": restarts_total, + } + ) + if gpu_errors is not UNSET: + field_dict["gpu_errors"] = gpu_errors + if last_failure is not UNSET: + field_dict["last_failure"] = last_failure + if last_frame_age_ms is not UNSET: + field_dict["last_frame_age_ms"] = last_frame_age_ms + + return field_dict + + @classmethod + def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: + from ..models.gpu_error_status import GpuErrorStatus + from ..models.render_failure_status import RenderFailureStatus + + d = dict(src_dict) + consecutive_failures = d.pop("consecutive_failures") + + max_restarts = d.pop("max_restarts") + + phase = d.pop("phase") + + restarts_total = d.pop("restarts_total") + + _gpu_errors = d.pop("gpu_errors", UNSET) + gpu_errors: GpuErrorStatus | Unset + if isinstance(_gpu_errors, Unset): + gpu_errors = UNSET + else: + gpu_errors = GpuErrorStatus.from_dict(_gpu_errors) + + def _parse_last_failure(data: object) -> None | RenderFailureStatus | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, dict): + raise TypeError() + last_failure_type_0 = RenderFailureStatus.from_dict(data) + + return last_failure_type_0 + except (TypeError, ValueError, AttributeError, KeyError): + pass + return cast(None | RenderFailureStatus | Unset, data) + + last_failure = _parse_last_failure(d.pop("last_failure", UNSET)) + + def _parse_last_frame_age_ms(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + last_frame_age_ms = _parse_last_frame_age_ms(d.pop("last_frame_age_ms", UNSET)) + + render_supervision_status = cls( + consecutive_failures=consecutive_failures, + max_restarts=max_restarts, + phase=phase, + restarts_total=restarts_total, + gpu_errors=gpu_errors, + last_failure=last_failure, + last_frame_age_ms=last_frame_age_ms, + ) + + render_supervision_status.additional_properties = d + return render_supervision_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/python/src/hypercolor/_generated/models/system_status.py b/python/src/hypercolor/_generated/models/system_status.py index cd869fd28..30445dceb 100644 --- a/python/src/hypercolor/_generated/models/system_status.py +++ b/python/src/hypercolor/_generated/models/system_status.py @@ -17,6 +17,7 @@ from ..models.preview_runtime_status import PreviewRuntimeStatus from ..models.render_acceleration_status import RenderAccelerationStatus from ..models.render_loop_status import RenderLoopStatus + from ..models.render_supervision_status import RenderSupervisionStatus from ..models.screen_capture_capacity_status import ScreenCaptureCapacityStatus from ..models.session_performance_status import SessionPerformanceStatus @@ -65,6 +66,12 @@ class SystemStatus: active_scene (None | str | Unset): latest_frame (LatestFrameStatus | None | Unset): macos_daemon_ownership (MacosDaemonOwnershipStatus | None | Unset): + render_supervision (RenderSupervisionStatus | Unset): Health and recovery state of the supervised render thread. + + The render thread restarts its pipeline in-process after a panic or an + error exit, within a bounded budget, and the daemon exits so its + service manager restarts it once that budget runs out or the compositor + GPU device is lost. """ active_scene_snapshot_locked: bool @@ -93,6 +100,7 @@ class SystemStatus: active_scene: None | str | Unset = UNSET latest_frame: LatestFrameStatus | None | Unset = UNSET macos_daemon_ownership: MacosDaemonOwnershipStatus | None | Unset = UNSET + render_supervision: RenderSupervisionStatus | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -171,6 +179,10 @@ def to_dict(self) -> dict[str, Any]: else: macos_daemon_ownership = self.macos_daemon_ownership + render_supervision: dict[str, Any] | Unset = UNSET + if not isinstance(self.render_supervision, Unset): + render_supervision = self.render_supervision.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -207,6 +219,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["latest_frame"] = latest_frame if macos_daemon_ownership is not UNSET: field_dict["macos_daemon_ownership"] = macos_daemon_ownership + if render_supervision is not UNSET: + field_dict["render_supervision"] = render_supervision return field_dict @@ -225,6 +239,9 @@ def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: RenderAccelerationStatus, ) from ..models.render_loop_status import RenderLoopStatus + from ..models.render_supervision_status import ( + RenderSupervisionStatus, + ) from ..models.screen_capture_capacity_status import ( ScreenCaptureCapacityStatus, ) @@ -341,6 +358,13 @@ def _parse_macos_daemon_ownership( d.pop("macos_daemon_ownership", UNSET) ) + _render_supervision = d.pop("render_supervision", UNSET) + render_supervision: RenderSupervisionStatus | Unset + if isinstance(_render_supervision, Unset): + render_supervision = UNSET + else: + render_supervision = RenderSupervisionStatus.from_dict(_render_supervision) + system_status = cls( active_scene_snapshot_locked=active_scene_snapshot_locked, audio_available=audio_available, @@ -368,6 +392,7 @@ def _parse_macos_daemon_ownership( active_scene=active_scene, latest_frame=latest_frame, macos_daemon_ownership=macos_daemon_ownership, + render_supervision=render_supervision, ) system_status.additional_properties = d From 4b8f617a2a2e0a8f7ded25923c3e84761fa64cca Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 01:49:38 -0700 Subject: [PATCH 12/26] feat(core): hold render time and skip wake work while a loop is parked RenderLoop::park excludes parked time from elapsed the way a pause does without leaving the Running state, so media layers and time bindings that read render time continue where they stopped instead of jumping by the parked span. A pause and a park that overlap are excluded once. RenderLoop::abandon_frame drops a ticked frame without recording it, so a park feeds no timing into tier decisions and does not advance the frame number. OutputDemandSignal now notifies its waiter list only while a producer has armed a wait, with full fences on both sides so a waiter armed before a state change always sees it or is woken. Every event publication notifies the signal and tokio's notify_waiters takes a lock on each call, so the armed check keeps the publish path free of that cost while nothing is parked. MediaProducer::is_live_stream lets the renderer drop live streams, whose decoder runs on its own worker, while it renders nothing. Servo comments now say what throttling does here: it stops the page's self-scheduled requestAnimationFrame ticks, while timers keep the 4 ms minimum Hypercolor configures. Coverage lands for every park and pause interleaving, the abandoned frame, routed device targets, live stream detection, effect pool park and warming, and Servo warming from load start to the first completed frame. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-core/src/bus/demand.rs | 53 +++++++-- crates/hypercolor-core/src/bus/mod.rs | 2 +- .../src/effect/builtin/web_viewport.rs | 4 +- .../src/effect/media/producer.rs | 15 +++ crates/hypercolor-core/src/effect/pool.rs | 60 ++++++++++ .../src/effect/servo/renderer.rs | 9 +- .../src/effect/servo/renderer/tests.rs | 59 ++++++++++ .../src/effect/servo/session.rs | 4 +- crates/hypercolor-core/src/engine/fps.rs | 6 + crates/hypercolor-core/src/engine/mod.rs | 89 ++++++++++++--- .../tests/backend_manager_tests.rs | 32 ++++++ crates/hypercolor-core/tests/bus_tests.rs | 15 ++- crates/hypercolor-core/tests/engine_tests.rs | 103 ++++++++++++++++++ .../tests/media_producer_tests.rs | 21 ++++ 14 files changed, 437 insertions(+), 35 deletions(-) diff --git a/crates/hypercolor-core/src/bus/demand.rs b/crates/hypercolor-core/src/bus/demand.rs index dd020270a..93807161c 100644 --- a/crates/hypercolor-core/src/bus/demand.rs +++ b/crates/hypercolor-core/src/bus/demand.rs @@ -9,16 +9,20 @@ //! changes, instead of polling for either. use std::sync::Arc; -use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering, fence}; use tokio::sync::Notify; use tokio::sync::futures::Notified; /// Wake signal for producers that idle while nothing consumes their output. +/// +/// Every event publication notifies it, so notification skips the waiter +/// list entirely unless a producer has armed a wait. #[derive(Debug, Default)] pub struct OutputDemandSignal { epoch: AtomicU64, holds: AtomicUsize, + armed: AtomicUsize, notify: Notify, } @@ -35,19 +39,27 @@ impl OutputDemandSignal { self.holds.load(Ordering::Acquire) } - /// Future that resolves on the next notification. - /// - /// Pin it and call [`Notified::enable`] before re-checking demand: a - /// notification that lands between the check and the await then still - /// wakes the waiter. - pub fn notified(&self) -> Notified<'_> { - self.notify.notified() + /// Arm a wait. Notifications reach waiters only while at least one + /// [`ArmedOutputDemand`] lives, so arm before re-checking demand. + pub fn arm(&self) -> ArmedOutputDemand<'_> { + self.armed.fetch_add(1, Ordering::SeqCst); + // Orders the arm before every demand read the waiter makes next. + fence(Ordering::SeqCst); + ArmedOutputDemand { signal: self } } /// Wake every current waiter. pub fn notify(&self) { self.epoch.fetch_add(1, Ordering::AcqRel); - self.notify.notify_waiters(); + // The state change that prompted this call (a new receiver, a hold, + // an event) may use weaker orderings, so a full fence orders it + // before the armed check. Paired with the fence in `arm`, either + // this load sees the waiter and wakes it, or the waiter's demand + // read sees the state change. + fence(Ordering::SeqCst); + if self.armed.load(Ordering::SeqCst) > 0 { + self.notify.notify_waiters(); + } } pub(super) fn acquire_hold(self: &Arc) -> OutputDemandHold { @@ -59,6 +71,29 @@ impl OutputDemandSignal { } } +/// An armed wait on an [`OutputDemandSignal`]. +#[derive(Debug)] +pub struct ArmedOutputDemand<'a> { + signal: &'a OutputDemandSignal, +} + +impl ArmedOutputDemand<'_> { + /// Future that resolves on the next notification. + /// + /// Pin it and call [`Notified::enable`] before re-checking demand: a + /// notification that lands between the check and the await then still + /// wakes the waiter. + pub fn notified(&self) -> Notified<'_> { + self.signal.notify.notified() + } +} + +impl Drop for ArmedOutputDemand<'_> { + fn drop(&mut self) { + self.signal.armed.fetch_sub(1, Ordering::SeqCst); + } +} + /// Keeps output demand registered for consumers that read the bus without /// holding a lane receiver, such as a borrow of a per-zone sender. /// diff --git a/crates/hypercolor-core/src/bus/mod.rs b/crates/hypercolor-core/src/bus/mod.rs index a04c8e0c2..b9b4b5c83 100644 --- a/crates/hypercolor-core/src/bus/mod.rs +++ b/crates/hypercolor-core/src/bus/mod.rs @@ -19,7 +19,7 @@ mod demand; mod filter; -pub use demand::{OutputDemandHold, OutputDemandSignal}; +pub use demand::{ArmedOutputDemand, OutputDemandHold, OutputDemandSignal}; pub use filter::{EventFilter, FilteredEventReceiver}; use std::collections::HashMap; diff --git a/crates/hypercolor-core/src/effect/builtin/web_viewport.rs b/crates/hypercolor-core/src/effect/builtin/web_viewport.rs index 3b47f9aa4..11f3022ea 100644 --- a/crates/hypercolor-core/src/effect/builtin/web_viewport.rs +++ b/crates/hypercolor-core/src/effect/builtin/web_viewport.rs @@ -287,8 +287,8 @@ impl EffectRenderer for WebViewportRenderer { } fn park(&mut self) { - // A live web page keeps its own timers and animations running between - // renders; throttling it stops that work until the next render. + // A live web page keeps animating between renders; throttling it + // stops its self-scheduled animation frames until the next render. if let Some(session) = self.session.as_ref() && let Err(error) = session.park() { diff --git a/crates/hypercolor-core/src/effect/media/producer.rs b/crates/hypercolor-core/src/effect/media/producer.rs index cc0d5ca07..a917d9214 100644 --- a/crates/hypercolor-core/src/effect/media/producer.rs +++ b/crates/hypercolor-core/src/effect/media/producer.rs @@ -285,6 +285,21 @@ impl MediaProducer { !self.frames.is_empty() } + /// Whether a background worker decodes a live stream for this + /// producer, which keeps running until the producer is dropped. + #[must_use] + pub const fn is_live_stream(&self) -> bool { + #[cfg(feature = "media-video")] + { + self.live_stream.is_some() + } + #[cfg(not(feature = "media-video"))] + { + let _ = self; + false + } + } + #[must_use] pub fn live_stream_error(&self) -> Option { #[cfg(feature = "media-video")] diff --git a/crates/hypercolor-core/src/effect/pool.rs b/crates/hypercolor-core/src/effect/pool.rs index efd707836..05fa5cb8e 100644 --- a/crates/hypercolor-core/src/effect/pool.rs +++ b/crates/hypercolor-core/src/effect/pool.rs @@ -962,6 +962,35 @@ mod tests { applied: Vec<(String, ControlValue)>, } + struct ParkSpyRenderer { + parks: Arc, + warming: Arc, + } + + impl EffectRenderer for ParkSpyRenderer { + fn init(&mut self, _metadata: &EffectMetadata) -> Result<()> { + Ok(()) + } + + fn render_into(&mut self, _input: &FrameInput<'_>, _target: &mut Canvas) -> Result<()> { + Ok(()) + } + + fn apply_controls(&mut self, _batch: &ControlDeltaBatch<'_>) -> Result<()> { + Ok(()) + } + + fn park(&mut self) { + self.parks.fetch_add(1, Ordering::SeqCst); + } + + fn is_warming(&self) -> bool { + self.warming.load(Ordering::SeqCst) + } + + fn destroy(&mut self) {} + } + impl DestroySpyRenderer { fn new(destroyed: Arc) -> Self { Self { destroyed } @@ -1814,6 +1843,37 @@ mod tests { ); } + #[test] + fn park_reaches_every_renderer_and_warming_tracks_any_of_them() { + let parks = Arc::new(AtomicU64::new(0)); + let first_warming = Arc::new(AtomicBool::new(false)); + let second_warming = Arc::new(AtomicBool::new(false)); + let mut pool = EffectPool::new(); + assert!(!pool.any_warming()); + pool.park(); + + for warming in [&first_warming, &second_warming] { + let effect_id = EffectId::new(uuid::Uuid::now_v7()); + let mut slot = advance_spy_slot(effect_id, Arc::new(AtomicU64::new(0))); + slot.renderer = Box::new(ParkSpyRenderer { + parks: Arc::clone(&parks), + warming: Arc::clone(warming), + }); + pool.slots + .insert(EffectSlotKey::new(ZoneId::new(), SceneLayerId::new()), slot); + } + + pool.park(); + assert_eq!(parks.load(Ordering::SeqCst), 2); + assert!(!pool.any_warming()); + + second_warming.store(true, Ordering::SeqCst); + assert!(pool.any_warming()); + second_warming.store(false, Ordering::SeqCst); + first_warming.store(true, Ordering::SeqCst); + assert!(pool.any_warming()); + } + #[test] fn advance_layer_output_ticks_renderer_without_rendering_canvas() { let advanced = Arc::new(AtomicU64::new(0)); diff --git a/crates/hypercolor-core/src/effect/servo/renderer.rs b/crates/hypercolor-core/src/effect/servo/renderer.rs index 948ed0d28..db306059d 100644 --- a/crates/hypercolor-core/src/effect/servo/renderer.rs +++ b/crates/hypercolor-core/src/effect/servo/renderer.rs @@ -269,10 +269,11 @@ impl EffectRenderer for ServoRenderer { } fn park(&mut self) { - // Servo's script thread schedules its own animation ticks for a page - // with live requestAnimationFrame callbacks and keeps its timers - // running, so an unobserved page keeps drawing WebGL until it is - // throttled. The next render request unthrottles the webview. + // Servo's script thread schedules its own animation ticks every 30 ms + // for a page with live requestAnimationFrame callbacks, so an + // unobserved page keeps drawing WebGL until its webview is + // throttled, which stops those ticks. Timers keep their own pace. + // The next render request unthrottles the webview. if let Some(session) = self.session.as_ref() && let Err(error) = session.park() { diff --git a/crates/hypercolor-core/src/effect/servo/renderer/tests.rs b/crates/hypercolor-core/src/effect/servo/renderer/tests.rs index a81740108..976d23080 100644 --- a/crates/hypercolor-core/src/effect/servo/renderer/tests.rs +++ b/crates/hypercolor-core/src/effect/servo/renderer/tests.rs @@ -707,6 +707,65 @@ fn init_with_canvas_size_reuses_previous_canvas_while_new_effect_loads() { assert!(stopped.load(Ordering::SeqCst)); } +#[test] +fn renderer_warms_from_load_start_until_the_new_page_renders_its_first_frame() { + let _lock = SHARED_WORKER_STATE_TEST_LOCK + .lock() + .expect("shared worker test lock"); + reset_shared_servo_worker_state(); + + let (worker, load_rx, release_tx, unload_rx, stopped) = spawn_blocking_load_test_worker(); + install_running_shared_worker(worker); + + let temp_dir = tempfile::tempdir().expect("temporary directory"); + let source_path = temp_dir.path().join("effect.html"); + std::fs::write(&source_path, "") + .expect("write source html"); + + let mut renderer = ServoRenderer::new(); + assert!( + !renderer.is_warming(), + "an uninitialized renderer has nothing to warm" + ); + renderer.park(); + + // A reload keeps the previous page's canvas on screen, so the canvas + // alone must not read as warm. + renderer.initialized = true; + renderer.last_canvas = Some(solid_canvas(640, 480, 12, 34, 56)); + renderer + .init_with_canvas_size(&html_metadata(source_path), 640, 480) + .expect("renderer should queue initialization"); + load_rx + .recv_timeout(Duration::from_millis(100)) + .expect("create-session command should be queued asynchronously"); + assert!(renderer.is_warming(), "a loading page is warming"); + + release_tx.send(()).expect("release create-session"); + wait_for_load_completion(&mut renderer); + assert!(renderer.session.is_some()); + assert!( + renderer.is_warming(), + "a loaded page stays warming until its first frame lands" + ); + renderer.park(); + + renderer.accept_completed_canvas(solid_canvas(640, 480, 1, 2, 3), false); + assert!( + !renderer.is_warming(), + "the first completed frame ends warming" + ); + + renderer.destroy(); + assert!(!renderer.is_warming()); + unload_rx + .recv_timeout(Duration::from_millis(100)) + .expect("destroy should unload test worker"); + + shutdown_shared_servo_worker().expect("shared worker shutdown should succeed"); + assert!(stopped.load(Ordering::SeqCst)); +} + #[test] fn destroy_discards_completed_load_task_before_it_is_polled() { let (worker, load_rx, unload_rx, stopped) = spawn_load_test_worker(); diff --git a/crates/hypercolor-core/src/effect/servo/session.rs b/crates/hypercolor-core/src/effect/servo/session.rs index 93697b592..e149e659d 100644 --- a/crates/hypercolor-core/src/effect/servo/session.rs +++ b/crates/hypercolor-core/src/effect/servo/session.rs @@ -274,8 +274,8 @@ impl ServoSessionHandle { } /// Throttle this session's page while the render loop requests no - /// frames, so its animation timers and requestAnimationFrame loop stop - /// running unobserved. The next render request unthrottles it. + /// frames, so its requestAnimationFrame loop stops scheduling itself + /// unobserved. The next render request unthrottles it. /// /// # Errors /// diff --git a/crates/hypercolor-core/src/engine/fps.rs b/crates/hypercolor-core/src/engine/fps.rs index 89b348f82..6095ceac9 100644 --- a/crates/hypercolor-core/src/engine/fps.rs +++ b/crates/hypercolor-core/src/engine/fps.rs @@ -337,6 +337,12 @@ impl FpsController { self.frame_start = Some(Instant::now()); } + /// Forget the frame begun by [`begin_frame`](Self::begin_frame) + /// without recording it. + pub fn abandon_frame(&mut self) { + self.frame_start = None; + } + /// Mark the end of the current frame. /// /// Updates the EWMA, consecutive miss counter, and checks for tier diff --git a/crates/hypercolor-core/src/engine/mod.rs b/crates/hypercolor-core/src/engine/mod.rs index 0a9af954e..5e2b1a9ce 100644 --- a/crates/hypercolor-core/src/engine/mod.rs +++ b/crates/hypercolor-core/src/engine/mod.rs @@ -108,11 +108,16 @@ pub struct RenderLoop { /// Increments whenever a live render loop enters the paused state. pause_generation: u64, - /// Wall-clock instant when the current pause began. - paused_at: Option, + /// Wall-clock instant the current span excluded from elapsed time + /// began, while the loop is paused or parked. + held_since: Option, + + /// Whether the loop is parked: running, but rendering nothing because + /// nothing consumes its frames. + parked: bool, /// Total wall-clock time excluded from render elapsed time. - paused_duration: Duration, + held_duration: Duration, /// Monotonically increasing frame counter. frame_number: u64, @@ -138,8 +143,9 @@ impl RenderLoop { running: Arc::new(AtomicBool::new(false)), state: RenderLoopState::Created, pause_generation: 0, - paused_at: None, - paused_duration: Duration::ZERO, + held_since: None, + parked: false, + held_duration: Duration::ZERO, frame_number: 0, start_time: None, } @@ -154,8 +160,9 @@ impl RenderLoop { running: Arc::new(AtomicBool::new(false)), state: RenderLoopState::Created, pause_generation: 0, - paused_at: None, - paused_duration: Duration::ZERO, + held_since: None, + parked: false, + held_duration: Duration::ZERO, frame_number: 0, start_time: None, } @@ -170,8 +177,9 @@ impl RenderLoop { self.running.store(true, Ordering::Release); self.state = RenderLoopState::Running; self.start_time = Some(Instant::now()); - self.paused_at = None; - self.paused_duration = Duration::ZERO; + self.held_since = None; + self.parked = false; + self.held_duration = Duration::ZERO; info!(tier = %self.fps_controller.tier(), "Render loop started"); } @@ -206,7 +214,7 @@ impl RenderLoop { if self.state == RenderLoopState::Running { self.state = RenderLoopState::Paused; self.pause_generation = self.pause_generation.wrapping_add(1); - self.paused_at = Some(Instant::now()); + self.begin_hold(); debug!("Render loop paused"); } } @@ -214,14 +222,54 @@ impl RenderLoop { /// Resume from paused state. pub fn resume(&mut self) { if self.state == RenderLoopState::Paused { - if let Some(paused_at) = self.paused_at.take() { - self.paused_duration += paused_at.elapsed(); + if !self.parked { + self.end_hold(); } self.state = RenderLoopState::Running; debug!("Render loop resumed"); } } + /// Park a running loop that renders nothing because nothing consumes + /// its frames. + /// + /// Parked time is excluded from [`elapsed`](Self::elapsed) the way + /// paused time is, so time-driven layers continue from where they + /// stopped instead of jumping by the parked span. The lifecycle state + /// stays [`Running`](RenderLoopState::Running). + pub fn park(&mut self) { + if self.state == RenderLoopState::Running && !self.parked { + self.parked = true; + self.begin_hold(); + } + } + + /// Leave a park started by [`park`](Self::park). + pub fn unpark(&mut self) { + if self.parked { + self.parked = false; + if self.state != RenderLoopState::Paused { + self.end_hold(); + } + } + } + + /// Whether the loop is parked. + #[must_use] + pub fn is_parked(&self) -> bool { + self.parked + } + + fn begin_hold(&mut self) { + self.held_since.get_or_insert_with(Instant::now); + } + + fn end_hold(&mut self) { + if let Some(held_since) = self.held_since.take() { + self.held_duration += held_since.elapsed(); + } + } + /// Get an `Arc` handle that can be used to stop the loop /// from another thread. #[must_use] @@ -251,6 +299,14 @@ impl RenderLoop { self.frame_complete_with_max_tier(None) } + /// Drop the frame begun by [`tick`](Self::tick) without completing it. + /// + /// The frame never rendered, so it records no timing, feeds no tier + /// decision, and does not advance [`frame_number`](Self::frame_number). + pub fn abandon_frame(&mut self) { + self.fps_controller.abandon_frame(); + } + /// Finalize the current frame and optionally apply a new runtime ceiling /// before evaluating automatic tier transitions for the next frame. pub fn frame_complete_with_max_tier( @@ -296,9 +352,9 @@ impl RenderLoop { #[must_use] pub fn elapsed(&self) -> Duration { self.start_time.map_or(Duration::ZERO, |started_at| { - let now = self.paused_at.unwrap_or_else(Instant::now); + let now = self.held_since.unwrap_or_else(Instant::now); now.duration_since(started_at) - .saturating_sub(self.paused_duration) + .saturating_sub(self.held_duration) }) } @@ -344,8 +400,9 @@ impl std::fmt::Debug for RenderLoop { f.debug_struct("RenderLoop") .field("state", &self.state) .field("pause_generation", &self.pause_generation) - .field("paused_at", &self.paused_at) - .field("paused_duration", &self.paused_duration) + .field("held_since", &self.held_since) + .field("parked", &self.parked) + .field("held_duration", &self.held_duration) .field("frame_number", &self.frame_number) .field("running", &self.running.load(Ordering::Relaxed)) .field("start_time", &self.start_time) diff --git a/crates/hypercolor-core/tests/backend_manager_tests.rs b/crates/hypercolor-core/tests/backend_manager_tests.rs index 8329afb77..05aec655f 100644 --- a/crates/hypercolor-core/tests/backend_manager_tests.rs +++ b/crates/hypercolor-core/tests/backend_manager_tests.rs @@ -1541,6 +1541,38 @@ fn map_and_unmap_device() { assert!(!manager.unmap_device("wled:strip_1")); } +#[test] +fn mapped_device_targets_list_each_binding_with_its_backend() { + let mut manager = BackendManager::new(); + let strip = DeviceId::new(); + let display = DeviceId::new(); + assert_eq!(manager.mapped_device_targets().count(), 0); + + manager.map_device("wled:strip_1", "wled", strip); + manager.map_device("wled:strip_1_alias", "wled", strip); + manager.map_device("simulator:lcd", "simulator", display); + + let mut targets = manager + .mapped_device_targets() + .map(|(backend_id, device_id)| (backend_id.to_owned(), device_id)) + .collect::>(); + targets.sort(); + let mut expected = vec![ + ("simulator".to_owned(), display), + ("wled".to_owned(), strip), + ("wled".to_owned(), strip), + ]; + expected.sort(); + assert_eq!(targets, expected); + + assert!(manager.unmap_device("simulator:lcd")); + assert!( + manager + .mapped_device_targets() + .all(|(backend_id, device_id)| backend_id == "wled" && device_id == strip) + ); +} + #[test] fn unmap_device_clears_cached_target_fps_when_last_mapping_is_removed() { let device_id = DeviceId::new(); diff --git a/crates/hypercolor-core/tests/bus_tests.rs b/crates/hypercolor-core/tests/bus_tests.rs index 8c8ec7e2e..1461caaec 100644 --- a/crates/hypercolor-core/tests/bus_tests.rs +++ b/crates/hypercolor-core/tests/bus_tests.rs @@ -1540,7 +1540,8 @@ async fn concurrent_input_status_publications_are_generation_ordered() { /// Arm a waiter on the bus output-demand signal the way an idle producer /// does, run `trigger`, and report whether the waiter woke. async fn output_demand_wakes_on(bus: &HypercolorBus, trigger: impl FnOnce()) -> bool { - let notified = bus.output_demand().notified(); + let armed = bus.output_demand().arm(); + let notified = armed.notified(); tokio::pin!(notified); notified.as_mut().enable(); trigger(); @@ -1615,6 +1616,18 @@ async fn published_events_wake_output_demand_waiters() { assert!(output_demand_wakes_on(&bus, || bus.publish(paused_event())).await); } +#[tokio::test] +async fn output_demand_notification_without_an_armed_waiter_only_advances_the_epoch() { + let bus = HypercolorBus::new(); + let epoch = bus.output_demand().epoch(); + + bus.publish(paused_event()); + let _canvas = bus.canvas_receiver(); + + assert_eq!(bus.output_demand().epoch(), epoch + 2); + assert!(output_demand_wakes_on(&bus, || bus.publish(paused_event())).await); +} + #[tokio::test] async fn output_demand_waiter_that_is_not_triggered_stays_asleep() { let bus = HypercolorBus::new(); diff --git a/crates/hypercolor-core/tests/engine_tests.rs b/crates/hypercolor-core/tests/engine_tests.rs index 082d71987..6c6389a04 100644 --- a/crates/hypercolor-core/tests/engine_tests.rs +++ b/crates/hypercolor-core/tests/engine_tests.rs @@ -694,6 +694,109 @@ fn render_loop_elapsed_excludes_every_pause_interval() { assert_eq!(rl.elapsed(), second_pause_elapsed); } +#[test] +fn render_loop_elapsed_excludes_parked_time_without_leaving_running() { + let mut rl = RenderLoop::new(60); + rl.start(); + std::thread::sleep(Duration::from_millis(10)); + + rl.park(); + assert!(rl.is_parked()); + assert_eq!(rl.state(), RenderLoopState::Running); + assert!(rl.is_running(), "a parked loop still ticks"); + let parked_elapsed = rl.elapsed(); + std::thread::sleep(Duration::from_millis(30)); + assert_eq!(rl.elapsed(), parked_elapsed); + + rl.unpark(); + assert!(!rl.is_parked()); + std::thread::sleep(Duration::from_millis(10)); + assert!(rl.elapsed() > parked_elapsed); +} + +#[test] +fn render_loop_pause_inside_a_park_excludes_the_whole_span_once() { + let mut rl = RenderLoop::new(60); + rl.start(); + std::thread::sleep(Duration::from_millis(10)); + + rl.park(); + let held_at = rl.elapsed(); + std::thread::sleep(Duration::from_millis(15)); + rl.pause(); + std::thread::sleep(Duration::from_millis(15)); + rl.resume(); + assert_eq!( + rl.elapsed(), + held_at, + "resuming a pause while still parked keeps the time held" + ); + std::thread::sleep(Duration::from_millis(15)); + rl.unpark(); + let resumed_at = rl.elapsed(); + assert!( + resumed_at >= held_at && resumed_at < held_at + Duration::from_millis(10), + "the park and the pause inside it are excluded exactly once: {held_at:?} -> {resumed_at:?}" + ); +} + +#[test] +fn render_loop_unpark_while_paused_keeps_time_held_until_resume() { + let mut rl = RenderLoop::new(60); + rl.start(); + std::thread::sleep(Duration::from_millis(10)); + + rl.park(); + let held_at = rl.elapsed(); + rl.pause(); + std::thread::sleep(Duration::from_millis(15)); + rl.unpark(); + assert!(!rl.is_parked()); + assert_eq!(rl.state(), RenderLoopState::Paused); + std::thread::sleep(Duration::from_millis(15)); + assert_eq!( + rl.elapsed(), + held_at, + "leaving the park while paused keeps the pause holding time" + ); + + rl.resume(); + let resumed_at = rl.elapsed(); + assert!( + resumed_at >= held_at && resumed_at < held_at + Duration::from_millis(10), + "the park and the pause after it are excluded exactly once: {held_at:?} -> {resumed_at:?}" + ); + std::thread::sleep(Duration::from_millis(10)); + assert!(rl.elapsed() > resumed_at); +} + +#[test] +fn render_loop_park_is_ignored_unless_running() { + let mut rl = RenderLoop::new(60); + rl.park(); + assert!(!rl.is_parked(), "a loop that never started cannot park"); + + rl.start(); + rl.pause(); + rl.park(); + assert!(!rl.is_parked(), "a paused loop is idle for its own reason"); +} + +#[test] +fn render_loop_abandoned_frame_records_nothing() { + let mut rl = RenderLoop::new(60); + rl.start(); + assert!(rl.tick()); + rl.abandon_frame(); + + assert_eq!(rl.frame_number(), 0); + assert!( + rl.frame_complete().is_none(), + "an abandoned frame has nothing left to complete" + ); + assert_eq!(rl.frame_number(), 0); +} + #[test] fn render_loop_debug_output_is_readable() { let rl = RenderLoop::new(60); diff --git a/crates/hypercolor-core/tests/media_producer_tests.rs b/crates/hypercolor-core/tests/media_producer_tests.rs index 34efb9e26..e7bb6fe45 100644 --- a/crates/hypercolor-core/tests/media_producer_tests.rs +++ b/crates/hypercolor-core/tests/media_producer_tests.rs @@ -425,6 +425,27 @@ fn webm_video_frames_decode_when_feature_is_enabled() { assert!(pixel.r >= 250 && pixel.g >= 250 && pixel.b >= 250 && pixel.a == 255); } +#[test] +fn decoded_media_is_not_a_live_stream() { + let producer = MediaProducer::from_bytes(&animated_gif_bytes(), "image/gif") + .expect("animated GIF should decode"); + + assert!(!producer.is_live_stream()); +} + +#[cfg(feature = "media-video")] +#[test] +fn stream_url_producer_reports_a_live_stream() { + let producer = MediaProducer::from_bytes_with_stream_policy( + b"http://1.1.1.1/hypercolor-missing-live.m3u8\n", + "application/vnd.hypercolor.stream-url", + &StreamUrlPolicy::default(), + ) + .expect("stream URL producer should start"); + + assert!(producer.is_live_stream()); +} + #[cfg(feature = "media-video")] #[test] fn stream_url_producer_returns_before_first_live_frame() { From bb20c6c20e351551a2dcda4757d973a2ddc940c9 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 01:49:38 -0700 Subject: [PATCH 13/26] fix(daemon): settle transitions and media before parking, exact telemetry A crossfade advances only as frames render, so parking mid transition froze it with the outgoing scene's producers alive; the loop now keeps rendering until a transition finishes or the settle limit passes, and until loading media has a frame. Live media streams kept decoding on their own worker while parked; parking now drops them, on a short-lived thread because dropping one joins a decoder that waits on its network source. Parking also calls RenderLoop::park and abandon_frame, so render time stops and the parked frame feeds no tier statistics. The parked flag cleared only when a frame was recorded, so a loop woken into the idle or sleep throttle, or paused while parked, kept reporting a park. The gate now clears it whenever it decides to render and when the loop leaves the running state. Status and diagnose measure frame age on render time, the clock frame timestamps use, rather than daemon uptime, which also stops pauses inflating frame_age_ms. One-off frame and spectrum reads borrow the lane instead of subscribing, so they no longer wake a parked loop, and the backend manager is locked only when nothing else proves demand. Tests cover a device routed without a registry entry, input event streams, pausing while parked, render time across a park, a transition finishing before the park, the display output hold, and the consumer counting helpers. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/src/api/system.rs | 17 +- .../src/display_output/mod.rs | 19 ++ .../src/domain/diagnostics.rs | 5 +- crates/hypercolor-daemon/src/domain/output.rs | 6 +- crates/hypercolor-daemon/src/mcp/resources.rs | 2 +- .../hypercolor-daemon/src/mcp/tools/system.rs | 2 +- crates/hypercolor-daemon/src/performance.rs | 7 + .../src/render_thread/frame_executor.rs | 21 +- .../src/render_thread/frame_policy.rs | 12 +- .../src/render_thread/output_gate.rs | 65 +++-- .../src/render_thread/pipeline_driver.rs | 10 +- .../src/render_thread/render_zones.rs | 28 +- .../tests/output_demand_tests.rs | 263 +++++++++++++++++- 13 files changed, 412 insertions(+), 45 deletions(-) diff --git a/crates/hypercolor-daemon/src/api/system.rs b/crates/hypercolor-daemon/src/api/system.rs index 00b6e5013..5fcf0f318 100644 --- a/crates/hypercolor-daemon/src/api/system.rs +++ b/crates/hypercolor-daemon/src/api/system.rs @@ -161,9 +161,12 @@ async fn system_status_with_privacy( let output_demand = performance.output_demand.status(); // Query the live render loop for timing data. - let render_loop_status = { + let (render_loop_status, render_elapsed_ms) = { let rl = state.render_loop.read().await; let snapshot = rl.stats(); + // Frame timestamps use render time, which stops while the loop is + // paused or parked, so frame age is measured on the same clock. + let render_elapsed_ms = rl.elapsed().as_secs_f64() * 1000.0; let capacity_fps = if snapshot.state == RenderLoopState::Running { round_1(paced_fps( snapshot.avg_frame_time.as_secs_f64(), @@ -174,7 +177,7 @@ async fn system_status_with_privacy( }; // A parked loop delivers no frames; `output_demand` says why. let delivering = snapshot.state == RenderLoopState::Running && !output_demand.parked; - RenderLoopStatus { + let status = RenderLoopStatus { state: snapshot.state.to_string(), fps_tier: snapshot.tier.to_string(), target_fps: snapshot.tier.fps(), @@ -189,13 +192,15 @@ async fn system_status_with_privacy( consecutive_misses: snapshot.consecutive_misses, total_frames: snapshot.total_frames, output_demand, - } + }; + (status, render_elapsed_ms) }; let running = render_loop_is_operational(render_loop_status.state.as_str()); let latest_frame = if render_loop_status.state == "running" { - performance.latest_frame.as_ref().map(|frame| { - latest_frame_status(frame, state.start_time.elapsed().as_secs_f64() * 1000.0) - }) + performance + .latest_frame + .as_ref() + .map(|frame| latest_frame_status(frame, render_elapsed_ms)) } else { None }; diff --git a/crates/hypercolor-daemon/src/display_output/mod.rs b/crates/hypercolor-daemon/src/display_output/mod.rs index 848061f79..f753d1251 100644 --- a/crates/hypercolor-daemon/src/display_output/mod.rs +++ b/crates/hypercolor-daemon/src/display_output/mod.rs @@ -1121,6 +1121,25 @@ mod tests { ))) } + #[test] + fn display_output_holds_render_demand_exactly_while_it_has_targets() { + let event_bus = HypercolorBus::new(); + let mut hold = None; + + sync_output_demand_hold(&mut hold, &event_bus, true); + assert_eq!(event_bus.output_demand().hold_count(), 1); + sync_output_demand_hold(&mut hold, &event_bus, true); + assert_eq!( + event_bus.output_demand().hold_count(), + 1, + "a held demand is not taken twice" + ); + + sync_output_demand_hold(&mut hold, &event_bus, false); + assert_eq!(event_bus.output_demand().hold_count(), 0); + assert!(hold.is_none()); + } + fn display_target(blend_mode: BlendMode) -> DisplayTarget { let device_id = DeviceId::new(); DisplayTarget { diff --git a/crates/hypercolor-daemon/src/domain/diagnostics.rs b/crates/hypercolor-daemon/src/domain/diagnostics.rs index 720ea6ec9..2271c804e 100644 --- a/crates/hypercolor-daemon/src/domain/diagnostics.rs +++ b/crates/hypercolor-daemon/src/domain/diagnostics.rs @@ -152,8 +152,11 @@ struct DiagnosticsInputs { } impl DiagnosticsInputs { + /// Render time now, on the clock frame timestamps are taken from. + /// Daemon uptime runs on through pauses and parks that render time + /// skips, so measuring frame age against it would count those spans. fn render_elapsed_ms(&self) -> f64 { - self.uptime.as_secs_f64() * 1000.0 + self.render_loop.elapsed.as_secs_f64() * 1000.0 } } diff --git a/crates/hypercolor-daemon/src/domain/output.rs b/crates/hypercolor-daemon/src/domain/output.rs index 5c3f1c70e..7f43c47e6 100644 --- a/crates/hypercolor-daemon/src/domain/output.rs +++ b/crates/hypercolor-daemon/src/domain/output.rs @@ -39,6 +39,9 @@ pub struct RenderLoopStatus { pub running: bool, /// Pacing state and tier at the moment of the read. pub stats: hypercolor_core::engine::RenderLoopStats, + /// Render time at the moment of the read: the clock frame timestamps + /// use, which stops while the loop is paused or parked. + pub elapsed: std::time::Duration, } /// Owning authority for global output power, brightness, and quiescence. @@ -136,6 +139,7 @@ impl OutputContext { RenderLoopStatus { running: guard.is_running(), stats: guard.stats(), + elapsed: guard.elapsed(), } } @@ -201,7 +205,7 @@ impl OutputContext { }; let frame_number = self .event_bus - .frame_receiver() + .frame_lane() .borrow() .frame_number .saturating_add(1); diff --git a/crates/hypercolor-daemon/src/mcp/resources.rs b/crates/hypercolor-daemon/src/mcp/resources.rs index 4cbbfe5d1..e1e348d9b 100644 --- a/crates/hypercolor-daemon/src/mcp/resources.rs +++ b/crates/hypercolor-daemon/src/mcp/resources.rs @@ -139,7 +139,7 @@ async fn read_scenes_with_state(state: &AppState) -> Value { } fn read_audio_with_state(state: &AppState) -> Value { - let spectrum = state.event_bus.spectrum_receiver().borrow().clone(); + let spectrum = state.event_bus.spectrum_lane().borrow().clone(); let (enabled, device, fft_size) = if let Some(config_manager) = state.config_manager.as_ref() { let config = config_manager.get(); ( diff --git a/crates/hypercolor-daemon/src/mcp/tools/system.rs b/crates/hypercolor-daemon/src/mcp/tools/system.rs index 7ccfe5507..9d91c7ada 100644 --- a/crates/hypercolor-daemon/src/mcp/tools/system.rs +++ b/crates/hypercolor-daemon/src/mcp/tools/system.rs @@ -175,7 +175,7 @@ pub(super) fn handle_get_sensor_data_with_state( } pub(super) fn handle_get_audio_state_with_state(state: &AppState) -> Result { - let spectrum = state.event_bus.spectrum_receiver().borrow().clone(); + let spectrum = state.event_bus.spectrum_lane().borrow().clone(); let enabled = state .config_manager .as_ref() diff --git a/crates/hypercolor-daemon/src/performance.rs b/crates/hypercolor-daemon/src/performance.rs index 7574214bc..96ebe03b1 100644 --- a/crates/hypercolor-daemon/src/performance.rs +++ b/crates/hypercolor-daemon/src/performance.rs @@ -455,6 +455,13 @@ impl PerformanceTracker { true } + /// Record that the loop left its park without completing a frame, such + /// as a frame that renders through a throttle path or a loop paused + /// while parked. + pub(crate) fn note_output_unparked(&mut self) { + self.output_demand.parked_since = None; + } + /// Record what woke a parked loop. The park ends with the next /// recorded frame. pub(crate) fn note_output_woken(&mut self, reason: &'static str) { diff --git a/crates/hypercolor-daemon/src/render_thread/frame_executor.rs b/crates/hypercolor-daemon/src/render_thread/frame_executor.rs index f4c9e9305..71723c87f 100644 --- a/crates/hypercolor-daemon/src/render_thread/frame_executor.rs +++ b/crates/hypercolor-daemon/src/render_thread/frame_executor.rs @@ -535,6 +535,9 @@ pub(crate) async fn execute_frame( }) .await; let render_us = render_stage.total_us; + frame_loop + .output_gate + .note_scene_transition(render_stage.scene_transition_active); let preview_advance_start = Instant::now(); { @@ -939,16 +942,20 @@ async fn gate_output_demand( .consumers(state, &frame_loop.input_demands) .await; let signature = ProducerSignature::observe(state).await; - let decision = frame_loop.output_gate.decide( - consumers.any(), - signature, - render.render_zone_runtime.producers_warming(), - Instant::now(), - ); + let warming = render.render_zone_runtime.producers_warming() + || frame_loop.output_gate.transition_active(); + let decision = + frame_loop + .output_gate + .decide(consumers.any(), signature, warming, Instant::now()); if decision == GateDecision::Render { + if frame_loop.output_gate.leave_park() { + state.performance.write().await.note_output_unparked(); + } return None; } + frame_loop.output_gate.enter_park(); frame_loop.clear_input_demands(); let _ = render.screen_queue.clear_latest(); render.sparkleflinger.release_native_screen_caches(); @@ -1023,7 +1030,7 @@ async fn force_static_sleep_snapshot( let frame_number = state .event_bus - .frame_receiver() + .frame_lane() .borrow() .frame_number .saturating_add(1); diff --git a/crates/hypercolor-daemon/src/render_thread/frame_policy.rs b/crates/hypercolor-daemon/src/render_thread/frame_policy.rs index 7b4e8ca43..9ed702d1d 100644 --- a/crates/hypercolor-daemon/src/render_thread/frame_policy.rs +++ b/crates/hypercolor-daemon/src/render_thread/frame_policy.rs @@ -137,9 +137,13 @@ impl FramePolicy { self.complete_throttled_frame(render_loop, SESSION_SLEEP_THROTTLE_DELAY) } - /// Close the admitted frame without rendering it and park the loop. + /// Drop the admitted frame without rendering it and park the loop. + /// + /// The frame is abandoned rather than completed, so parking feeds no + /// timing into tier decisions and does not advance the frame number. pub(crate) fn park_execution(&mut self, render_loop: &mut RenderLoop) -> FrameExecution { - let _ = render_loop.frame_complete(); + render_loop.abandon_frame(); + render_loop.park(); FrameExecution { next_wake: NextWake::Park, next_skip_decision: SkipDecision::None, @@ -334,7 +338,9 @@ mod tests { assert!(execution.parks()); assert_eq!(execution.next_skip_decision, SkipDecision::None); - assert_eq!(render_loop.frame_number(), 1); + assert_eq!(render_loop.frame_number(), 0); + assert!(render_loop.is_parked()); + assert!(render_loop.frame_complete().is_none()); } #[test] diff --git a/crates/hypercolor-daemon/src/render_thread/output_gate.rs b/crates/hypercolor-daemon/src/render_thread/output_gate.rs index ada227576..738131cdb 100644 --- a/crates/hypercolor-daemon/src/render_thread/output_gate.rs +++ b/crates/hypercolor-daemon/src/render_thread/output_gate.rs @@ -13,8 +13,9 @@ //! //! Parking waits for the producers to settle first. After the scene, the //! effect catalog, or the canvas changes, the loop keeps rendering until -//! every producer is warm, so an effect applied while nobody watches is -//! loaded and showing its own content by the time a consumer arrives. +//! every producer is warm and any scene transition has finished, so an +//! effect applied while nobody watches is loaded and showing its own +//! content by the time a consumer arrives. use std::time::{Duration, Instant}; @@ -35,8 +36,8 @@ pub(crate) const PARKED_SAFETY_TICK: Duration = Duration::from_secs(1); const SETTLE_WARM_FRAMES: u32 = 3; /// Longest the loop keeps rendering for producers that never finish -/// warming, such as a page that fails to load. Matches the Servo page-load -/// timeout. +/// warming, such as a page that never loads. Matches the Servo worker's +/// ready timeout, the longest a session creation or page load may block. const SETTLE_LIMIT: Duration = Duration::from_secs(10); /// Everything the producers render from, so a change to any of it is work @@ -118,16 +119,21 @@ pub(crate) struct OutputGate { /// Consumers last recorded into render telemetry. recorded: Option, streak: Option, + /// Whether the last rendered frame was mid scene transition. + transition_active: bool, + /// Whether the loop is parked, from the park until a frame renders or + /// the loop leaves the running state. + parked: bool, } impl OutputGate { /// Count consumers and record them for status surfaces when they /// changed. /// - /// The device registry is read only when its generation moved. A rig - /// with a registered device stops there; otherwise the backend - /// manager's routes are counted too, so a device mapped for output - /// counts even before the registry reports it renderable. + /// The device registry is read only when its generation moved. When + /// neither a registered device nor any bus-side consumer exists, the + /// backend manager's routes are counted too, so a device mapped for + /// output counts even before the registry reports it renderable. pub(crate) async fn consumers( &mut self, state: &RenderThreadState, @@ -144,16 +150,14 @@ impl OutputGate { count } }; - let output_devices = if registry_devices > 0 { - registry_devices - } else { - routed_output_device_count(state).await - }; let input_event_streams = u32::try_from(input_demands.stream_registrations_requesting(SourceKind::Interaction)) .unwrap_or(u32::MAX); - let consumers = - OutputConsumers::from_bus(&state.event_bus, output_devices, input_event_streams); + let mut consumers = + OutputConsumers::from_bus(&state.event_bus, registry_devices, input_event_streams); + if !consumers.any() { + consumers.output_devices = routed_output_device_count(state).await; + } if self.recorded != Some(consumers) { self.recorded = Some(consumers); state @@ -165,6 +169,34 @@ impl OutputGate { consumers } + /// Record whether the frame just rendered was mid scene transition. + /// + /// A transition advances only as frames render, so the loop keeps + /// rendering until it completes rather than freezing it mid crossfade + /// with the outgoing scene's producers still alive. Only frames that + /// reach composition update the flag, and the settle limit still + /// bounds the wait, so a crossfade longer than that limit pauses + /// partway and finishes once a consumer returns. + pub(crate) fn note_scene_transition(&mut self, active: bool) { + self.transition_active = active; + } + + /// Whether the previous rendered frame was mid scene transition. + pub(crate) const fn transition_active(&self) -> bool { + self.transition_active + } + + /// Record that the loop parked. + pub(crate) fn enter_park(&mut self) { + self.parked = true; + } + + /// Record that the loop left its park, returning whether it was + /// parked. + pub(crate) fn leave_park(&mut self) -> bool { + std::mem::take(&mut self.parked) + } + /// Decide whether this frame renders. /// /// `producers_warming` describes the producers as the previous frame @@ -256,8 +288,9 @@ pub(crate) async fn wait_for_output_demand( state: &RenderThreadState, frame_loop: &mut FrameLoopState, ) -> RenderWakeReason { + let armed_output_demand = state.event_bus.output_demand().arm(); loop { - let output_demand = state.event_bus.output_demand().notified(); + let output_demand = armed_output_demand.notified(); let devices = state.device_registry.changed(); let transactions = state.scene_transactions.queued(); let mut input_demand_revisions = frame_loop.input_demands.subscribe_revision(); diff --git a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs index 2c82fab36..36b9df5a9 100644 --- a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs +++ b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs @@ -84,9 +84,17 @@ async fn resume_from_park( ) { let now = Instant::now(); let (interval, resume_frame) = { - let render_loop = state.render_loop.read().await; + let mut render_loop = state.render_loop.write().await; + render_loop.unpark(); (render_loop.target_interval(), render_loop.frame_number()) }; + if reason == RenderWakeReason::RenderLoopState { + // A paused or stopped loop is idle for its own reason, not for + // lack of consumers; the gate parks it again if it resumes with + // nothing to render for. + runtime.frame_loop.output_gate.leave_park(); + state.performance.write().await.note_output_unparked(); + } runtime .frame_loop .clock diff --git a/crates/hypercolor-daemon/src/render_thread/render_zones.rs b/crates/hypercolor-daemon/src/render_thread/render_zones.rs index b69ae5e60..32bdae96d 100644 --- a/crates/hypercolor-daemon/src/render_thread/render_zones.rs +++ b/crates/hypercolor-daemon/src/render_thread/render_zones.rs @@ -349,14 +349,38 @@ impl ZoneRuntime { Ok(runtime) } - /// Park every effect producer while the render loop requests no frames. + /// Park every producer while the render loop requests no frames. + /// + /// Effect producers quiesce in place. Live media streams decode on a + /// worker of their own, so they are dropped instead and reconnect when + /// a layer next renders; a live stream has no past frame worth keeping. + /// Dropping one joins its decoder, which waits on the stream source, so + /// the drop runs on a short-lived thread rather than the render thread. pub(crate) fn park_producers(&mut self) { self.effect_pool.park(); + let live_streams = self + .media_producers + .extract_if(|_, cached| cached.producer.is_live_stream()) + .map(|(_, cached)| cached) + .collect::>(); + if live_streams.is_empty() { + return; + } + if let Err(error) = std::thread::Builder::new() + .name("hypercolor-media-park".to_owned()) + .spawn(move || drop(live_streams)) + { + tracing::warn!(%error, "Could not spawn the live stream teardown thread"); + } } - /// Whether any effect producer has not yet shown its current content. + /// Whether any producer has not yet shown its current content. pub(crate) fn producers_warming(&self) -> bool { self.effect_pool.any_warming() + || self.media_producers.values().any(|cached| { + !cached.producer.has_renderable_frame() + && cached.producer.live_stream_error().is_none() + }) } pub(crate) fn drain_layer_runtime_events(&mut self) -> Vec { diff --git a/crates/hypercolor-daemon/tests/output_demand_tests.rs b/crates/hypercolor-daemon/tests/output_demand_tests.rs index 90c219fed..71492c30e 100644 --- a/crates/hypercolor-daemon/tests/output_demand_tests.rs +++ b/crates/hypercolor-daemon/tests/output_demand_tests.rs @@ -18,25 +18,33 @@ use hypercolor_core::device::mock::{MockDeviceBackend, MockDeviceConfig}; use hypercolor_core::device::{BackendManager, DeviceRegistry}; use hypercolor_core::effect::{EffectRegistry, builtin::register_builtin_effects}; use hypercolor_core::engine::{FpsTier, RenderLoop}; -use hypercolor_core::input::InputManager; -use hypercolor_core::scene::SceneManager; +use hypercolor_core::input::{InputManager, SourceKind}; +use hypercolor_core::scene::{SceneManager, make_scene}; use hypercolor_core::spatial::SpatialEngine; use hypercolor_daemon::SceneTransactionQueue; use hypercolor_daemon::canvas_capture::{CanvasCaptureRequest, CanvasImageFormat, capture_canvas}; use hypercolor_daemon::domain::scene::SceneService; use hypercolor_daemon::domain::spatial::SpatialService; -use hypercolor_daemon::output_demand::{OutputDemandTelemetry, is_output_device}; +use hypercolor_daemon::output_demand::{ + OutputConsumers, OutputDemandTelemetry, is_output_backend, is_output_device, + output_device_count, +}; use hypercolor_daemon::output_power::OutputPowerState; use hypercolor_daemon::performance::PerformanceTracker; use hypercolor_daemon::preview_runtime::PreviewRuntime; -use hypercolor_daemon::render_thread::{CanvasDims, RenderThread, RenderThreadState}; +use hypercolor_daemon::render_thread::{ + CanvasDims, InputPublicationConsumer, InputPublicationDemand, RenderThread, RenderThreadState, +}; use hypercolor_daemon::simulators::SimulatedDisplayExt; +use hypercolor_driver_api::DeviceBackend; use hypercolor_types::api::simulators::SimulatedDisplay; use hypercolor_types::config::RenderAccelerationMode; use hypercolor_types::control::ControlValue; use hypercolor_types::device::{DeviceId, DeviceInfo, DeviceState}; -use hypercolor_types::effect::EffectMetadata; -use hypercolor_types::event::FrameData; +use hypercolor_types::effect::{EffectId, EffectMetadata}; +use hypercolor_types::event::{FrameData, SceneChangeReason}; +use hypercolor_types::layer::{SceneLayer, SceneLayerId}; +use hypercolor_types::scene::{Zone, ZoneId, ZoneRole}; use hypercolor_types::spatial::{ EdgeBehavior, LedTopology, NormalizedPosition, Output, SamplingMode, SpatialLayout, }; @@ -112,6 +120,37 @@ fn builtin_effect_metadata(stem: &str) -> EffectMetadata { .expect("builtin effect should exist") } +fn builtin_effect_id(stem: &str) -> EffectId { + builtin_effect_metadata(stem).id +} + +fn primary_zone( + effect_id: EffectId, + controls: HashMap, + layout: SpatialLayout, +) -> Zone { + Zone { + id: ZoneId::new(), + name: "Primary".into(), + description: None, + layers: vec![SceneLayer::from_effect( + SceneLayerId::new(), + effect_id, + controls, + HashMap::new(), + None, + )], + layout, + brightness: 1.0, + enabled: true, + color: None, + display_target: None, + role: ZoneRole::Primary, + controls_version: 0, + layers_version: 0, + } +} + fn solid_color_controls(r: u8, g: u8, b: u8) -> HashMap { HashMap::from([( "color".into(), @@ -609,3 +648,215 @@ async fn scene_change_while_parked_renders_the_new_effect_then_parks_again() { stop(&state, render_thread).await; } + +#[tokio::test] +async fn consumer_counts_cover_every_kind_and_skip_simulated_displays() { + let bus = HypercolorBus::new(); + assert!(!OutputConsumers::from_bus(&bus, 0, 0).any()); + assert!(OutputConsumers::from_bus(&bus, 1, 0).any()); + assert!(OutputConsumers::from_bus(&bus, 0, 1).any()); + let canvas = bus.canvas_receiver(); + assert_eq!(OutputConsumers::from_bus(&bus, 0, 0).stream_receivers, 1); + drop(canvas); + let hold = bus.hold_output_demand(); + assert_eq!(OutputConsumers::from_bus(&bus, 0, 0).demand_holds, 1); + drop(hold); + assert!(!OutputConsumers::from_bus(&bus, 0, 0).any()); + + assert!(is_output_backend("wled")); + assert!(!is_output_backend("simulator")); + + let registry = DeviceRegistry::new(); + assert_eq!(output_device_count(®istry).await, 0); + let strip = connect_device(®istry, mock_device_info("Strip")).await; + let _known = registry.add(mock_device_info("Known Only")).await; + let simulator = SimulatedDisplay { + id: DeviceId::new(), + name: "Virtual LCD".into(), + width: 240, + height: 240, + circular: false, + enabled: true, + } + .device_info(); + connect_device(®istry, simulator).await; + assert_eq!( + output_device_count(®istry).await, + 1, + "only the connected physical device counts" + ); + registry.set_state(&strip, DeviceState::Disabled).await; + assert_eq!(output_device_count(®istry).await, 0); +} + +#[tokio::test] +async fn routed_device_missing_from_the_registry_keeps_the_loop_rendering() { + let mut state = make_render_state("rainbow", HashMap::new()); + let device_id = DeviceId::new(); + let backend = Arc::new(MockDeviceBackend::new().with_device(&MockDeviceConfig { + name: "Routed Strip".into(), + led_count: 1, + topology: LedTopology::Point, + id: Some(device_id), + })); + backend + .connect(&device_id) + .await + .expect("mock device should connect"); + let mut manager = BackendManager::new(); + manager.register_backend(backend); + manager.map_device("mock:main", "mock", device_id); + state.backend_manager = Arc::new(Mutex::new(manager)); + let render_thread = start(&state).await; + + assert_renders_at_full_tier(&state, "a routed device").await; + assert_eq!(telemetry(&state).await.parks_total, 0); + assert_eq!(telemetry(&state).await.consumers.output_devices, 1); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn input_event_stream_keeps_the_loop_republishing_host_input() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + + let stream = render_thread.input_publication_demands().register( + InputPublicationConsumer::PassiveStream, + InputPublicationDemand::default().with_source(SourceKind::Interaction, 60), + ); + let latency = wait_until_rendering_past(&state, parked_at).await; + assert!( + latency < WAKE_LATENCY_BOUND, + "an input event stream should wake the loop at once, took {latency:?}" + ); + assert_renders_at_full_tier(&state, "an input event stream").await; + assert_eq!(telemetry(&state).await.consumers.input_event_streams, 1); + + drop(stream); + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn pausing_a_parked_loop_ends_the_park_and_resuming_parks_again() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + wait_until_parked(&state, 1).await; + + state.render_loop.write().await.pause(); + tokio::time::timeout(WAIT_DEADLINE, async { + while telemetry(&state).await.parked_since.is_some() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("a paused loop should stop reporting a park"); + assert_eq!( + telemetry(&state).await.last_wake_reason, + Some("render_loop_state") + ); + assert!(!state.render_loop.read().await.is_parked()); + + state.render_loop.write().await.resume(); + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn render_time_holds_still_while_parked() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + wait_until_parked(&state, 1).await; + + let parked_elapsed = state.render_loop.read().await.elapsed(); + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!( + state.render_loop.read().await.elapsed(), + parked_elapsed, + "media layers and time bindings read render time, which must not run on while parked" + ); + + let frames = state.event_bus.frame_receiver(); + wait_until_rendering_past(&state, frame_number(&state).await).await; + tokio::time::sleep(Duration::from_millis(100)).await; + let resumed_elapsed = state.render_loop.read().await.elapsed(); + assert!(resumed_elapsed > parked_elapsed); + assert!( + resumed_elapsed < parked_elapsed + Duration::from_millis(400), + "render time should resume from the park, not jump by it: {parked_elapsed:?} -> {resumed_elapsed:?}" + ); + drop(frames); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn scene_transition_finishes_before_the_loop_parks() { + let state = make_render_state("solid_color", solid_color_controls(255, 0, 0)); + let layout = test_layout(vec![point_zone("zone_main", "mock:main", 0.5, 0.5)]); + let solid_id = builtin_effect_id("solid_color"); + let mut scene_a = make_scene("Red"); + scene_a.transition.duration_ms = 0; + scene_a.zones = vec![primary_zone( + solid_id, + solid_color_controls(255, 0, 0), + layout.clone(), + )]; + let mut scene_b = make_scene("Blue"); + scene_b.transition.duration_ms = 600; + scene_b.zones = vec![primary_zone( + solid_id, + solid_color_controls(0, 0, 255), + layout, + )]; + let mut mutation = state.scene_manager.begin_mutation().await; + mutation + .create_scene(scene_a.clone()) + .expect("scene should stage"); + mutation + .create_scene(scene_b.clone()) + .expect("scene should stage"); + mutation + .activate(scene_a.id, None, SceneChangeReason::UserActivate) + .expect("scene should activate"); + state + .scene_manager + .commit_mutation(mutation) + .await + .expect("scenes should commit"); + + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + assert_eq!( + zone_main_color(&state.event_bus.frame_lane().borrow()), + [255, 0, 0] + ); + + let mut mutation = state.scene_manager.begin_mutation().await; + mutation + .activate(scene_b.id, None, SceneChangeReason::UserActivate) + .expect("scene should activate"); + state + .scene_manager + .commit_mutation(mutation) + .await + .expect("scene activation should commit"); + + let parked_again_at = wait_until_parked(&state, 2).await; + assert_eq!( + zone_main_color(&state.event_bus.frame_lane().borrow()), + [0, 0, 255], + "the crossfade should finish on the new scene before the loop parks" + ); + assert!( + parked_again_at - parked_at >= 15, + "a 600 ms crossfade renders through to its end, rendered {} frames", + parked_again_at - parked_at + ); + + stop(&state, render_thread).await; +} From 89a30b024a50f7574e5fb3345dd76ec3fc7dd6f9 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 01:49:38 -0700 Subject: [PATCH 14/26] docs: correct how a parked render loop wakes and keeps time Bus events make a parked loop re-check its work rather than wake it to render. Throttled pages stop their self-scheduled requestAnimationFrame loops but keep their timers, so an effect that animates from timers keeps running while parked. Parking waits for transitions within the settle limit and drops live media streams, and render time stops while parked, so media layers and time bindings continue where they left off. The architecture page and the SparkleFlinger notes now say so. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/content/architecture/render-pipeline.md | 4 +-- .../30-sparkleflinger-implementation.md | 31 +++++++++++-------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/docs/content/architecture/render-pipeline.md b/docs/content/architecture/render-pipeline.md index 4c8fd8321..04e2c9b4d 100644 --- a/docs/content/architecture/render-pipeline.md +++ b/docs/content/architecture/render-pipeline.md @@ -175,9 +175,9 @@ Every frame costs producer, compositor, and GPU work whether or not anything rec - WebSocket and diagnostic subscribers to host input events, which the loop republishes; - explicit output demand holds, taken by anything that reads published frames without a receiver. -With none of them, the loop **parks**: it renders nothing, withdraws its audio, screen, and interaction capture demand, and throttles HTML effect pages so their own timers and `requestAnimationFrame` loops stop too. A parked loop sleeps until something can observe a frame. Subscribing to a stream, taking a demand hold, connecting a device, queuing a layout change, or publishing any bus event wakes it at once, and it resumes at its full tier. A one second safety tick re-checks in case a consumer arrived without signalling. No observable cadence changes, because the moment anything can see a frame, it renders at full rate. +With none of them, the loop **parks**: it renders nothing, withdraws its audio, screen, and interaction capture demand, drops live media streams, and throttles HTML effect pages, which stops the `requestAnimationFrame` loops they schedule for themselves. Page timers keep their own pace, so an effect that animates from `setTimeout` or `setInterval` rather than `requestAnimationFrame` keeps running while parked; the bundled effects all animate from `requestAnimationFrame`. A parked loop sleeps until something can observe a frame. Subscribing to a stream, taking a demand hold, connecting a device, or queuing a layout change wakes it at once, and it resumes at its full tier; any other bus event makes it re-check whether it has work. A one second safety tick re-checks in case a consumer arrived without signalling. No observable cadence changes, because the moment anything can see a frame, it renders at full rate. -Parking waits for the producers to settle. After the scene, an effect, or the canvas changes, the loop keeps rendering until every producer has shown its new content (bounded at ten seconds), so an effect applied while nobody watches is already loaded when a client or capture arrives. Effect time carries across the park: the first frame after a wake advances effect time by one frame, not by the parked span, and no catch-up frames follow. A canvas capture that wakes the loop skips the wake frame itself, whose HTML producers still show output requested before the park, and returns the next one. +Parking waits for the producers to settle. After the scene, an effect, or the canvas changes, the loop keeps rendering until every producer has shown its new content and any scene transition has finished (bounded at ten seconds, so a longer crossfade pauses partway and finishes when a consumer returns), so an effect applied while nobody watches is already loaded when a client or capture arrives. Time carries across the park: render time stops while parked, so media layers and time bindings continue where they left off, and the first frame after a wake advances effect time by one frame, not by the parked span, with no catch-up frames. A canvas capture that wakes the loop skips the wake frame itself, whose HTML producers still show output requested before the park, and returns the next one. `/api/v1/status` reports the state under `render_loop.output_demand` (`parked`, `idle_reason`, `parked_ms`, and the consumer counts the loop last saw), with `delivered_fps` and `actual_fps` at zero while parked. `/api/v1/diagnose` carries the same object under `snapshot.render.output_demand`. Rigs with a connected device never park. diff --git a/docs/design/30-sparkleflinger-implementation.md b/docs/design/30-sparkleflinger-implementation.md index 052b444d9..72ab65e49 100644 --- a/docs/design/30-sparkleflinger-implementation.md +++ b/docs/design/30-sparkleflinger-implementation.md @@ -225,21 +225,26 @@ With no consumer the gate parks the loop instead of rendering. It withdraws the authoritative input capture demand, releases the screen queue, calls `EffectRenderer::park` on every producer (which throttles Servo webviews so the page's script thread stops scheduling its own -animation ticks and slows its timers), and returns `NextWake::Park`. The -pipeline driver then sleeps on the bus output-demand signal (notified by -lane subscriptions, demand holds, and event publication), the device -registry change signal, the input demand revision, and the scene -transaction queue, with a one second safety tick. On wake it rebases the -frame clock one target interval back, so effect time steps by one frame, -records the wake frame for capture freshness, and continues at the -current tier. +animation ticks; page timers keep their pace), drops live media streams +on a teardown thread, abandons +the admitted frame without recording it, calls `RenderLoop::park` so +render time stops like it does for a pause, and returns `NextWake::Park`. +The pipeline driver then sleeps on the bus output-demand signal (notified +by lane subscriptions, demand holds, and event publication, and armed only +while the loop waits), the device registry change signal, the input demand +revision, and the scene transaction queue, with a one second safety tick. +On wake it unparks the render loop, rebases the frame clock one target +interval back so effect time steps by one frame, records the wake frame +for capture freshness, and continues at the current tier. The gate parks only once the producers are settled: three consecutive -frame starts with no producer reporting `is_warming` under an unchanged -producer signature (scene plan generation, zones revision, effect registry -generation, zone layout preview generation, canvas size), bounded at ten -seconds. Frames that consumers drive count toward settling, so losing the -last consumer parks at once when the producers are warm. +frame starts with no producer reporting `is_warming`, no loading media, +and no scene transition in flight, under an unchanged producer signature +(scene plan generation, zones revision, effect registry generation, zone +layout preview generation, canvas size), bounded at ten seconds, so a +crossfade longer than that pauses partway until a consumer returns. Frames +that consumers drive count toward settling, so losing the last consumer +parks at once when the producers are warm. ### 3.5 Input sampling From ef27ce572c02080e6306ccc270983c65a441e7ed Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Tue, 20 Oct 2026 01:58:30 -0700 Subject: [PATCH 15/26] feat(diagnose): name what paused the render loop in the render checks A paused render loop renders nothing by design, and from the outside that looks like a hang: diagnose showed a healthy running supervision with a minute-old last frame and every device at 0 fps, and the pause itself only appeared as a terse state=paused warning. A manual pause also persists across restarts, so a daemon can start paused long after whatever paused it. The render_loop and render_thread details now name the cause when the loop is paused: a manual output pause (which persists until power is set back to running), a stopped last effect, or a sleeping session. render_thread still passes while paused, since no frames are due, and says so along with how recently the thread last polled. The troubleshooting guide notes that a manual pause, including one from an integration such as Home Assistant, survives a restart. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/domain/diagnostics.rs | 102 ++++++++++++++++-- crates/hypercolor-daemon/src/domain/output.rs | 6 ++ docs/content/troubleshooting/performance.md | 4 +- 3 files changed, 101 insertions(+), 11 deletions(-) diff --git a/crates/hypercolor-daemon/src/domain/diagnostics.rs b/crates/hypercolor-daemon/src/domain/diagnostics.rs index ff3806427..470c5475f 100644 --- a/crates/hypercolor-daemon/src/domain/diagnostics.rs +++ b/crates/hypercolor-daemon/src/domain/diagnostics.rs @@ -23,6 +23,7 @@ use crate::domain::display::DisplayContext; use crate::domain::input_status::{actionable_input_diagnostics, input_status_snapshot}; use crate::domain::output::{OutputContext, RenderLoopStatus}; use crate::domain::spatial::SpatialService; +use crate::output_power::{OutputOverride, OutputPowerState}; use crate::performance::{LatestFrameMetrics, PerformanceSnapshot}; use crate::render_thread::gpu_errors::GpuErrorTotals; use crate::render_thread::{RenderSupervisionSnapshot, RenderThreadPhase}; @@ -158,6 +159,7 @@ struct DiagnosticsInputs { config_available: bool, config_persistence: Option<(PathBuf, ConfigPersistence)>, render_supervision: RenderSupervisionSnapshot, + output_power: OutputPowerState, } impl DiagnosticsInputs { @@ -349,6 +351,7 @@ impl DiagnosticsContext { config_available: self.authorities.platform.config_available(), config_persistence: self.authorities.platform.config_persistence(), render_supervision: self.authorities.output.render_supervision().snapshot(), + output_power: self.authorities.output.power_state(), } } } @@ -371,21 +374,25 @@ fn render_checks(inputs: &DiagnosticsInputs) -> Vec { let render_elapsed_ms = inputs.render_elapsed_ms(); let running = inputs.render_loop.running; let render_loop_stats = inputs.render_loop.stats; + let pause = render_loop_pause_reason(render_loop_stats.state, inputs.output_power) + .map_or_else(String::new, |reason| format!("; {reason}")); + let mut render_thread = render_thread_check( + &inputs.render_supervision, + render_loop_stats.state, + RENDER_THREAD_STALL_FAIL, + ); + render_thread.detail.push_str(&pause); let mut checks = vec![ DiagnoseCheck { category: "render".to_owned(), name: "render_loop".to_owned(), status: if running { "pass" } else { "warning" }.to_owned(), detail: format!( - "state={}, tier={}", + "state={}, tier={}{pause}", render_loop_stats.state, render_loop_stats.tier ), }, - render_thread_check( - &inputs.render_supervision, - render_loop_stats.state, - RENDER_THREAD_STALL_FAIL, - ), + render_thread, gpu_errors_check(&inputs.render_supervision.gpu_errors), ]; if !running { @@ -433,6 +440,28 @@ fn render_checks(inputs: &DiagnosticsInputs) -> Vec { checks } +/// What paused the render loop, for the render checks' details. +/// +/// A paused loop renders nothing by design, which looks like a hang from +/// the outside, so the details say where the pause came from. +fn render_loop_pause_reason( + loop_state: RenderLoopState, + power: OutputPowerState, +) -> Option<&'static str> { + if loop_state != RenderLoopState::Paused { + return None; + } + Some(match power.output_override { + OutputOverride::Paused => { + "paused by output power: a manual pause (PATCH /api/v1/output or a pause \ + control), which persists across restarts until power is set back to running" + } + OutputOverride::Stopped => "paused by output power: the last effect was stopped", + OutputOverride::None if power.session_sleeping => "paused while the session sleeps", + OutputOverride::None => "paused, though output power reports running", + }) +} + /// Whether the supervised render thread is alive and producing frames. /// /// Fails while a pipeline is restarting or the daemon is terminating after @@ -514,13 +543,22 @@ fn render_thread_status( supervision.restarts_total ), ), + _ if loop_state == RenderLoopState::Paused => ( + "pass", + format!( + "phase=running, the render loop is paused so no frames are due, \ + last_poll_age_ms={}, last_frame_age_ms={}, restarts={}, \ + last_failure={failure}", + millis_or_none(supervision.last_heartbeat_age), + millis_or_none(supervision.last_frame_age), + supervision.restarts_total + ), + ), _ => ( "pass", format!( "phase=running, last_frame_age_ms={}, restarts={}, last_failure={failure}", - supervision - .last_frame_age - .map_or_else(|| "none".to_owned(), |age| age.as_millis().to_string()), + millis_or_none(supervision.last_frame_age), supervision.restarts_total ), ), @@ -555,6 +593,10 @@ fn render_thread_status( /// `gpu_errors` check to warn. Older errors stay in the counts. pub const GPU_ERROR_WARNING_WINDOW: Duration = Duration::from_mins(10); +fn millis_or_none(age: Option) -> String { + age.map_or_else(|| "none".to_owned(), |age| age.as_millis().to_string()) +} + /// Whether a GPU error went uncaptured recently. /// /// Uncaptured validation, out-of-memory, and internal errors no longer @@ -1410,6 +1452,48 @@ mod tests { } } + #[test] + fn a_paused_loop_passes_the_render_thread_check_and_names_its_pause() { + use crate::output_power::{OutputOverride, OutputPowerState}; + use crate::render_thread::RenderThreadPhase; + use hypercolor_core::engine::RenderLoopState; + + let paused_for_a_minute = supervision(RenderThreadPhase::Running, 60_000, 40); + let check = super::render_thread_check( + &paused_for_a_minute, + RenderLoopState::Paused, + std::time::Duration::from_secs(3), + ); + assert_eq!(check.status, "pass", "{}", check.detail); + assert!( + check.detail.contains("no frames are due"), + "{}", + check.detail + ); + + let manual = OutputPowerState { + output_override: OutputOverride::Paused, + ..OutputPowerState::default() + }; + assert!( + super::render_loop_pause_reason(RenderLoopState::Paused, manual) + .is_some_and(|reason| reason.contains("manual pause") + && reason.contains("persists across restarts")) + ); + let stopped = OutputPowerState { + output_override: OutputOverride::Stopped, + ..OutputPowerState::default() + }; + assert!( + super::render_loop_pause_reason(RenderLoopState::Paused, stopped) + .is_some_and(|reason| reason.contains("last effect was stopped")) + ); + assert_eq!( + super::render_loop_pause_reason(RenderLoopState::Running, manual), + None + ); + } + #[test] fn gpu_errors_check_warns_on_recent_errors_and_fails_on_device_loss() { use crate::render_thread::gpu_errors::{GpuErrorSample, GpuErrorTotals}; diff --git a/crates/hypercolor-daemon/src/domain/output.rs b/crates/hypercolor-daemon/src/domain/output.rs index 4c9b02862..06f759e25 100644 --- a/crates/hypercolor-daemon/src/domain/output.rs +++ b/crates/hypercolor-daemon/src/domain/output.rs @@ -128,6 +128,12 @@ impl OutputContext { &self.render_supervision } + /// The current output power state. + #[must_use] + pub(crate) fn power_state(&self) -> crate::output_power::OutputPowerState { + self.output_power.snapshot() + } + /// How long the daemon's subsystems have been up. #[must_use] pub fn uptime(&self) -> std::time::Duration { diff --git a/docs/content/troubleshooting/performance.md b/docs/content/troubleshooting/performance.md index b051bf876..4dcf884b0 100644 --- a/docs/content/troubleshooting/performance.md +++ b/docs/content/troubleshooting/performance.md @@ -49,7 +49,7 @@ The `render` category has seven checks for FPS and LED output problems. render_loop state=running, tier=medium ``` -The `tier` field shows where the FPS controller has landed. Compare it against your configured `daemon.target_fps`: on the default config, `medium` (30 FPS) is the healthy steady state. A tier sitting below your configured target means the loop has downshifted under budget pressure. If `state` is anything other than `running`, the render loop has stopped; see [common issues](@/troubleshooting/common-issues.md). +The `tier` field shows where the FPS controller has landed. Compare it against your configured `daemon.target_fps`: on the default config, `medium` (30 FPS) is the healthy steady state. A tier sitting below your configured target means the loop has downshifted under budget pressure. If `state` is anything other than `running`, the render loop is not rendering; see [common issues](@/troubleshooting/common-issues.md). A `paused` loop renders nothing on purpose, so the detail names what paused it. A manual output pause (from `PATCH /api/v1/output`, a pause control, or an integration such as Home Assistant) persists across daemon restarts until output power is set back to running. **Tier reference:** @@ -69,7 +69,7 @@ The controller downshifts after 2 consecutive budget misses and only upshifts af render_thread phase=running, last_frame_age_ms=12, restarts=0, last_failure=none ``` -The render thread supervises its own pipeline. When a frame panics or the pipeline loses its input feed, the daemon logs the failure with a backtrace, rebuilds the pipeline with fresh GPU and render state, and keeps going. This check fails while that restart is in progress, when the thread has gone without completing a frame for 3 seconds while the loop runs, and when the thread is gone while the loop still says it is running. +The render thread supervises its own pipeline. When a frame panics or the pipeline loses its input feed, the daemon logs the failure with a backtrace, rebuilds the pipeline with fresh GPU and render state, and keeps going. This check fails while that restart is in progress, when the thread has gone without completing a frame for 3 seconds while the loop runs, and when the thread is gone while the loop still says it is running. A paused loop passes, since no frames are due, and the detail names what paused it. `restarts` counts in-process restarts since the daemon started. After five restarts within five minutes, or when the GPU device is lost, the daemon stops recovering in-process and exits with an error so its service manager starts a fresh process. Under systemd, a render thread stuck inside one frame for 20 seconds also stops the service watchdog keepalive, so systemd restarts the daemon. Every failure is also written to `render-failure.json` in the daemon state directory (`~/.local/state/hypercolor/` on Linux), so it survives a log that was lost. The next run reports that record here as `(earlier daemon run)`. From 68e0ffc577a40a440b914a5ca9743bee102bb061 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 08:33:18 -0700 Subject: [PATCH 16/26] docs(render): point the parked-loop status note at /api/v1/system The render pipeline page sent readers to /api/v1/status, which was retired, so the retired-route docs check failed. The output demand object lives under render_loop in /api/v1/system. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/content/architecture/render-pipeline.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/architecture/render-pipeline.md b/docs/content/architecture/render-pipeline.md index 04e2c9b4d..5e5f5343f 100644 --- a/docs/content/architecture/render-pipeline.md +++ b/docs/content/architecture/render-pipeline.md @@ -179,7 +179,7 @@ With none of them, the loop **parks**: it renders nothing, withdraws its audio, Parking waits for the producers to settle. After the scene, an effect, or the canvas changes, the loop keeps rendering until every producer has shown its new content and any scene transition has finished (bounded at ten seconds, so a longer crossfade pauses partway and finishes when a consumer returns), so an effect applied while nobody watches is already loaded when a client or capture arrives. Time carries across the park: render time stops while parked, so media layers and time bindings continue where they left off, and the first frame after a wake advances effect time by one frame, not by the parked span, with no catch-up frames. A canvas capture that wakes the loop skips the wake frame itself, whose HTML producers still show output requested before the park, and returns the next one. -`/api/v1/status` reports the state under `render_loop.output_demand` (`parked`, `idle_reason`, `parked_ms`, and the consumer counts the loop last saw), with `delivered_fps` and `actual_fps` at zero while parked. `/api/v1/diagnose` carries the same object under `snapshot.render.output_demand`. Rigs with a connected device never park. +`/api/v1/system` reports the state under `render_loop.output_demand` (`parked`, `idle_reason`, `parked_ms`, and the consumer counts the loop last saw), with `delivered_fps` and `actual_fps` at zero while parked. `/api/v1/diagnose` carries the same object under `snapshot.render.output_demand`. Rigs with a connected device never park. ## Canvas resize and layout updates From 5063a4d631faf936569c709057e09d6f7d7c1d60 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 08:38:58 -0700 Subject: [PATCH 17/26] fix(render): publish a restart only together with the failure behind it record_failure bumped restarts_total first and stored the failure and the new phase afterward, so a snapshot taken in between saw one restart, the dead attempt's Running phase, and no failure. The Windows GPU job caught that window in the injected panic test. The failure, both counters, and the phase now change together under the last_failure lock, and snapshot reads them under the same lock. Logging and persisting the record stay outside it, so a status read never waits on the state directory. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/render_thread/supervision.rs | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/crates/hypercolor-daemon/src/render_thread/supervision.rs b/crates/hypercolor-daemon/src/render_thread/supervision.rs index aa8a0504a..9287d9cc7 100644 --- a/crates/hypercolor-daemon/src/render_thread/supervision.rs +++ b/crates/hypercolor-daemon/src/render_thread/supervision.rs @@ -703,18 +703,21 @@ impl RenderSupervision { #[must_use] pub fn snapshot(&self) -> RenderSupervisionSnapshot { let now = Instant::now(); - let current = self + let failure_guard = self .inner .last_failure .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone(); - let last_failure_in_this_process = current.is_some(); + .unwrap_or_else(PoisonError::into_inner); + let current = failure_guard.clone(); let phase = self.phase(); + let restarts_total = self.inner.restarts_total.load(Ordering::Relaxed); + let consecutive_failures = self.inner.consecutive_failures.load(Ordering::Relaxed); + drop(failure_guard); + let last_failure_in_this_process = current.is_some(); RenderSupervisionSnapshot { phase, - restarts_total: self.inner.restarts_total.load(Ordering::Relaxed), - consecutive_failures: self.inner.consecutive_failures.load(Ordering::Relaxed), + restarts_total, + consecutive_failures, max_restarts: self.inner.policy.max_restarts, last_heartbeat_age: self.age(&self.inner.last_heartbeat_us, now), last_frame_age: self.age(&self.inner.last_frame_us, now), @@ -849,17 +852,14 @@ impl RenderSupervision { RenderThreadPhase::Stopped, ), }; + // Only the render thread records failures, so the counter cannot + // move between this read and the commit below. + let previous_restarts = self.inner.restarts_total.load(Ordering::Relaxed); let restarts_total = if action == RenderRecoveryAction::Restart { - self.inner - .restarts_total - .fetch_add(1, Ordering::Relaxed) - .saturating_add(1) + previous_restarts.saturating_add(1) } else { - self.inner.restarts_total.load(Ordering::Relaxed) + previous_restarts }; - self.inner - .consecutive_failures - .store(consecutive_failures, Ordering::Relaxed); let record = failure_record(failure, action, restart_attempt, restarts_total); log_failure(&record, plan, self.inner.policy.max_restarts); if let Some(store) = self.inner.store.as_ref() @@ -867,12 +867,23 @@ impl RenderSupervision { { tracing::warn!(path = %store.path.display(), error = %format!("{error:#}"), "could not persist the render failure record"); } - *self + // The failure, its counters, and the phase change together under + // the failure lock, which `snapshot` holds while it reads them, so + // no reader sees a counted restart without the failure behind it. + let mut last_failure = self .inner .last_failure .lock() - .unwrap_or_else(PoisonError::into_inner) = Some(record.clone()); + .unwrap_or_else(PoisonError::into_inner); + *last_failure = Some(record.clone()); + self.inner + .consecutive_failures + .store(consecutive_failures, Ordering::Relaxed); + self.inner + .restarts_total + .store(restarts_total, Ordering::Relaxed); self.set_phase(phase); + drop(last_failure); record } From 340dd2d894d52ea0c9cf0907baa816985770e593 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 08:38:58 -0700 Subject: [PATCH 18/26] refactor(diagnose): name the output power read output_power_snapshot The accessor already read through OutputPower::snapshot, but its name, power_state, matched the architecture test that keeps control-plane modules off the raw power watch, so the daemon suite failed on a false positive. The new name says what it returns and clears the guard. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/src/domain/diagnostics.rs | 2 +- crates/hypercolor-daemon/src/domain/output.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/hypercolor-daemon/src/domain/diagnostics.rs b/crates/hypercolor-daemon/src/domain/diagnostics.rs index 470c5475f..50c25494a 100644 --- a/crates/hypercolor-daemon/src/domain/diagnostics.rs +++ b/crates/hypercolor-daemon/src/domain/diagnostics.rs @@ -351,7 +351,7 @@ impl DiagnosticsContext { config_available: self.authorities.platform.config_available(), config_persistence: self.authorities.platform.config_persistence(), render_supervision: self.authorities.output.render_supervision().snapshot(), - output_power: self.authorities.output.power_state(), + output_power: self.authorities.output.output_power_snapshot(), } } } diff --git a/crates/hypercolor-daemon/src/domain/output.rs b/crates/hypercolor-daemon/src/domain/output.rs index 06f759e25..3028219ce 100644 --- a/crates/hypercolor-daemon/src/domain/output.rs +++ b/crates/hypercolor-daemon/src/domain/output.rs @@ -130,7 +130,7 @@ impl OutputContext { /// The current output power state. #[must_use] - pub(crate) fn power_state(&self) -> crate::output_power::OutputPowerState { + pub(crate) fn output_power_snapshot(&self) -> crate::output_power::OutputPowerState { self.output_power.snapshot() } From 967a73663c45ba65fb6d3a506b2271a827b25394 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 09:10:32 -0700 Subject: [PATCH 19/26] fix(render): never start a pipeline inside a park it did not enter The park flag and its time hold live on the shared render loop, but the output gate that ends a park belongs to the pipeline. When the supervisor replaces a pipeline that failed while parked, the new one gets a fresh gate that knows nothing of the old park. render_loop.tick ignores the flag, so the new pipeline rendered with effect time held, and status kept reporting a park that was no longer happening. A pipeline now leaves any inherited park before its first frame and clears the parked telemetry with it. The new test plants a stale park before spawning and fails without the fix. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/render_thread/pipeline_driver.rs | 19 ++++++++++++++ .../tests/render_thread_tests.rs | 26 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs index 2156a414b..c81131227 100644 --- a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs +++ b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs @@ -21,6 +21,7 @@ pub(crate) async fn run_pipeline( mode = ?state.render_acceleration_mode, "render pipeline started" ); + clear_stale_park(&state).await; let mut skip_decision = SkipDecision::None; let mut next_frame_at = Instant::now(); let mut rebase_clock_on_resume = false; @@ -81,6 +82,24 @@ pub(crate) async fn run_pipeline( info!("render pipeline exited"); } +/// Leave a park that a previous pipeline entered and never left. +/// +/// The park flag and its hold live on the shared render loop, but the +/// output gate that would end them belongs to the pipeline. A pipeline that +/// replaces one that failed while parked starts with a fresh gate, so it +/// would otherwise render with effect time still held. +async fn clear_stale_park(state: &RenderThreadState) { + let was_parked = { + let mut render_loop = state.render_loop.write().await; + let was_parked = render_loop.is_parked(); + render_loop.unpark(); + was_parked + }; + if was_parked { + state.performance.write().await.note_output_unparked(); + } +} + /// Pick the loop back up after a park without a jump in effect time. /// /// Effects advance by the frame clock's delta, so the clock restarts one diff --git a/crates/hypercolor-daemon/tests/render_thread_tests.rs b/crates/hypercolor-daemon/tests/render_thread_tests.rs index c3449b28c..21e7422a2 100644 --- a/crates/hypercolor-daemon/tests/render_thread_tests.rs +++ b/crates/hypercolor-daemon/tests/render_thread_tests.rs @@ -8225,3 +8225,29 @@ async fn a_parked_loop_keeps_its_supervisor_heartbeat_fresh() { state.render_loop.write().await.stop(); rt.shutdown().await.expect("shutdown while parked"); } + +#[tokio::test] +async fn a_new_pipeline_never_inherits_a_park_it_did_not_enter() { + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let state = solid_strip_state(); + let _frame_consumer = state.event_bus.frame_receiver(); + { + let mut render_loop = state.render_loop.write().await; + render_loop.start(); + // A park left behind by a pipeline that died while parked. + render_loop.park(); + } + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + + wait_for_render_frames(&state, 3).await; + assert!( + !state.render_loop.read().await.is_parked(), + "a consumed loop must render unparked so effect time advances" + ); + + state.render_loop.write().await.stop(); + rt.shutdown() + .await + .expect("shutdown after clearing a stale park"); +} From 6854715ebe35cba34358d43b223547787cfac66c Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 09:50:55 -0700 Subject: [PATCH 20/26] fix(diagnose): measure a render stall from when frames were due again The render_thread check judged a running loop by the age of its last frame. Right after a long pause or park, that frame is as old as the idle span, so the check reported a stall from the moment the loop was resumed until its first frame completed: the gap before the render thread noticed a resume, and the first frame after a wake, whose gate ends the park before the frame finishes. Every idle heartbeat (a paused poll or a pass of the park wait) now marks that no frame was due up to that moment, and the check measures a running loop from the later of that mark and its last frame. The reported last_frame_age is unchanged. The test helper reads the park from the same telemetry diagnose uses. Both new tests fail when the check goes back to the last frame's age. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/domain/diagnostics.rs | 3 +- .../src/render_thread/supervision.rs | 26 +++++- .../tests/render_thread_tests.rs | 83 ++++++++++++++++++- 3 files changed, 103 insertions(+), 9 deletions(-) diff --git a/crates/hypercolor-daemon/src/domain/diagnostics.rs b/crates/hypercolor-daemon/src/domain/diagnostics.rs index 7bac78156..6dcff6f24 100644 --- a/crates/hypercolor-daemon/src/domain/diagnostics.rs +++ b/crates/hypercolor-daemon/src/domain/diagnostics.rs @@ -563,7 +563,7 @@ fn render_thread_status( let (age, activity) = if idle { (supervision.last_heartbeat_age, "polled") } else { - (supervision.last_frame_age, "completed a frame") + (supervision.frame_due_age, "completed a frame") }; match age { Some(age) if loop_active && age >= stall_after => ( @@ -1397,6 +1397,7 @@ mod tests { max_restarts: 5, last_heartbeat_age: Some(std::time::Duration::from_millis(heartbeat_age_ms)), last_frame_age: Some(std::time::Duration::from_millis(frame_age_ms)), + frame_due_age: Some(std::time::Duration::from_millis(frame_age_ms)), current_run: None, last_failure: None, last_failure_in_this_process: false, diff --git a/crates/hypercolor-daemon/src/render_thread/supervision.rs b/crates/hypercolor-daemon/src/render_thread/supervision.rs index 9287d9cc7..17e87a9b0 100644 --- a/crates/hypercolor-daemon/src/render_thread/supervision.rs +++ b/crates/hypercolor-daemon/src/render_thread/supervision.rs @@ -394,6 +394,10 @@ pub struct RenderSupervisionSnapshot { pub last_heartbeat_age: Option, /// Time since the render loop last completed a frame. pub last_frame_age: Option, + /// Time since a frame was last due: the later of the last completed + /// frame and the last idle iteration. A loop that just left a pause or a + /// park is measured from when it left, not from its last frame before. + pub frame_due_age: Option, /// How long the current pipeline has been running. pub current_run: Option, /// The latest failure in this process, or else the failure the previous @@ -577,6 +581,8 @@ struct SupervisionInner { attempt_started_us: AtomicU64, last_heartbeat_us: AtomicU64, last_frame_us: AtomicU64, + /// The latest idle iteration, after which frames were due again. + idle_until_us: AtomicU64, last_failure: Mutex>, previous_failure: Option, termination: watch::Sender>>, @@ -653,6 +659,7 @@ impl RenderSupervision { attempt_started_us: AtomicU64::new(0), last_heartbeat_us: AtomicU64::new(0), last_frame_us: AtomicU64::new(0), + idle_until_us: AtomicU64::new(0), last_failure: Mutex::new(None), previous_failure, termination: watch::Sender::new(None), @@ -721,6 +728,7 @@ impl RenderSupervision { max_restarts: self.inner.policy.max_restarts, last_heartbeat_age: self.age(&self.inner.last_heartbeat_us, now), last_frame_age: self.age(&self.inner.last_frame_us, now), + frame_due_age: self.frame_due_age(now), current_run: (phase == RenderThreadPhase::Running) .then(|| self.age(&self.inner.attempt_started_us, now)) .flatten(), @@ -798,11 +806,12 @@ impl RenderSupervision { self.set_phase(RenderThreadPhase::Running); } - /// The render loop completed an idle or paused iteration at `now`. + /// The render loop completed an idle iteration at `now`: a paused poll + /// or a pass of the park wait. No frame was due up to this moment. pub(crate) fn heartbeat(&self, now: Instant) { - self.inner - .last_heartbeat_us - .store(self.stamp(now), Ordering::Relaxed); + let stamp = self.stamp(now); + self.inner.last_heartbeat_us.store(stamp, Ordering::Relaxed); + self.inner.idle_until_us.store(stamp, Ordering::Relaxed); } /// The render loop completed a frame at `now`. @@ -1005,6 +1014,15 @@ impl RenderSupervision { .saturating_add(1) } + fn frame_due_age(&self, now: Instant) -> Option { + let due_since = self + .inner + .last_frame_us + .load(Ordering::Relaxed) + .max(self.inner.idle_until_us.load(Ordering::Relaxed)); + (due_since != 0).then(|| Duration::from_micros(self.stamp(now).saturating_sub(due_since))) + } + fn age(&self, stamp: &AtomicU64, now: Instant) -> Option { let stamp = stamp.load(Ordering::Relaxed); (stamp != 0).then(|| Duration::from_micros(self.stamp(now).saturating_sub(stamp))) diff --git a/crates/hypercolor-daemon/tests/render_thread_tests.rs b/crates/hypercolor-daemon/tests/render_thread_tests.rs index 21e7422a2..301af9927 100644 --- a/crates/hypercolor-daemon/tests/render_thread_tests.rs +++ b/crates/hypercolor-daemon/tests/render_thread_tests.rs @@ -8077,10 +8077,16 @@ async fn render_thread_check_now( supervision: &RenderSupervision, stall_after: Duration, ) -> hypercolor_types::api::diagnose::DiagnoseCheck { - let (loop_state, parked) = { - let render_loop = state.render_loop.read().await; - (render_loop.state(), render_loop.is_parked()) - }; + let loop_state = state.render_loop.read().await.state(); + // Diagnose reads the park from the performance telemetry, not from the + // render loop's flag; the two differ briefly around a wake. + let parked = state + .performance + .read() + .await + .output_demand_telemetry() + .parked_since + .is_some(); render_thread_check(&supervision.snapshot(), loop_state, parked, stall_after) } @@ -8251,3 +8257,72 @@ async fn a_new_pipeline_never_inherits_a_park_it_did_not_enter() { .await .expect("shutdown after clearing a stale park"); } + +#[tokio::test] +async fn a_loop_resumed_from_a_long_pause_is_not_reported_as_stalled() { + let stall_after = Duration::from_millis(1_000); + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let state = solid_strip_state(); + let _frame_consumer = state.event_bus.frame_receiver(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 3).await; + + state.render_loop.write().await.pause(); + tokio::time::sleep(Duration::from_millis(1_500)).await; + state.render_loop.write().await.resume(); + + // Read before the render thread can notice the resume: the last frame is + // a whole pause old, but frames have only been due since the last poll. + let check = render_thread_check_now(&state, &supervision, stall_after).await; + assert_eq!(check.status, "pass", "{}", check.detail); + let resumed_from = state.render_loop.read().await.frame_number(); + wait_for_render_frames(&state, resumed_from + 3).await; + let check = render_thread_check_now(&state, &supervision, stall_after).await; + assert_eq!(check.status, "pass", "{}", check.detail); + + state.render_loop.write().await.stop(); + rt.shutdown().await.expect("shutdown after a long pause"); +} + +#[tokio::test] +async fn a_loop_woken_from_a_long_park_is_not_reported_as_stalled() { + let stall_after = Duration::from_millis(1_000); + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let state = solid_strip_state(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + while !state.render_loop.read().await.is_parked() { + assert!( + tokio::time::Instant::now() < deadline, + "a loop with no consumer should park" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + tokio::time::sleep(Duration::from_millis(1_500)).await; + + let parked_frame = state.render_loop.read().await.frame_number(); + let _frame_consumer = state.event_bus.frame_receiver(); + // Sample across the wake, including the first frame whose gate ends the + // park before that frame completes. + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + loop { + let check = render_thread_check_now(&state, &supervision, stall_after).await; + assert_eq!(check.status, "pass", "{}", check.detail); + if state.render_loop.read().await.frame_number() >= parked_frame + 3 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "the woken loop should render" + ); + tokio::task::yield_now().await; + } + + state.render_loop.write().await.stop(); + rt.shutdown().await.expect("shutdown after a long park"); +} From b5610274c9d50fb0b152396a53ffa09b742a4108 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 09:51:28 -0700 Subject: [PATCH 21/26] fix(render): clear park telemetry whenever a pipeline starts clear_stale_park reset the parked telemetry only when the render loop's own flag was still set. A pipeline that fails after it unparks the loop but before its next recorded frame leaves the telemetry saying parked, and if the loop is then paused before the new pipeline renders, status reports a park instead of the pause. Clearing it is idempotent, so the new pipeline now always does. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../src/render_thread/pipeline_driver.rs | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs index c81131227..5a507e030 100644 --- a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs +++ b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs @@ -89,15 +89,10 @@ pub(crate) async fn run_pipeline( /// replaces one that failed while parked starts with a fresh gate, so it /// would otherwise render with effect time still held. async fn clear_stale_park(state: &RenderThreadState) { - let was_parked = { - let mut render_loop = state.render_loop.write().await; - let was_parked = render_loop.is_parked(); - render_loop.unpark(); - was_parked - }; - if was_parked { - state.performance.write().await.note_output_unparked(); - } + state.render_loop.write().await.unpark(); + // The telemetry can outlive the loop's flag: a pipeline that failed + // between unparking and its next recorded frame leaves it set. + state.performance.write().await.note_output_unparked(); } /// Pick the loop back up after a park without a jump in effect time. From c0947cfa8f2885364180e367287595f618629319 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 09:51:37 -0700 Subject: [PATCH 22/26] fix(cli): show a parked render loop as parked, not as a failing rate A parked loop reports zero FPS on purpose, but hypercolor status still painted that as an error-colored 0% next to the explanation line. The render line now ends in a muted "parked" while the loop is parked and keeps the FPS health rating otherwise, and the status test asserts both. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-cli/src/commands/status.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/hypercolor-cli/src/commands/status.rs b/crates/hypercolor-cli/src/commands/status.rs index e77566025..111bdf88a 100644 --- a/crates/hypercolor-cli/src/commands/status.rs +++ b/crates/hypercolor-cli/src/commands/status.rs @@ -361,7 +361,12 @@ fn status_table_lines(data: &SystemStatus, p: &Painter) -> Vec { let fps_pct = ratio_percent(fps_ratio); let bar = render_fps_bar(fps_ratio, 24, p); let fps_display = format!("{actual_fps:>4.1} {} {target_fps}", p.muted("/")); - let health_word = format_fps_health(fps_pct, p); + // A parked loop reports zero FPS by design, not as a failing rate. + let health_word = if render_loop.output_demand.parked { + p.muted("parked") + } else { + format_fps_health(fps_pct, p) + }; lines.push(format!( " {} {} {bar} {health_word}", @@ -995,6 +1000,18 @@ mod tests { assert!(parked_lines.contains("parked: nothing consumes frames")); assert!(!rendering_lines.contains("parked")); + + // Zero FPS reads as a failing rate only while the loop renders. + let fps_line = |lines: &str| { + lines + .lines() + .find(|line| line.contains("/ 60")) + .map(str::to_owned) + .expect("status should print the render rate") + }; + assert!(fps_line(&parked_lines).ends_with("parked")); + assert!(!fps_line(&parked_lines).contains("0%")); + assert!(fps_line(&rendering_lines).ends_with("0%")); } #[test] From 74d06c41ddebfd556f7a7758b029ddddef1b384f Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 09:51:41 -0700 Subject: [PATCH 23/26] docs(render): say a woken loop resumes at its current tier The parking docs promised that a woken loop resumes at its full tier, but unpark keeps whatever tier the FPS controller held before the park, so a loop that downshifted resumes downshifted. The type docs, the diagnose detail, the REST, architecture, and troubleshooting pages, and the regenerated Python client now say current tier. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/src/domain/diagnostics.rs | 2 +- crates/hypercolor-daemon/src/render_thread/output_gate.rs | 2 +- crates/hypercolor-daemon/tests/output_demand_tests.rs | 2 +- crates/hypercolor-types/src/api/system.rs | 2 +- docs/content/api/rest.md | 2 +- docs/content/architecture/render-pipeline.md | 2 +- docs/content/troubleshooting/performance.md | 2 +- .../hypercolor/_generated/models/diagnose_render_snapshot.py | 2 +- python/src/hypercolor/_generated/models/render_loop_status.py | 2 +- .../hypercolor/_generated/models/render_output_demand_status.py | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/hypercolor-daemon/src/domain/diagnostics.rs b/crates/hypercolor-daemon/src/domain/diagnostics.rs index 6dcff6f24..a35c834c0 100644 --- a/crates/hypercolor-daemon/src/domain/diagnostics.rs +++ b/crates/hypercolor-daemon/src/domain/diagnostics.rs @@ -419,7 +419,7 @@ fn render_checks(inputs: &DiagnosticsInputs) -> Vec { name: "frame_liveness".to_owned(), status: "pass".to_owned(), detail: format!( - "parked: nothing consumes rendered frames (no output devices, streams, or captures); the next consumer resumes rendering at the full tier. parks_total={}", + "parked: nothing consumes rendered frames (no output devices, streams, or captures); the next consumer resumes rendering at the current tier. parks_total={}", performance.output_demand.parks_total ), }); diff --git a/crates/hypercolor-daemon/src/render_thread/output_gate.rs b/crates/hypercolor-daemon/src/render_thread/output_gate.rs index efd3ad148..bf1347fa1 100644 --- a/crates/hypercolor-daemon/src/render_thread/output_gate.rs +++ b/crates/hypercolor-daemon/src/render_thread/output_gate.rs @@ -8,7 +8,7 @@ //! the loop instead of rendering. A parked loop sleeps on the bus output //! demand signal, the device registry, the input demand registry, and the //! scene transaction queue, so the first consumer to appear wakes it at -//! once and it resumes at its full tier. A long safety tick re-evaluates in +//! once and it resumes at its current tier. A long safety tick re-evaluates in //! case some consumer arrived without signalling. //! //! Parking waits for the producers to settle first. After the scene, the diff --git a/crates/hypercolor-daemon/tests/output_demand_tests.rs b/crates/hypercolor-daemon/tests/output_demand_tests.rs index 71492c30e..cc2a13d21 100644 --- a/crates/hypercolor-daemon/tests/output_demand_tests.rs +++ b/crates/hypercolor-daemon/tests/output_demand_tests.rs @@ -1,7 +1,7 @@ //! Integration tests for output-demand gating. //! //! The render loop parks while nothing consumes its frames and resumes at -//! its full tier the moment a consumer appears. These tests drive the real +//! its current tier the moment a consumer appears. These tests drive the real //! render thread and prove each consumer kind keeps or wakes it, that a //! capture after a park returns a freshly rendered frame, and that effect //! time carries across the park without a jump. diff --git a/crates/hypercolor-types/src/api/system.rs b/crates/hypercolor-types/src/api/system.rs index d6093d01b..fc5066fd8 100644 --- a/crates/hypercolor-types/src/api/system.rs +++ b/crates/hypercolor-types/src/api/system.rs @@ -337,7 +337,7 @@ pub struct RenderLoopStatus { /// The render loop renders only while something consumes its output: a /// connected device, a display, a preview or canvas stream, a capture, or /// an explicit demand hold. With none of those it parks, renders nothing, -/// and resumes at its full tier the moment a consumer appears. +/// and resumes at its current tier the moment a consumer appears. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] #[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] pub struct RenderOutputDemandStatus { diff --git a/docs/content/api/rest.md b/docs/content/api/rest.md index 5f23b4405..dc0e4ffae 100644 --- a/docs/content/api/rest.md +++ b/docs/content/api/rest.md @@ -262,7 +262,7 @@ keyed daemon receive the public identity without the status block. `render_loop.output_demand` says what consumes rendered frames. With no device, stream, preview, or capture attached the loop parks: `parked` turns true, `idle_reason` reads `no_output_consumers`, `parked_ms` counts up, and `delivered_fps` and `actual_fps` read -zero until a consumer appears and the loop resumes at its full tier. +zero until a consumer appears and the loop resumes at its current tier. `status.effect_count` reflects whatever the registry holds at request time (native built-ins plus discovered HTML effects); treat it as live, not a fixed product diff --git a/docs/content/architecture/render-pipeline.md b/docs/content/architecture/render-pipeline.md index 5e5f5343f..fed9e0744 100644 --- a/docs/content/architecture/render-pipeline.md +++ b/docs/content/architecture/render-pipeline.md @@ -175,7 +175,7 @@ Every frame costs producer, compositor, and GPU work whether or not anything rec - WebSocket and diagnostic subscribers to host input events, which the loop republishes; - explicit output demand holds, taken by anything that reads published frames without a receiver. -With none of them, the loop **parks**: it renders nothing, withdraws its audio, screen, and interaction capture demand, drops live media streams, and throttles HTML effect pages, which stops the `requestAnimationFrame` loops they schedule for themselves. Page timers keep their own pace, so an effect that animates from `setTimeout` or `setInterval` rather than `requestAnimationFrame` keeps running while parked; the bundled effects all animate from `requestAnimationFrame`. A parked loop sleeps until something can observe a frame. Subscribing to a stream, taking a demand hold, connecting a device, or queuing a layout change wakes it at once, and it resumes at its full tier; any other bus event makes it re-check whether it has work. A one second safety tick re-checks in case a consumer arrived without signalling. No observable cadence changes, because the moment anything can see a frame, it renders at full rate. +With none of them, the loop **parks**: it renders nothing, withdraws its audio, screen, and interaction capture demand, drops live media streams, and throttles HTML effect pages, which stops the `requestAnimationFrame` loops they schedule for themselves. Page timers keep their own pace, so an effect that animates from `setTimeout` or `setInterval` rather than `requestAnimationFrame` keeps running while parked; the bundled effects all animate from `requestAnimationFrame`. A parked loop sleeps until something can observe a frame. Subscribing to a stream, taking a demand hold, connecting a device, or queuing a layout change wakes it at once, and it resumes at its current tier; any other bus event makes it re-check whether it has work. A one second safety tick re-checks in case a consumer arrived without signalling. No observable cadence changes, because the moment anything can see a frame, it renders at the rate it would have without the park. Parking waits for the producers to settle. After the scene, an effect, or the canvas changes, the loop keeps rendering until every producer has shown its new content and any scene transition has finished (bounded at ten seconds, so a longer crossfade pauses partway and finishes when a consumer returns), so an effect applied while nobody watches is already loaded when a client or capture arrives. Time carries across the park: render time stops while parked, so media layers and time bindings continue where they left off, and the first frame after a wake advances effect time by one frame, not by the parked span, with no catch-up frames. A canvas capture that wakes the loop skips the wake frame itself, whose HTML producers still show output requested before the park, and returns the next one. diff --git a/docs/content/troubleshooting/performance.md b/docs/content/troubleshooting/performance.md index 240e88ddc..b261392ec 100644 --- a/docs/content/troubleshooting/performance.md +++ b/docs/content/troubleshooting/performance.md @@ -67,7 +67,7 @@ The controller downshifts after 2 consecutive budget misses and only upshifts af render_loop state=running, tier=full, parked=no_output_consumers ``` -`parked` means nothing consumes rendered frames: no connected device, no open preview or stream, no capture. The loop renders nothing until one appears, then resumes at its full tier, so `0 fps` here is expected rather than a stall. `frame_liveness` passes while parked, and `hypercolor status` shows the same state on its render line. `snapshot.render.output_demand` in the JSON report lists the consumer counts the loop last saw and what last woke it. +`parked` means nothing consumes rendered frames: no connected device, no open preview or stream, no capture. The loop renders nothing until one appears, then resumes at its current tier, so `0 fps` here is expected rather than a stall. `frame_liveness` passes while parked, and `hypercolor status` shows the same state on its render line. `snapshot.render.output_demand` in the JSON report lists the consumer counts the loop last saw and what last woke it. ### `render_thread` diff --git a/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py b/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py index 7e9b61776..487ec8b55 100644 --- a/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py +++ b/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py @@ -30,7 +30,7 @@ class DiagnoseRenderSnapshot: The render loop renders only while something consumes its output: a connected device, a display, a preview or canvas stream, a capture, or an explicit demand hold. With none of those it parks, renders nothing, - and resumes at its full tier the moment a consumer appears. + and resumes at its current tier the moment a consumer appears. supervision (RenderSupervisionStatus | Unset): Health and recovery state of the supervised render thread. The render thread restarts its pipeline in-process after a panic or an diff --git a/python/src/hypercolor/_generated/models/render_loop_status.py b/python/src/hypercolor/_generated/models/render_loop_status.py index fa96480a0..68f7f2cfa 100644 --- a/python/src/hypercolor/_generated/models/render_loop_status.py +++ b/python/src/hypercolor/_generated/models/render_loop_status.py @@ -34,7 +34,7 @@ class RenderLoopStatus: The render loop renders only while something consumes its output: a connected device, a display, a preview or canvas stream, a capture, or an explicit demand hold. With none of those it parks, renders nothing, - and resumes at its full tier the moment a consumer appears. + and resumes at its current tier the moment a consumer appears. """ actual_fps: float diff --git a/python/src/hypercolor/_generated/models/render_output_demand_status.py b/python/src/hypercolor/_generated/models/render_output_demand_status.py index 0d3b61ed9..bccff0573 100644 --- a/python/src/hypercolor/_generated/models/render_output_demand_status.py +++ b/python/src/hypercolor/_generated/models/render_output_demand_status.py @@ -19,7 +19,7 @@ class RenderOutputDemandStatus: The render loop renders only while something consumes its output: a connected device, a display, a preview or canvas stream, a capture, or an explicit demand hold. With none of those it parks, renders nothing, - and resumes at its full tier the moment a consumer appears. + and resumes at its current tier the moment a consumer appears. Attributes: demand_holds (int): Explicit demand holds, such as display output for previewed From 2a978e7b8a461987a228677d5c0fcd2d362b3b31 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 10:09:06 -0700 Subject: [PATCH 24/26] fix(panic): start backtrace excerpts at the panicking frame on Windows The excerpt skipped every leading frame that looked like panic machinery, including anything under std::panicking. MSVC symbols keep their generic arguments, so the code that panicked runs inside frames such as std::panicking::catch_unwind::do_call, and on Windows the skip ran all the way to thread_start. The Windows CI job caught it: the recorded excerpt held nothing but thread startup frames. The excerpt now starts right after the last panic entry point near the top of the backtrace (core::panicking, begin_panic_handler, panic_with_hook, __rust_end_short_backtrace, and the unwrap and expect failure paths). Without one it skips only the capture frames. A unit test pins an MSVC-shaped backtrace. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/src/panic_hook.rs | 122 ++++++++++++++++++--- 1 file changed, 106 insertions(+), 16 deletions(-) diff --git a/crates/hypercolor-daemon/src/panic_hook.rs b/crates/hypercolor-daemon/src/panic_hook.rs index d0b1a09e6..ec417acfb 100644 --- a/crates/hypercolor-daemon/src/panic_hook.rs +++ b/crates/hypercolor-daemon/src/panic_hook.rs @@ -201,18 +201,7 @@ fn remember(report: &PanicReport) { #[must_use] pub fn backtrace_excerpt(backtrace: &str) -> String { let frames = split_frames(backtrace); - let start = frames - .iter() - .take(PANIC_MACHINERY_SCAN_FRAMES) - .position(|frame| is_panic_machinery(frame)) - .map_or(0, |entry| { - entry - + frames - .iter() - .skip(entry) - .take_while(|frame| is_panic_machinery(frame)) - .count() - }); + let start = panicking_frame_index(&frames); let mut excerpt = String::new(); for frame in frames.iter().skip(start).take(BACKTRACE_EXCERPT_FRAMES) { if excerpt.len() + frame.len() > BACKTRACE_EXCERPT_BYTES { @@ -257,19 +246,61 @@ fn starts_frame(line: &str) -> bool { digits > 0 && trimmed[digits..].starts_with(": ") } -fn is_panic_machinery(frame: &str) -> bool { - let symbol = frame.lines().next().unwrap_or_default(); +/// Index of the first frame of the code that panicked. +/// +/// That is the frame right after the last panic entry point near the top. +/// Matching entry points, rather than every `std::panicking` frame, matters +/// on MSVC targets: their symbols keep generic arguments, so the code that +/// panicked can run inside a frame such as +/// `std::panicking::catch_unwind::do_call`. +/// Without an entry point, the excerpt skips only the capture frames. +fn panicking_frame_index(frames: &[&str]) -> usize { + frames + .iter() + .take(PANIC_MACHINERY_SCAN_FRAMES) + .rposition(|frame| is_panic_entry(frame)) + .map_or_else( + || { + frames + .iter() + .take_while(|frame| is_capture_frame(frame)) + .count() + }, + |entry| entry + 1, + ) +} + +fn frame_symbol(frame: &str) -> &str { + frame.lines().next().unwrap_or_default() +} + +/// Frames of the backtrace capture and this hook. +fn is_capture_frame(frame: &str) -> bool { + let symbol = frame_symbol(frame); [ "std::backtrace_rs::", "std::backtrace::", "hypercolor_daemon::panic_hook::", " as core::ops::function::Fn>::call", - "std::panicking::", - "std::sys::backtrace::", + ] + .iter() + .any(|needle| symbol.contains(needle)) +} + +/// Frames of the runtime path from a `panic!` to the hook. +fn is_panic_entry(frame: &str) -> bool { + let symbol = frame_symbol(frame); + [ "rust_begin_unwind", "core::panicking::", "core::option::expect_failed", + "core::option::unwrap_failed", "core::result::unwrap_failed", + "std::panicking::begin_panic", + "std::panicking::panic_with_hook", + "std::panicking::rust_panic_with_hook", + "std::rt::begin_panic", + "std::sys::backtrace::__rust_end_short_backtrace", ] .iter() .any(|needle| symbol.contains(needle)) @@ -307,6 +338,65 @@ mod tests { assert!(!excerpt.contains("force_capture")); } + /// An MSVC backtrace: symbols keep their generic arguments, and the + /// panicking closure runs inside a `std::panicking` frame. + const CAPTURED_MSVC: &str = " 0: std::backtrace_rs::backtrace::win64::trace + at /rustc/library\\std\\src\\..\\..\\backtrace\\src\\backtrace\\win64.rs:85 + 1: std::backtrace::Backtrace::create + at /rustc/library\\std\\src\\backtrace.rs:331 + 2: hypercolor_daemon::panic_hook::PanicReport::capture + at .\\src\\panic_hook.rs:60 + 3: hypercolor_daemon::panic_hook::install::closure$0::closure$0 + at .\\src\\panic_hook.rs:120 + 4: alloc::boxed::impl$30::call >,dyn$ > > >,alloc::alloc::Global> + 5: std::panicking::panic_with_hook + at /rustc/library\\std\\src\\panicking.rs:842 + 6: std::panicking::begin_panic_handler::closure$0 + at /rustc/library\\std\\src\\panicking.rs:707 + 7: std::sys::backtrace::__rust_end_short_backtrace + 8: std::panicking::begin_panic_handler + at /rustc/library\\std\\src\\panicking.rs:698 + 9: core::panicking::panic_fmt + at /rustc/library\\core\\src\\panicking.rs:75 + 10: std::panicking::catch_unwind::do_call > + at .\\src\\render_thread\\pipeline_driver.rs:40 + 11: std::panicking::catch_unwind,hypercolor_daemon::render_thread::pipeline_driver::run_pipeline::closure_env$0> + 12: std::sys::backtrace::__rust_begin_short_backtrace > + 13: std::sys::thread::windows::impl$0::new::thread_start +"; + + #[test] + fn excerpt_starts_at_the_panicking_frame_with_msvc_symbols() { + let excerpt = backtrace_excerpt(CAPTURED_MSVC); + + assert!( + excerpt.trim_start().starts_with( + "10: std::panicking::catch_unwind::do_call + 3: some::caller::run +", + ); + + assert!( + excerpt + .trim_start() + .starts_with("2: std::panicking::catch_unwind"), + "{excerpt}" + ); + } + #[test] fn excerpt_without_recognizable_frames_keeps_the_raw_text() { assert_eq!( From 68359c1b4e2091f47d1a1d85d4eea36798f72706 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 11:22:03 -0700 Subject: [PATCH 25/26] fix(panic): stop the excerpt skip at the first unrecognized frame The entry-point rule still produced an excerpt of thread startup frames on the Windows CI job, which builds tests without debuginfo, so its frames symbolize differently from any fixture here. Searching for the last entry point can land past the panic site when an unexpected frame matches. The excerpt now skips only the contiguous run of capture and panic runtime frames at the top and stops at the first frame it does not recognize: an unfamiliar frame costs a few extra lines instead of hiding the panic site. The MSVC form of the boxed hook call is matched by its PanicHookInfo argument. The hook test prints the full backtrace when it fails, so a Windows miss shows the real frames. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/src/panic_hook.rs | 25 ++++++++----------- .../tests/panic_hook_tests.rs | 5 ++-- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/crates/hypercolor-daemon/src/panic_hook.rs b/crates/hypercolor-daemon/src/panic_hook.rs index ec417acfb..d67c7e1cd 100644 --- a/crates/hypercolor-daemon/src/panic_hook.rs +++ b/crates/hypercolor-daemon/src/panic_hook.rs @@ -248,26 +248,19 @@ fn starts_frame(line: &str) -> bool { /// Index of the first frame of the code that panicked. /// -/// That is the frame right after the last panic entry point near the top. -/// Matching entry points, rather than every `std::panicking` frame, matters -/// on MSVC targets: their symbols keep generic arguments, so the code that -/// panicked can run inside a frame such as +/// The excerpt skips the contiguous run of capture and panic runtime frames +/// at the top and stops at the first frame it does not recognize, so an +/// unfamiliar runtime frame costs a few extra lines rather than hiding the +/// panic site. The runtime is matched by its entry points, not by every +/// `std::panicking` frame: MSVC symbols keep generic arguments, so the code +/// that panicked can run inside a frame such as /// `std::panicking::catch_unwind::do_call`. -/// Without an entry point, the excerpt skips only the capture frames. fn panicking_frame_index(frames: &[&str]) -> usize { frames .iter() .take(PANIC_MACHINERY_SCAN_FRAMES) - .rposition(|frame| is_panic_entry(frame)) - .map_or_else( - || { - frames - .iter() - .take_while(|frame| is_capture_frame(frame)) - .count() - }, - |entry| entry + 1, - ) + .take_while(|frame| is_capture_frame(frame) || is_panic_entry(frame)) + .count() } fn frame_symbol(frame: &str) -> &str { @@ -282,6 +275,8 @@ fn is_capture_frame(frame: &str) -> bool { "std::backtrace::", "hypercolor_daemon::panic_hook::", " as core::ops::function::Fn>::call", + // The boxed hook's call on MSVC, which names its argument type. + "PanicHookInfo", ] .iter() .any(|needle| symbol.contains(needle)) diff --git a/crates/hypercolor-daemon/tests/panic_hook_tests.rs b/crates/hypercolor-daemon/tests/panic_hook_tests.rs index b5a80e522..d82dc2c46 100644 --- a/crates/hypercolor-daemon/tests/panic_hook_tests.rs +++ b/crates/hypercolor-daemon/tests/panic_hook_tests.rs @@ -76,8 +76,9 @@ fn hook_reports_a_named_thread_panic_to_observers_the_catcher_and_the_previous_h ); assert!( recent.backtrace_excerpt().contains("panic_hook_tests"), - "the excerpt should start near the panicking frame:\n{}", - recent.backtrace_excerpt() + "the excerpt should start near the panicking frame:\n{}\nfull backtrace:\n{}", + recent.backtrace_excerpt(), + recent.backtrace ); assert!( PREVIOUS_HOOK_CALLS.load(Ordering::SeqCst) > previous_calls, From d8cadbf65c66c2dce2f605b9886f860ad8294180 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Wed, 21 Oct 2026 12:22:01 -0700 Subject: [PATCH 26/26] fix(panic): cut backtrace excerpts at std's short backtrace marker The Windows CI job printed the real backtrace: its tests build without debuginfo, so the caller's frames take the nearest public symbol, and here every one of them read core::panicking::assert_failed. Any rule that skips panic runtime frames by name then skips the caller too, and the backtrace never names the test at all. The excerpt now starts right after __rust_end_short_backtrace, the cut std's own short backtraces make, so it opens with the panic entry and then the caller however its frames are labeled. Without the marker it falls back to skipping the recognized capture and runtime frames at the top. A fixture pins that Windows backtrace. The hook test now holds the excerpt to the frames the backtrace actually names, and always to skipping the capture frames. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/hypercolor-daemon/src/panic_hook.rs | 80 +++++++++++++++---- .../tests/panic_hook_tests.rs | 12 ++- 2 files changed, 73 insertions(+), 19 deletions(-) diff --git a/crates/hypercolor-daemon/src/panic_hook.rs b/crates/hypercolor-daemon/src/panic_hook.rs index d67c7e1cd..780c7cce2 100644 --- a/crates/hypercolor-daemon/src/panic_hook.rs +++ b/crates/hypercolor-daemon/src/panic_hook.rs @@ -246,21 +246,37 @@ fn starts_frame(line: &str) -> bool { digits > 0 && trimmed[digits..].starts_with(": ") } -/// Index of the first frame of the code that panicked. +/// The frame std's own short backtraces start after: everything above it +/// is the backtrace capture, this hook, and the hook dispatch. +const SHORT_BACKTRACE_END: &str = "__rust_end_short_backtrace"; + +/// Index of the first frame worth keeping. /// -/// The excerpt skips the contiguous run of capture and panic runtime frames -/// at the top and stops at the first frame it does not recognize, so an -/// unfamiliar runtime frame costs a few extra lines rather than hiding the -/// panic site. The runtime is matched by its entry points, not by every -/// `std::panicking` frame: MSVC symbols keep generic arguments, so the code -/// that panicked can run inside a frame such as -/// `std::panicking::catch_unwind::do_call`. +/// That is the frame after std's short backtrace marker, the same cut a +/// default panic message makes, so the excerpt opens with the panic entry +/// (`begin_panic_handler`, `panic_fmt`) and then the code that panicked. +/// Frame names cannot place the cut more precisely: a build without +/// debuginfo, such as the Windows CI tests, labels the caller's frames with +/// the nearest public symbol, which can be a `core::panicking` function. +/// Without the marker, the excerpt skips the contiguous run of capture and +/// panic runtime frames at the top and stops at the first frame it does not +/// recognize, so an unfamiliar frame costs a few extra lines rather than +/// hiding the panic site. fn panicking_frame_index(frames: &[&str]) -> usize { frames .iter() .take(PANIC_MACHINERY_SCAN_FRAMES) - .take_while(|frame| is_capture_frame(frame) || is_panic_entry(frame)) - .count() + .position(|frame| frame_symbol(frame).contains(SHORT_BACKTRACE_END)) + .map_or_else( + || { + frames + .iter() + .take(PANIC_MACHINERY_SCAN_FRAMES) + .take_while(|frame| is_capture_frame(frame) || is_panic_entry(frame)) + .count() + }, + |marker| marker + 1, + ) } fn frame_symbol(frame: &str) -> &str { @@ -295,7 +311,6 @@ fn is_panic_entry(frame: &str) -> bool { "std::panicking::panic_with_hook", "std::panicking::rust_panic_with_hook", "std::rt::begin_panic", - "std::sys::backtrace::__rust_end_short_backtrace", ] .iter() .any(|needle| symbol.contains(needle)) @@ -361,17 +376,50 @@ mod tests { "; #[test] - fn excerpt_starts_at_the_panicking_frame_with_msvc_symbols() { + fn excerpt_starts_after_the_short_backtrace_marker_with_msvc_symbols() { let excerpt = backtrace_excerpt(CAPTURED_MSVC); assert!( - excerpt.trim_start().starts_with( - "10: std::panicking::catch_unwind::do_call previous_calls, "the hook installed before the daemon hook should still run"