diff --git a/crates/hypercolor-app/tests/packaging_tests.rs b/crates/hypercolor-app/tests/packaging_tests.rs index 0d489861d..942b4a374 100644 --- a/crates/hypercolor-app/tests/packaging_tests.rs +++ b/crates/hypercolor-app/tests/packaging_tests.rs @@ -1804,6 +1804,37 @@ fn pawnio_lookups_see_the_64_bit_install_from_any_powershell() { } } +/// The daemon exits nonzero when its render thread cannot recover in +/// process. Every launcher that runs it as a service must restart it on +/// that exit, not only on a crash: systemd on failure, and the Windows +/// service manager only once the failure flag extends its recovery actions +/// to non-crash failures. +#[test] +fn daemon_service_launchers_restart_a_daemon_that_exits_with_an_error() { + for (name, unit) in [ + ("hypercolor.service", SYSTEMD_USER_UNIT), + ("hypercolor.service.system", SYSTEMD_PACKAGED_USER_UNIT), + ] { + assert!( + unit.lines().any(|line| line.trim() == "Restart=on-failure"), + "{name} must restart the daemon after an error exit" + ); + } + let code: Vec<&str> = INSTALL_WINDOWS_SERVICE_PS1 + .lines() + .filter(|line| !line.trim_start().starts_with('#')) + .collect(); + for step in [ + "sc.exe failure $ServiceName reset= 86400 actions= restart/", + "sc.exe failureflag $ServiceName 1", + ] { + assert!( + code.iter().any(|line| line.contains(step)), + "the daemon service installer is missing {step}" + ); + } +} + /// `Start-Service` and `Stop-Service` wait without a bound while a service /// is pending, which would hang the installer. The broker installer bounds /// every wait, checks the broker stayed up, and on failure prints the diff --git a/crates/hypercolor-cli/src/commands/status.rs b/crates/hypercolor-cli/src/commands/status.rs index 298688eae..111bdf88a 100644 --- a/crates/hypercolor-cli/src/commands/status.rs +++ b/crates/hypercolor-cli/src/commands/status.rs @@ -361,7 +361,12 @@ fn status_table_lines(data: &SystemStatus, p: &Painter) -> Vec { let fps_pct = ratio_percent(fps_ratio); let bar = render_fps_bar(fps_ratio, 24, p); let fps_display = format!("{actual_fps:>4.1} {} {target_fps}", p.muted("/")); - let health_word = format_fps_health(fps_pct, p); + // A parked loop reports zero FPS by design, not as a failing rate. + let health_word = if render_loop.output_demand.parked { + p.muted("parked") + } else { + format_fps_health(fps_pct, p) + }; lines.push(format!( " {} {} {bar} {health_word}", @@ -381,6 +386,13 @@ fn status_table_lines(data: &SystemStatus, p: &Painter) -> Vec { p.muted(&format!("{} frames", format_count(total_frames))), p.muted(""), )); + if render_loop.output_demand.parked { + lines.push(format!( + " {} {}", + p.muted(&pad("", 10)), + p.muted("parked: nothing consumes frames, so nothing renders until a device, preview, or capture appears"), + )); + } // ── Frame budget ──────────────────────────────────────────────── if let Some(latest_frame) = &data.latest_frame { @@ -611,8 +623,8 @@ mod tests { use hypercolor_types::api::system::{ LatestFrameStatus, MacosCapabilityOwner, MacosDaemonHandoverPhase, MacosDaemonOwnerConflictStatus, MacosDaemonOwnerRecoveryRequiredStatus, - MacosDaemonOwnershipStatus, PreviewRuntimeStatus, RenderLoopStatus, RenderSurfaceStatus, - ServerInfo, SystemResource, SystemStatus, + MacosDaemonOwnershipStatus, PreviewRuntimeStatus, RenderLoopStatus, + RenderOutputDemandStatus, RenderSurfaceStatus, ServerInfo, SystemResource, SystemStatus, }; use serde_json::json; @@ -958,6 +970,50 @@ mod tests { assert_eq!(format_count(1_234_567), "1,234,567"); } + #[test] + fn status_lines_explain_a_parked_render_loop() { + let painter = Painter::plain(); + let parked = SystemStatus { + render_loop: RenderLoopStatus { + state: "running".to_owned(), + target_fps: 60, + output_demand: RenderOutputDemandStatus { + parked: true, + idle_reason: Some("no_output_consumers".to_owned()), + ..RenderOutputDemandStatus::default() + }, + ..RenderLoopStatus::default() + }, + ..SystemStatus::default() + }; + let rendering = SystemStatus { + render_loop: RenderLoopStatus { + state: "running".to_owned(), + target_fps: 60, + ..RenderLoopStatus::default() + }, + ..SystemStatus::default() + }; + + let parked_lines = status_table_lines(&parked, &painter).join("\n"); + let rendering_lines = status_table_lines(&rendering, &painter).join("\n"); + + assert!(parked_lines.contains("parked: nothing consumes frames")); + assert!(!rendering_lines.contains("parked")); + + // Zero FPS reads as a failing rate only while the loop renders. + let fps_line = |lines: &str| { + lines + .lines() + .find(|line| line.contains("/ 60")) + .map(str::to_owned) + .expect("status should print the render rate") + }; + assert!(fps_line(&parked_lines).ends_with("parked")); + assert!(!fps_line(&parked_lines).contains("0%")); + assert!(fps_line(&rendering_lines).ends_with("0%")); + } + #[test] fn status_lines_include_core_fields() { let data = SystemStatus { diff --git a/crates/hypercolor-core/src/bus/demand.rs b/crates/hypercolor-core/src/bus/demand.rs new file mode 100644 index 000000000..93807161c --- /dev/null +++ b/crates/hypercolor-core/src/bus/demand.rs @@ -0,0 +1,113 @@ +//! Output-demand signalling for the latest-value bus lanes. +//! +//! A producer that publishes onto watch lanes can stop producing while +//! nothing consumes its output, but watch channels never announce a new +//! subscriber. [`OutputDemandSignal`] closes that gap: every lane +//! subscription, every [`OutputDemandHold`], and every discrete event +//! published on the bus notifies it, so an idle producer can sleep on the +//! signal and re-evaluate the moment a consumer appears or observable state +//! changes, instead of polling for either. + +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering, fence}; + +use tokio::sync::Notify; +use tokio::sync::futures::Notified; + +/// Wake signal for producers that idle while nothing consumes their output. +/// +/// Every event publication notifies it, so notification skips the waiter +/// list entirely unless a producer has armed a wait. +#[derive(Debug, Default)] +pub struct OutputDemandSignal { + epoch: AtomicU64, + holds: AtomicUsize, + armed: AtomicUsize, + notify: Notify, +} + +impl OutputDemandSignal { + /// Monotonic count of notifications, for observing that one happened. + #[must_use] + pub fn epoch(&self) -> u64 { + self.epoch.load(Ordering::Acquire) + } + + /// Number of live [`OutputDemandHold`]s. + #[must_use] + pub fn hold_count(&self) -> usize { + self.holds.load(Ordering::Acquire) + } + + /// Arm a wait. Notifications reach waiters only while at least one + /// [`ArmedOutputDemand`] lives, so arm before re-checking demand. + pub fn arm(&self) -> ArmedOutputDemand<'_> { + self.armed.fetch_add(1, Ordering::SeqCst); + // Orders the arm before every demand read the waiter makes next. + fence(Ordering::SeqCst); + ArmedOutputDemand { signal: self } + } + + /// Wake every current waiter. + pub fn notify(&self) { + self.epoch.fetch_add(1, Ordering::AcqRel); + // The state change that prompted this call (a new receiver, a hold, + // an event) may use weaker orderings, so a full fence orders it + // before the armed check. Paired with the fence in `arm`, either + // this load sees the waiter and wakes it, or the waiter's demand + // read sees the state change. + fence(Ordering::SeqCst); + if self.armed.load(Ordering::SeqCst) > 0 { + self.notify.notify_waiters(); + } + } + + pub(super) fn acquire_hold(self: &Arc) -> OutputDemandHold { + self.holds.fetch_add(1, Ordering::AcqRel); + self.notify(); + OutputDemandHold { + signal: Arc::clone(self), + } + } +} + +/// An armed wait on an [`OutputDemandSignal`]. +#[derive(Debug)] +pub struct ArmedOutputDemand<'a> { + signal: &'a OutputDemandSignal, +} + +impl ArmedOutputDemand<'_> { + /// Future that resolves on the next notification. + /// + /// Pin it and call [`Notified::enable`] before re-checking demand: a + /// notification that lands between the check and the await then still + /// wakes the waiter. + pub fn notified(&self) -> Notified<'_> { + self.signal.notify.notified() + } +} + +impl Drop for ArmedOutputDemand<'_> { + fn drop(&mut self) { + self.signal.armed.fetch_sub(1, Ordering::SeqCst); + } +} + +/// Keeps output demand registered for consumers that read the bus without +/// holding a lane receiver, such as a borrow of a per-zone sender. +/// +/// Dropping the hold withdraws the demand. Release needs no notification: +/// a producer only loses demand on its own next evaluation. +#[derive(Debug)] +#[must_use = "dropping the hold withdraws the output demand it registers"] +pub struct OutputDemandHold { + signal: Arc, +} + +impl Drop for OutputDemandHold { + fn drop(&mut self) { + let previous = self.signal.holds.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "output demand hold underflow"); + } +} diff --git a/crates/hypercolor-core/src/bus/mod.rs b/crates/hypercolor-core/src/bus/mod.rs index 57f18f567..b9b4b5c83 100644 --- a/crates/hypercolor-core/src/bus/mod.rs +++ b/crates/hypercolor-core/src/bus/mod.rs @@ -9,12 +9,17 @@ //! - **Canvas previews** — latest render and screen-source canvases via `tokio::sync::watch`. //! - **Authoritative scene canvases** — latest full-scene display surface for non-preview consumers. //! - **Per-zone canvases** — latest render-zone canvases via per-zone `tokio::sync::watch`. +//! - **Output demand**: [`OutputDemandSignal`] wakes a producer that idles while +//! nothing consumes its output, whenever a lane gains a receiver, a demand hold is +//! taken, or an event is published. //! //! The bus is `Send + Sync` and cloneable. Channel operations are lock-free; //! low-frequency status-event deduplication uses a short critical section. +mod demand; mod filter; +pub use demand::{ArmedOutputDemand, OutputDemandHold, OutputDemandSignal}; pub use filter::{EventFilter, FilteredEventReceiver}; use std::collections::HashMap; @@ -632,24 +637,41 @@ pub struct WatchLane { sender: watch::Sender, published: AtomicU64, revision: AtomicU64, + demand: Arc, } impl WatchLane { /// Create a lane with its initial latest value. #[must_use] pub fn new(initial: T) -> Self { + Self::with_demand_signal(initial, Arc::default()) + } + + fn with_demand_signal(initial: T, demand: Arc) -> Self { let (sender, _) = watch::channel(initial); Self { sender, published: AtomicU64::new(0), revision: AtomicU64::new(0), + demand, } } /// Subscribe to the latest value. + /// + /// A new receiver is new output demand, so subscribing notifies the + /// lane's [`OutputDemandSignal`]. #[must_use] pub fn subscribe(&self) -> watch::Receiver { - self.sender.subscribe() + let receiver = self.sender.subscribe(); + self.demand.notify(); + receiver + } + + /// Number of live receivers. + #[must_use] + pub fn receiver_count(&self) -> usize { + self.sender.receiver_count() } /// Borrow the latest published value. @@ -739,16 +761,37 @@ pub struct BusLanes { } impl BusLanes { - fn new() -> Self { + fn new(demand: &Arc) -> Self { Self { - frame: WatchLane::new(FrameData::empty()), - spectrum: WatchLane::new(SpectrumData::empty()), - previews: std::array::from_fn(|_| WatchLane::new(CanvasFrame::empty())), - screen_zones: WatchLane::new(ScreenZonesFrame::default()), - zone_preview: WatchLane::new(Arc::from([])), + frame: WatchLane::with_demand_signal(FrameData::empty(), Arc::clone(demand)), + spectrum: WatchLane::with_demand_signal(SpectrumData::empty(), Arc::clone(demand)), + previews: std::array::from_fn(|_| { + WatchLane::with_demand_signal(CanvasFrame::empty(), Arc::clone(demand)) + }), + screen_zones: WatchLane::with_demand_signal( + ScreenZonesFrame::default(), + Arc::clone(demand), + ), + zone_preview: WatchLane::with_demand_signal(Arc::from([]), Arc::clone(demand)), } } + /// Live receivers across every fixed lane. + #[must_use] + pub fn receiver_count(&self) -> usize { + self.frame + .receiver_count() + .saturating_add(self.spectrum.receiver_count()) + .saturating_add( + self.previews + .iter() + .map(WatchLane::receiver_count) + .fold(0, usize::saturating_add), + ) + .saturating_add(self.screen_zones.receiver_count()) + .saturating_add(self.zone_preview.receiver_count()) + } + #[must_use] pub const fn frame(&self) -> &WatchLane { &self.frame @@ -812,6 +855,9 @@ pub struct HypercolorBus { /// Display-output-authored final display surface metadata keyed by zone id. display_zone_output_routes: Arc>, + /// Notified when output demand appears or observable state changes. + output_demand: Arc, + /// Clock every event publication is stamped against. clock: PublicationClock, } @@ -831,15 +877,17 @@ impl HypercolorBus { #[must_use] pub fn with_clock(clock: PublicationClock) -> Self { let (events, _) = broadcast::channel(EVENT_CHANNEL_CAPACITY); + let output_demand = Arc::new(OutputDemandSignal::default()); Self { events, input_status_events: Arc::new(Mutex::new(HashMap::new())), - lanes: Arc::new(BusLanes::new()), + lanes: Arc::new(BusLanes::new(&output_demand)), zone_canvases: Arc::new(Mutex::new(HashMap::new())), display_zone_targets: Arc::new(Mutex::new(DisplayZoneTargetRegistry::default())), display_zone_output_routes: Arc::new(Mutex::new( DisplayZoneOutputRouteRegistry::default(), )), + output_demand, clock, } } @@ -868,6 +916,9 @@ impl HypercolorBus { }; // Ignore send errors -- they mean no subscribers exist. let _ = self.events.send(timestamped); + // Discrete events mark state changes an idle producer may need to + // render before anyone asks for a frame. + self.output_demand.notify(); } /// Publish a source-health transition when its operational state changed. @@ -1126,7 +1177,44 @@ impl HypercolorBus { /// Subscribe to a zone's canvas updates. #[must_use] pub fn zone_canvas_receiver(&self, id: ZoneId) -> watch::Receiver { - self.zone_canvas_sender(id).subscribe() + let receiver = self.zone_canvas_sender(id).subscribe(); + self.output_demand.notify(); + receiver + } + + /// Live receivers across every per-zone canvas stream. + #[must_use] + pub fn zone_canvas_receiver_count(&self) -> usize { + self.zone_canvases + .lock() + .expect("zone canvas registry should not be poisoned") + .values() + .map(watch::Sender::receiver_count) + .fold(0, usize::saturating_add) + } + + // ── Output demand ──────────────────────────────────────────────── + + /// The signal notified by lane subscriptions, demand holds, and event + /// publication. + #[must_use] + pub fn output_demand(&self) -> &OutputDemandSignal { + &self.output_demand + } + + /// Register output demand for a consumer that reads published frames + /// without holding a lane receiver. + pub fn hold_output_demand(&self) -> OutputDemandHold { + self.output_demand.acquire_hold() + } + + /// Live receivers on every latest-value output stream: the fixed lanes + /// and the per-zone canvas streams. + #[must_use] + pub fn output_receiver_count(&self) -> usize { + self.lanes + .receiver_count() + .saturating_add(self.zone_canvas_receiver_count()) } /// Number of tracked per-zone canvas streams. diff --git a/crates/hypercolor-core/src/device/manager/mapping.rs b/crates/hypercolor-core/src/device/manager/mapping.rs index 69b331867..978ed1ef9 100644 --- a/crates/hypercolor-core/src/device/manager/mapping.rs +++ b/crates/hypercolor-core/src/device/manager/mapping.rs @@ -148,6 +148,16 @@ impl BackendManager { self.device_map.len() } + /// Every routed physical device as `(backend_id, device_id)`. + /// + /// A device bound under several layout ids appears once per binding; + /// callers that count devices deduplicate. + pub fn mapped_device_targets(&self) -> impl Iterator + '_ { + self.device_map + .values() + .map(|mapping| (mapping.backend_id.as_str(), mapping.device_id)) + } + /// Each spatial layout `device_id` with the physical device it routes to. /// /// Anything derived from these bindings stays valid until diff --git a/crates/hypercolor-core/src/device/registry.rs b/crates/hypercolor-core/src/device/registry.rs index 7816c177a..139ad94a3 100644 --- a/crates/hypercolor-core/src/device/registry.rs +++ b/crates/hypercolor-core/src/device/registry.rs @@ -9,7 +9,8 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; use hypercolor_driver_api::{DiscoveredDevice, DiscoveryConnectBehavior}; -use tokio::sync::RwLock; +use tokio::sync::futures::Notified; +use tokio::sync::{Notify, RwLock}; use tracing::{debug, info, trace, warn}; use hypercolor_types::device::{ @@ -67,6 +68,7 @@ impl TrackedDevice { pub struct DeviceRegistry { inner: Arc>, generation: Arc, + changed: Arc, } #[derive(Debug, Default)] @@ -152,6 +154,7 @@ impl DeviceRegistry { Self { inner: Arc::new(RwLock::new(RegistryInner::default())), generation: Arc::new(AtomicU64::new(0)), + changed: Arc::new(Notify::new()), } } @@ -561,6 +564,15 @@ impl DeviceRegistry { self.generation.load(Ordering::Relaxed) } + /// Future that resolves the next time [`Self::generation`] moves. + /// + /// Pin it and call [`Notified::enable`] before reading the registry, so + /// a mutation that lands between the read and the await still wakes the + /// waiter. + pub fn changed(&self) -> Notified<'_> { + self.changed.notified() + } + /// Number of devices currently tracked. pub async fn len(&self) -> usize { let inner = self.inner.read().await; @@ -805,6 +817,7 @@ impl DeviceRegistry { fn bump_generation(&self) { self.generation.fetch_add(1, Ordering::Relaxed); + self.changed.notify_waiters(); } } diff --git a/crates/hypercolor-core/src/effect/builtin/web_viewport.rs b/crates/hypercolor-core/src/effect/builtin/web_viewport.rs index fa93735bc..11f3022ea 100644 --- a/crates/hypercolor-core/src/effect/builtin/web_viewport.rs +++ b/crates/hypercolor-core/src/effect/builtin/web_viewport.rs @@ -286,6 +286,16 @@ impl EffectRenderer for WebViewportRenderer { Ok(()) } + fn park(&mut self) { + // A live web page keeps animating between renders; throttling it + // stops its self-scheduled animation frames until the next render. + if let Some(session) = self.session.as_ref() + && let Err(error) = session.park() + { + tracing::debug!(%error, "Could not throttle the parked web viewport page"); + } + } + fn apply_controls(&mut self, batch: &ControlDeltaBatch<'_>) -> anyhow::Result<()> { for (control_id, value) in batch.changes { match control_id.as_str() { diff --git a/crates/hypercolor-core/src/effect/media/producer.rs b/crates/hypercolor-core/src/effect/media/producer.rs index cc0d5ca07..a917d9214 100644 --- a/crates/hypercolor-core/src/effect/media/producer.rs +++ b/crates/hypercolor-core/src/effect/media/producer.rs @@ -285,6 +285,21 @@ impl MediaProducer { !self.frames.is_empty() } + /// Whether a background worker decodes a live stream for this + /// producer, which keeps running until the producer is dropped. + #[must_use] + pub const fn is_live_stream(&self) -> bool { + #[cfg(feature = "media-video")] + { + self.live_stream.is_some() + } + #[cfg(not(feature = "media-video"))] + { + let _ = self; + false + } + } + #[must_use] pub fn live_stream_error(&self) -> Option { #[cfg(feature = "media-video")] diff --git a/crates/hypercolor-core/src/effect/pool.rs b/crates/hypercolor-core/src/effect/pool.rs index dbf6222aa..05fa5cb8e 100644 --- a/crates/hypercolor-core/src/effect/pool.rs +++ b/crates/hypercolor-core/src/effect/pool.rs @@ -241,6 +241,21 @@ impl EffectPool { } } + /// Park every renderer because the render loop stopped requesting + /// frames. See [`EffectRenderer::park`]. + pub fn park(&mut self) { + for slot in self.slots.values_mut() { + slot.renderer.park(); + } + } + + /// Whether any renderer is still warming up its current content. See + /// [`EffectRenderer::is_warming`]. + #[must_use] + pub fn any_warming(&self) -> bool { + self.slots.values().any(|slot| slot.renderer.is_warming()) + } + pub fn remove_zone(&mut self, zone_id: ZoneId) { if self.slots.keys().any(|key| key.zone_id == zone_id) { let next_generation = self.next_generation(); @@ -947,6 +962,35 @@ mod tests { applied: Vec<(String, ControlValue)>, } + struct ParkSpyRenderer { + parks: Arc, + warming: Arc, + } + + impl EffectRenderer for ParkSpyRenderer { + fn init(&mut self, _metadata: &EffectMetadata) -> Result<()> { + Ok(()) + } + + fn render_into(&mut self, _input: &FrameInput<'_>, _target: &mut Canvas) -> Result<()> { + Ok(()) + } + + fn apply_controls(&mut self, _batch: &ControlDeltaBatch<'_>) -> Result<()> { + Ok(()) + } + + fn park(&mut self) { + self.parks.fetch_add(1, Ordering::SeqCst); + } + + fn is_warming(&self) -> bool { + self.warming.load(Ordering::SeqCst) + } + + fn destroy(&mut self) {} + } + impl DestroySpyRenderer { fn new(destroyed: Arc) -> Self { Self { destroyed } @@ -1799,6 +1843,37 @@ mod tests { ); } + #[test] + fn park_reaches_every_renderer_and_warming_tracks_any_of_them() { + let parks = Arc::new(AtomicU64::new(0)); + let first_warming = Arc::new(AtomicBool::new(false)); + let second_warming = Arc::new(AtomicBool::new(false)); + let mut pool = EffectPool::new(); + assert!(!pool.any_warming()); + pool.park(); + + for warming in [&first_warming, &second_warming] { + let effect_id = EffectId::new(uuid::Uuid::now_v7()); + let mut slot = advance_spy_slot(effect_id, Arc::new(AtomicU64::new(0))); + slot.renderer = Box::new(ParkSpyRenderer { + parks: Arc::clone(&parks), + warming: Arc::clone(warming), + }); + pool.slots + .insert(EffectSlotKey::new(ZoneId::new(), SceneLayerId::new()), slot); + } + + pool.park(); + assert_eq!(parks.load(Ordering::SeqCst), 2); + assert!(!pool.any_warming()); + + second_warming.store(true, Ordering::SeqCst); + assert!(pool.any_warming()); + second_warming.store(false, Ordering::SeqCst); + first_warming.store(true, Ordering::SeqCst); + assert!(pool.any_warming()); + } + #[test] fn advance_layer_output_ticks_renderer_without_rendering_canvas() { let advanced = Arc::new(AtomicU64::new(0)); diff --git a/crates/hypercolor-core/src/effect/servo/renderer.rs b/crates/hypercolor-core/src/effect/servo/renderer.rs index bc60f717c..db306059d 100644 --- a/crates/hypercolor-core/src/effect/servo/renderer.rs +++ b/crates/hypercolor-core/src/effect/servo/renderer.rs @@ -16,6 +16,7 @@ use hypercolor_types::effect::{ControlKind, EffectMetadata, EffectSource}; use std::collections::HashMap; use std::path::PathBuf; use std::time::Duration; +use tracing::debug; use super::ServoSessionHandle; #[cfg(test)] @@ -80,6 +81,10 @@ pub struct ServoRenderer { animation_cadence: AnimationCadence, host_driven_animation: bool, last_submit_time_secs: Option, + /// Set when a page starts loading and cleared by its first completed + /// frame. A reload keeps the previous page's canvas on screen, so the + /// canvas alone cannot tell whether the current page has rendered. + awaiting_first_frame: bool, producer_role: ServoProducerRole, display_descriptor: Option, } @@ -133,6 +138,7 @@ impl ServoRenderer { animation_cadence: AnimationCadence::MatchRenderLoop, host_driven_animation: false, last_submit_time_secs: None, + awaiting_first_frame: false, producer_role, display_descriptor: None, } @@ -262,6 +268,26 @@ impl EffectRenderer for ServoRenderer { self.display_descriptor = descriptor; } + fn park(&mut self) { + // Servo's script thread schedules its own animation ticks every 30 ms + // for a page with live requestAnimationFrame callbacks, so an + // unobserved page keeps drawing WebGL until its webview is + // throttled, which stops those ticks. Timers keep their own pace. + // The next render request unthrottles the webview. + if let Some(session) = self.session.as_ref() + && let Err(error) = session.park() + { + debug!(%error, "Could not throttle the parked Servo page"); + } + } + + fn is_warming(&self) -> bool { + self.initialized + && self.load_failed.is_none() + && self.hard_stall.is_none() + && (self.load_task.is_some() || (self.session.is_some() && self.awaiting_first_frame)) + } + fn destroy(&mut self) { self.load_task = None; if let Some(session) = self.session.take() { @@ -302,6 +328,7 @@ impl EffectRenderer for ServoRenderer { self.animation_cadence = AnimationCadence::MatchRenderLoop; self.host_driven_animation = false; self.last_submit_time_secs = None; + self.awaiting_first_frame = false; } } diff --git a/crates/hypercolor-core/src/effect/servo/renderer/frame_poll.rs b/crates/hypercolor-core/src/effect/servo/renderer/frame_poll.rs index ea5d4d371..b65b930cf 100644 --- a/crates/hypercolor-core/src/effect/servo/renderer/frame_poll.rs +++ b/crates/hypercolor-core/src/effect/servo/renderer/frame_poll.rs @@ -49,6 +49,7 @@ impl ServoRenderer { self.clear_stall_state(); self.last_gpu_frame = Some(frame); self.warned_fallback_frame = false; + self.awaiting_first_frame = false; } Ok(Some(EffectRenderOutput::Pending)) => {} Ok(None) => self.update_stall_state(pending, soft_stall_timeout), @@ -75,6 +76,7 @@ impl ServoRenderer { pub(super) fn accept_completed_canvas(&mut self, canvas: Canvas, clear_gpu_frame: bool) { self.clear_stall_state(); + self.awaiting_first_frame = false; self.last_canvas = Some(canvas); self.clear_cached_gpu_frame_if_needed(clear_gpu_frame); self.warned_fallback_frame = false; diff --git a/crates/hypercolor-core/src/effect/servo/renderer/load.rs b/crates/hypercolor-core/src/effect/servo/renderer/load.rs index 1ad304cea..e764ebf89 100644 --- a/crates/hypercolor-core/src/effect/servo/renderer/load.rs +++ b/crates/hypercolor-core/src/effect/servo/renderer/load.rs @@ -162,6 +162,7 @@ impl ServoRenderer { let display_descriptor = (self.producer_role == ServoProducerRole::DisplayFaceHtml) .then(|| self.display_descriptor.clone()) .flatten(); + self.awaiting_first_frame = true; self.load_task = Some(start_servo_load_task( metadata.name.clone(), path.clone(), diff --git a/crates/hypercolor-core/src/effect/servo/renderer/tests.rs b/crates/hypercolor-core/src/effect/servo/renderer/tests.rs index b9fff5c72..26ef5137f 100644 --- a/crates/hypercolor-core/src/effect/servo/renderer/tests.rs +++ b/crates/hypercolor-core/src/effect/servo/renderer/tests.rs @@ -707,6 +707,65 @@ fn init_with_canvas_size_reuses_previous_canvas_while_new_effect_loads() { assert!(stopped.load(Ordering::SeqCst)); } +#[test] +fn renderer_warms_from_load_start_until_the_new_page_renders_its_first_frame() { + let _lock = SHARED_WORKER_STATE_TEST_LOCK + .lock() + .expect("shared worker test lock"); + reset_shared_servo_worker_state(); + + let (worker, load_rx, release_tx, unload_rx, stopped) = spawn_blocking_load_test_worker(); + install_running_shared_worker(worker); + + let temp_dir = tempfile::tempdir().expect("temporary directory"); + let source_path = temp_dir.path().join("effect.html"); + std::fs::write(&source_path, "") + .expect("write source html"); + + let mut renderer = ServoRenderer::new(); + assert!( + !renderer.is_warming(), + "an uninitialized renderer has nothing to warm" + ); + renderer.park(); + + // A reload keeps the previous page's canvas on screen, so the canvas + // alone must not read as warm. + renderer.initialized = true; + renderer.last_canvas = Some(solid_canvas(640, 480, 12, 34, 56)); + renderer + .init_with_canvas_size(&html_metadata(source_path), 640, 480) + .expect("renderer should queue initialization"); + load_rx + .recv_timeout(Duration::from_millis(100)) + .expect("create-session command should be queued asynchronously"); + assert!(renderer.is_warming(), "a loading page is warming"); + + release_tx.send(()).expect("release create-session"); + wait_for_load_completion(&mut renderer); + assert!(renderer.session.is_some()); + assert!( + renderer.is_warming(), + "a loaded page stays warming until its first frame lands" + ); + renderer.park(); + + renderer.accept_completed_canvas(solid_canvas(640, 480, 1, 2, 3), false); + assert!( + !renderer.is_warming(), + "the first completed frame ends warming" + ); + + renderer.destroy(); + assert!(!renderer.is_warming()); + unload_rx + .recv_timeout(Duration::from_millis(100)) + .expect("destroy should unload test worker"); + + shutdown_shared_servo_worker().expect("shared worker shutdown should succeed"); + assert!(stopped.load(Ordering::SeqCst)); +} + #[test] fn destroy_discards_completed_load_task_before_it_is_polled() { let (worker, load_rx, unload_rx, stopped) = spawn_load_test_worker(); diff --git a/crates/hypercolor-core/src/effect/servo/session.rs b/crates/hypercolor-core/src/effect/servo/session.rs index 29bc49c85..e149e659d 100644 --- a/crates/hypercolor-core/src/effect/servo/session.rs +++ b/crates/hypercolor-core/src/effect/servo/session.rs @@ -273,6 +273,18 @@ impl ServoSessionHandle { } } + /// Throttle this session's page while the render loop requests no + /// frames, so its requestAnimationFrame loop stops scheduling itself + /// unobserved. The next render request unthrottles it. + /// + /// # Errors + /// + /// Returns an error when the worker's control queue cannot accept the + /// command. + pub fn park(&self) -> Result<()> { + self.worker.set_session_throttled(self.session_id, true) + } + pub fn resize(&mut self, width: u32, height: u32) { self.render_width = width.max(1); self.render_height = height.max(1); diff --git a/crates/hypercolor-core/src/effect/servo/worker.rs b/crates/hypercolor-core/src/effect/servo/worker.rs index eaaa48cb4..e8b8da192 100644 --- a/crates/hypercolor-core/src/effect/servo/worker.rs +++ b/crates/hypercolor-core/src/effect/servo/worker.rs @@ -507,6 +507,18 @@ impl ServoWorkerRuntime { record_servo_destroy_wait(destroy_started.elapsed()); let _ = response_tx.send(result); } + ScheduledServoWork::Command(WorkerCommand::SetThrottled { + session_id, + throttled, + }) => { + if let Err(error) = self.set_webview_throttled(session_id, throttled) { + debug!(?session_id, throttled, %error, "Skipped Servo webview throttle"); + } else { + // Deliver the throttle to the page's script thread now + // rather than on the next render, which may be far off. + self.servo.spin_event_loop(); + } + } ScheduledServoWork::Command(WorkerCommand::MemoryReport { response_tx }) => { let result = self.memory_report(); let _ = response_tx.send(result); diff --git a/crates/hypercolor-core/src/effect/servo/worker/test_support.rs b/crates/hypercolor-core/src/effect/servo/worker/test_support.rs index b595a8e4d..3880127b9 100644 --- a/crates/hypercolor-core/src/effect/servo/worker/test_support.rs +++ b/crates/hypercolor-core/src/effect/servo/worker/test_support.rs @@ -69,6 +69,7 @@ pub fn spawn_test_worker() -> (ServoWorker, Arc) { let _ = response_tx.send(()); break; } + WorkerCommand::SetThrottled { .. } => {} WorkerCommand::MemoryReport { response_tx } => { let _ = response_tx.send(Ok(empty_memory_report())); } @@ -170,6 +171,7 @@ pub fn spawn_render_test_worker() -> ( WorkerCommand::LoadUrl { response_tx, .. } => { let _ = response_tx.send(Ok(())); } + WorkerCommand::SetThrottled { .. } => {} WorkerCommand::MemoryReport { response_tx } => { let _ = response_tx.send(Ok(empty_memory_report())); } @@ -251,6 +253,7 @@ pub fn spawn_contended_render_test_worker() -> ContendedRenderTestWorker { let _ = response_tx.send(()); break; } + WorkerCommand::SetThrottled { .. } => {} WorkerCommand::MemoryReport { response_tx } => { let _ = response_tx.send(Ok(empty_memory_report())); } @@ -322,6 +325,7 @@ pub fn spawn_load_test_worker() -> ( let _ = response_tx.send(()); break; } + WorkerCommand::SetThrottled { .. } => {} WorkerCommand::MemoryReport { response_tx } => { let _ = response_tx.send(Ok(empty_memory_report())); } @@ -391,6 +395,7 @@ pub fn spawn_blocking_load_test_worker() -> ( let _ = response_tx.send(()); break; } + WorkerCommand::SetThrottled { .. } => {} WorkerCommand::MemoryReport { response_tx } => { let _ = response_tx.send(Ok(empty_memory_report())); } diff --git a/crates/hypercolor-core/src/effect/servo/worker_client.rs b/crates/hypercolor-core/src/effect/servo/worker_client.rs index c867bc87d..d197b038c 100644 --- a/crates/hypercolor-core/src/effect/servo/worker_client.rs +++ b/crates/hypercolor-core/src/effect/servo/worker_client.rs @@ -199,6 +199,12 @@ pub(super) enum WorkerCommand { session_id: ServoSessionId, response_tx: SyncSender>, }, + /// Throttle or unthrottle one session's webview without waiting for + /// the worker to apply it. + SetThrottled { + session_id: ServoSessionId, + throttled: bool, + }, MemoryReport { response_tx: SyncSender>, }, @@ -799,6 +805,28 @@ impl ServoWorkerClient { Ok(()) } + /// Queue a webview throttle change for one session. + /// + /// Commands are ordered barriers on the worker, so a throttle queued + /// behind an in-flight render applies after that render finishes, and + /// the next render command unthrottles the webview again. + pub(super) fn set_session_throttled( + &self, + session_id: ServoSessionId, + throttled: bool, + ) -> Result<()> { + if !self.has_session(session_id) { + return Ok(()); + } + self.try_send_command( + WorkerCommand::SetThrottled { + session_id, + throttled, + }, + "set-throttled", + ) + } + pub(super) fn memory_report(&self) -> Result { let (response_tx, response_rx) = mpsc::sync_channel(1); self.try_send_command(WorkerCommand::MemoryReport { response_tx }, "memory-report")?; @@ -921,6 +949,41 @@ mod tests { assert_eq!(queued_id, session_id); } + #[test] + fn session_throttle_queues_without_waiting_and_skips_unknown_sessions() { + let (command_tx, command_rx) = mpsc::sync_channel(4); + let client = + ServoWorkerClient::new(command_tx, Arc::new(ServoWorkerClientSharedState::new())); + let session_id = ServoSessionId(11); + + client + .set_session_throttled(session_id, true) + .expect("an unknown session has nothing to throttle"); + assert!(command_rx.try_recv().is_err()); + + client.with_sessions(|sessions| { + sessions.insert( + session_id, + ClientStateSlot::new(ServoProducerRole::SceneHtml), + ); + }); + client + .set_session_throttled(session_id, true) + .expect("throttle should queue without waiting for the worker"); + + let WorkerCommand::SetThrottled { + session_id: queued_id, + throttled, + } = command_rx + .try_recv() + .expect("throttle command should be queued") + else { + panic!("expected a throttle command"); + }; + assert_eq!(queued_id, session_id); + assert!(throttled); + } + #[test] fn render_submission_retains_work_without_blocking_when_command_queue_is_full() { let (command_tx, _command_rx) = mpsc::sync_channel(1); diff --git a/crates/hypercolor-core/src/effect/traits.rs b/crates/hypercolor-core/src/effect/traits.rs index ef65ceb8f..a6d369870 100644 --- a/crates/hypercolor-core/src/effect/traits.rs +++ b/crates/hypercolor-core/src/effect/traits.rs @@ -295,6 +295,28 @@ pub trait EffectRenderer: Send { None } + /// The render loop stopped requesting frames because nothing consumes + /// them. + /// + /// Renderers that keep working between frames on their own, such as a + /// web page whose animation timers fire without being asked, should + /// quiesce that work here. The next render call resumes the renderer, + /// so parking needs no counterpart. The default no-op covers renderers + /// that only work when called. + fn park(&mut self) {} + + /// Whether the renderer has not yet produced output for its current + /// content, for example while a page loads or before its first frame + /// lands. + /// + /// The render loop keeps rendering while any producer warms up, so a + /// renderer that is parked once warm shows its real content the moment + /// a consumer arrives. Renderers that produce output synchronously keep + /// the default. + fn is_warming(&self) -> bool { + false + } + /// Tear down the renderer and release all resources. /// /// Called when the effect transitions to `Destroying`. After this call, diff --git a/crates/hypercolor-core/src/engine/fps.rs b/crates/hypercolor-core/src/engine/fps.rs index 89b348f82..6095ceac9 100644 --- a/crates/hypercolor-core/src/engine/fps.rs +++ b/crates/hypercolor-core/src/engine/fps.rs @@ -337,6 +337,12 @@ impl FpsController { self.frame_start = Some(Instant::now()); } + /// Forget the frame begun by [`begin_frame`](Self::begin_frame) + /// without recording it. + pub fn abandon_frame(&mut self) { + self.frame_start = None; + } + /// Mark the end of the current frame. /// /// Updates the EWMA, consecutive miss counter, and checks for tier diff --git a/crates/hypercolor-core/src/engine/mod.rs b/crates/hypercolor-core/src/engine/mod.rs index 0a9af954e..5e2b1a9ce 100644 --- a/crates/hypercolor-core/src/engine/mod.rs +++ b/crates/hypercolor-core/src/engine/mod.rs @@ -108,11 +108,16 @@ pub struct RenderLoop { /// Increments whenever a live render loop enters the paused state. pause_generation: u64, - /// Wall-clock instant when the current pause began. - paused_at: Option, + /// Wall-clock instant the current span excluded from elapsed time + /// began, while the loop is paused or parked. + held_since: Option, + + /// Whether the loop is parked: running, but rendering nothing because + /// nothing consumes its frames. + parked: bool, /// Total wall-clock time excluded from render elapsed time. - paused_duration: Duration, + held_duration: Duration, /// Monotonically increasing frame counter. frame_number: u64, @@ -138,8 +143,9 @@ impl RenderLoop { running: Arc::new(AtomicBool::new(false)), state: RenderLoopState::Created, pause_generation: 0, - paused_at: None, - paused_duration: Duration::ZERO, + held_since: None, + parked: false, + held_duration: Duration::ZERO, frame_number: 0, start_time: None, } @@ -154,8 +160,9 @@ impl RenderLoop { running: Arc::new(AtomicBool::new(false)), state: RenderLoopState::Created, pause_generation: 0, - paused_at: None, - paused_duration: Duration::ZERO, + held_since: None, + parked: false, + held_duration: Duration::ZERO, frame_number: 0, start_time: None, } @@ -170,8 +177,9 @@ impl RenderLoop { self.running.store(true, Ordering::Release); self.state = RenderLoopState::Running; self.start_time = Some(Instant::now()); - self.paused_at = None; - self.paused_duration = Duration::ZERO; + self.held_since = None; + self.parked = false; + self.held_duration = Duration::ZERO; info!(tier = %self.fps_controller.tier(), "Render loop started"); } @@ -206,7 +214,7 @@ impl RenderLoop { if self.state == RenderLoopState::Running { self.state = RenderLoopState::Paused; self.pause_generation = self.pause_generation.wrapping_add(1); - self.paused_at = Some(Instant::now()); + self.begin_hold(); debug!("Render loop paused"); } } @@ -214,14 +222,54 @@ impl RenderLoop { /// Resume from paused state. pub fn resume(&mut self) { if self.state == RenderLoopState::Paused { - if let Some(paused_at) = self.paused_at.take() { - self.paused_duration += paused_at.elapsed(); + if !self.parked { + self.end_hold(); } self.state = RenderLoopState::Running; debug!("Render loop resumed"); } } + /// Park a running loop that renders nothing because nothing consumes + /// its frames. + /// + /// Parked time is excluded from [`elapsed`](Self::elapsed) the way + /// paused time is, so time-driven layers continue from where they + /// stopped instead of jumping by the parked span. The lifecycle state + /// stays [`Running`](RenderLoopState::Running). + pub fn park(&mut self) { + if self.state == RenderLoopState::Running && !self.parked { + self.parked = true; + self.begin_hold(); + } + } + + /// Leave a park started by [`park`](Self::park). + pub fn unpark(&mut self) { + if self.parked { + self.parked = false; + if self.state != RenderLoopState::Paused { + self.end_hold(); + } + } + } + + /// Whether the loop is parked. + #[must_use] + pub fn is_parked(&self) -> bool { + self.parked + } + + fn begin_hold(&mut self) { + self.held_since.get_or_insert_with(Instant::now); + } + + fn end_hold(&mut self) { + if let Some(held_since) = self.held_since.take() { + self.held_duration += held_since.elapsed(); + } + } + /// Get an `Arc` handle that can be used to stop the loop /// from another thread. #[must_use] @@ -251,6 +299,14 @@ impl RenderLoop { self.frame_complete_with_max_tier(None) } + /// Drop the frame begun by [`tick`](Self::tick) without completing it. + /// + /// The frame never rendered, so it records no timing, feeds no tier + /// decision, and does not advance [`frame_number`](Self::frame_number). + pub fn abandon_frame(&mut self) { + self.fps_controller.abandon_frame(); + } + /// Finalize the current frame and optionally apply a new runtime ceiling /// before evaluating automatic tier transitions for the next frame. pub fn frame_complete_with_max_tier( @@ -296,9 +352,9 @@ impl RenderLoop { #[must_use] pub fn elapsed(&self) -> Duration { self.start_time.map_or(Duration::ZERO, |started_at| { - let now = self.paused_at.unwrap_or_else(Instant::now); + let now = self.held_since.unwrap_or_else(Instant::now); now.duration_since(started_at) - .saturating_sub(self.paused_duration) + .saturating_sub(self.held_duration) }) } @@ -344,8 +400,9 @@ impl std::fmt::Debug for RenderLoop { f.debug_struct("RenderLoop") .field("state", &self.state) .field("pause_generation", &self.pause_generation) - .field("paused_at", &self.paused_at) - .field("paused_duration", &self.paused_duration) + .field("held_since", &self.held_since) + .field("parked", &self.parked) + .field("held_duration", &self.held_duration) .field("frame_number", &self.frame_number) .field("running", &self.running.load(Ordering::Relaxed)) .field("start_time", &self.start_time) diff --git a/crates/hypercolor-core/tests/backend_manager_tests.rs b/crates/hypercolor-core/tests/backend_manager_tests.rs index 8329afb77..05aec655f 100644 --- a/crates/hypercolor-core/tests/backend_manager_tests.rs +++ b/crates/hypercolor-core/tests/backend_manager_tests.rs @@ -1541,6 +1541,38 @@ fn map_and_unmap_device() { assert!(!manager.unmap_device("wled:strip_1")); } +#[test] +fn mapped_device_targets_list_each_binding_with_its_backend() { + let mut manager = BackendManager::new(); + let strip = DeviceId::new(); + let display = DeviceId::new(); + assert_eq!(manager.mapped_device_targets().count(), 0); + + manager.map_device("wled:strip_1", "wled", strip); + manager.map_device("wled:strip_1_alias", "wled", strip); + manager.map_device("simulator:lcd", "simulator", display); + + let mut targets = manager + .mapped_device_targets() + .map(|(backend_id, device_id)| (backend_id.to_owned(), device_id)) + .collect::>(); + targets.sort(); + let mut expected = vec![ + ("simulator".to_owned(), display), + ("wled".to_owned(), strip), + ("wled".to_owned(), strip), + ]; + expected.sort(); + assert_eq!(targets, expected); + + assert!(manager.unmap_device("simulator:lcd")); + assert!( + manager + .mapped_device_targets() + .all(|(backend_id, device_id)| backend_id == "wled" && device_id == strip) + ); +} + #[test] fn unmap_device_clears_cached_target_fps_when_last_mapping_is_removed() { let device_id = DeviceId::new(); diff --git a/crates/hypercolor-core/tests/bus_tests.rs b/crates/hypercolor-core/tests/bus_tests.rs index 0ec84ddc2..1461caaec 100644 --- a/crates/hypercolor-core/tests/bus_tests.rs +++ b/crates/hypercolor-core/tests/bus_tests.rs @@ -1534,3 +1534,103 @@ async fn concurrent_input_status_publications_are_generation_ordered() { .all(|pair| pair[0] < pair[1]) ); } + +// ── Output demand ───────────────────────────────────────────────────────── + +/// Arm a waiter on the bus output-demand signal the way an idle producer +/// does, run `trigger`, and report whether the waiter woke. +async fn output_demand_wakes_on(bus: &HypercolorBus, trigger: impl FnOnce()) -> bool { + let armed = bus.output_demand().arm(); + let notified = armed.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + trigger(); + timeout(Duration::from_millis(200), notified).await.is_ok() +} + +#[tokio::test] +async fn lane_subscriptions_wake_output_demand_waiters() { + let bus = HypercolorBus::new(); + let epoch = bus.output_demand().epoch(); + + let mut receivers = Vec::new(); + assert!(output_demand_wakes_on(&bus, || receivers.push(bus.canvas_receiver())).await); + assert!(output_demand_wakes_on(&bus, || receivers.push(bus.scene_canvas_receiver())).await); + assert!(output_demand_wakes_on(&bus, || receivers.push(bus.screen_canvas_receiver())).await); + let mut frames = None; + assert!(output_demand_wakes_on(&bus, || frames = Some(bus.frame_receiver())).await); + let mut zone_canvas = None; + assert!( + output_demand_wakes_on(&bus, || zone_canvas = + Some(bus.zone_canvas_receiver(ZoneId::new()))) + .await + ); + + assert!(bus.output_demand().epoch() >= epoch + 5); +} + +#[tokio::test] +async fn output_receiver_count_spans_every_output_stream() { + let bus = HypercolorBus::new(); + assert_eq!(bus.output_receiver_count(), 0); + + let frames = bus.frame_receiver(); + let spectrum = bus.spectrum_receiver(); + let canvas = bus.canvas_receiver(); + let scene = bus.scene_canvas_receiver(); + let screen = bus.screen_canvas_receiver(); + let web = bus.web_viewport_canvas_receiver(); + let zones = bus.screen_zones_receiver(); + let previews = bus.zone_preview_receiver(); + let zone_canvas = bus.zone_canvas_receiver(ZoneId::new()); + assert_eq!(bus.output_receiver_count(), 9); + assert_eq!(bus.zone_canvas_receiver_count(), 1); + + drop(( + frames, spectrum, canvas, scene, screen, web, zones, previews, + )); + assert_eq!(bus.output_receiver_count(), 1); + drop(zone_canvas); + assert_eq!(bus.output_receiver_count(), 0); +} + +#[tokio::test] +async fn output_demand_holds_count_until_dropped_and_wake_waiters() { + let bus = HypercolorBus::new(); + let mut hold = None; + + assert!(output_demand_wakes_on(&bus, || hold = Some(bus.hold_output_demand())).await); + assert_eq!(bus.output_demand().hold_count(), 1); + let second = bus.hold_output_demand(); + assert_eq!(bus.output_demand().hold_count(), 2); + + drop(hold); + drop(second); + assert_eq!(bus.output_demand().hold_count(), 0); +} + +#[tokio::test] +async fn published_events_wake_output_demand_waiters() { + let bus = HypercolorBus::new(); + + assert!(output_demand_wakes_on(&bus, || bus.publish(paused_event())).await); +} + +#[tokio::test] +async fn output_demand_notification_without_an_armed_waiter_only_advances_the_epoch() { + let bus = HypercolorBus::new(); + let epoch = bus.output_demand().epoch(); + + bus.publish(paused_event()); + let _canvas = bus.canvas_receiver(); + + assert_eq!(bus.output_demand().epoch(), epoch + 2); + assert!(output_demand_wakes_on(&bus, || bus.publish(paused_event())).await); +} + +#[tokio::test] +async fn output_demand_waiter_that_is_not_triggered_stays_asleep() { + let bus = HypercolorBus::new(); + + assert!(!output_demand_wakes_on(&bus, || {}).await); +} diff --git a/crates/hypercolor-core/tests/device_tests.rs b/crates/hypercolor-core/tests/device_tests.rs index e7a992022..143ca3620 100644 --- a/crates/hypercolor-core/tests/device_tests.rs +++ b/crates/hypercolor-core/tests/device_tests.rs @@ -405,6 +405,40 @@ async fn registry_add_and_get() { assert_eq!(tracked.state, DeviceState::Known); } +#[tokio::test] +async fn registry_changes_wake_waiters_armed_before_they_land() { + let registry = DeviceRegistry::new(); + let device = mock_device_info("Case Fans"); + let id = device.id; + + let changed = registry.changed(); + tokio::pin!(changed); + changed.as_mut().enable(); + registry.add(device).await; + tokio::time::timeout(Duration::from_millis(200), changed) + .await + .expect("adding a device should wake a waiter armed before it"); + + let changed = registry.changed(); + tokio::pin!(changed); + changed.as_mut().enable(); + assert!(registry.set_state(&id, DeviceState::Connected).await); + tokio::time::timeout(Duration::from_millis(200), changed) + .await + .expect("a state change should wake a waiter armed before it"); + + let changed = registry.changed(); + tokio::pin!(changed); + changed.as_mut().enable(); + assert!(registry.set_state(&id, DeviceState::Connected).await); + assert!( + tokio::time::timeout(Duration::from_millis(100), changed) + .await + .is_err(), + "a no-op state write should not wake waiters" + ); +} + #[tokio::test] async fn registry_add_returns_existing_id() { let registry = DeviceRegistry::new(); diff --git a/crates/hypercolor-core/tests/engine_tests.rs b/crates/hypercolor-core/tests/engine_tests.rs index 082d71987..6c6389a04 100644 --- a/crates/hypercolor-core/tests/engine_tests.rs +++ b/crates/hypercolor-core/tests/engine_tests.rs @@ -694,6 +694,109 @@ fn render_loop_elapsed_excludes_every_pause_interval() { assert_eq!(rl.elapsed(), second_pause_elapsed); } +#[test] +fn render_loop_elapsed_excludes_parked_time_without_leaving_running() { + let mut rl = RenderLoop::new(60); + rl.start(); + std::thread::sleep(Duration::from_millis(10)); + + rl.park(); + assert!(rl.is_parked()); + assert_eq!(rl.state(), RenderLoopState::Running); + assert!(rl.is_running(), "a parked loop still ticks"); + let parked_elapsed = rl.elapsed(); + std::thread::sleep(Duration::from_millis(30)); + assert_eq!(rl.elapsed(), parked_elapsed); + + rl.unpark(); + assert!(!rl.is_parked()); + std::thread::sleep(Duration::from_millis(10)); + assert!(rl.elapsed() > parked_elapsed); +} + +#[test] +fn render_loop_pause_inside_a_park_excludes_the_whole_span_once() { + let mut rl = RenderLoop::new(60); + rl.start(); + std::thread::sleep(Duration::from_millis(10)); + + rl.park(); + let held_at = rl.elapsed(); + std::thread::sleep(Duration::from_millis(15)); + rl.pause(); + std::thread::sleep(Duration::from_millis(15)); + rl.resume(); + assert_eq!( + rl.elapsed(), + held_at, + "resuming a pause while still parked keeps the time held" + ); + std::thread::sleep(Duration::from_millis(15)); + rl.unpark(); + let resumed_at = rl.elapsed(); + assert!( + resumed_at >= held_at && resumed_at < held_at + Duration::from_millis(10), + "the park and the pause inside it are excluded exactly once: {held_at:?} -> {resumed_at:?}" + ); +} + +#[test] +fn render_loop_unpark_while_paused_keeps_time_held_until_resume() { + let mut rl = RenderLoop::new(60); + rl.start(); + std::thread::sleep(Duration::from_millis(10)); + + rl.park(); + let held_at = rl.elapsed(); + rl.pause(); + std::thread::sleep(Duration::from_millis(15)); + rl.unpark(); + assert!(!rl.is_parked()); + assert_eq!(rl.state(), RenderLoopState::Paused); + std::thread::sleep(Duration::from_millis(15)); + assert_eq!( + rl.elapsed(), + held_at, + "leaving the park while paused keeps the pause holding time" + ); + + rl.resume(); + let resumed_at = rl.elapsed(); + assert!( + resumed_at >= held_at && resumed_at < held_at + Duration::from_millis(10), + "the park and the pause after it are excluded exactly once: {held_at:?} -> {resumed_at:?}" + ); + std::thread::sleep(Duration::from_millis(10)); + assert!(rl.elapsed() > resumed_at); +} + +#[test] +fn render_loop_park_is_ignored_unless_running() { + let mut rl = RenderLoop::new(60); + rl.park(); + assert!(!rl.is_parked(), "a loop that never started cannot park"); + + rl.start(); + rl.pause(); + rl.park(); + assert!(!rl.is_parked(), "a paused loop is idle for its own reason"); +} + +#[test] +fn render_loop_abandoned_frame_records_nothing() { + let mut rl = RenderLoop::new(60); + rl.start(); + assert!(rl.tick()); + rl.abandon_frame(); + + assert_eq!(rl.frame_number(), 0); + assert!( + rl.frame_complete().is_none(), + "an abandoned frame has nothing left to complete" + ); + assert_eq!(rl.frame_number(), 0); +} + #[test] fn render_loop_debug_output_is_readable() { let rl = RenderLoop::new(60); diff --git a/crates/hypercolor-core/tests/media_producer_tests.rs b/crates/hypercolor-core/tests/media_producer_tests.rs index 34efb9e26..e7bb6fe45 100644 --- a/crates/hypercolor-core/tests/media_producer_tests.rs +++ b/crates/hypercolor-core/tests/media_producer_tests.rs @@ -425,6 +425,27 @@ fn webm_video_frames_decode_when_feature_is_enabled() { assert!(pixel.r >= 250 && pixel.g >= 250 && pixel.b >= 250 && pixel.a == 255); } +#[test] +fn decoded_media_is_not_a_live_stream() { + let producer = MediaProducer::from_bytes(&animated_gif_bytes(), "image/gif") + .expect("animated GIF should decode"); + + assert!(!producer.is_live_stream()); +} + +#[cfg(feature = "media-video")] +#[test] +fn stream_url_producer_reports_a_live_stream() { + let producer = MediaProducer::from_bytes_with_stream_policy( + b"http://1.1.1.1/hypercolor-missing-live.m3u8\n", + "application/vnd.hypercolor.stream-url", + &StreamUrlPolicy::default(), + ) + .expect("stream URL producer should start"); + + assert!(producer.is_live_stream()); +} + #[cfg(feature = "media-video")] #[test] fn stream_url_producer_returns_before_first_live_frame() { diff --git a/crates/hypercolor-daemon/Cargo.toml b/crates/hypercolor-daemon/Cargo.toml index d7ab0e044..1f88628ea 100644 --- a/crates/hypercolor-daemon/Cargo.toml +++ b/crates/hypercolor-daemon/Cargo.toml @@ -43,6 +43,10 @@ required-features = ["persistence-test-hooks"] name = "render_thread_tests" required-features = ["persistence-test-hooks"] +[[test]] +name = "output_demand_tests" +required-features = ["persistence-test-hooks"] + [[test]] name = "simulator_tests" required-features = ["persistence-test-hooks"] diff --git a/crates/hypercolor-daemon/src/api/system.rs b/crates/hypercolor-daemon/src/api/system.rs index fd8166542..c72d98436 100644 --- a/crates/hypercolor-daemon/src/api/system.rs +++ b/crates/hypercolor-daemon/src/api/system.rs @@ -158,10 +158,15 @@ async fn system_status_with_privacy( ) }; + let output_demand = performance.output_demand.status(); + // Query the live render loop for timing data. - let render_loop_status = { + let (render_loop_status, render_elapsed_ms) = { let rl = state.render_loop.read().await; let snapshot = rl.stats(); + // Frame timestamps use render time, which stops while the loop is + // paused or parked, so frame age is measured on the same clock. + let render_elapsed_ms = rl.elapsed().as_secs_f64() * 1000.0; let capacity_fps = if snapshot.state == RenderLoopState::Running { round_1(paced_fps( snapshot.avg_frame_time.as_secs_f64(), @@ -170,27 +175,32 @@ async fn system_status_with_privacy( } else { 0.0 }; - RenderLoopStatus { + // A parked loop delivers no frames; `output_demand` says why. + let delivering = snapshot.state == RenderLoopState::Running && !output_demand.parked; + let status = RenderLoopStatus { state: snapshot.state.to_string(), fps_tier: snapshot.tier.to_string(), target_fps: snapshot.tier.fps(), ceiling_fps: snapshot.max_tier.fps(), capacity_fps, - delivered_fps: if snapshot.state == RenderLoopState::Running { + delivered_fps: if delivering { round_1(performance.delivered_fps) } else { 0.0 }, - actual_fps: capacity_fps, + actual_fps: if delivering { capacity_fps } else { 0.0 }, consecutive_misses: snapshot.consecutive_misses, total_frames: snapshot.total_frames, - } + output_demand, + }; + (status, render_elapsed_ms) }; let running = render_loop_is_operational(render_loop_status.state.as_str()); let latest_frame = if render_loop_status.state == "running" { - performance.latest_frame.as_ref().map(|frame| { - latest_frame_status(frame, state.start_time.elapsed().as_secs_f64() * 1000.0) - }) + performance + .latest_frame + .as_ref() + .map(|frame| latest_frame_status(frame, render_elapsed_ms)) } else { None }; @@ -300,6 +310,12 @@ async fn system_status_with_privacy( macos_daemon_ownership, compositor_acceleration: render_acceleration_status(&state.render_acceleration), render_loop: render_loop_status, + render_supervision: state + .domains + .output + .render_supervision() + .snapshot() + .status(include_private_diagnostics), session_performance, latest_frame, effect_health, diff --git a/crates/hypercolor-daemon/src/app_state.rs b/crates/hypercolor-daemon/src/app_state.rs index aab5e5563..edc2515b8 100644 --- a/crates/hypercolor-daemon/src/app_state.rs +++ b/crates/hypercolor-daemon/src/app_state.rs @@ -590,6 +590,8 @@ impl AppState { display_frames: Arc::clone(&display_frames), device_metrics: Arc::clone(&device_metrics), input_manager: input_manager.clone(), + // An isolated AppState never spawns a render thread. + render_supervision: crate::render_thread::RenderSupervision::detached(), }, |layout| { let discovery_runtime = crate::discovery::DiscoveryRuntime { diff --git a/crates/hypercolor-daemon/src/canvas_capture.rs b/crates/hypercolor-daemon/src/canvas_capture.rs index 67daf9db0..e8973ce61 100644 --- a/crates/hypercolor-daemon/src/canvas_capture.rs +++ b/crates/hypercolor-daemon/src/canvas_capture.rs @@ -173,7 +173,7 @@ pub async fn capture_canvas( if index > 0 && !request.interval.is_zero() { tokio::time::sleep(request.interval).await; } - let (frame, fresh) = next_frame(&mut receiver).await; + let (frame, fresh) = next_frame(preview, &mut receiver).await; if frame.width == 0 || frame.height == 0 { return Err(DomainError::service_unavailable( "No canvas frame has been rendered yet; apply an effect and capture again", @@ -190,13 +190,28 @@ pub async fn capture_canvas( /// Take the next frame the receiver has not seen yet, waiting for one when /// necessary, and fall back to the latest published frame when none arrives /// in time. -async fn next_frame(receiver: &mut PreviewFrameReceiver) -> (CanvasFrame, bool) { - let fresh = matches!( - tokio::time::timeout(FRESH_FRAME_TIMEOUT, receiver.changed()).await, - Ok(Ok(())) - ); - let frame = receiver.borrow_and_update().clone(); - (frame, fresh) +/// +/// A capture of an idle daemon wakes a parked render loop. The first frame +/// after that wake still composes producer output requested before the +/// park, so the capture waits for the frame after it, which is rendered +/// from the capture's own request. +async fn next_frame( + preview: &PreviewRuntime, + receiver: &mut PreviewFrameReceiver, +) -> (CanvasFrame, bool) { + let deadline = tokio::time::Instant::now() + FRESH_FRAME_TIMEOUT; + let mut published = false; + loop { + let changed = tokio::time::timeout_at(deadline, receiver.changed()).await; + let frame = receiver.borrow_and_update().clone(); + if !matches!(changed, Ok(Ok(()))) { + return (frame, published); + } + published = true; + if !preview.is_render_resume_frame(frame.frame_number) { + return (frame, true); + } + } } async fn encode( diff --git a/crates/hypercolor-daemon/src/daemon.rs b/crates/hypercolor-daemon/src/daemon.rs index b02b71a26..fa38e4b72 100644 --- a/crates/hypercolor-daemon/src/daemon.rs +++ b/crates/hypercolor-daemon/src/daemon.rs @@ -18,7 +18,7 @@ use tokio::net::TcpListener; use tokio::sync::watch; use tokio::task::JoinSet; use tokio::time::{Duration, sleep}; -use tracing::{info, warn}; +use tracing::{error, info, warn}; use tracing_subscriber::EnvFilter; use crate::api; @@ -26,6 +26,7 @@ use crate::api::security::{CredentialAuthority, CredentialTier, RemoteClientFami use crate::app_state::AppState; use crate::macos_owner::{MacosDaemonOwner, MacosDaemonSessionAttestation, MacosOwnerSnapshot}; use crate::mdns::MdnsPublisher; +use crate::render_thread::{RenderFailureRecord, RenderSupervision}; use crate::startup::{DaemonState, StartupProgress, config_sources}; const MAIN_RUNTIME_WORKERS: usize = 4; @@ -34,6 +35,18 @@ const MAIN_RUNTIME_THREAD_KEEP_ALIVE: std::time::Duration = std::time::Duration: const API_LISTEN_BACKLOG: i32 = 1024; const API_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(3); +/// How long the graceful shutdown after an unrecoverable render failure may +/// take before the process exits regardless. +/// +/// The daemon must not keep serving without a renderer, and a failure bad +/// enough to exhaust the restart budget can also wedge the teardown, so +/// this backstop bounds the gap a service manager sees. +const RENDER_FAILURE_EXIT_DEADLINE: Duration = Duration::from_secs(20); + +/// Exit code of a process exit forced by an unrecoverable render failure. +/// `EX_SOFTWARE`: an internal failure. +pub(crate) const RENDER_FAILURE_EXIT_CODE: i32 = 70; + /// Runtime options for one daemon process. #[derive(Default)] pub struct DaemonRunOptions { @@ -167,10 +180,16 @@ impl PreparedDaemon { let progress = StartupProgress::default(); let handoff = api::startup::ApiHandoff::starting(progress.clone(), env!("CARGO_PKG_VERSION")); + // The API stops on the caller's shutdown signal and also when the + // render thread gives up, so the server listens to a channel this + // function owns rather than the caller's. + let (api_shutdown_tx, api_shutdown_rx) = watch::channel(*shutdown_rx.borrow()); + let _shutdown_relay = + BackgroundTask::spawn(relay_shutdown_signal(shutdown_rx, api_shutdown_tx.clone())); let server = ApiServerTask::spawn(serve_api_handoff_with_shutdown_timeout( listeners, handoff.clone(), - shutdown_rx, + api_shutdown_rx, API_GRACEFUL_SHUTDOWN_TIMEOUT, )); info!(binds = %self.listen_addr, "API listeners answering startup probes"); @@ -185,6 +204,12 @@ impl PreparedDaemon { self.options.service_status.take(), progress.clone(), )?; + let render_supervision = daemon_state.domains.output.render_supervision().clone(); + let _render_failure_watch = BackgroundTask::spawn(shut_down_after_render_failure( + render_supervision.clone(), + api_shutdown_tx.clone(), + Some(RENDER_FAILURE_EXIT_DEADLINE), + )); let ui_dir = resolve_ui_dir(self.options.ui_dir.clone()); daemon_state.session_monitors = self.options.session_monitors.take(); record_api_binding( @@ -255,9 +280,30 @@ impl PreparedDaemon { ); hypercolor_linux_session::notify_ready(); - hypercolor_linux_session::spawn_watchdog(); + // A render thread wedged inside a frame cannot be interrupted, so it + // stops the keepalives and systemd restarts the daemon instead. + let watchdog_supervision = render_supervision.clone(); + hypercolor_linux_session::spawn_watchdog_while(move || { + watchdog_supervision.watchdog_healthy() + }); - server.join().await?; + let server_result = server.join().await; + if let Some(record) = render_supervision.termination() { + // The termination error must win over any teardown error: it is + // what tells a process host, such as the Windows service, to + // exit the way its service manager restarts. + if let Err(error) = server_result { + warn!(error = %format!("{error:#}"), "API server failed while stopping after the render failure"); + } + if let Some(publisher) = mdns_publisher { + publisher.shutdown().await; + } + if let Err(error) = daemon_state.shutdown().await { + warn!(error = %format!("{error:#}"), "daemon shutdown after the render failure did not finish cleanly"); + } + return Err(render_failure_exit_error(&record)); + } + server_result?; if let Some(publisher) = mdns_publisher { publisher.shutdown().await; @@ -265,11 +311,199 @@ impl PreparedDaemon { daemon_state.shutdown().await?; + if let Some(record) = render_supervision.termination() { + return Err(render_failure_exit_error(&record)); + } info!("Hypercolor daemon exited cleanly"); Ok(()) } } +/// A spawned task that is aborted when its owner goes away. +struct BackgroundTask(tokio::task::JoinHandle<()>); + +impl BackgroundTask { + fn spawn(task: impl Future + Send + 'static) -> Self { + Self(tokio::spawn(task)) + } +} + +impl Drop for BackgroundTask { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// Forward the caller's shutdown signal into the API shutdown channel. +/// +/// A dropped caller sender counts as shutdown, which is how the API server +/// treated the caller's channel directly. +async fn relay_shutdown_signal( + mut external: watch::Receiver, + api_shutdown: watch::Sender, +) { + let _ = external.wait_for(|shutdown| *shutdown).await; + api_shutdown.send_replace(true); +} + +/// Shut the API down once the render thread gives up. +/// +/// The daemon then runs its normal graceful shutdown and returns an error, +/// so the process exits nonzero and its service manager (the systemd unit, +/// launchd, the Windows service recovery actions, or the desktop app's +/// supervisor) starts a fresh one. `exit_deadline` arms a backstop that +/// exits the process if that shutdown does not finish in time. +async fn shut_down_after_render_failure( + supervision: RenderSupervision, + api_shutdown: watch::Sender, + exit_deadline: Option, +) { + let record = supervision.terminated().await; + error!( + kind = record.kind.as_str(), + message = %record.message, + restarts_total = record.restarts_total, + failure_record = %supervision + .failure_record_path() + .map_or_else(|| "not persisted".to_owned(), |path| path.display().to_string()), + "render thread cannot recover; stopping the daemon so its service manager restarts it" + ); + if let Some(deadline) = exit_deadline { + arm_render_failure_exit_backstop(deadline); + } + api_shutdown.send_replace(true); +} + +fn arm_render_failure_exit_backstop(deadline: Duration) { + let spawned = std::thread::Builder::new() + .name("hypercolor-exit-backstop".to_owned()) + .spawn(move || { + std::thread::sleep(deadline); + error!( + deadline_ms = u64::try_from(deadline.as_millis()).unwrap_or(u64::MAX), + exit_code = RENDER_FAILURE_EXIT_CODE, + "graceful shutdown after the render failure did not finish; exiting now" + ); + std::process::exit(RENDER_FAILURE_EXIT_CODE); + }); + if let Err(spawn_error) = spawned { + error!(%spawn_error, "could not arm the exit backstop for the render failure shutdown"); + } +} + +/// A daemon run that ended because the render thread could not recover. +#[derive(Debug, thiserror::Error)] +#[error( + "render thread could not recover ({kind}: {message}); exiting so the service manager restarts the daemon" +)] +pub(crate) struct RenderTerminationError { + kind: &'static str, + message: String, +} + +fn render_failure_exit_error(record: &RenderFailureRecord) -> anyhow::Error { + anyhow::Error::new(RenderTerminationError { + kind: record.kind.as_str(), + message: record.message.clone(), + }) +} + +/// Whether a daemon run ended because the render thread could not recover. +/// +/// A process host that reports its own exit, such as the Windows service, +/// uses this to exit in a way its service manager restarts. +#[must_use] +pub fn is_render_termination(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() +} + +#[cfg(test)] +mod render_failure_shutdown_tests { + use std::sync::Arc; + use std::time::Duration; + + use tokio::sync::watch; + + use super::{ + is_render_termination, relay_shutdown_signal, render_failure_exit_error, + shut_down_after_render_failure, + }; + use crate::render_thread::{ + RenderFailureKind, RenderFailureRecord, RenderRecoveryAction, RenderSupervision, + RenderThreadPhase, + }; + + fn exhausted_record() -> RenderFailureRecord { + RenderFailureRecord { + kind: RenderFailureKind::Panic, + message: "frame panicked again".to_owned(), + thread: Some("hypercolor-render".to_owned()), + location: None, + occurred_at: "2026-10-19T22:30:00.000Z".to_owned(), + occurred_at_unix_ms: 0, + pid: std::process::id(), + process_instance: String::new(), + daemon_version: env!("CARGO_PKG_VERSION").to_owned(), + restart_attempt: 6, + restarts_total: 5, + action: RenderRecoveryAction::TerminateProcess, + backtrace: None, + } + } + + #[tokio::test] + async fn an_unrecoverable_render_failure_shuts_the_api_down_and_fails_the_run() { + let supervision = RenderSupervision::detached(); + let (api_shutdown_tx, mut api_shutdown_rx) = watch::channel(false); + let watcher = tokio::spawn(shut_down_after_render_failure( + supervision.clone(), + api_shutdown_tx, + None, + )); + tokio::task::yield_now().await; + assert!(!*api_shutdown_rx.borrow(), "nothing failed yet"); + + supervision.request_termination(Arc::new(exhausted_record())); + + tokio::time::timeout( + Duration::from_secs(5), + api_shutdown_rx.wait_for(|stop| *stop), + ) + .await + .expect("the API should be told to stop") + .expect("the watcher holds the sender until it fires"); + watcher.await.expect("watcher should finish"); + assert_eq!(supervision.phase(), RenderThreadPhase::Terminating); + let record = supervision + .termination() + .expect("termination should stay recorded for the exit path"); + let error = render_failure_exit_error(&record); + assert!(is_render_termination(&error)); + assert!(!is_render_termination(&anyhow::anyhow!( + "an unrelated failure" + ))); + let error = format!("{error:#}"); + assert!(error.contains("frame panicked again"), "{error}"); + } + + #[tokio::test] + async fn the_caller_shutdown_signal_and_its_disappearance_both_stop_the_api() { + let (external_tx, external_rx) = watch::channel(false); + let (api_tx, mut api_rx) = watch::channel(false); + let relay = tokio::spawn(relay_shutdown_signal(external_rx, api_tx)); + external_tx.send_replace(true); + relay.await.expect("relay should finish"); + assert!(*api_rx.borrow_and_update()); + + let (external_tx, external_rx) = watch::channel(false); + let (api_tx, api_rx) = watch::channel(false); + let relay = tokio::spawn(relay_shutdown_signal(external_rx, api_tx)); + drop(external_tx); + relay.await.expect("relay should finish"); + assert!(*api_rx.borrow()); + } +} + /// Publish where the API listens, before installers read it. fn record_api_binding( daemon: &mut DaemonState, diff --git a/crates/hypercolor-daemon/src/display_output/mod.rs b/crates/hypercolor-daemon/src/display_output/mod.rs index c8a0db449..f753d1251 100644 --- a/crates/hypercolor-daemon/src/display_output/mod.rs +++ b/crates/hypercolor-daemon/src/display_output/mod.rs @@ -22,6 +22,7 @@ use tracing::{debug, info, warn}; use hypercolor_core::bus::{ CanvasFrame, DisplayZoneFrame, DisplayZoneOutputRoute, DisplayZoneViewport, HypercolorBus, + OutputDemandHold, }; use hypercolor_core::device::{BackendManager, DeviceRegistry}; use hypercolor_core::spatial::is_display_zone; @@ -394,6 +395,12 @@ async fn run_display_output(state: DisplayOutputState, mut shutdown_rx: oneshot: .await; let mut scene_canvas_rx = display_requires_scene_canvas(initial_targets.targets.as_ref()) .then(|| state.preview_runtime.scene_canvas_receiver()); + let mut output_demand_hold = None; + sync_output_demand_hold( + &mut output_demand_hold, + &state.event_bus, + !initial_targets.targets.is_empty(), + ); reconcile_display_workers(&state, &mut workers, initial_targets.targets.as_ref()).await; let mut last_reconciled_target_version = Some(initial_targets.version); let mut last_dispatched_sources = @@ -450,6 +457,11 @@ async fn run_display_output(state: DisplayOutputState, mut shutdown_rx: oneshot: state.preview_runtime.as_ref(), display_requires_scene_canvas(targets.targets.as_ref()), ); + sync_output_demand_hold( + &mut output_demand_hold, + &state.event_bus, + !targets.targets.is_empty(), + ); if targets.targets.is_empty() { last_dispatched_sources.clear(); continue; @@ -596,6 +608,26 @@ fn sync_display_canvas_receiver( } } +/// Hold render output demand while any display target exists. +/// +/// A face rendered straight to its zone is read from the zone's canvas +/// sender, not through a receiver, so the render loop would not count it as +/// a consumer. Physical displays are already counted as devices; this hold +/// is what keeps a previewed simulated display rendering. +fn sync_output_demand_hold( + hold: &mut Option, + event_bus: &HypercolorBus, + has_targets: bool, +) { + if has_targets { + if hold.is_none() { + *hold = Some(event_bus.hold_output_demand()); + } + } else { + let _ = hold.take(); + } +} + fn display_requires_scene_canvas(targets: &[Arc]) -> bool { targets.iter().any(|target| target.requires_scene_canvas()) } @@ -1089,6 +1121,25 @@ mod tests { ))) } + #[test] + fn display_output_holds_render_demand_exactly_while_it_has_targets() { + let event_bus = HypercolorBus::new(); + let mut hold = None; + + sync_output_demand_hold(&mut hold, &event_bus, true); + assert_eq!(event_bus.output_demand().hold_count(), 1); + sync_output_demand_hold(&mut hold, &event_bus, true); + assert_eq!( + event_bus.output_demand().hold_count(), + 1, + "a held demand is not taken twice" + ); + + sync_output_demand_hold(&mut hold, &event_bus, false); + assert_eq!(event_bus.output_demand().hold_count(), 0); + assert!(hold.is_none()); + } + fn display_target(blend_mode: BlendMode) -> DisplayTarget { let device_id = DeviceId::new(); DisplayTarget { diff --git a/crates/hypercolor-daemon/src/domain/diagnostics.rs b/crates/hypercolor-daemon/src/domain/diagnostics.rs index bee1e6f4f..a35c834c0 100644 --- a/crates/hypercolor-daemon/src/domain/diagnostics.rs +++ b/crates/hypercolor-daemon/src/domain/diagnostics.rs @@ -23,10 +23,19 @@ use crate::domain::display::DisplayContext; use crate::domain::input_status::{actionable_input_diagnostics, input_status_snapshot}; use crate::domain::output::{OutputContext, RenderLoopStatus}; use crate::domain::spatial::SpatialService; +use crate::output_power::{OutputOverride, OutputPowerState}; use crate::performance::{LatestFrameMetrics, PerformanceSnapshot}; +use crate::render_thread::gpu_errors::GpuErrorTotals; +use crate::render_thread::{RenderSupervisionSnapshot, RenderThreadPhase}; +use hypercolor_core::engine::RenderLoopState; const RENDER_FRAME_STALE_WARNING_MS: f64 = 2_000.0; const RENDER_FRAME_STALE_FAIL_MS: f64 = 10_000.0; +/// How long the render thread may go without completing a frame (or, when +/// paused, without polling) before the `render_thread` check fails. A +/// healthy loop completes a throttled idle frame at least every quarter +/// second, so a few seconds of silence means it is wedged or gone. +pub const RENDER_THREAD_STALL_FAIL: Duration = Duration::from_secs(3); const DEFAULT_SAFE_CHECKS: [&str; 8] = [ "daemon", "render", @@ -149,11 +158,16 @@ struct DiagnosticsInputs { tracked_devices: usize, config_available: bool, config_persistence: Option<(PathBuf, ConfigPersistence)>, + render_supervision: RenderSupervisionSnapshot, + output_power: OutputPowerState, } impl DiagnosticsInputs { + /// Render time now, on the clock frame timestamps are taken from. + /// Daemon uptime runs on through pauses and parks that render time + /// skips, so measuring frame age against it would count those spans. fn render_elapsed_ms(&self) -> f64 { - self.uptime.as_secs_f64() * 1000.0 + self.render_loop.elapsed.as_secs_f64() * 1000.0 } } @@ -339,6 +353,8 @@ impl DiagnosticsContext { tracked_devices: self.authorities.devices.device_registry().len().await, config_available: self.authorities.platform.config_available(), config_persistence: self.authorities.platform.config_persistence(), + render_supervision: self.authorities.output.render_supervision().snapshot(), + output_power: self.authorities.output.output_power_snapshot(), } } } @@ -361,18 +377,54 @@ fn render_checks(inputs: &DiagnosticsInputs) -> Vec { let render_elapsed_ms = inputs.render_elapsed_ms(); let running = inputs.render_loop.running; let render_loop_stats = inputs.render_loop.stats; - let mut checks = vec![DiagnoseCheck { - category: "render".to_owned(), - name: "render_loop".to_owned(), - status: if running { "pass" } else { "warning" }.to_owned(), - detail: format!( - "state={}, tier={}", - render_loop_stats.state, render_loop_stats.tier - ), - }]; + let parked = performance.output_demand.parked_since.is_some(); + let pause = render_loop_pause_reason(render_loop_stats.state, inputs.output_power) + .map_or_else(String::new, |reason| format!("; {reason}")); + let park = if parked { + format!( + ", parked={}", + crate::output_demand::IDLE_REASON_NO_OUTPUT_CONSUMERS + ) + } else { + String::new() + }; + let mut render_thread = render_thread_check( + &inputs.render_supervision, + render_loop_stats.state, + parked, + RENDER_THREAD_STALL_FAIL, + ); + render_thread.detail.push_str(&pause); + let mut checks = vec![ + DiagnoseCheck { + category: "render".to_owned(), + name: "render_loop".to_owned(), + status: if running { "pass" } else { "warning" }.to_owned(), + detail: format!( + "state={}, tier={}{park}{pause}", + render_loop_stats.state, render_loop_stats.tier + ), + }, + render_thread, + gpu_errors_check(&inputs.render_supervision.gpu_errors), + ]; if !running { return checks; } + if parked { + // A parked loop renders nothing by design, so frame age and LED + // freshness describe the last frame before the park, not a stall. + checks.push(DiagnoseCheck { + category: "render".to_owned(), + name: "frame_liveness".to_owned(), + status: "pass".to_owned(), + detail: format!( + "parked: nothing consumes rendered frames (no output devices, streams, or captures); the next consumer resumes rendering at the current tier. parks_total={}", + performance.output_demand.parks_total + ), + }); + return checks; + } let (status, detail) = render_frame_liveness_status(performance.latest_frame.as_ref(), render_elapsed_ms); @@ -415,6 +467,223 @@ fn render_checks(inputs: &DiagnosticsInputs) -> Vec { checks } +/// What paused the render loop, for the render checks' details. +/// +/// A paused loop renders nothing by design, which looks like a hang from +/// the outside, so the details say where the pause came from. +fn render_loop_pause_reason( + loop_state: RenderLoopState, + power: OutputPowerState, +) -> Option<&'static str> { + if loop_state != RenderLoopState::Paused { + return None; + } + Some(match power.output_override { + OutputOverride::Paused => { + "paused by output power: a manual pause (PATCH /api/v1/output or a pause \ + control), which persists across restarts until power is set back to running" + } + OutputOverride::Stopped => "paused by output power: the last effect was stopped", + OutputOverride::None if power.session_sleeping => "paused while the session sleeps", + OutputOverride::None => "paused, though output power reports running", + }) +} + +/// Whether the supervised render thread is alive and producing frames. +/// +/// Fails while a pipeline is restarting or the daemon is terminating after +/// an unrecoverable failure, when the render thread is gone while the loop +/// still runs, and when a running loop has completed no frame (or a paused +/// loop has not polled) for `stall_after`. +#[must_use] +pub fn render_thread_check( + supervision: &RenderSupervisionSnapshot, + loop_state: RenderLoopState, + parked: bool, + stall_after: Duration, +) -> DiagnoseCheck { + let (status, detail) = render_thread_status(supervision, loop_state, parked, stall_after); + DiagnoseCheck { + category: "render".to_owned(), + name: "render_thread".to_owned(), + status: status.to_owned(), + detail, + } +} + +fn render_thread_status( + supervision: &RenderSupervisionSnapshot, + loop_state: RenderLoopState, + parked: bool, + stall_after: Duration, +) -> (&'static str, String) { + let failure = supervision.last_failure.as_ref().map_or_else( + || "none".to_owned(), + |record| { + format!( + "{}{} at {}: {}", + record.kind.as_str(), + if supervision.last_failure_in_this_process { + "" + } else { + " (earlier daemon run)" + }, + record.occurred_at, + record.message + ) + }, + ); + let loop_active = matches!( + loop_state, + RenderLoopState::Running | RenderLoopState::Paused + ); + match supervision.phase { + RenderThreadPhase::Terminating => ( + "fail", + format!( + "the render thread could not recover after {} restart(s); the daemon is \ + exiting so its service manager restarts it. Last failure: {failure}", + supervision.restarts_total + ), + ), + RenderThreadPhase::Restarting => ( + "fail", + format!( + "the render pipeline is restarting after {} failure(s) in a row ({} restart(s) \ + so far, {} allowed per window). Last failure: {failure}", + supervision.consecutive_failures, + supervision.restarts_total, + supervision.max_restarts + ), + ), + RenderThreadPhase::Running => { + // A paused or parked loop renders nothing by design, so only + // its polling shows the thread is alive. + let idle = loop_state == RenderLoopState::Paused || parked; + let (age, activity) = if idle { + (supervision.last_heartbeat_age, "polled") + } else { + (supervision.frame_due_age, "completed a frame") + }; + match age { + Some(age) if loop_active && age >= stall_after => ( + "fail", + format!( + "the render thread has not {activity} for {} ms while the render loop \ + is {loop_state}; restarts={}, last_failure={failure}", + age.as_millis(), + supervision.restarts_total + ), + ), + _ if idle => ( + "pass", + format!( + "phase=running, the render loop is {} so no frames are due, \ + last_poll_age_ms={}, last_frame_age_ms={}, restarts={}, \ + last_failure={failure}", + if parked { + "parked with no output consumers" + } else { + "paused" + }, + millis_or_none(supervision.last_heartbeat_age), + millis_or_none(supervision.last_frame_age), + supervision.restarts_total + ), + ), + _ => ( + "pass", + format!( + "phase=running, last_frame_age_ms={}, restarts={}, last_failure={failure}", + millis_or_none(supervision.last_frame_age), + supervision.restarts_total + ), + ), + } + } + phase @ (RenderThreadPhase::NotStarted + | RenderThreadPhase::Starting + | RenderThreadPhase::Stopped) => { + if loop_active { + ( + "fail", + format!( + "the render thread is {} while the render loop is {loop_state}; nothing \ + is rendering. Last failure: {failure}", + phase.as_str() + ), + ) + } else { + ( + "pass", + format!( + "phase={}, render loop {loop_state}, last_failure={failure}", + phase.as_str() + ), + ) + } + } + } +} + +/// How recent the latest uncaptured GPU error must be for the +/// `gpu_errors` check to warn. Older errors stay in the counts. +pub const GPU_ERROR_WARNING_WINDOW: Duration = Duration::from_mins(10); + +fn millis_or_none(age: Option) -> String { + age.map_or_else(|| "none".to_owned(), |age| age.as_millis().to_string()) +} + +/// Whether a GPU error went uncaptured recently. +/// +/// Uncaptured validation, out-of-memory, and internal errors no longer +/// panic the thread that raised them, so this check is where they stay +/// visible: the counts since startup are always in the detail, an error +/// within [`GPU_ERROR_WARNING_WINDOW`] warns, and a lost device fails. +#[must_use] +pub fn gpu_errors_check(totals: &GpuErrorTotals) -> DiagnoseCheck { + let now_unix_ms = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }); + gpu_errors_check_at(totals, now_unix_ms) +} + +fn gpu_errors_check_at(totals: &GpuErrorTotals, now_unix_ms: u64) -> DiagnoseCheck { + let last_age_ms = totals + .last + .as_ref() + .map(|last| now_unix_ms.saturating_sub(last.at_unix_ms)); + let last = totals.last.as_ref().map_or_else(String::new, |last| { + format!( + "; last ({} ms ago): {}: {}: {}", + last_age_ms.unwrap_or_default(), + last.device, + last.kind, + last.message + ) + }); + let counts = format!( + "validation={}, out_of_memory={}, internal={}, device_lost={}", + totals.validation, totals.out_of_memory, totals.internal, totals.device_lost + ); + let window_ms = u64::try_from(GPU_ERROR_WARNING_WINDOW.as_millis()).unwrap_or(u64::MAX); + let status = if totals.device_lost > 0 { + "fail" + } else if totals.uncaptured() > 0 && last_age_ms.is_none_or(|age| age < window_ms) { + "warning" + } else { + "pass" + }; + DiagnoseCheck { + category: "render".to_owned(), + name: "gpu_errors".to_owned(), + status: status.to_owned(), + detail: format!("{counts}{last}"), + } +} + fn device_checks(inputs: &DiagnosticsInputs, snapshot: &DiagnoseSnapshot) -> Vec { let count = inputs.tracked_devices; let output_status = if snapshot.device_output.worker_finished_queues > 0 @@ -724,7 +993,11 @@ fn render_multi_zone_sampling_status( fn build_diagnose_snapshot(inputs: &DiagnosticsInputs) -> DiagnoseSnapshot { DiagnoseSnapshot { input: inputs.input.clone(), - render: build_render_snapshot(&inputs.performance, inputs.render_elapsed_ms()), + render: build_render_snapshot( + &inputs.performance, + inputs.render_elapsed_ms(), + &inputs.render_supervision, + ), usb: build_usb_actor_snapshot(inputs.usb_actor), display_output: build_display_output_snapshot(inputs.display_output.clone()), device_output: build_device_output_snapshot(inputs.device_metrics.as_ref()), @@ -735,9 +1008,11 @@ fn build_diagnose_snapshot(inputs: &DiagnosticsInputs) -> DiagnoseSnapshot { fn build_render_snapshot( performance: &PerformanceSnapshot, render_elapsed_ms: f64, + supervision: &RenderSupervisionSnapshot, ) -> DiagnoseRenderSnapshot { let pacing = performance.pacing; DiagnoseRenderSnapshot { + output_demand: performance.output_demand.status(), latest_frame: performance .latest_frame .as_ref() @@ -797,6 +1072,7 @@ fn build_render_snapshot( publish_avg_ms: round_2(pacing.publish_avg_ms), publish_p95_ms: round_2(pacing.publish_p95_ms), }, + supervision: supervision.status(true), } } @@ -1096,6 +1372,7 @@ mod tests { ..PerformanceSnapshot::default() }, 0.0, + &crate::render_thread::RenderSupervision::detached().snapshot(), ); let frame = snapshot @@ -1108,6 +1385,230 @@ mod tests { assert_eq!(frame.zone_composition_skips, 2); } + fn supervision( + phase: crate::render_thread::RenderThreadPhase, + frame_age_ms: u64, + heartbeat_age_ms: u64, + ) -> crate::render_thread::RenderSupervisionSnapshot { + crate::render_thread::RenderSupervisionSnapshot { + phase, + restarts_total: 0, + consecutive_failures: 0, + max_restarts: 5, + last_heartbeat_age: Some(std::time::Duration::from_millis(heartbeat_age_ms)), + last_frame_age: Some(std::time::Duration::from_millis(frame_age_ms)), + frame_due_age: Some(std::time::Duration::from_millis(frame_age_ms)), + current_run: None, + last_failure: None, + last_failure_in_this_process: false, + gpu_errors: crate::render_thread::gpu_errors::GpuErrorTotals::default(), + } + } + + #[test] + fn render_thread_check_fails_on_stalls_restarts_and_a_missing_thread() { + use crate::render_thread::RenderThreadPhase; + use hypercolor_core::engine::RenderLoopState; + + let stall = std::time::Duration::from_secs(3); + let cases = [ + ( + RenderThreadPhase::Running, + RenderLoopState::Running, + 20, + 20, + "pass", + ), + ( + RenderThreadPhase::Running, + RenderLoopState::Running, + 5_000, + 20, + "fail", + ), + ( + RenderThreadPhase::Running, + RenderLoopState::Paused, + 60_000, + 40, + "pass", + ), + ( + RenderThreadPhase::Running, + RenderLoopState::Paused, + 60_000, + 4_000, + "fail", + ), + ( + RenderThreadPhase::Restarting, + RenderLoopState::Running, + 20, + 20, + "fail", + ), + ( + RenderThreadPhase::Terminating, + RenderLoopState::Running, + 20, + 20, + "fail", + ), + ( + RenderThreadPhase::Stopped, + RenderLoopState::Running, + 20, + 20, + "fail", + ), + ( + RenderThreadPhase::Stopped, + RenderLoopState::Stopped, + 9_000, + 9_000, + "pass", + ), + ( + RenderThreadPhase::NotStarted, + RenderLoopState::Created, + 0, + 0, + "pass", + ), + ]; + for (phase, loop_state, frame_age, heartbeat_age, expected) in cases { + let check = super::render_thread_check( + &supervision(phase, frame_age, heartbeat_age), + loop_state, + false, + stall, + ); + assert_eq!( + check.status, expected, + "{phase:?} with the loop {loop_state} and a {frame_age} ms frame age: {}", + check.detail + ); + assert_eq!(check.name, "render_thread"); + } + } + + #[test] + fn a_parked_loop_is_judged_by_its_polling_not_its_last_frame() { + use crate::render_thread::RenderThreadPhase; + use hypercolor_core::engine::RenderLoopState; + + let stall = std::time::Duration::from_secs(3); + let parked_for_a_minute = supervision(RenderThreadPhase::Running, 60_000, 400); + let check = + super::render_thread_check(&parked_for_a_minute, RenderLoopState::Running, true, stall); + assert_eq!(check.status, "pass", "{}", check.detail); + assert!( + check.detail.contains("parked with no output consumers"), + "{}", + check.detail + ); + + let wedged_while_parked = supervision(RenderThreadPhase::Running, 60_000, 5_000); + let check = + super::render_thread_check(&wedged_while_parked, RenderLoopState::Running, true, stall); + assert_eq!(check.status, "fail", "{}", check.detail); + assert!(check.detail.contains("polled"), "{}", check.detail); + + let unparked = super::render_thread_check( + &parked_for_a_minute, + RenderLoopState::Running, + false, + stall, + ); + assert_eq!(unparked.status, "fail", "{}", unparked.detail); + } + + #[test] + fn a_paused_loop_passes_the_render_thread_check_and_names_its_pause() { + use crate::output_power::{OutputOverride, OutputPowerState}; + use crate::render_thread::RenderThreadPhase; + use hypercolor_core::engine::RenderLoopState; + + let paused_for_a_minute = supervision(RenderThreadPhase::Running, 60_000, 40); + let check = super::render_thread_check( + &paused_for_a_minute, + RenderLoopState::Paused, + false, + std::time::Duration::from_secs(3), + ); + assert_eq!(check.status, "pass", "{}", check.detail); + assert!( + check.detail.contains("no frames are due"), + "{}", + check.detail + ); + + let manual = OutputPowerState { + output_override: OutputOverride::Paused, + ..OutputPowerState::default() + }; + assert!( + super::render_loop_pause_reason(RenderLoopState::Paused, manual) + .is_some_and(|reason| reason.contains("manual pause") + && reason.contains("persists across restarts")) + ); + let stopped = OutputPowerState { + output_override: OutputOverride::Stopped, + ..OutputPowerState::default() + }; + assert!( + super::render_loop_pause_reason(RenderLoopState::Paused, stopped) + .is_some_and(|reason| reason.contains("last effect was stopped")) + ); + assert_eq!( + super::render_loop_pause_reason(RenderLoopState::Running, manual), + None + ); + } + + #[test] + fn gpu_errors_check_warns_on_recent_errors_and_fails_on_device_loss() { + use crate::render_thread::gpu_errors::{GpuErrorSample, GpuErrorTotals}; + + let now_ms = 50_000_000; + assert_eq!( + super::gpu_errors_check_at(&GpuErrorTotals::default(), now_ms).status, + "pass" + ); + let validation = GpuErrorTotals { + validation: 2, + last: Some(GpuErrorSample { + device: "SparkleFlinger GPU compositor".to_owned(), + kind: "validation".to_owned(), + message: "Usage flags are invalid".to_owned(), + at_unix_ms: now_ms - 1_000, + }), + ..GpuErrorTotals::default() + }; + let check = super::gpu_errors_check_at(&validation, now_ms); + assert_eq!(check.status, "warning"); + let window_ms = u64::try_from(super::GPU_ERROR_WARNING_WINDOW.as_millis()) + .expect("the window fits in milliseconds"); + let settled = super::gpu_errors_check_at(&validation, now_ms + window_ms); + assert_eq!(settled.status, "pass"); + assert!( + settled.detail.contains("validation=2"), + "{}", + settled.detail + ); + assert!(check.detail.contains("validation=2"), "{}", check.detail); + assert!( + check.detail.contains("Usage flags are invalid"), + "{}", + check.detail + ); + let lost = GpuErrorTotals { + device_lost: 1, + ..validation + }; + assert_eq!(super::gpu_errors_check_at(&lost, now_ms).status, "fail"); + } + #[test] fn multi_zone_sampling_passes_when_every_zone_samples_on_the_gpu() { let (status, detail) = render_multi_zone_sampling_status(Some(&LatestFrameMetrics { diff --git a/crates/hypercolor-daemon/src/domain/output.rs b/crates/hypercolor-daemon/src/domain/output.rs index 5c3f1c70e..d62d52560 100644 --- a/crates/hypercolor-daemon/src/domain/output.rs +++ b/crates/hypercolor-daemon/src/domain/output.rs @@ -39,6 +39,9 @@ pub struct RenderLoopStatus { pub running: bool, /// Pacing state and tier at the moment of the read. pub stats: hypercolor_core::engine::RenderLoopStats, + /// Render time at the moment of the read: the clock frame timestamps + /// use, which stops while the loop is paused or parked. + pub elapsed: std::time::Duration, } /// Owning authority for global output power, brightness, and quiescence. @@ -54,6 +57,7 @@ pub struct OutputContext { preview_runtime: Arc, devices: DeviceContext, start_time: Instant, + render_supervision: crate::render_thread::RenderSupervision, } impl OutputContext { @@ -104,6 +108,7 @@ impl OutputContext { preview_runtime: Arc, devices: DeviceContext, start_time: Instant, + render_supervision: crate::render_thread::RenderSupervision, ) -> Self { Self { output_power, @@ -116,9 +121,22 @@ impl OutputContext { preview_runtime, devices, start_time, + render_supervision, } } + /// Health and recovery state of the supervised render thread. + #[must_use] + pub const fn render_supervision(&self) -> &crate::render_thread::RenderSupervision { + &self.render_supervision + } + + /// The current output power state. + #[must_use] + pub(crate) fn output_power_snapshot(&self) -> crate::output_power::OutputPowerState { + self.output_power.snapshot() + } + /// How long the daemon's subsystems have been up. #[must_use] pub fn uptime(&self) -> std::time::Duration { @@ -136,6 +154,7 @@ impl OutputContext { RenderLoopStatus { running: guard.is_running(), stats: guard.stats(), + elapsed: guard.elapsed(), } } @@ -201,7 +220,7 @@ impl OutputContext { }; let frame_number = self .event_bus - .frame_receiver() + .frame_lane() .borrow() .frame_number .saturating_add(1); diff --git a/crates/hypercolor-daemon/src/lib.rs b/crates/hypercolor-daemon/src/lib.rs index a506fc149..3d5befec1 100644 --- a/crates/hypercolor-daemon/src/lib.rs +++ b/crates/hypercolor-daemon/src/lib.rs @@ -42,7 +42,9 @@ pub mod macos_tcc_canary; pub mod mcp; pub mod mdns; pub mod network; +pub mod output_demand; pub mod output_power; +pub mod panic_hook; pub mod path_migration; pub mod performance; pub mod release_notifications; diff --git a/crates/hypercolor-daemon/src/mcp/resources.rs b/crates/hypercolor-daemon/src/mcp/resources.rs index 4cbbfe5d1..e1e348d9b 100644 --- a/crates/hypercolor-daemon/src/mcp/resources.rs +++ b/crates/hypercolor-daemon/src/mcp/resources.rs @@ -139,7 +139,7 @@ async fn read_scenes_with_state(state: &AppState) -> Value { } fn read_audio_with_state(state: &AppState) -> Value { - let spectrum = state.event_bus.spectrum_receiver().borrow().clone(); + let spectrum = state.event_bus.spectrum_lane().borrow().clone(); let (enabled, device, fft_size) = if let Some(config_manager) = state.config_manager.as_ref() { let config = config_manager.get(); ( diff --git a/crates/hypercolor-daemon/src/mcp/tools/system.rs b/crates/hypercolor-daemon/src/mcp/tools/system.rs index 7ccfe5507..9d91c7ada 100644 --- a/crates/hypercolor-daemon/src/mcp/tools/system.rs +++ b/crates/hypercolor-daemon/src/mcp/tools/system.rs @@ -175,7 +175,7 @@ pub(super) fn handle_get_sensor_data_with_state( } pub(super) fn handle_get_audio_state_with_state(state: &AppState) -> Result { - let spectrum = state.event_bus.spectrum_receiver().borrow().clone(); + let spectrum = state.event_bus.spectrum_lane().borrow().clone(); let enabled = state .config_manager .as_ref() diff --git a/crates/hypercolor-daemon/src/output_demand.rs b/crates/hypercolor-daemon/src/output_demand.rs new file mode 100644 index 000000000..277d2f33b --- /dev/null +++ b/crates/hypercolor-daemon/src/output_demand.rs @@ -0,0 +1,136 @@ +//! Output demand: what consumes the frames the render loop produces. +//! +//! Rendering costs CPU and GPU every frame whether or not anything receives +//! the result. The render loop therefore asks, before each frame, whether +//! any consumer exists, and parks while none does (see +//! `render_thread::output_gate`). This module owns the definition of a +//! consumer so the render thread and the status surfaces count the same +//! things: +//! +//! - a connected device, LED or display, that receives frames: one the +//! device registry reports renderable, or one the backend manager routes +//! frames to. Simulated displays are excluded here: they consume frames +//! only while previewed, and display output holds demand for them then; +//! - a live receiver on any latest-value output stream of the bus: frame +//! data, spectrum, the canvas and scene-canvas previews (WebSocket +//! streams, display backdrops, and canvas captures all subscribe one), +//! screen canvas, screen zones, web viewport, zone previews, and the +//! per-zone canvas streams; +//! - a stream or diagnostic consumer of host input events, which the +//! render loop republishes from the interaction input it routes; +//! - an explicit [`hypercolor_core::bus::OutputDemandHold`], taken by +//! consumers that read published frames without a receiver. + +use std::time::Instant; + +use hypercolor_core::bus::HypercolorBus; +use hypercolor_core::device::DeviceRegistry; +use hypercolor_types::api::system::RenderOutputDemandStatus; +use hypercolor_types::device::DeviceInfo; + +use crate::simulators::SIMULATED_DISPLAY_BACKEND_ID; + +/// Why a parked render loop is producing no frames. +pub const IDLE_REASON_NO_OUTPUT_CONSUMERS: &str = "no_output_consumers"; + +/// Consumers of rendered output observed at one instant. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct OutputConsumers { + /// Connected devices that receive frames. + pub output_devices: u32, + /// Live receivers on the bus output streams. + pub stream_receivers: u32, + /// Stream and diagnostic consumers of republished host input events. + pub input_event_streams: u32, + /// Explicit output-demand holds. + pub demand_holds: u32, +} + +impl OutputConsumers { + /// Whether anything consumes rendered frames. + #[must_use] + pub const fn any(self) -> bool { + self.output_devices > 0 + || self.stream_receivers > 0 + || self.input_event_streams > 0 + || self.demand_holds > 0 + } + + /// Combine the bus-side consumers with counts the caller observed. + #[must_use] + pub fn from_bus(bus: &HypercolorBus, output_devices: u32, input_event_streams: u32) -> Self { + Self { + output_devices, + stream_receivers: saturating_u32(bus.output_receiver_count()), + input_event_streams, + demand_holds: saturating_u32(bus.output_demand().hold_count()), + } + } +} + +/// Whether frames routed to `backend_id` reach a consumer on their own, +/// without anyone watching them. +#[must_use] +pub fn is_output_backend(backend_id: &str) -> bool { + backend_id != SIMULATED_DISPLAY_BACKEND_ID +} + +/// Whether a registry device in `renderable` state consumes frames on its +/// own, without anyone watching it. +#[must_use] +pub fn is_output_device(info: &DeviceInfo, renderable: bool) -> bool { + renderable && is_output_backend(info.output_backend_id()) +} + +/// Count the connected devices that receive frames. +pub async fn output_device_count(registry: &DeviceRegistry) -> u32 { + let count = registry + .list() + .await + .iter() + .filter(|tracked| is_output_device(&tracked.info, tracked.state.is_renderable())) + .count(); + saturating_u32(count) +} + +/// Park bookkeeping the render thread records for status surfaces. +#[derive(Debug, Clone, Copy, Default)] +pub struct OutputDemandTelemetry { + /// When the current park began; `None` while the loop renders. + pub parked_since: Option, + /// Parks since the daemon started. + pub parks_total: u64, + /// What last woke a parked loop. + pub last_wake_reason: Option<&'static str>, + /// The consumers the render loop counted at its latest evaluation: + /// every frame while it renders, every wake signal while it is parked. + pub consumers: OutputConsumers, +} + +impl OutputDemandTelemetry { + /// Project the recorded state onto the status contract. + #[must_use] + pub fn status(self) -> RenderOutputDemandStatus { + let consumers = self.consumers; + let parked_ms = self + .parked_since + .map(|since| u64::try_from(since.elapsed().as_millis()).unwrap_or(u64::MAX)); + RenderOutputDemandStatus { + parked: self.parked_since.is_some(), + idle_reason: self + .parked_since + .map(|_| IDLE_REASON_NO_OUTPUT_CONSUMERS.to_owned()), + parked_ms, + output_devices: consumers.output_devices, + stream_receivers: consumers.stream_receivers, + input_event_streams: consumers.input_event_streams, + demand_holds: consumers.demand_holds, + parks_total: self.parks_total, + last_wake_reason: self.last_wake_reason.map(str::to_owned), + } + } +} + +fn saturating_u32(value: usize) -> u32 { + u32::try_from(value).unwrap_or(u32::MAX) +} diff --git a/crates/hypercolor-daemon/src/panic_hook.rs b/crates/hypercolor-daemon/src/panic_hook.rs new file mode 100644 index 000000000..780c7cce2 --- /dev/null +++ b/crates/hypercolor-daemon/src/panic_hook.rs @@ -0,0 +1,470 @@ +//! Process-wide panic reporting. +//! +//! The default hook writes a panic only to stderr, and only prints a +//! backtrace when `RUST_BACKTRACE` asks for one. When stderr shares a sink +//! that stopped accepting writes (a full tmpfs, a closed pipe), a panicking +//! thread leaves no trace at all and the daemon keeps serving without it. +//! +//! This hook captures every panic on every thread: the thread name, the +//! payload, the source location, and a backtrace taken on the spot. It +//! hands the report to every registered observer (each render supervisor +//! persists its render thread's panics to the state directory, which +//! matters most in `panic = "abort"` builds where nothing runs after the +//! hook), keeps a few recent reports for code that catches the unwind, +//! chains the previous hook so stderr output stays exactly as before, and +//! records the report through `tracing` at error level. + +use std::backtrace::Backtrace; +use std::collections::VecDeque; +use std::panic::PanicHookInfo; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, Once, RwLock}; +use std::thread::ThreadId; +use std::time::SystemTime; + +/// How many recent reports stay available to [`take_recent_report`]. +const RECENT_REPORT_CAPACITY: usize = 8; + +/// Frames kept in [`PanicReport::backtrace_excerpt`]. +const BACKTRACE_EXCERPT_FRAMES: usize = 48; + +/// Upper bound on [`PanicReport::backtrace_excerpt`] in bytes. +const BACKTRACE_EXCERPT_BYTES: usize = 12 * 1024; + +/// Frames near the top of a captured backtrace that may still belong to +/// the capture and the panic machinery rather than the code that panicked. +const PANIC_MACHINERY_SCAN_FRAMES: usize = 40; + +/// Everything one panic left behind, captured inside the hook. +#[derive(Debug, Clone)] +pub struct PanicReport { + /// Identity of the panicking thread. + pub thread_id: ThreadId, + /// Name of the panicking thread, or ``. + pub thread_name: String, + /// The panic message, when the payload is a string. + pub payload: String, + /// `file:line:column` of the panic, when the runtime reports one. + pub location: Option, + /// The full backtrace captured in the hook. + pub backtrace: String, + /// Wall-clock time the hook ran. + pub occurred_at: SystemTime, +} + +impl PanicReport { + fn capture(info: &PanicHookInfo<'_>) -> Self { + let thread = std::thread::current(); + Self { + thread_id: thread.id(), + thread_name: thread.name().unwrap_or("").to_owned(), + payload: info + .payload_as_str() + .unwrap_or("non-string panic payload") + .to_owned(), + location: info.location().map(|location| { + format!( + "{}:{}:{}", + location.file(), + location.line(), + location.column() + ) + }), + backtrace: Backtrace::force_capture().to_string(), + occurred_at: SystemTime::now(), + } + } + + /// The backtrace without the capture and panic machinery frames, + /// bounded in frames and bytes so it fits a persisted record. + #[must_use] + pub fn backtrace_excerpt(&self) -> String { + backtrace_excerpt(&self.backtrace) + } +} + +/// Called with every panic report before the previous hook runs. +pub type PanicObserver = Arc; + +/// Keeps an observer registered until it is dropped. +#[must_use = "the observer is removed as soon as its registration drops"] +pub struct PanicObserverRegistration { + id: u64, +} + +impl Drop for PanicObserverRegistration { + fn drop(&mut self) { + OBSERVERS + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .retain(|(id, _)| *id != self.id); + } +} + +static INSTALL: Once = Once::new(); +static RECENT_REPORTS: Mutex> = Mutex::new(VecDeque::new()); +static OBSERVERS: RwLock> = RwLock::new(Vec::new()); +static NEXT_OBSERVER_ID: AtomicU64 = AtomicU64::new(1); + +/// Install the process-wide panic hook. +/// +/// Call this before any subsystem starts threads, and in particular before +/// Servo starts: its background hang monitor briefly swaps the hook and +/// restores whatever it found, so a hook installed while it holds the swap +/// would be lost. The previous hook is chained, never replaced. Calling +/// this more than once is a no-op. +pub fn install() { + INSTALL.call_once(|| { + let previous = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + let report = PanicReport::capture(info); + notify_observers(&report); + remember(&report); + previous(info); + tracing::error!( + target: "hypercolor_daemon::panic", + thread = %report.thread_name, + location = report.location.as_deref().unwrap_or("unknown"), + payload = %report.payload, + backtrace = %report.backtrace, + "thread panicked" + ); + })); + }); +} + +/// Register `observer` to see every panic report until the returned +/// registration drops. +pub fn add_observer(observer: PanicObserver) -> PanicObserverRegistration { + let id = NEXT_OBSERVER_ID.fetch_add(1, Ordering::Relaxed); + OBSERVERS + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push((id, observer)); + PanicObserverRegistration { id } +} + +/// Take the newest report recorded on `thread`, if one is still held. +pub fn take_recent_report(thread: ThreadId) -> Option { + take_recent_report_where(|report| report.thread_id == thread) +} + +/// Take the newest held report that `matches`. +pub fn take_recent_report_where(matches: impl Fn(&PanicReport) -> bool) -> Option { + let mut reports = RECENT_REPORTS + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let index = reports.iter().rposition(matches)?; + reports.remove(index) +} + +fn notify_observers(report: &PanicReport) { + // `try_read` keeps a panic raised while the observer list is being + // changed from deadlocking inside its own hook. + let observers: Vec = match OBSERVERS.try_read() { + Ok(observers) => observers + .iter() + .map(|(_, observer)| Arc::clone(observer)) + .collect(), + Err(std::sync::TryLockError::Poisoned(poisoned)) => poisoned + .into_inner() + .iter() + .map(|(_, observer)| Arc::clone(observer)) + .collect(), + Err(std::sync::TryLockError::WouldBlock) => Vec::new(), + }; + for observer in observers { + observer(report); + } +} + +fn remember(report: &PanicReport) { + let mut reports = match RECENT_REPORTS.try_lock() { + Ok(reports) => reports, + Err(std::sync::TryLockError::Poisoned(poisoned)) => poisoned.into_inner(), + Err(std::sync::TryLockError::WouldBlock) => return, + }; + if reports.len() == RECENT_REPORT_CAPACITY { + reports.pop_front(); + } + reports.push_back(report.clone()); +} + +/// Trim a captured backtrace to the frames worth persisting. +/// +/// `Backtrace::force_capture` inside a hook records the capture itself, the +/// hook, and the panic machinery above the frame that panicked. The excerpt +/// skips that contiguous run near the top and keeps a bounded number of +/// the frames after it. Machinery frames further down (a `catch_unwind` +/// that will catch this panic, say) are never reached, because the skip +/// stops at the first frame that is not machinery. +#[must_use] +pub fn backtrace_excerpt(backtrace: &str) -> String { + let frames = split_frames(backtrace); + let start = panicking_frame_index(&frames); + let mut excerpt = String::new(); + for frame in frames.iter().skip(start).take(BACKTRACE_EXCERPT_FRAMES) { + if excerpt.len() + frame.len() > BACKTRACE_EXCERPT_BYTES { + break; + } + excerpt.push_str(frame); + } + if excerpt.is_empty() { + let mut end = backtrace.len().min(BACKTRACE_EXCERPT_BYTES); + while !backtrace.is_char_boundary(end) { + end -= 1; + } + excerpt.push_str(&backtrace[..end]); + } + excerpt +} + +/// Split backtrace text into frames, each frame being its numbered symbol +/// line plus any `at file:line` lines that follow it. +fn split_frames(backtrace: &str) -> Vec<&str> { + let mut frames = Vec::new(); + let mut frame_start: Option = None; + let mut offset = 0; + for line in backtrace.split_inclusive('\n') { + if starts_frame(line) { + if let Some(start) = frame_start { + frames.push(&backtrace[start..offset]); + } + frame_start = Some(offset); + } + offset += line.len(); + } + if let Some(start) = frame_start { + frames.push(&backtrace[start..]); + } + frames +} + +fn starts_frame(line: &str) -> bool { + let trimmed = line.trim_start(); + let digits = trimmed.bytes().take_while(u8::is_ascii_digit).count(); + digits > 0 && trimmed[digits..].starts_with(": ") +} + +/// The frame std's own short backtraces start after: everything above it +/// is the backtrace capture, this hook, and the hook dispatch. +const SHORT_BACKTRACE_END: &str = "__rust_end_short_backtrace"; + +/// Index of the first frame worth keeping. +/// +/// That is the frame after std's short backtrace marker, the same cut a +/// default panic message makes, so the excerpt opens with the panic entry +/// (`begin_panic_handler`, `panic_fmt`) and then the code that panicked. +/// Frame names cannot place the cut more precisely: a build without +/// debuginfo, such as the Windows CI tests, labels the caller's frames with +/// the nearest public symbol, which can be a `core::panicking` function. +/// Without the marker, the excerpt skips the contiguous run of capture and +/// panic runtime frames at the top and stops at the first frame it does not +/// recognize, so an unfamiliar frame costs a few extra lines rather than +/// hiding the panic site. +fn panicking_frame_index(frames: &[&str]) -> usize { + frames + .iter() + .take(PANIC_MACHINERY_SCAN_FRAMES) + .position(|frame| frame_symbol(frame).contains(SHORT_BACKTRACE_END)) + .map_or_else( + || { + frames + .iter() + .take(PANIC_MACHINERY_SCAN_FRAMES) + .take_while(|frame| is_capture_frame(frame) || is_panic_entry(frame)) + .count() + }, + |marker| marker + 1, + ) +} + +fn frame_symbol(frame: &str) -> &str { + frame.lines().next().unwrap_or_default() +} + +/// Frames of the backtrace capture and this hook. +fn is_capture_frame(frame: &str) -> bool { + let symbol = frame_symbol(frame); + [ + "std::backtrace_rs::", + "std::backtrace::", + "hypercolor_daemon::panic_hook::", + " as core::ops::function::Fn>::call", + // The boxed hook's call on MSVC, which names its argument type. + "PanicHookInfo", + ] + .iter() + .any(|needle| symbol.contains(needle)) +} + +/// Frames of the runtime path from a `panic!` to the hook. +fn is_panic_entry(frame: &str) -> bool { + let symbol = frame_symbol(frame); + [ + "rust_begin_unwind", + "core::panicking::", + "core::option::expect_failed", + "core::option::unwrap_failed", + "core::result::unwrap_failed", + "std::panicking::begin_panic", + "std::panicking::panic_with_hook", + "std::panicking::rust_panic_with_hook", + "std::rt::begin_panic", + ] + .iter() + .any(|needle| symbol.contains(needle)) +} + +#[cfg(test)] +mod tests { + use super::backtrace_excerpt; + + const CAPTURED: &str = " 0: std::backtrace::Backtrace::force_capture + at /rustc/library/std/src/backtrace.rs:312:9 + 1: hypercolor_daemon::panic_hook::PanicReport::capture + at ./src/panic_hook.rs:60:24 + 2: std::panicking::panic_with_hook + at /rustc/library/std/src/panicking.rs:842:13 + 3: core::panicking::panic_fmt + at /rustc/library/core/src/panicking.rs:75:14 + 4: hypercolor_daemon::render_thread::pipeline_driver::run_pipeline + at ./src/render_thread/pipeline_driver.rs:40:13 + 5: tokio::runtime::park::CachedParkThread::block_on +"; + + #[test] + fn excerpt_starts_at_the_frame_that_panicked() { + let excerpt = backtrace_excerpt(CAPTURED); + + assert!( + excerpt + .trim_start() + .starts_with("4: hypercolor_daemon::render_thread::pipeline_driver::run_pipeline"), + "{excerpt}" + ); + assert!(excerpt.contains("pipeline_driver.rs:40:13")); + assert!(excerpt.contains("5: tokio::runtime::park")); + assert!(!excerpt.contains("force_capture")); + } + + /// An MSVC backtrace: symbols keep their generic arguments, and the + /// panicking closure runs inside a `std::panicking` frame. + const CAPTURED_MSVC: &str = " 0: std::backtrace_rs::backtrace::win64::trace + at /rustc/library\\std\\src\\..\\..\\backtrace\\src\\backtrace\\win64.rs:85 + 1: std::backtrace::Backtrace::create + at /rustc/library\\std\\src\\backtrace.rs:331 + 2: hypercolor_daemon::panic_hook::PanicReport::capture + at .\\src\\panic_hook.rs:60 + 3: hypercolor_daemon::panic_hook::install::closure$0::closure$0 + at .\\src\\panic_hook.rs:120 + 4: alloc::boxed::impl$30::call >,dyn$ > > >,alloc::alloc::Global> + 5: std::panicking::panic_with_hook + at /rustc/library\\std\\src\\panicking.rs:842 + 6: std::panicking::begin_panic_handler::closure$0 + at /rustc/library\\std\\src\\panicking.rs:707 + 7: std::sys::backtrace::__rust_end_short_backtrace + 8: std::panicking::begin_panic_handler + at /rustc/library\\std\\src\\panicking.rs:698 + 9: core::panicking::panic_fmt + at /rustc/library\\core\\src\\panicking.rs:75 + 10: std::panicking::catch_unwind::do_call > + at .\\src\\render_thread\\pipeline_driver.rs:40 + 11: std::panicking::catch_unwind,hypercolor_daemon::render_thread::pipeline_driver::run_pipeline::closure_env$0> + 12: std::sys::backtrace::__rust_begin_short_backtrace > + 13: std::sys::thread::windows::impl$0::new::thread_start +"; + + #[test] + fn excerpt_starts_after_the_short_backtrace_marker_with_msvc_symbols() { + let excerpt = backtrace_excerpt(CAPTURED_MSVC); + + assert!( + excerpt + .trim_start() + .starts_with("8: std::panicking::begin_panic_handler"), + "{excerpt}" + ); + assert!(excerpt.contains( + "10: std::panicking::catch_unwind::do_call + 3: some::caller::run +", + ); + + assert!( + excerpt + .trim_start() + .starts_with("2: std::panicking::catch_unwind"), + "{excerpt}" + ); + } + + #[test] + fn excerpt_without_recognizable_frames_keeps_the_raw_text() { + assert_eq!( + backtrace_excerpt("disabled backtrace"), + "disabled backtrace" + ); + } + + #[test] + fn excerpt_is_bounded_in_frames_and_bytes() { + use std::fmt::Write as _; + + let mut long = String::new(); + for index in 0..500 { + writeln!( + long, + " {index}: some::deeply::nested::frame_{index}\n at ./src/lib.rs:{index}:1" + ) + .expect("writing to a String cannot fail"); + } + + let excerpt = backtrace_excerpt(&long); + + assert!(excerpt.len() <= super::BACKTRACE_EXCERPT_BYTES); + assert!(excerpt.contains("0: some::deeply::nested::frame_0")); + assert!(!excerpt.contains("frame_48\n")); + } +} diff --git a/crates/hypercolor-daemon/src/performance.rs b/crates/hypercolor-daemon/src/performance.rs index 02df8e2a6..96ebe03b1 100644 --- a/crates/hypercolor-daemon/src/performance.rs +++ b/crates/hypercolor-daemon/src/performance.rs @@ -3,6 +3,8 @@ use std::collections::VecDeque; use std::time::{Duration, Instant}; +use crate::output_demand::OutputDemandTelemetry; + const FRAME_HISTORY_CAPACITY: usize = 120; const LATENCY_BUCKET_WIDTH_US: u32 = 100; const LATENCY_BUCKET_COUNT: usize = 4096; @@ -306,6 +308,7 @@ pub(crate) struct PerformanceSnapshot { pub full_frame_copy_count_total: u64, pub full_frame_copy_frames_total: u64, pub full_frame_copy_bytes_total: u64, + pub output_demand: OutputDemandTelemetry, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -427,11 +430,62 @@ pub struct PerformanceTracker { full_frame_copy_count_total: u64, full_frame_copy_frames_total: u64, full_frame_copy_bytes_total: u64, + output_demand: OutputDemandTelemetry, } impl PerformanceTracker { + /// Record that the render loop parked because nothing consumes its + /// frames. + /// + /// A loop that wakes and parks again without rendering stays in the + /// same park, so only the first call of a park counts it. The interval + /// history is dropped because a parked loop delivers nothing: delivered + /// FPS then reads zero, and the parked gap stays out of the first + /// interval measured after the loop resumes. + /// + /// Returns whether this call began a new park. + pub(crate) fn note_output_parked(&mut self, at: Instant) -> bool { + self.frame_intervals_us.clear(); + self.last_frame_recorded_at = None; + if self.output_demand.parked_since.is_some() { + return false; + } + self.output_demand.parked_since = Some(at); + self.output_demand.parks_total = self.output_demand.parks_total.saturating_add(1); + true + } + + /// Record that the loop left its park without completing a frame, such + /// as a frame that renders through a throttle path or a loop paused + /// while parked. + pub(crate) fn note_output_unparked(&mut self) { + self.output_demand.parked_since = None; + } + + /// Record what woke a parked loop. The park ends with the next + /// recorded frame. + pub(crate) fn note_output_woken(&mut self, reason: &'static str) { + self.output_demand.last_wake_reason = Some(reason); + } + + /// Record the consumers the render loop counted at its latest + /// evaluation. + pub(crate) fn note_output_consumers( + &mut self, + consumers: crate::output_demand::OutputConsumers, + ) { + self.output_demand.consumers = consumers; + } + + /// Park state the render thread last recorded. + #[must_use] + pub fn output_demand_telemetry(&self) -> OutputDemandTelemetry { + self.output_demand + } + /// Record one completed frame. pub(crate) fn record_frame(&mut self, metrics: &LatestFrameMetrics) { + self.output_demand.parked_since = None; let recorded_at = Instant::now(); if let Some(previous) = self.last_frame_recorded_at.replace(recorded_at) { self.frame_intervals_us.push_back(duration_micros_u64( @@ -550,6 +604,7 @@ impl PerformanceTracker { full_frame_copy_count_total: self.full_frame_copy_count_total, full_frame_copy_frames_total: self.full_frame_copy_frames_total, full_frame_copy_bytes_total: self.full_frame_copy_bytes_total, + output_demand: self.output_demand, } } diff --git a/crates/hypercolor-daemon/src/preview_runtime.rs b/crates/hypercolor-daemon/src/preview_runtime.rs index 3e698685a..bf64f5144 100644 --- a/crates/hypercolor-daemon/src/preview_runtime.rs +++ b/crates/hypercolor-daemon/src/preview_runtime.rs @@ -84,8 +84,14 @@ struct PreviewRuntimeTelemetry { screen_canvas: PreviewObservation, web_viewport_canvas: PreviewObservation, zone_preview: PreviewObservation, + /// Frame number of the first frame after the render loop last woke + /// from a park, tagged with [`RENDER_RESUME_FRAME_SET`]; zero until the + /// loop first wakes. + render_resume_frame: AtomicU64, } +const RENDER_RESUME_FRAME_SET: u64 = 1 << 32; + impl PreviewRuntimeTelemetry { fn preview(&self, kind: PreviewKind) -> &PreviewObservation { match kind { @@ -288,6 +294,28 @@ impl PreviewRuntime { self.interactive_executor.store(None); } + /// Record the frame number the render loop renders first after waking + /// from an output-demand park. + pub fn note_render_resume(&self, frame_number: u32) { + self.telemetry.render_resume_frame.store( + RENDER_RESUME_FRAME_SET | u64::from(frame_number), + Ordering::Release, + ); + } + + /// Whether `frame_number` is the first frame after the most recent wake + /// from a park. + /// + /// Asynchronous producers deliver one tick late, so that frame composes + /// producer output requested before the park. Consumers that need output + /// rendered after they asked, such as canvas captures, wait one more + /// frame. + #[must_use] + pub fn is_render_resume_frame(&self, frame_number: u32) -> bool { + self.telemetry.render_resume_frame.load(Ordering::Acquire) + == RENDER_RESUME_FRAME_SET | u64::from(frame_number) + } + pub fn note_canvas_frame(&self, frame_number: u32, timestamp_ms: u32) { self.telemetry .preview(PreviewKind::Canvas) diff --git a/crates/hypercolor-daemon/src/process.rs b/crates/hypercolor-daemon/src/process.rs index 25de159b8..bf49a6081 100644 --- a/crates/hypercolor-daemon/src/process.rs +++ b/crates/hypercolor-daemon/src/process.rs @@ -298,6 +298,9 @@ impl From for ServoGpuImportMode { /// Returns an error when process bootstrap, daemon startup, serving, or /// graceful shutdown fails. pub fn run(extension_installers: &'static [&'static dyn DaemonExtensionInstaller]) -> Result<()> { + // First, before any thread exists, so every thread's panic is recorded + // and no dependency that saves and restores the hook can drop it. + crate::panic_hook::install(); #[cfg(target_os = "macos")] let (mut args, macos_owner_argument) = { let matches = DaemonArgs::command().get_matches(); diff --git a/crates/hypercolor-daemon/src/render_thread.rs b/crates/hypercolor-daemon/src/render_thread.rs index b005018d6..7169cdc6b 100644 --- a/crates/hypercolor-daemon/src/render_thread.rs +++ b/crates/hypercolor-daemon/src/render_thread.rs @@ -36,9 +36,11 @@ mod frame_throttle; #[cfg(feature = "wgpu")] #[doc(hidden)] pub mod gpu_device; +pub mod gpu_errors; mod input_publication; mod layer_runtime; mod lighting_feed; +mod output_gate; mod pipeline_driver; mod pipeline_runtime; mod producer_queue; @@ -51,6 +53,8 @@ mod screen_canvas; mod screen_parity_diagnostics; #[doc(hidden)] pub mod sparkleflinger; +pub mod supervision; +mod supervisor; mod unassigned_output; mod zone_previews; @@ -61,17 +65,17 @@ use std::sync::mpsc; use std::time::{Duration, Instant}; use anyhow::{Context, Result, anyhow}; +use arc_swap::ArcSwapOption; use tokio::sync::{Mutex, RwLock, watch}; use tokio_util::sync::CancellationToken; use tracing::info; +use self::input_publication::InputPublicationMonitor; pub(crate) use self::input_publication::InputScreenBranchRequest; pub use self::input_publication::{ InputPublicationConsumer, InputPublicationDemand, InputPublicationDemandHandle, InputPublicationDemandRegistration, InputPublicationStatus, InputScreenBranchDemand, }; -use self::input_publication::{InputPublicationMonitor, InputPublicationPump}; -use self::pipeline_driver::run_pipeline; pub(crate) use self::producer_queue::ProducerFrame; pub(crate) use self::render_zones::{RenderSceneContext, ZoneFrameInputs}; pub(crate) use self::scene_dependency::SceneDependencyKey; @@ -83,6 +87,10 @@ pub(crate) use self::scene_dependency::SceneDependencyKey; pub(crate) use self::screen_parity_diagnostics::{ ScreenParityDiagnosticHandle, ScreenParityLiveSnapshot, ScreenParitySnapshotError, }; +pub use self::supervision::{ + RenderFailureKind, RenderFailureRecord, RenderRecoveryAction, RenderRestartPolicy, + RenderSupervision, RenderSupervisionSnapshot, RenderThreadPhase, +}; use crate::discovery::DiscoveryRuntime; use crate::domain::scene::{ScenePlanReader, SceneService}; use crate::domain::spatial::SpatialService; @@ -91,7 +99,7 @@ use crate::output_power::OutputPowerState; use crate::performance::PerformanceTracker; use crate::preview_runtime::PreviewRuntime; use crate::scene_transactions::SceneTransactionQueue; -use crate::startup::{StartupProgress, startup_step}; +use crate::startup::StartupProgress; use crate::zone_layout_preview::ZoneLayoutPreviewStore; use hypercolor_core::asset::AssetLibrary; use hypercolor_core::bus::HypercolorBus; @@ -246,11 +254,20 @@ const fn u8_to_fps_tier(value: u8) -> FpsTier { /// Call [`shutdown`](Self::shutdown) to stop the thread gracefully. /// The render loop must be stopped first (via `RenderLoop::stop()`) — the /// thread will exit on the next `tick()` returning `false`. +/// +/// The thread supervises its own pipeline: a panic or error exit is logged, +/// recorded in the [`RenderSupervision`] it was spawned with, and followed +/// by an in-process restart, until the restart budget runs out and the +/// supervision asks the daemon to exit. pub struct RenderThread { join_handle: Option>>, cancel: CancellationToken, input_publication_demands: InputPublicationDemandHandle, - input_publication_monitor: InputPublicationMonitor, + /// The current input pump's monitor, replaced when a failed pump is + /// restarted. + input_publication_monitor: Arc>, + supervision: RenderSupervision, + _panic_observer: Option, #[cfg(all(target_os = "macos", feature = "wgpu", feature = "screen-capture"))] screen_parity_diagnostics: ScreenParityDiagnosticHandle, } @@ -358,7 +375,16 @@ impl RenderThread { } pub fn try_spawn(state: RenderThreadState) -> Result { - Self::try_spawn_with_runtime_builder(state, build_render_runtime) + Self::try_spawn_supervised(state, RenderSupervision::detached()) + } + + /// Spawn the render thread under `supervision`, which carries the + /// restart policy and receives its health and failures. + pub fn try_spawn_supervised( + state: RenderThreadState, + supervision: RenderSupervision, + ) -> Result { + Self::spawn_inner(state, build_render_runtime, None, supervision) } /// Spawn the render thread while reporting each completed startup step @@ -375,8 +401,9 @@ impl RenderThread { pub fn try_spawn_reporting( state: RenderThreadState, progress: StartupProgress, + supervision: RenderSupervision, ) -> Result { - Self::spawn_inner(state, build_render_runtime, Some(progress)) + Self::spawn_inner(state, build_render_runtime, Some(progress), supervision) } #[doc(hidden)] @@ -387,24 +414,23 @@ impl RenderThread { where F: FnOnce() -> Result + Send + 'static, { - Self::spawn_inner(state, build_runtime, None) + Self::spawn_inner(state, build_runtime, None, RenderSupervision::detached()) } fn spawn_inner( state: RenderThreadState, build_runtime: F, progress: Option, + supervision: RenderSupervision, ) -> Result where F: FnOnce() -> Result + Send + 'static, { let input_publication_demands = InputPublicationDemandHandle::new(state.input_manager.screen_native_execution_policy()); - let pump_demands = input_publication_demands.clone(); - let pipeline_demands = input_publication_demands.clone(); let cancel = CancellationToken::new(); - let worker_cancel = cancel.clone(); - let (ready_tx, ready_rx) = mpsc::sync_channel::>(1); + let input_publication_monitor = Arc::new(ArcSwapOption::empty()); + let (ready_tx, ready_rx) = mpsc::sync_channel::>(1); #[cfg(feature = "wgpu")] let (screen_parity_diagnostics, screen_parity_mailbox) = screen_parity_diagnostics::screen_parity_diagnostic_channel(); @@ -413,118 +439,67 @@ impl RenderThread { not(all(target_os = "macos", feature = "screen-capture")) ))] let _ = screen_parity_diagnostics; - let join_handle = std::thread::Builder::new() - .name("hypercolor-render".to_owned()) - .spawn(move || -> Result<()> { - let _scene_transaction_consumer = state.scene_transactions.consumer(); - crate::process::configure_render_thread_priority(); - let runtime = - match startup_step(progress.as_ref(), "render runtime", build_runtime) { - Ok(runtime) => runtime, - Err(error) => { - let _ = ready_tx.send(Err(error)); - return Ok(()); - } - }; - let mut input_pump = match startup_step( - progress.as_ref(), - "input publication pump", - || { - runtime.block_on(InputPublicationPump::start( - state.input_manager.clone(), - pump_demands, - Some(state.interaction_routing.device_input().clone()), - )) - }, - ) { - Ok(pump) => pump, - Err(error) => { - let _ = ready_tx.send(Err(error)); - return Ok(()); - } - }; - #[cfg(all( - target_os = "macos", - feature = "wgpu", - feature = "screen-capture" - ))] - let pipeline = pipeline_runtime::PipelineRuntime::from_state( - &state, - input_pump.reader(), - pipeline_demands, - #[cfg(feature = "wgpu")] - screen_parity_mailbox, - progress.as_ref(), - ); - #[cfg(not(all( - target_os = "macos", - feature = "wgpu", - feature = "screen-capture" - )))] - let pipeline = pipeline_runtime::PipelineRuntime::from_state( - &state, - input_pump.reader(), - pipeline_demands, - #[cfg(feature = "wgpu")] - screen_parity_mailbox, - progress.as_ref(), - ); - // Startup reporting ends with the pipeline build. - drop(progress); - match pipeline { - Ok(runtime_state) => { - let monitor = input_pump.monitor(); - if ready_tx.send(Ok(monitor.clone())).is_err() { - return runtime.block_on(input_pump.shutdown()); - } - let pipeline_result = runtime.block_on(async { - tokio::select! { - () = run_pipeline(state.clone(), runtime_state) => Ok(()), - status = monitor.wait_for_terminal() => { - tracing::error!(?status, "input publication pump terminated while rendering"); - state.render_loop.write().await.stop(); - Err(anyhow!("input publication pump terminated: {status:?}")) - } - () = worker_cancel.cancelled() => { - state.render_loop.write().await.stop(); - Ok(()) - } - } - }); - let shutdown_result = runtime.block_on(input_pump.shutdown()); - pipeline_result.and(shutdown_result) - } - Err(error) => { - if let Err(shutdown_error) = runtime.block_on(input_pump.shutdown()) { - tracing::warn!(%shutdown_error, "input publication cleanup failed after render startup error"); - } - let _ = ready_tx.send(Err(error)); - Ok(()) - } - } - }) - .context("failed to spawn render thread")?; - let ready = ready_rx - .recv() - .context("render thread exited before startup completed")?; - let input_publication_monitor = match ready { - Ok(monitor) => monitor, + let panic_observer = supervision + .panic_observer() + .map(crate::panic_hook::add_observer); + supervision.set_phase(RenderThreadPhase::Starting); + let launch = supervisor::RenderThreadLaunch { + state, + build_runtime, + progress, + ready: ready_tx, + cancel: cancel.clone(), + input_demands: input_publication_demands.clone(), + input_monitor: Arc::clone(&input_publication_monitor), + supervision: supervision.clone(), + #[cfg(feature = "wgpu")] + screen_parity_mailbox, + }; + let spawned = std::thread::Builder::new() + .name(supervision::RENDER_THREAD_NAME.to_owned()) + .spawn(move || supervisor::run(launch)) + .context("failed to spawn render thread"); + let join_handle = match spawned { + Ok(join_handle) => join_handle, Err(error) => { - let _ = join_handle.join(); + supervision.set_phase(RenderThreadPhase::Stopped); return Err(error); } }; + let startup = ready_rx + .recv() + .context("render thread exited before startup completed") + .and_then(|ready| ready); + if let Err(error) = startup { + let _ = join_handle.join(); + return Err(match supervision.termination() { + Some(record) => error.context(format!( + "render thread failed during startup ({}): {}", + record.kind.as_str(), + record.message + )), + None => error, + }); + } info!("render thread spawned"); Ok(Self { join_handle: Some(join_handle), cancel, input_publication_demands, input_publication_monitor, + supervision, + _panic_observer: panic_observer, #[cfg(all(target_os = "macos", feature = "wgpu", feature = "screen-capture"))] screen_parity_diagnostics, }) } + /// The supervision this thread reports its health and failures to. + #[must_use] + pub fn supervision(&self) -> RenderSupervision { + self.supervision.clone() + } + /// Clone the lock-free demand publication used by input consumers. pub fn input_publication_demands(&self) -> InputPublicationDemandHandle { self.input_publication_demands.clone() @@ -532,7 +507,10 @@ impl RenderThread { /// Read the input-publication worker lifecycle state without blocking. pub fn input_publication_status(&self) -> InputPublicationStatus { - self.input_publication_monitor.status() + self.input_publication_monitor + .load() + .as_ref() + .map_or(InputPublicationStatus::Starting, |monitor| monitor.status()) } #[cfg(all(target_os = "macos", feature = "wgpu", feature = "screen-capture"))] @@ -583,7 +561,7 @@ fn build_render_runtime() -> Result { .worker_threads(RENDER_RUNTIME_WORKERS) .max_blocking_threads(RENDER_RUNTIME_MAX_BLOCKING_THREADS) .thread_keep_alive(RENDER_RUNTIME_THREAD_KEEP_ALIVE) - .thread_name("hypercolor-render-rt") + .thread_name(supervision::RENDER_RUNTIME_THREAD_NAME) .enable_all() .build() .context("failed to initialize render thread runtime") diff --git a/crates/hypercolor-daemon/src/render_thread/frame_executor.rs b/crates/hypercolor-daemon/src/render_thread/frame_executor.rs index 29411c281..71723c87f 100644 --- a/crates/hypercolor-daemon/src/render_thread/frame_executor.rs +++ b/crates/hypercolor-daemon/src/render_thread/frame_executor.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; -use tracing::{info, warn}; +use tracing::{debug, info, warn}; use hypercolor_core::bus::{CanvasFrame, DisplayZoneFrame}; use hypercolor_core::device::BackendManager; @@ -14,10 +14,11 @@ use hypercolor_types::session::OffOutputBehavior; use super::frame_composer::{ComposeRequest, compose_frame}; use super::frame_io::{FramePublicationRequest, FramePublicationSurfaces, publish_frame_updates}; use super::frame_metrics::{ActiveFrameMetricsInput, summarize_active_frame}; -use super::frame_policy::{FrameExecution, SkipDecision}; +use super::frame_policy::{FrameExecution, FramePolicy, SkipDecision}; use super::frame_reporting::{FrameCompletionReport, report_active_frame_completion}; use super::frame_sampling::{LedSamplingOutcome, resolve_led_sampling}; use super::frame_throttle::{maybe_idle_throttle, maybe_sleep_throttle}; +use super::output_gate::{GateDecision, ProducerSignature}; use super::pipeline_runtime::PreparedCanvasResize; use super::pipeline_runtime::{ FrameLoopState, OutputFrameSource, OutputReuseKey, PendingSamplingWork, PipelineRuntime, @@ -339,6 +340,11 @@ pub(crate) async fn execute_frame( .frame_policy .complete_throttled_frame(&mut render_loop, Duration::ZERO); } + if let Some(parked) = + gate_output_demand(state, frame_loop, render, &mut runtime.frame_policy).await + { + return parked; + } render.sparkleflinger.wait_for_gpu_frame_budget(); let mut scene_snapshot = build_frame_scene_snapshot( state, @@ -529,6 +535,9 @@ pub(crate) async fn execute_frame( }) .await; let render_us = render_stage.total_us; + frame_loop + .output_gate + .note_scene_transition(render_stage.scene_transition_active); let preview_advance_start = Instant::now(); { @@ -916,6 +925,54 @@ pub(crate) async fn execute_frame( } } +/// Park the loop instead of rendering a frame nothing consumes. +/// +/// The frame-rate inputs (audio, screen, interaction capture) feed only the +/// frames, so their capture demand is withdrawn with them and comes back +/// with the next rendered frame, and the effect producers quiesce work +/// they would otherwise keep doing between frames. +async fn gate_output_demand( + state: &RenderThreadState, + frame_loop: &mut FrameLoopState, + render: &mut RenderCaches, + frame_policy: &mut FramePolicy, +) -> Option { + let consumers = frame_loop + .output_gate + .consumers(state, &frame_loop.input_demands) + .await; + let signature = ProducerSignature::observe(state).await; + let warming = render.render_zone_runtime.producers_warming() + || frame_loop.output_gate.transition_active(); + let decision = + frame_loop + .output_gate + .decide(consumers.any(), signature, warming, Instant::now()); + if decision == GateDecision::Render { + if frame_loop.output_gate.leave_park() { + state.performance.write().await.note_output_unparked(); + } + return None; + } + + frame_loop.output_gate.enter_park(); + frame_loop.clear_input_demands(); + let _ = render.screen_queue.clear_latest(); + render.sparkleflinger.release_native_screen_caches(); + render.render_zone_runtime.park_producers(); + render.sparkleflinger.end_gpu_frame(); + let began = state + .performance + .write() + .await + .note_output_parked(Instant::now()); + if began { + debug!("Render loop parked: nothing consumes its frames"); + } + let mut render_loop = state.render_loop.write().await; + Some(frame_policy.park_execution(&mut render_loop)) +} + fn publish_input_demands( frame_loop: &mut super::pipeline_runtime::FrameLoopState, effect_demand: super::scene_snapshot::EffectDemand, @@ -973,7 +1030,7 @@ async fn force_static_sleep_snapshot( let frame_number = state .event_bus - .frame_receiver() + .frame_lane() .borrow() .frame_number .saturating_add(1); diff --git a/crates/hypercolor-daemon/src/render_thread/frame_policy.rs b/crates/hypercolor-daemon/src/render_thread/frame_policy.rs index 364cb034d..9ed702d1d 100644 --- a/crates/hypercolor-daemon/src/render_thread/frame_policy.rs +++ b/crates/hypercolor-daemon/src/render_thread/frame_policy.rs @@ -49,6 +49,11 @@ impl FrameExecution { pub(crate) fn resolve_deadline(self, scheduled_start: Instant, now: Instant) -> Instant { self.next_wake.resolve_deadline(scheduled_start, now) } + + /// Whether the loop parked because nothing consumes its frames. + pub(crate) const fn parks(&self) -> bool { + matches!(self.next_wake, NextWake::Park) + } } /// Scheduler decision for when the next render iteration should begin. @@ -57,6 +62,8 @@ pub(crate) enum NextWake { Interval(Duration), /// Hold the loop for a fixed delay before checking again. Delay(Duration), + /// Sleep until output demand appears; the pipeline driver owns the wait. + Park, } impl NextWake { @@ -64,6 +71,7 @@ impl NextWake { match self { Self::Interval(interval) => advance_deadline(scheduled_start, interval, now), Self::Delay(delay) => now.checked_add(delay).unwrap_or(now), + Self::Park => now, } } } @@ -129,6 +137,19 @@ impl FramePolicy { self.complete_throttled_frame(render_loop, SESSION_SLEEP_THROTTLE_DELAY) } + /// Drop the admitted frame without rendering it and park the loop. + /// + /// The frame is abandoned rather than completed, so parking feeds no + /// timing into tier decisions and does not advance the frame number. + pub(crate) fn park_execution(&mut self, render_loop: &mut RenderLoop) -> FrameExecution { + render_loop.abandon_frame(); + render_loop.park(); + FrameExecution { + next_wake: NextWake::Park, + next_skip_decision: SkipDecision::None, + } + } + pub(crate) fn inactive_loop_execution( &self, loop_state: RenderLoopState, @@ -306,6 +327,22 @@ mod tests { assert_eq!(execution.next_skip_decision, SkipDecision::None); } + #[test] + fn park_execution_parks_without_a_skip_decision() { + let mut render_loop = RenderLoop::new(60); + render_loop.start(); + assert!(render_loop.tick()); + let mut policy = FramePolicy::new(FpsTier::Full); + + let execution = policy.park_execution(&mut render_loop); + + assert!(execution.parks()); + assert_eq!(execution.next_skip_decision, SkipDecision::None); + assert_eq!(render_loop.frame_number(), 0); + assert!(render_loop.is_parked()); + assert!(render_loop.frame_complete().is_none()); + } + #[test] fn sleep_throttle_execution_uses_sleep_delay() { let mut render_loop = RenderLoop::new(60); diff --git a/crates/hypercolor-daemon/src/render_thread/gpu_device.rs b/crates/hypercolor-daemon/src/render_thread/gpu_device.rs index 1db097624..47de62be9 100644 --- a/crates/hypercolor-daemon/src/render_thread/gpu_device.rs +++ b/crates/hypercolor-daemon/src/render_thread/gpu_device.rs @@ -2,6 +2,10 @@ use std::sync::Arc; use anyhow::{Context, Result}; +use super::gpu_errors::{ + DeviceLossSignal, GpuDeviceRole, GpuErrorKind, record_device_lost, record_uncaptured_error, +}; + const GPU_RESOURCE_CREATION_BUDGET_PERCENT: u8 = 80; const GPU_DEVICE_LOSS_BUDGET_PERCENT: u8 = 95; @@ -17,6 +21,7 @@ struct GpuRenderDeviceInner { device: wgpu::Device, queue: wgpu::Queue, info: GpuRenderDeviceInfo, + loss: DeviceLossSignal, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -125,6 +130,7 @@ impl GpuRenderDevice { trace: wgpu::Trace::Off, })) .with_context(|| format!("failed to create a {label} wgpu device"))?; + let loss = install_error_handlers(&device, label, GpuDeviceRole::Compositor); let limits = device.limits(); let info = GpuRenderDeviceInfo { @@ -147,6 +153,7 @@ impl GpuRenderDevice { device, queue, info, + loss, }), }) } @@ -193,6 +200,7 @@ impl GpuRenderDevice { trace: wgpu::Trace::Off, })) .with_context(|| format!("failed to create an independent {label} wgpu device"))?; + let loss = install_error_handlers(&device, label, GpuDeviceRole::Auxiliary); let adapter_info = adapter.get_info(); let limits = device.limits(); @@ -216,6 +224,7 @@ impl GpuRenderDevice { device, queue, info, + loss, }), }) } @@ -240,6 +249,11 @@ impl GpuRenderDevice { self.inner.info.clone() } + /// Raised once wgpu reports this device lost. + pub(crate) fn loss_signal(&self) -> DeviceLossSignal { + self.inner.loss.clone() + } + pub(crate) fn require_texture_usage( &self, format: wgpu::TextureFormat, @@ -257,6 +271,54 @@ impl GpuRenderDevice { } } +/// Route every error and loss `device` reports into the daemon's GPU +/// error accounting, and return the signal its loss raises. +/// +/// Without a handler wgpu panics the calling thread on the first +/// uncaptured validation error, which on the render thread takes the whole +/// pipeline down. With it the error is counted, logged at error level with +/// the device label and wgpu's description, and kept for diagnostics. +fn install_error_handlers( + device: &wgpu::Device, + label: &'static str, + role: GpuDeviceRole, +) -> DeviceLossSignal { + device.on_uncaptured_error(Arc::new(move |error: wgpu::Error| { + let kind = match &error { + wgpu::Error::Validation { .. } => GpuErrorKind::Validation, + wgpu::Error::OutOfMemory { .. } => GpuErrorKind::OutOfMemory, + wgpu::Error::Internal { .. } => GpuErrorKind::Internal, + }; + record_uncaptured_error(label, kind, &error.to_string()); + })); + let loss = DeviceLossSignal::default(); + let raised = loss.clone(); + device.set_device_lost_callback(move |reason, message| { + let reason = match reason { + wgpu::DeviceLostReason::Unknown => "unknown", + wgpu::DeviceLostReason::Destroyed => "destroyed", + }; + record_device_lost(label, role, reason, &message); + raised.raise(format!("{label} lost ({reason}: {message})")); + }); + loss +} + +/// Raise one uncaptured validation error on `device` on purpose. +/// +/// A buffer that is both read- and write-mappable is invalid without +/// `MAPPABLE_PRIMARY_BUFFERS`, which no daemon device requests, so wgpu +/// reports it synchronously through the uncaptured-error handler. +#[cfg(any(test, feature = "persistence-test-hooks"))] +pub(crate) fn raise_uncaptured_validation_error(device: &wgpu::Device) { + let _invalid = device.create_buffer(&wgpu::BufferDescriptor { + label: Some("injected validation error"), + size: 4, + usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::MAP_WRITE, + mapped_at_creation: false, + }); +} + /// Make test devices compile shaders the way shipped builds do on users' /// machines. /// @@ -657,6 +719,45 @@ mod tests { assert!(llvmpipe.software_adapter_reason().is_some()); } + #[test] + fn uncaptured_validation_errors_are_counted_instead_of_panicking() { + let Ok(device) = GpuRenderDevice::new_with_backend_preference( + "validation handler test device", + GpuBackendPreference::Default, + ) else { + return; + }; + let before = crate::render_thread::gpu_errors::gpu_error_totals(); + + raise_uncaptured_validation_error(device.device()); + + let after = crate::render_thread::gpu_errors::gpu_error_totals(); + assert!(after.validation > before.validation); + } + + #[test] + fn a_destroyed_device_raises_only_its_own_loss_signal() { + let Ok(device) = GpuRenderDevice::new_with_backend_preference( + "loss signal test device", + GpuBackendPreference::Default, + ) else { + return; + }; + let sibling = device + .independent_device("loss signal sibling device") + .expect("selected adapter should support a second logical device"); + + device.device().destroy(); + let _ = device.device().poll(wgpu::PollType::Poll); + + let lost = device + .loss_signal() + .lost() + .expect("destroying the device should run its device-lost callback"); + assert!(lost.contains("loss signal test device"), "{lost}"); + assert!(sibling.loss_signal().lost().is_none()); + } + #[test] fn independent_device_reuses_adapter_without_sharing_device_or_queue() { let Ok(authoritative) = GpuRenderDevice::new_with_backend_preference( diff --git a/crates/hypercolor-daemon/src/render_thread/gpu_errors.rs b/crates/hypercolor-daemon/src/render_thread/gpu_errors.rs new file mode 100644 index 000000000..071326751 --- /dev/null +++ b/crates/hypercolor-daemon/src/render_thread/gpu_errors.rs @@ -0,0 +1,338 @@ +//! Process-wide accounting for wgpu errors no error scope captured. +//! +//! wgpu's default uncaptured-error handler panics the calling thread, which +//! on the render thread takes the whole pipeline down for a validation +//! mistake. Every device the daemon opens routes those errors here instead: +//! each one is counted, the latest is kept for diagnostics, and they are +//! logged at error level under a burst limit so a cascade of follow-on +//! errors from one invalid resource cannot flood the log. Device loss is +//! counted and logged too, and each device raises its own +//! [`DeviceLossSignal`], which the render supervisor waits on for the +//! device its pipeline draws with. +//! +//! The accounting has no wgpu types in it, so builds without the `wgpu` +//! feature still report (all zero) totals. +#![cfg_attr( + not(feature = "wgpu"), + allow( + dead_code, + reason = "only wgpu devices report errors; builds without wgpu still report zero totals" + ) +)] + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use tokio::sync::watch; + +/// Errors logged per [`LOG_WINDOW`] before the rest are only counted. +const LOG_BURST: u32 = 10; + +/// Window the [`LOG_BURST`] limit applies to. +const LOG_WINDOW: Duration = Duration::from_secs(1); + +/// Which kind of uncaptured error wgpu raised. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum GpuErrorKind { + Validation, + OutOfMemory, + Internal, +} + +impl GpuErrorKind { + const fn as_str(self) -> &'static str { + match self { + Self::Validation => "validation", + Self::OutOfMemory => "out_of_memory", + Self::Internal => "internal", + } + } +} + +/// What a device does for the daemon, reported with its loss. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum GpuDeviceRole { + /// A compositor device a render pipeline draws on. + Compositor, + /// Any other logical device, such as the interactive preview executor's. + Auxiliary, +} + +/// The most recent uncaptured error or device loss. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GpuErrorSample { + /// Label of the device that reported it. + pub device: String, + /// `validation`, `out_of_memory`, `internal`, or `device_lost`. + pub kind: String, + /// wgpu's description of the error. + pub message: String, + /// Wall-clock time it was recorded, in Unix milliseconds. + pub at_unix_ms: u64, +} + +/// Counts of every uncaptured error and device loss since process start. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct GpuErrorTotals { + pub validation: u64, + pub out_of_memory: u64, + pub internal: u64, + pub device_lost: u64, + pub last: Option, +} + +impl GpuErrorTotals { + /// Uncaptured errors of every kind, device loss excluded. + #[must_use] + pub const fn uncaptured(&self) -> u64 { + self.validation + .saturating_add(self.out_of_memory) + .saturating_add(self.internal) + } +} + +#[derive(Debug, Default)] +struct LogWindow { + started_at: Option, + logged: u32, + suppressed: u64, +} + +struct GpuErrorTelemetry { + validation: AtomicU64, + out_of_memory: AtomicU64, + internal: AtomicU64, + device_lost: AtomicU64, + last: Mutex>, + log_window: Mutex, +} + +static TELEMETRY: GpuErrorTelemetry = GpuErrorTelemetry { + validation: AtomicU64::new(0), + out_of_memory: AtomicU64::new(0), + internal: AtomicU64::new(0), + device_lost: AtomicU64::new(0), + last: Mutex::new(None), + log_window: Mutex::new(LogWindow { + started_at: None, + logged: 0, + suppressed: 0, + }), +}; + +/// Sticky "this device was lost" flag for one wgpu device. +/// +/// Clones share the flag. It is set once, by the device-lost callback, and +/// stays set: a lost device never comes back, so anything that checks it +/// later, such as a pipeline rebuilt on the same device, sees the loss too. +#[derive(Debug, Clone)] +pub(crate) struct DeviceLossSignal { + sender: Arc>>>, +} + +impl Default for DeviceLossSignal { + fn default() -> Self { + Self { + sender: Arc::new(watch::Sender::new(None)), + } + } +} + +impl DeviceLossSignal { + /// Mark the device lost, keeping the first description. + pub(crate) fn raise(&self, description: String) { + self.sender.send_if_modified(|lost| { + if lost.is_some() { + return false; + } + *lost = Some(Arc::from(description)); + true + }); + } + + /// The loss description, once the device is lost. + pub(crate) fn lost(&self) -> Option> { + self.sender.borrow().clone() + } + + /// Wait until the device is lost, and return the description. + pub(crate) async fn wait(&self) -> String { + let mut receiver = self.sender.subscribe(); + let lost = receiver + .wait_for(Option::is_some) + .await + .map(|lost| lost.as_deref().unwrap_or("device lost").to_owned()); + match lost { + Ok(lost) => lost, + // This signal holds its own sender, so the channel cannot close + // while the wait borrows it. + Err(_) => std::future::pending().await, + } + } +} + +/// Record one uncaptured error from `device`. +pub(crate) fn record_uncaptured_error(device: &str, kind: GpuErrorKind, message: &str) { + let counter = match kind { + GpuErrorKind::Validation => &TELEMETRY.validation, + GpuErrorKind::OutOfMemory => &TELEMETRY.out_of_memory, + GpuErrorKind::Internal => &TELEMETRY.internal, + }; + let total = counter.fetch_add(1, Ordering::Relaxed).saturating_add(1); + store_last(device, kind.as_str(), message); + + if let Some(suppressed) = admit_log() { + tracing::error!( + device, + kind = kind.as_str(), + thread = std::thread::current().name().unwrap_or(""), + total, + suppressed_since_last_log = suppressed, + error = message, + "uncaptured wgpu error" + ); + } +} + +/// Count and log that `device` was lost. +pub(crate) fn record_device_lost(device: &str, role: GpuDeviceRole, reason: &str, message: &str) { + let total = TELEMETRY + .device_lost + .fetch_add(1, Ordering::Relaxed) + .saturating_add(1); + let detail = if message.is_empty() { + reason.to_owned() + } else { + format!("{reason}: {message}") + }; + store_last(device, "device_lost", &detail); + tracing::error!( + device, + ?role, + reason, + total, + error = message, + "wgpu device lost" + ); +} + +/// Snapshot the totals since process start. +#[must_use] +pub fn gpu_error_totals() -> GpuErrorTotals { + GpuErrorTotals { + validation: TELEMETRY.validation.load(Ordering::Relaxed), + out_of_memory: TELEMETRY.out_of_memory.load(Ordering::Relaxed), + internal: TELEMETRY.internal.load(Ordering::Relaxed), + device_lost: TELEMETRY.device_lost.load(Ordering::Relaxed), + last: TELEMETRY + .last + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone(), + } +} + +fn store_last(device: &str, kind: &str, message: &str) { + let at_unix_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }); + let sample = GpuErrorSample { + device: device.to_owned(), + kind: kind.to_owned(), + message: message.to_owned(), + at_unix_ms, + }; + *TELEMETRY + .last + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(sample); +} + +/// Admit one log line under the burst limit, returning how many errors +/// went unlogged since the last admitted one. +fn admit_log() -> Option { + let mut window = TELEMETRY + .log_window + .lock() + .unwrap_or_else(PoisonError::into_inner); + let now = Instant::now(); + if window + .started_at + .is_none_or(|started| now.saturating_duration_since(started) >= LOG_WINDOW) + { + window.started_at = Some(now); + window.logged = 0; + } + if window.logged >= LOG_BURST { + window.suppressed = window.suppressed.saturating_add(1); + return None; + } + window.logged += 1; + Some(std::mem::take(&mut window.suppressed)) +} + +#[cfg(test)] +mod tests { + use super::{ + DeviceLossSignal, GpuDeviceRole, GpuErrorKind, gpu_error_totals, record_device_lost, + record_uncaptured_error, + }; + + #[test] + fn uncaptured_errors_count_by_kind_and_keep_the_latest() { + let before = gpu_error_totals(); + + record_uncaptured_error("unit test device", GpuErrorKind::Validation, "bad usage"); + record_uncaptured_error( + "unit test device", + GpuErrorKind::OutOfMemory, + "Out of Memory", + ); + + let after = gpu_error_totals(); + assert!(after.validation > before.validation); + assert!(after.out_of_memory > before.out_of_memory); + assert!(after.uncaptured() >= before.uncaptured() + 2); + assert!(after.last.is_some(), "the latest error should be kept"); + } + + #[test] + fn device_loss_is_counted() { + let before = gpu_error_totals(); + + record_device_lost( + "preview device", + GpuDeviceRole::Auxiliary, + "unknown", + "reset", + ); + + assert!(gpu_error_totals().device_lost > before.device_lost); + } + + #[tokio::test] + async fn a_raised_loss_wakes_a_waiter_and_stays_raised() { + let signal = DeviceLossSignal::default(); + assert!(signal.lost().is_none()); + let waiter = tokio::spawn({ + let signal = signal.clone(); + async move { signal.wait().await } + }); + + signal.raise("compositor lost (unknown: reset)".to_owned()); + signal.raise("a later loss report".to_owned()); + + assert_eq!( + waiter.await.expect("waiter should finish"), + "compositor lost (unknown: reset)" + ); + assert_eq!( + signal.lost().as_deref(), + Some("compositor lost (unknown: reset)") + ); + assert_eq!(signal.wait().await, "compositor lost (unknown: reset)"); + } +} diff --git a/crates/hypercolor-daemon/src/render_thread/input_publication.rs b/crates/hypercolor-daemon/src/render_thread/input_publication.rs index 69c365cae..6a43d2725 100644 --- a/crates/hypercolor-daemon/src/render_thread/input_publication.rs +++ b/crates/hypercolor-daemon/src/render_thread/input_publication.rs @@ -729,6 +729,25 @@ impl InputPublicationDemandHandle { self.snapshot().registration_count(consumer) } + /// Count live stream and diagnostic registrations that request `source`. + /// + /// These consumers render nothing themselves; they read what the + /// authoritative renderer republishes from that source, such as host + /// input events, so they need the renderer running. + #[must_use] + pub fn stream_registrations_requesting(&self, source: SourceKind) -> usize { + self.snapshot() + .entries + .iter() + .filter(|entry| { + matches!( + entry.consumer, + InputPublicationConsumer::PassiveStream | InputPublicationConsumer::Diagnostic + ) && entry.demand.requested_hz(source) > 0 + }) + .count() + } + /// Read the current aggregate rate for one source domain. #[must_use] pub fn requested_hz(&self, source: SourceKind) -> u32 { @@ -751,7 +770,7 @@ impl InputPublicationDemandHandle { self.registry.latest.load_full() } - fn subscribe_revision(&self) -> watch::Receiver { + pub(super) fn subscribe_revision(&self) -> watch::Receiver { self.registry.revision_tx.subscribe() } @@ -1054,7 +1073,7 @@ pub enum InputPublicationStatus { } impl InputPublicationStatus { - const fn is_terminal(&self) -> bool { + pub(crate) const fn is_terminal(&self) -> bool { matches!(self, Self::Stopped | Self::Failed(_)) } } diff --git a/crates/hypercolor-daemon/src/render_thread/output_gate.rs b/crates/hypercolor-daemon/src/render_thread/output_gate.rs new file mode 100644 index 000000000..bf1347fa1 --- /dev/null +++ b/crates/hypercolor-daemon/src/render_thread/output_gate.rs @@ -0,0 +1,449 @@ +//! Output-demand gate: park the render loop while nothing consumes frames. +//! +//! Every frame costs producer, compositor, and GPU work, so a loop that +//! renders with no device, display, preview, or capture attached spends a +//! fixed per-frame cost on output no one receives. The gate removes that +//! cost without changing any observable cadence: before each frame it +//! counts consumers (see [`crate::output_demand`]), and with none it parks +//! the loop instead of rendering. A parked loop sleeps on the bus output +//! demand signal, the device registry, the input demand registry, and the +//! scene transaction queue, so the first consumer to appear wakes it at +//! once and it resumes at its current tier. A long safety tick re-evaluates in +//! case some consumer arrived without signalling. +//! +//! Parking waits for the producers to settle first. After the scene, the +//! effect catalog, or the canvas changes, the loop keeps rendering until +//! every producer is warm and any scene transition has finished, so an +//! effect applied while nobody watches is loaded and showing its own +//! content by the time a consumer arrives. + +use std::time::{Duration, Instant}; + +use hypercolor_core::input::SourceKind; +use tokio::pin; + +use super::RenderThreadState; +use super::input_publication::InputPublicationDemandHandle; +use super::pipeline_runtime::FrameLoopState; +use super::supervision::RenderSupervision; +use crate::output_demand::{OutputConsumers, is_output_backend, output_device_count}; + +/// How often a parked loop re-evaluates demand without any wake signal. +pub(crate) const PARKED_SAFETY_TICK: Duration = Duration::from_secs(1); + +/// Consecutive frame starts with no producer warming before the loop may +/// park. The first frame after a change reconciles the producers, so one +/// clean start alone does not prove a new producer is warm. +const SETTLE_WARM_FRAMES: u32 = 3; + +/// Longest the loop keeps rendering for producers that never finish +/// warming, such as a page that never loads. Matches the Servo worker's +/// ready timeout, the longest a session creation or page load may block. +const SETTLE_LIMIT: Duration = Duration::from_secs(10); + +/// Everything the producers render from, so a change to any of it is work +/// the loop must render before it parks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct ProducerSignature { + scene_generation: u64, + zones_revision: u64, + effect_registry_generation: u64, + zone_layout_preview_generation: u64, + canvas_width: u32, + canvas_height: u32, +} + +impl ProducerSignature { + pub(crate) async fn observe(state: &RenderThreadState) -> Self { + let (scene_generation, zones_revision) = { + let plan = state.scene_plan.load(); + (plan.generation, plan.zones_revision) + }; + let effect_registry_generation = state.effect_registry.read().await.generation(); + Self { + scene_generation, + zones_revision, + effect_registry_generation, + zone_layout_preview_generation: state.zone_layout_previews.generation(), + canvas_width: state.canvas_dims.width(), + canvas_height: state.canvas_dims.height(), + } + } +} + +/// Whether the next frame renders or the loop parks. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum GateDecision { + Render, + Park, +} + +/// Why a parked loop woke. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum RenderWakeReason { + /// A device, stream receiver, or demand hold appeared. + OutputConsumer, + /// A queued scene transaction has a caller waiting on it. + SceneTransaction, + /// The scene, effect catalog, or canvas changed, so the producers must + /// settle on the new state. + ProducerState, + /// The loop was paused or stopped, which the main loop handles. + RenderLoopState, +} + +impl RenderWakeReason { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::OutputConsumer => "output_consumer", + Self::SceneTransaction => "scene_transaction", + Self::ProducerState => "producer_state", + Self::RenderLoopState => "render_loop_state", + } + } +} + +/// Run of frame starts under one producer signature. +#[derive(Clone, Copy, Debug)] +struct WarmStreak { + signature: ProducerSignature, + warm_frames: u32, + since: Instant, +} + +/// Render-thread state for the output-demand gate. +#[derive(Debug, Default)] +pub(crate) struct OutputGate { + /// Registry output device count keyed by the registry generation it + /// was read at. + registry_devices: Option<(u64, u32)>, + /// Consumers last recorded into render telemetry. + recorded: Option, + streak: Option, + /// Whether the last rendered frame was mid scene transition. + transition_active: bool, + /// Whether the loop is parked, from the park until a frame renders or + /// the loop leaves the running state. + parked: bool, +} + +impl OutputGate { + /// Count consumers and record them for status surfaces when they + /// changed. + /// + /// The device registry is read only when its generation moved. When + /// neither a registered device nor any bus-side consumer exists, the + /// backend manager's routes are counted too, so a device mapped for + /// output counts even before the registry reports it renderable. + pub(crate) async fn consumers( + &mut self, + state: &RenderThreadState, + input_demands: &InputPublicationDemandHandle, + ) -> OutputConsumers { + let generation = state.device_registry.generation(); + let registry_devices = match self.registry_devices { + Some((counted_at, count)) if counted_at == generation => count, + _ => { + // A mutation racing the count leaves the cache keyed to the + // older generation, so the next evaluation counts again. + let count = output_device_count(&state.device_registry).await; + self.registry_devices = Some((generation, count)); + count + } + }; + let input_event_streams = + u32::try_from(input_demands.stream_registrations_requesting(SourceKind::Interaction)) + .unwrap_or(u32::MAX); + let mut consumers = + OutputConsumers::from_bus(&state.event_bus, registry_devices, input_event_streams); + if !consumers.any() { + consumers.output_devices = routed_output_device_count(state).await; + } + if self.recorded != Some(consumers) { + self.recorded = Some(consumers); + state + .performance + .write() + .await + .note_output_consumers(consumers); + } + consumers + } + + /// Record whether the frame just rendered was mid scene transition. + /// + /// A transition advances only as frames render, so the loop keeps + /// rendering until it completes rather than freezing it mid crossfade + /// with the outgoing scene's producers still alive. Only frames that + /// reach composition update the flag, and the settle limit still + /// bounds the wait, so a crossfade longer than that limit pauses + /// partway and finishes once a consumer returns. + pub(crate) fn note_scene_transition(&mut self, active: bool) { + self.transition_active = active; + } + + /// Whether the previous rendered frame was mid scene transition. + pub(crate) const fn transition_active(&self) -> bool { + self.transition_active + } + + /// Record that the loop parked. + pub(crate) fn enter_park(&mut self) { + self.parked = true; + } + + /// Record that the loop left its park, returning whether it was + /// parked. + pub(crate) fn leave_park(&mut self) -> bool { + std::mem::take(&mut self.parked) + } + + /// Decide whether this frame renders. + /// + /// `producers_warming` describes the producers as the previous frame + /// left them. Frames that consumers drive count toward settling too, so + /// losing the last consumer parks at once when the producers are warm. + pub(crate) fn decide( + &mut self, + consumed: bool, + signature: ProducerSignature, + producers_warming: bool, + now: Instant, + ) -> GateDecision { + let streak = match &mut self.streak { + Some(streak) if streak.signature == signature => streak, + _ => self.streak.insert(WarmStreak { + signature, + warm_frames: 0, + since: now, + }), + }; + streak.warm_frames = if producers_warming { + 0 + } else { + streak.warm_frames.saturating_add(1) + }; + if consumed { + return GateDecision::Render; + } + let settled = streak.warm_frames >= SETTLE_WARM_FRAMES + || now.saturating_duration_since(streak.since) >= SETTLE_LIMIT; + if settled { + GateDecision::Park + } else { + GateDecision::Render + } + } + + /// Why a parked loop should resume now, if it should. + pub(crate) async fn resume_reason( + &mut self, + state: &RenderThreadState, + input_demands: &InputPublicationDemandHandle, + ) -> Option { + if !state.render_loop.read().await.is_running() { + return Some(RenderWakeReason::RenderLoopState); + } + if state.scene_transactions.has_pending() { + return Some(RenderWakeReason::SceneTransaction); + } + if self.consumers(state, input_demands).await.any() { + return Some(RenderWakeReason::OutputConsumer); + } + let signature = ProducerSignature::observe(state).await; + if self + .streak + .is_none_or(|streak| streak.signature != signature) + { + return Some(RenderWakeReason::ProducerState); + } + None + } +} + +/// Count devices the backend manager routes frames to, simulated displays +/// excluded, each counted once however many layout ids bind it. +/// +/// Rigs carry a handful of routes, so the quadratic dedupe stays cheaper +/// than allocating a set on every frame. +async fn routed_output_device_count(state: &RenderThreadState) -> u32 { + let manager = state.backend_manager.lock().await; + let routed = || { + manager + .mapped_device_targets() + .filter(|(backend_id, _)| is_output_backend(backend_id)) + .map(|(_, device_id)| device_id) + }; + let count = routed() + .enumerate() + .filter(|(index, device_id)| !routed().take(*index).any(|seen| seen == *device_id)) + .count(); + u32::try_from(count).unwrap_or(u32::MAX) +} + +/// Sleep until a parked loop has a reason to run a frame. +/// +/// Each signal is armed before demand is re-evaluated, so a consumer that +/// appears between the evaluation and the sleep still wakes the loop. +pub(crate) async fn wait_for_output_demand( + state: &RenderThreadState, + frame_loop: &mut FrameLoopState, + supervision: &RenderSupervision, +) -> RenderWakeReason { + let armed_output_demand = state.event_bus.output_demand().arm(); + loop { + // Every pass, the safety tick included, proves the parked thread is + // alive to the stall check and the service watchdog. + supervision.heartbeat(Instant::now()); + let output_demand = armed_output_demand.notified(); + let devices = state.device_registry.changed(); + let transactions = state.scene_transactions.queued(); + let mut input_demand_revisions = frame_loop.input_demands.subscribe_revision(); + pin!(output_demand, devices, transactions); + output_demand.as_mut().enable(); + devices.as_mut().enable(); + transactions.as_mut().enable(); + + if let Some(reason) = frame_loop + .output_gate + .resume_reason(state, &frame_loop.input_demands) + .await + { + return reason; + } + + tokio::select! { + () = &mut output_demand => {} + () = &mut devices => {} + () = &mut transactions => {} + _ = input_demand_revisions.changed() => {} + () = tokio::time::sleep(PARKED_SAFETY_TICK) => {} + } + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use super::{GateDecision, OutputGate, ProducerSignature, SETTLE_LIMIT}; + + fn signature(scene_generation: u64) -> ProducerSignature { + ProducerSignature { + scene_generation, + zones_revision: 1, + effect_registry_generation: 1, + zone_layout_preview_generation: 0, + canvas_width: 640, + canvas_height: 480, + } + } + + #[test] + fn unconsumed_loop_renders_until_producers_settle_then_parks() { + let mut gate = OutputGate::default(); + let now = Instant::now(); + + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Park + ); + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Park + ); + } + + #[test] + fn warming_producer_holds_the_loop_awake_until_it_warms() { + let mut gate = OutputGate::default(); + let now = Instant::now(); + + for _ in 0..8 { + assert_eq!( + gate.decide(false, signature(1), true, now), + GateDecision::Render + ); + } + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Park + ); + } + + #[test] + fn producer_that_never_warms_parks_after_the_settle_limit() { + let mut gate = OutputGate::default(); + let start = Instant::now(); + + assert_eq!( + gate.decide(false, signature(1), true, start), + GateDecision::Render + ); + let late = start + SETTLE_LIMIT + Duration::from_millis(1); + assert_eq!( + gate.decide(false, signature(1), true, late), + GateDecision::Park + ); + } + + #[test] + fn consumers_always_render_and_count_toward_settling() { + let mut gate = OutputGate::default(); + let now = Instant::now(); + + for _ in 0..4 { + assert_eq!( + gate.decide(true, signature(1), false, now), + GateDecision::Render + ); + } + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Park, + "producers already warm under consumers should park as soon as the last consumer leaves" + ); + } + + #[test] + fn producer_change_while_unconsumed_settles_again() { + let mut gate = OutputGate::default(); + let now = Instant::now(); + for _ in 0..3 { + let _ = gate.decide(false, signature(1), false, now); + } + assert_eq!( + gate.decide(false, signature(1), false, now), + GateDecision::Park + ); + + assert_eq!( + gate.decide(false, signature(2), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(2), false, now), + GateDecision::Render + ); + assert_eq!( + gate.decide(false, signature(2), false, now), + GateDecision::Park + ); + } +} diff --git a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs index 4097308d3..5a507e030 100644 --- a/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs +++ b/crates/hypercolor-daemon/src/render_thread/pipeline_driver.rs @@ -5,16 +5,23 @@ use tracing::{debug, info}; use crate::deadline::wait_until_deadline; use hypercolor_core::engine::RenderLoopState; -use super::RenderThreadState; use super::frame_executor::{execute_frame, service_scene_transactions}; use super::frame_policy::SkipDecision; +use super::output_gate::{RenderWakeReason, wait_for_output_demand}; use super::pipeline_runtime::PipelineRuntime; +use super::supervision::RenderSupervision; +use super::{RenderThreadState, u64_to_u32}; -pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: PipelineRuntime) { +pub(crate) async fn run_pipeline( + state: RenderThreadState, + mut runtime: PipelineRuntime, + supervision: RenderSupervision, +) { info!( mode = ?state.render_acceleration_mode, "render pipeline started" ); + clear_stale_park(&state).await; let mut skip_decision = SkipDecision::None; let mut next_frame_at = Instant::now(); let mut rebase_clock_on_resume = false; @@ -23,6 +30,8 @@ pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: Pipeline loop { let scheduled_start = next_frame_at; wait_until_deadline(scheduled_start).await; + #[cfg(feature = "persistence-test-hooks")] + raise_injected_fault(&supervision, &mut runtime); let tick = { let mut render_loop = state.render_loop.write().await; @@ -42,7 +51,9 @@ pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: Pipeline if let Some(execution) = handle_inactive_render_loop(&state, &mut runtime, tick.state).await { - next_frame_at = execution.resolve_deadline(scheduled_start, Instant::now()); + let now = Instant::now(); + supervision.heartbeat(now); + next_frame_at = execution.resolve_deadline(scheduled_start, now); continue; } @@ -54,13 +65,101 @@ pub(crate) async fn run_pipeline(state: RenderThreadState, mut runtime: Pipeline runtime.frame_loop.clock.rebase(scheduled_start); } let frame = execute_frame(&state, &mut runtime, scheduled_start, skip_decision).await; + if frame.parks() { + let reason = + wait_for_output_demand(&state, &mut runtime.frame_loop, &supervision).await; + resume_from_park(&state, &mut runtime, reason).await; + skip_decision = SkipDecision::None; + next_frame_at = Instant::now(); + continue; + } + let now = Instant::now(); + supervision.frame_completed(now); skip_decision = frame.next_skip_decision; - next_frame_at = frame.resolve_deadline(scheduled_start, Instant::now()); + next_frame_at = frame.resolve_deadline(scheduled_start, now); } info!("render pipeline exited"); } +/// Leave a park that a previous pipeline entered and never left. +/// +/// The park flag and its hold live on the shared render loop, but the +/// output gate that would end them belongs to the pipeline. A pipeline that +/// replaces one that failed while parked starts with a fresh gate, so it +/// would otherwise render with effect time still held. +async fn clear_stale_park(state: &RenderThreadState) { + state.render_loop.write().await.unpark(); + // The telemetry can outlive the loop's flag: a pipeline that failed + // between unparking and its next recorded frame leaves it set. + state.performance.write().await.note_output_unparked(); +} + +/// Pick the loop back up after a park without a jump in effect time. +/// +/// Effects advance by the frame clock's delta, so the clock restarts one +/// target interval back: the first frame after the wake steps effect time +/// forward by exactly one frame instead of by the whole parked span, and no +/// catch-up frames follow. Capture freshness skips the wake frame itself, +/// whose asynchronous producers still show output requested before the +/// park. +async fn resume_from_park( + state: &RenderThreadState, + runtime: &mut PipelineRuntime, + reason: RenderWakeReason, +) { + let now = Instant::now(); + let (interval, resume_frame) = { + let mut render_loop = state.render_loop.write().await; + render_loop.unpark(); + (render_loop.target_interval(), render_loop.frame_number()) + }; + if reason == RenderWakeReason::RenderLoopState { + // A paused or stopped loop is idle for its own reason, not for + // lack of consumers; the gate parks it again if it resumes with + // nothing to render for. + runtime.frame_loop.output_gate.leave_park(); + state.performance.write().await.note_output_unparked(); + } + runtime + .frame_loop + .clock + .rebase(now.checked_sub(interval).unwrap_or(now)); + state + .preview_runtime + .note_render_resume(u64_to_u32(resume_frame)); + state + .performance + .write() + .await + .note_output_woken(reason.as_str()); + debug!(reason = reason.as_str(), "Render loop woke from a park"); +} + +/// Raise a fault a test queued, on the render thread, exactly where a real +/// one would surface: between frames, with the pipeline's state live. +#[cfg(feature = "persistence-test-hooks")] +fn raise_injected_fault(supervision: &RenderSupervision, runtime: &mut PipelineRuntime) { + use super::supervision::InjectedRenderFault; + + match supervision.take_injected_fault() { + None => {} + Some(InjectedRenderFault::Panic(message)) => panic!("{message}"), + Some(InjectedRenderFault::Stall(duration)) => std::thread::sleep(duration), + Some(InjectedRenderFault::GpuValidationError) => { + let raised = runtime + .render + .sparkleflinger + .raise_uncaptured_validation_error(); + if !raised { + tracing::warn!( + "injected GPU validation error skipped: the compositor runs on the CPU" + ); + } + } + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct RenderTickSnapshot { should_render: bool, diff --git a/crates/hypercolor-daemon/src/render_thread/pipeline_runtime.rs b/crates/hypercolor-daemon/src/render_thread/pipeline_runtime.rs index 908d2e9fb..4d4ddd9d4 100644 --- a/crates/hypercolor-daemon/src/render_thread/pipeline_runtime.rs +++ b/crates/hypercolor-daemon/src/render_thread/pipeline_runtime.rs @@ -1228,6 +1228,7 @@ pub(crate) struct FrameLoopState { pub(crate) clock: FrameClockState, pub(crate) inputs: InputReuseState, pub(crate) throttle: ThrottleState, + pub(crate) output_gate: super::output_gate::OutputGate, pub(crate) publication_cadence: PublicationCadenceState, pub(crate) output_reuse: OutputReuseState, pub(crate) lighting_feed: super::lighting_feed::LightingFeedState, @@ -2355,6 +2356,7 @@ impl PipelineRuntime { clock: FrameClockState::default(), inputs: InputReuseState::with_routing(input_reader, interaction_routing), throttle: ThrottleState::default(), + output_gate: super::output_gate::OutputGate::default(), publication_cadence: PublicationCadenceState::default(), output_reuse: OutputReuseState::default(), lighting_feed: super::lighting_feed::LightingFeedState::default(), diff --git a/crates/hypercolor-daemon/src/render_thread/render_zones.rs b/crates/hypercolor-daemon/src/render_thread/render_zones.rs index 981341c3b..32bdae96d 100644 --- a/crates/hypercolor-daemon/src/render_thread/render_zones.rs +++ b/crates/hypercolor-daemon/src/render_thread/render_zones.rs @@ -349,6 +349,40 @@ impl ZoneRuntime { Ok(runtime) } + /// Park every producer while the render loop requests no frames. + /// + /// Effect producers quiesce in place. Live media streams decode on a + /// worker of their own, so they are dropped instead and reconnect when + /// a layer next renders; a live stream has no past frame worth keeping. + /// Dropping one joins its decoder, which waits on the stream source, so + /// the drop runs on a short-lived thread rather than the render thread. + pub(crate) fn park_producers(&mut self) { + self.effect_pool.park(); + let live_streams = self + .media_producers + .extract_if(|_, cached| cached.producer.is_live_stream()) + .map(|(_, cached)| cached) + .collect::>(); + if live_streams.is_empty() { + return; + } + if let Err(error) = std::thread::Builder::new() + .name("hypercolor-media-park".to_owned()) + .spawn(move || drop(live_streams)) + { + tracing::warn!(%error, "Could not spawn the live stream teardown thread"); + } + } + + /// Whether any producer has not yet shown its current content. + pub(crate) fn producers_warming(&self) -> bool { + self.effect_pool.any_warming() + || self.media_producers.values().any(|cached| { + !cached.producer.has_renderable_frame() + && cached.producer.live_stream_error().is_none() + }) + } + pub(crate) fn drain_layer_runtime_events(&mut self) -> Vec { self.layer_runtime.drain_events() } diff --git a/crates/hypercolor-daemon/src/render_thread/render_zones/tests/pipeline_acceptance.rs b/crates/hypercolor-daemon/src/render_thread/render_zones/tests/pipeline_acceptance.rs index 64e37b664..c02114ce4 100644 --- a/crates/hypercolor-daemon/src/render_thread/render_zones/tests/pipeline_acceptance.rs +++ b/crates/hypercolor-daemon/src/render_thread/render_zones/tests/pipeline_acceptance.rs @@ -50,6 +50,10 @@ struct SteppedPipeline { state: RenderThreadState, /// Boxed so the futures that hold the pipeline stay small. runtime: Box, + /// The harness reads each published frame off the frame lane without a + /// receiver, so it holds output demand to keep the pipeline rendering + /// whether or not a case attaches a device. + _output_demand: hypercolor_core::bus::OutputDemandHold, } impl SteppedPipeline { @@ -101,7 +105,12 @@ impl SteppedPipeline { .await .expect("test scenes should commit"); state.render_loop.write().await.start(); - Some(Self { state, runtime }) + let output_demand = state.event_bus.hold_output_demand(); + Some(Self { + state, + runtime, + _output_demand: output_demand, + }) } async fn activate(&self, scene_id: SceneId) { diff --git a/crates/hypercolor-daemon/src/render_thread/screen_parity_diagnostics.rs b/crates/hypercolor-daemon/src/render_thread/screen_parity_diagnostics.rs index 11765d2c8..e89fb4b5b 100644 --- a/crates/hypercolor-daemon/src/render_thread/screen_parity_diagnostics.rs +++ b/crates/hypercolor-daemon/src/render_thread/screen_parity_diagnostics.rs @@ -13,7 +13,7 @@ reason = "consumed by platform diagnostic endpoints; targets without one carry the seam unused" )] -use std::sync::{Arc, mpsc}; +use std::sync::{Arc, Mutex, PoisonError, mpsc}; use anyhow::{Context, Result, anyhow}; use hypercolor_core::input::screen::implementer::{ @@ -37,8 +37,14 @@ pub(crate) struct ScreenParityDiagnosticHandle { sender: tokio_mpsc::Sender, } +/// The render thread's end of the parity mailbox. +/// +/// Clones share one receiver, so the supervisor keeps the mailbox across a +/// pipeline restart and the diagnostics handle stays connected to whichever +/// pipeline is live. +#[derive(Clone)] pub(crate) struct ScreenParityDiagnosticMailbox { - receiver: tokio_mpsc::Receiver, + receiver: Arc>>, } struct ScreenParityRequest { @@ -81,7 +87,9 @@ pub(crate) fn screen_parity_diagnostic_channel() let (sender, receiver) = tokio_mpsc::channel(REQUEST_CAPACITY); ( ScreenParityDiagnosticHandle { sender }, - ScreenParityDiagnosticMailbox { receiver }, + ScreenParityDiagnosticMailbox { + receiver: Arc::new(Mutex::new(receiver)), + }, ) } @@ -109,8 +117,12 @@ impl ScreenParityDiagnosticMailbox { descriptor: Option<&ResolvedScreenPublicationDescriptor>, spatial_engine: &SpatialEngine, ) { - let Ok(request) = self.receiver.try_recv() else { - return; + let request = { + let mut receiver = self.receiver.lock().unwrap_or_else(PoisonError::into_inner); + let Ok(request) = receiver.try_recv() else { + return; + }; + request }; let result = capture_active_snapshot( render_device, diff --git a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu.rs b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu.rs index 4acc2639d..1118c4052 100644 --- a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu.rs +++ b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu.rs @@ -191,7 +191,7 @@ fn reject_injected_gpu_preparation(fail_after_prepare: bool, resource: &'static } pub(crate) struct GpuSparkleFlinger { - _render_device: GpuRenderDevice, + render_device: GpuRenderDevice, device: wgpu::Device, queue: wgpu::Queue, /// The queue position each recent render frame ended at, oldest first, diff --git a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu/runtime.rs b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu/runtime.rs index 76b3ab071..4cb3aaff3 100644 --- a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu/runtime.rs +++ b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/gpu/runtime.rs @@ -117,7 +117,7 @@ impl GpuSparkleFlinger { })?; Ok(Self { - _render_device: render_device, + render_device, device, queue, probe, @@ -233,6 +233,16 @@ impl GpuSparkleFlinger { self.fail_next_screen_upload_pool_saturation = true; } + /// Raised once the device this compositor draws with is lost. + pub(crate) fn device_loss_signal(&self) -> crate::render_thread::gpu_errors::DeviceLossSignal { + self.render_device.loss_signal() + } + + #[cfg(feature = "persistence-test-hooks")] + pub(crate) fn raise_uncaptured_validation_error(&self) { + crate::render_thread::gpu_device::raise_uncaptured_validation_error(&self.device); + } + pub(crate) fn supports_plan(&self, plan: &CompositionPlan) -> bool { self.canvas_gpu_admitted && matches!( diff --git a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/mod.rs b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/mod.rs index cdde0ac8d..38c3fe2a5 100644 --- a/crates/hypercolor-daemon/src/render_thread/sparkleflinger/mod.rs +++ b/crates/hypercolor-daemon/src/render_thread/sparkleflinger/mod.rs @@ -1142,6 +1142,32 @@ impl SparkleFlinger { } } + /// The loss signal of the GPU device this compositor draws with, or + /// `None` on the CPU backend. + pub(crate) fn device_loss_signal( + &self, + ) -> Option { + match &self.backend { + #[cfg(feature = "wgpu")] + SparkleFlingerBackend::Gpu { gpu, .. } => Some(gpu.device_loss_signal()), + SparkleFlingerBackend::Cpu(_) => None, + } + } + + /// Raise one uncaptured validation error on the compositor's GPU + /// device, returning `false` on the CPU backend, which has none. + #[cfg(feature = "persistence-test-hooks")] + pub(crate) fn raise_uncaptured_validation_error(&self) -> bool { + match &self.backend { + #[cfg(feature = "wgpu")] + SparkleFlingerBackend::Gpu { gpu, .. } => { + gpu.raise_uncaptured_validation_error(); + true + } + SparkleFlingerBackend::Cpu(_) => false, + } + } + /// Wait for the GPU to finish an earlier frame before this one queues /// more work, so GPU latency stays bounded on a device slower than the /// frame cadence. A no-op on the CPU backend. diff --git a/crates/hypercolor-daemon/src/render_thread/supervision.rs b/crates/hypercolor-daemon/src/render_thread/supervision.rs new file mode 100644 index 000000000..17e87a9b0 --- /dev/null +++ b/crates/hypercolor-daemon/src/render_thread/supervision.rs @@ -0,0 +1,1238 @@ +//! Shared health and recovery state for the supervised render thread. +//! +//! The render thread restarts its pipeline in-process after a panic or an +//! error exit, within a bounded budget, and asks the daemon to exit once the +//! budget runs out or the compositor GPU device is lost. Everything the rest +//! of the daemon needs to see about that lives here: the lifecycle phase, a +//! heartbeat and frame-completion clock the render loop stamps every +//! iteration, restart counters, the last failure, and the termination +//! request the daemon waits on. +//! +//! Failures are also persisted to `render-failure.json` in the daemon state +//! directory, so the record survives a log that was lost (a full tmpfs, a +//! closed pipe) and a `panic = "abort"` build that never returns from the +//! panic hook. The next daemon run loads the previous record and reports it. + +use std::collections::VecDeque; +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU8, AtomicU32, AtomicU64, Ordering}; +use std::sync::{Arc, LazyLock, Mutex, PoisonError, Weak}; +use std::thread::ThreadId; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use tokio::sync::watch; + +#[cfg(feature = "persistence-test-hooks")] +use super::gpu_errors::DeviceLossSignal; +use super::gpu_errors::{GpuErrorTotals, gpu_error_totals}; +use crate::panic_hook::{PanicObserver, PanicReport}; + +/// Name of the OS thread that runs the render pipeline. +pub const RENDER_THREAD_NAME: &str = "hypercolor-render"; + +/// Name of the render runtime's Tokio worker threads. +pub const RENDER_RUNTIME_THREAD_NAME: &str = "hypercolor-render-rt"; + +/// File in the daemon state directory that holds the last render failure. +pub const RENDER_FAILURE_FILE: &str = "render-failure.json"; + +/// Identity of this daemon process, written into every failure record. +/// +/// A pid cannot tell runs apart: a container restarts the daemon as pid 1 +/// every time, and pids are reused across boots. This id is unique per +/// process, so a record from an earlier run is always recognized as one. +static PROCESS_INSTANCE: LazyLock = LazyLock::new(|| uuid::Uuid::now_v7().to_string()); + +/// How long a running render loop may go without completing an iteration +/// before the daemon reports itself unhealthy to its service manager's +/// watchdog. A healthy loop iterates at least every quarter second, so this +/// only trips on a thread that is wedged inside a frame, which nothing can +/// interrupt in-process. +pub const RENDER_WEDGED_AFTER: Duration = Duration::from_secs(20); + +/// Whether `name` belongs to the render thread or its runtime workers. +#[must_use] +pub fn is_render_thread_name(name: &str) -> bool { + name == RENDER_THREAD_NAME || name.starts_with(RENDER_RUNTIME_THREAD_NAME) +} + +// ── Policy ────────────────────────────────────────────────────────────────── + +/// How often and how fast a failed render pipeline restarts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RenderRestartPolicy { + /// Restarts allowed within [`window`](Self::window). The failure after + /// the last allowed restart terminates the daemon instead. + pub max_restarts: u32, + /// Rolling window the restart budget is counted over. + pub window: Duration, + /// Delay before the first restart in a streak. + pub initial_backoff: Duration, + /// Upper bound on the doubling restart delay. + pub max_backoff: Duration, + /// A pipeline that runs this long before failing starts a fresh streak, + /// so its restart waits the initial backoff again. + pub stable_uptime: Duration, +} + +impl Default for RenderRestartPolicy { + fn default() -> Self { + Self { + max_restarts: 5, + window: Duration::from_secs(300), + initial_backoff: Duration::from_millis(250), + max_backoff: Duration::from_secs(8), + stable_uptime: Duration::from_secs(60), + } + } +} + +impl RenderRestartPolicy { + fn backoff(&self, consecutive_failures: u32) -> Duration { + let doublings = consecutive_failures.saturating_sub(1).min(31); + self.initial_backoff + .saturating_mul(1_u32 << doublings) + .min(self.max_backoff) + } +} + +/// What the supervisor does after a failure. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum RecoveryPlan { + /// Rebuild the pipeline after `backoff`; `attempt` counts restarts in + /// the current window. + Restart { attempt: u32, backoff: Duration }, + /// Stop rendering and ask the daemon to exit. + Terminate(TerminationCause), + /// The daemon is already stopping, so nothing restarts. + DaemonStopping, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TerminationCause { + RestartBudgetExhausted { + failures: u32, + }, + GpuDeviceLost, + /// The supervisor itself failed outside a pipeline attempt. + SupervisorFailed, +} + +/// Rolling restart budget for one render thread. +#[derive(Debug)] +pub(crate) struct RestartBudget { + policy: RenderRestartPolicy, + failures: VecDeque, + consecutive: u32, +} + +impl RestartBudget { + pub(crate) const fn new(policy: RenderRestartPolicy) -> Self { + Self { + policy, + failures: VecDeque::new(), + consecutive: 0, + } + } + + /// Account for a failure at `now` of an attempt that had run for + /// `uptime`, and decide whether to restart. + pub(crate) fn record(&mut self, now: Instant, uptime: Duration) -> RecoveryPlan { + if uptime >= self.policy.stable_uptime { + self.consecutive = 0; + } + self.consecutive = self.consecutive.saturating_add(1); + self.failures.push_back(now); + while self + .failures + .front() + .is_some_and(|failed| now.saturating_duration_since(*failed) > self.policy.window) + { + self.failures.pop_front(); + } + let failures = u32::try_from(self.failures.len()).unwrap_or(u32::MAX); + if failures > self.policy.max_restarts { + return RecoveryPlan::Terminate(TerminationCause::RestartBudgetExhausted { failures }); + } + RecoveryPlan::Restart { + attempt: failures, + backoff: self.policy.backoff(self.consecutive), + } + } + + pub(crate) const fn consecutive(&self) -> u32 { + self.consecutive + } +} + +// ── Records ───────────────────────────────────────────────────────────────── + +/// Where the render thread is in its lifecycle. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(u8)] +pub enum RenderThreadPhase { + /// No render thread has been spawned for this supervision. + NotStarted = 0, + /// The thread is building its runtime, input pump, and first pipeline. + Starting = 1, + /// A pipeline is running. + Running = 2, + /// A pipeline failed and the supervisor is rebuilding it. + Restarting = 3, + /// The supervisor gave up; the daemon is exiting so its service manager + /// restarts it. + Terminating = 4, + /// The thread exited cleanly (daemon shutdown or a loop never started). + Stopped = 5, +} + +impl RenderThreadPhase { + const fn from_u8(value: u8) -> Self { + match value { + 1 => Self::Starting, + 2 => Self::Running, + 3 => Self::Restarting, + 4 => Self::Terminating, + 5 => Self::Stopped, + _ => Self::NotStarted, + } + } + + /// Stable snake-case name used by diagnostics and status. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::NotStarted => "not_started", + Self::Starting => "starting", + Self::Running => "running", + Self::Restarting => "restarting", + Self::Terminating => "terminating", + Self::Stopped => "stopped", + } + } +} + +/// What ended a render pipeline run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RenderFailureKind { + /// The pipeline panicked. + Panic, + /// The input publication pump the pipeline reads from stopped. + InputPumpTerminated, + /// A replacement input publication pump failed to start. + InputPumpRestartFailed, + /// A replacement pipeline failed to build. + PipelineRebuildFailed, + /// The compositor GPU device was lost. + GpuDeviceLost, +} + +impl RenderFailureKind { + /// Stable snake-case name used by diagnostics and status. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Panic => "panic", + Self::InputPumpTerminated => "input_pump_terminated", + Self::InputPumpRestartFailed => "input_pump_restart_failed", + Self::PipelineRebuildFailed => "pipeline_rebuild_failed", + Self::GpuDeviceLost => "gpu_device_lost", + } + } +} + +/// What the daemon did about a render failure. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RenderRecoveryAction { + /// The pipeline was rebuilt in-process with fresh GPU and render state. + Restart, + /// The daemon exits with an error so its service manager restarts it. + TerminateProcess, + /// The daemon was already stopping, so nothing was restarted. + DaemonStopping, + /// Written by the panic hook in a `panic = "abort"` build, which aborts + /// the process right after. + ProcessAbort, + /// Written by the panic hook before the supervisor caught the unwind. + /// The supervisor replaces it with its decision; a record still in + /// this state means the supervisor never got control back. + AwaitingSupervisor, +} + +impl RenderRecoveryAction { + /// Stable snake-case name used by diagnostics and status. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Restart => "restart", + Self::TerminateProcess => "terminate_process", + Self::DaemonStopping => "daemon_stopping", + Self::ProcessAbort => "process_abort", + Self::AwaitingSupervisor => "awaiting_supervisor", + } + } +} + +/// One render failure, as logged and persisted. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RenderFailureRecord { + pub kind: RenderFailureKind, + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thread: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub location: Option, + /// RFC 3339 UTC time of the failure. + pub occurred_at: String, + pub occurred_at_unix_ms: u64, + /// Process that recorded the failure. + pub pid: u32, + /// Unique id of the daemon process that recorded the failure. Absent + /// from records written before it existed. + #[serde(default)] + pub process_instance: String, + pub daemon_version: String, + /// Restarts in the budget window including the one this failure + /// triggered, or the failures counted when the budget ran out. + pub restart_attempt: u32, + /// Restarts since the process started, after this failure. + pub restarts_total: u64, + pub action: RenderRecoveryAction, + /// Backtrace frames from the panic site, when the failure was a panic. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backtrace: Option, +} + +/// A failure before the supervisor decides what to do with it. +#[derive(Debug, Clone)] +pub(crate) struct RenderFailure { + pub(crate) kind: RenderFailureKind, + pub(crate) message: String, + pub(crate) thread: Option, + pub(crate) location: Option, + pub(crate) backtrace: Option, +} + +impl RenderFailure { + pub(crate) fn panic(payload: String, report: Option) -> Self { + let thread = std::thread::current().name().map(str::to_owned); + match report { + Some(report) => Self { + kind: RenderFailureKind::Panic, + message: report.payload.clone(), + thread: Some(report.thread_name.clone()), + location: report.location.clone(), + backtrace: Some(report.backtrace_excerpt()), + }, + None => Self { + kind: RenderFailureKind::Panic, + message: payload, + thread, + location: None, + backtrace: None, + }, + } + } + + /// Attach a panic report's thread, location, and backtrace, keeping + /// this failure's kind and message. + pub(crate) fn with_panic_report(mut self, report: Option) -> Self { + if let Some(report) = report { + self.message = format!( + "{} (after a panic on {}: {})", + self.message, report.thread_name, report.payload + ); + self.thread = Some(report.thread_name.clone()); + self.location.clone_from(&report.location); + self.backtrace = Some(report.backtrace_excerpt()); + } + self + } + + pub(crate) fn from_error(kind: RenderFailureKind, error: &anyhow::Error) -> Self { + Self::new(kind, format!("{error:#}")) + } + + pub(crate) fn new(kind: RenderFailureKind, message: String) -> Self { + Self { + kind, + message, + thread: std::thread::current().name().map(str::to_owned), + location: None, + backtrace: None, + } + } + + /// Reclassify a failure that happened after the compositor device was + /// lost: a lost device is why it failed, and why it cannot restart. + pub(crate) fn after_device_loss(mut self, lost: &str) -> Self { + self.message = format!( + "{lost}; the pipeline then failed with {}: {}", + self.kind.as_str(), + self.message + ); + self.kind = RenderFailureKind::GpuDeviceLost; + self + } +} + +/// Everything diagnostics and status read about the render thread. +#[derive(Debug, Clone)] +pub struct RenderSupervisionSnapshot { + pub phase: RenderThreadPhase, + pub restarts_total: u64, + /// Failures since the pipeline last ran past the stable uptime. + pub consecutive_failures: u32, + pub max_restarts: u32, + /// Time since the render loop last completed an iteration of any kind, + /// paused polls included. + pub last_heartbeat_age: Option, + /// Time since the render loop last completed a frame. + pub last_frame_age: Option, + /// Time since a frame was last due: the later of the last completed + /// frame and the last idle iteration. A loop that just left a pause or a + /// park is measured from when it left, not from its last frame before. + pub frame_due_age: Option, + /// How long the current pipeline has been running. + pub current_run: Option, + /// The latest failure in this process, or else the failure the previous + /// daemon run persisted. + pub last_failure: Option, + /// Whether [`last_failure`](Self::last_failure) happened in this process. + pub last_failure_in_this_process: bool, + pub gpu_errors: GpuErrorTotals, +} + +impl RenderSupervisionSnapshot { + /// The API projection of this snapshot. + /// + /// `include_private` gates the failure message and the last GPU error + /// text, which can carry source paths and resource labels. + #[must_use] + pub fn status( + &self, + include_private: bool, + ) -> hypercolor_types::api::system::RenderSupervisionStatus { + use hypercolor_types::api::system::{ + GpuErrorStatus, RenderFailureStatus, RenderSupervisionStatus, + }; + + RenderSupervisionStatus { + phase: self.phase.as_str().to_owned(), + restarts_total: self.restarts_total, + consecutive_failures: self.consecutive_failures, + max_restarts: self.max_restarts, + last_frame_age_ms: self.last_frame_age.map(duration_millis), + last_failure: self + .last_failure + .as_ref() + .map(|record| RenderFailureStatus { + kind: record.kind.as_str().to_owned(), + message: if include_private { + record.message.clone() + } else { + String::new() + }, + thread: record.thread.clone(), + location: include_private.then(|| record.location.clone()).flatten(), + occurred_at: record.occurred_at.clone(), + action: record.action.as_str().to_owned(), + restart_attempt: record.restart_attempt, + current_process: self.last_failure_in_this_process, + }), + gpu_errors: GpuErrorStatus { + validation_errors: self.gpu_errors.validation, + out_of_memory_errors: self.gpu_errors.out_of_memory, + internal_errors: self.gpu_errors.internal, + device_lost: self.gpu_errors.device_lost, + last_error: include_private + .then(|| { + self.gpu_errors + .last + .as_ref() + .map(|last| format!("{}: {}: {}", last.device, last.kind, last.message)) + }) + .flatten(), + }, + } + } +} + +fn duration_millis(duration: Duration) -> u64 { + u64::try_from(duration.as_millis()).unwrap_or(u64::MAX) +} + +// ── Persistence ───────────────────────────────────────────────────────────── + +#[derive(Debug, Clone)] +struct RenderFailureStore { + path: PathBuf, +} + +/// Prefix and suffix of a record staged before its durable replace. +const STAGING_PREFIX: &str = ".render-failure."; +const STAGING_SUFFIX: &str = ".tmp"; + +impl RenderFailureStore { + /// Remove records a killed process staged and never replaced. + fn remove_stale_staging(&self) { + let Some(directory) = self.path.parent() else { + return; + }; + let Ok(entries) = std::fs::read_dir(directory) else { + return; + }; + for entry in entries.filter_map(std::result::Result::ok) { + let name = entry.file_name(); + let name = name.to_string_lossy(); + if name.starts_with(STAGING_PREFIX) && name.ends_with(STAGING_SUFFIX) { + let _ = std::fs::remove_file(entry.path()); + } + } + } + + fn load(&self) -> Option { + let bytes = match std::fs::read(&self.path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return None, + Err(error) => { + tracing::warn!(path = %self.path.display(), %error, "could not read the previous render failure record"); + return None; + } + }; + match serde_json::from_slice(&bytes) { + Ok(record) => Some(record), + Err(error) => { + tracing::warn!(path = %self.path.display(), %error, "ignoring an unreadable render failure record"); + None + } + } + } + + /// Replace the record durably. Safe to call from the panic hook: it + /// takes no lock the render thread could be holding. + fn persist(&self, record: &RenderFailureRecord) -> Result<()> { + let directory = self + .path + .parent() + .context("render failure record path has no parent directory")?; + std::fs::create_dir_all(directory).with_context(|| { + format!( + "failed to create the render failure directory {}", + directory.display() + ) + })?; + let payload = serde_json::to_vec_pretty(record) + .context("failed to serialize the render failure record")?; + let mut staged = tempfile::Builder::new() + .prefix(STAGING_PREFIX) + .suffix(STAGING_SUFFIX) + .tempfile_in(directory) + .context("failed to stage the render failure record")?; + staged + .write_all(&payload) + .and_then(|()| staged.as_file().sync_all()) + .context("failed to write the staged render failure record")?; + let (_file, staged_path) = staged + .keep() + .context("failed to keep the staged render failure record")?; + if let Err(error) = hypercolor_platform_fs::durable_replace(&staged_path, &self.path) { + let _ = std::fs::remove_file(&staged_path); + return Err(error).with_context(|| { + format!( + "failed to replace the render failure record {}", + self.path.display() + ) + }); + } + Ok(()) + } +} + +// ── Supervision handle ────────────────────────────────────────────────────── + +/// A fault the render thread raises on itself, for tests of the recovery +/// paths. +#[cfg(feature = "persistence-test-hooks")] +#[doc(hidden)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum InjectedRenderFault { + /// Panic with this message at the top of the next render loop + /// iteration. + Panic(&'static str), + /// Block the render thread for the given time, like a wedged frame. + Stall(Duration), + /// Raise one uncaptured validation error on the compositor GPU device. + GpuValidationError, +} + +struct SupervisionInner { + policy: RenderRestartPolicy, + store: Option, + epoch: Instant, + phase: AtomicU8, + restarts_total: AtomicU64, + consecutive_failures: AtomicU32, + attempt_started_us: AtomicU64, + last_heartbeat_us: AtomicU64, + last_frame_us: AtomicU64, + /// The latest idle iteration, after which frames were due again. + idle_until_us: AtomicU64, + last_failure: Mutex>, + previous_failure: Option, + termination: watch::Sender>>, + /// The `hypercolor-render` thread this supervision watches, so its + /// panic observer persists only that thread's panics. + render_thread: Mutex>, + #[cfg(feature = "persistence-test-hooks")] + injected_device_loss: DeviceLossSignal, + #[cfg(feature = "persistence-test-hooks")] + injected_faults: Mutex>, + #[cfg(feature = "persistence-test-hooks")] + injected_fault_pending: std::sync::atomic::AtomicBool, +} + +/// Shared handle to one render thread's health and recovery state. +/// +/// Cloning shares the same state. The daemon creates one at startup and +/// hands it to the render thread, diagnostics, and status. +#[derive(Clone)] +pub struct RenderSupervision { + inner: Arc, +} + +impl std::fmt::Debug for RenderSupervision { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RenderSupervision") + .field("phase", &self.phase()) + .field("policy", &self.inner.policy) + .finish_non_exhaustive() + } +} + +impl Default for RenderSupervision { + fn default() -> Self { + Self::detached() + } +} + +impl RenderSupervision { + /// Supervision with `policy` that persists failures to + /// `failure_record_path` when one is given. + /// + /// A record left by an earlier process at that path is loaded and + /// reported until this process records its own failure. + #[must_use] + pub fn new(policy: RenderRestartPolicy, failure_record_path: Option) -> Self { + let store = failure_record_path.map(|path| RenderFailureStore { path }); + if let Some(store) = store.as_ref() { + store.remove_stale_staging(); + } + let previous_failure = store + .as_ref() + .and_then(RenderFailureStore::load) + .filter(|record| record.process_instance != *PROCESS_INSTANCE); + if let Some(record) = previous_failure.as_ref() { + tracing::warn!( + kind = record.kind.as_str(), + message = %record.message, + occurred_at = %record.occurred_at, + pid = record.pid, + action = record.action.as_str(), + "an earlier daemon run recorded a render failure" + ); + } + Self { + inner: Arc::new(SupervisionInner { + policy, + store, + epoch: Instant::now(), + phase: AtomicU8::new(RenderThreadPhase::NotStarted as u8), + restarts_total: AtomicU64::new(0), + consecutive_failures: AtomicU32::new(0), + attempt_started_us: AtomicU64::new(0), + last_heartbeat_us: AtomicU64::new(0), + last_frame_us: AtomicU64::new(0), + idle_until_us: AtomicU64::new(0), + last_failure: Mutex::new(None), + previous_failure, + termination: watch::Sender::new(None), + render_thread: Mutex::new(None), + #[cfg(feature = "persistence-test-hooks")] + injected_device_loss: DeviceLossSignal::default(), + #[cfg(feature = "persistence-test-hooks")] + injected_faults: Mutex::new(VecDeque::new()), + #[cfg(feature = "persistence-test-hooks")] + injected_fault_pending: std::sync::atomic::AtomicBool::new(false), + }), + } + } + + /// Default policy, persisting to [`RENDER_FAILURE_FILE`] in + /// `state_dir`. + #[must_use] + pub fn for_state_dir(state_dir: &Path) -> Self { + Self::new( + RenderRestartPolicy::default(), + Some(state_dir.join(RENDER_FAILURE_FILE)), + ) + } + + /// Default policy without persistence. + #[must_use] + pub fn detached() -> Self { + Self::new(RenderRestartPolicy::default(), None) + } + + #[must_use] + pub fn policy(&self) -> RenderRestartPolicy { + self.inner.policy + } + + /// Where failures are persisted, if anywhere. + #[must_use] + pub fn failure_record_path(&self) -> Option<&Path> { + self.inner.store.as_ref().map(|store| store.path.as_path()) + } + + #[must_use] + pub fn phase(&self) -> RenderThreadPhase { + RenderThreadPhase::from_u8(self.inner.phase.load(Ordering::Acquire)) + } + + /// Read every health field at once. + #[must_use] + pub fn snapshot(&self) -> RenderSupervisionSnapshot { + let now = Instant::now(); + let failure_guard = self + .inner + .last_failure + .lock() + .unwrap_or_else(PoisonError::into_inner); + let current = failure_guard.clone(); + let phase = self.phase(); + let restarts_total = self.inner.restarts_total.load(Ordering::Relaxed); + let consecutive_failures = self.inner.consecutive_failures.load(Ordering::Relaxed); + drop(failure_guard); + let last_failure_in_this_process = current.is_some(); + RenderSupervisionSnapshot { + phase, + restarts_total, + consecutive_failures, + max_restarts: self.inner.policy.max_restarts, + last_heartbeat_age: self.age(&self.inner.last_heartbeat_us, now), + last_frame_age: self.age(&self.inner.last_frame_us, now), + frame_due_age: self.frame_due_age(now), + current_run: (phase == RenderThreadPhase::Running) + .then(|| self.age(&self.inner.attempt_started_us, now)) + .flatten(), + last_failure: current.or_else(|| self.inner.previous_failure.clone()), + last_failure_in_this_process, + gpu_errors: gpu_error_totals(), + } + } + + /// Whether the render thread is healthy enough for a service manager + /// watchdog: `false` only while a running pipeline has gone + /// [`RENDER_WEDGED_AFTER`] without completing a loop iteration. + #[must_use] + pub fn watchdog_healthy(&self) -> bool { + self.watchdog_healthy_at(Instant::now()) + } + + fn watchdog_healthy_at(&self, now: Instant) -> bool { + self.phase() != RenderThreadPhase::Running + || self + .age(&self.inner.last_heartbeat_us, now) + .is_none_or(|age| age < RENDER_WEDGED_AFTER) + } + + /// The failure the render thread terminated on, once it gave up. + #[must_use] + pub fn termination(&self) -> Option> { + self.inner.termination.borrow().clone() + } + + /// Wait until the render thread gives up, and return why. + pub async fn terminated(&self) -> Arc { + let mut receiver = self.inner.termination.subscribe(); + let record = receiver + .wait_for(Option::is_some) + .await + .map(|record| record.clone()); + match record { + Ok(Some(record)) => record, + // The sender lives as long as this handle, so the wait cannot + // end any other way. + Ok(None) | Err(_) => std::future::pending().await, + } + } + + /// Bind this supervision to the render thread calling it. + pub(crate) fn bind_render_thread(&self) { + *self + .inner + .render_thread + .lock() + .unwrap_or_else(PoisonError::into_inner) = Some(std::thread::current().id()); + } + + /// The loss a test injected for this supervision's compositor device. + #[cfg(feature = "persistence-test-hooks")] + pub(crate) fn injected_device_loss(&self) -> DeviceLossSignal { + self.inner.injected_device_loss.clone() + } + + pub(crate) fn set_phase(&self, phase: RenderThreadPhase) { + self.inner.phase.store(phase as u8, Ordering::Release); + } + + /// A pipeline attempt starts running at `now`. + pub(crate) fn attempt_started(&self, now: Instant) { + let stamp = self.stamp(now); + self.inner + .attempt_started_us + .store(stamp, Ordering::Relaxed); + // A fresh pipeline gets the same grace a fresh frame would, so the + // stall check measures this attempt rather than the dead one. + self.inner.last_heartbeat_us.store(stamp, Ordering::Relaxed); + self.inner.last_frame_us.store(stamp, Ordering::Relaxed); + self.set_phase(RenderThreadPhase::Running); + } + + /// The render loop completed an idle iteration at `now`: a paused poll + /// or a pass of the park wait. No frame was due up to this moment. + pub(crate) fn heartbeat(&self, now: Instant) { + let stamp = self.stamp(now); + self.inner.last_heartbeat_us.store(stamp, Ordering::Relaxed); + self.inner.idle_until_us.store(stamp, Ordering::Relaxed); + } + + /// The render loop completed a frame at `now`. + pub(crate) fn frame_completed(&self, now: Instant) { + let stamp = self.stamp(now); + self.inner.last_heartbeat_us.store(stamp, Ordering::Relaxed); + self.inner.last_frame_us.store(stamp, Ordering::Relaxed); + if self.inner.consecutive_failures.load(Ordering::Relaxed) != 0 { + let started = self.inner.attempt_started_us.load(Ordering::Relaxed); + let stable_us = + u64::try_from(self.inner.policy.stable_uptime.as_micros()).unwrap_or(u64::MAX); + if stamp.saturating_sub(started) >= stable_us { + self.inner.consecutive_failures.store(0, Ordering::Relaxed); + } + } + } + + /// Record `failure` and the plan for it: log it, persist it, update the + /// counters and the phase. + pub(crate) fn record_failure( + &self, + failure: RenderFailure, + plan: RecoveryPlan, + consecutive_failures: u32, + ) -> RenderFailureRecord { + let (action, restart_attempt, phase) = match plan { + RecoveryPlan::Restart { attempt, .. } => ( + RenderRecoveryAction::Restart, + attempt, + RenderThreadPhase::Restarting, + ), + RecoveryPlan::Terminate(TerminationCause::RestartBudgetExhausted { failures }) => ( + RenderRecoveryAction::TerminateProcess, + failures, + RenderThreadPhase::Terminating, + ), + RecoveryPlan::Terminate( + TerminationCause::GpuDeviceLost | TerminationCause::SupervisorFailed, + ) => ( + RenderRecoveryAction::TerminateProcess, + consecutive_failures, + RenderThreadPhase::Terminating, + ), + RecoveryPlan::DaemonStopping => ( + RenderRecoveryAction::DaemonStopping, + consecutive_failures, + RenderThreadPhase::Stopped, + ), + }; + // Only the render thread records failures, so the counter cannot + // move between this read and the commit below. + let previous_restarts = self.inner.restarts_total.load(Ordering::Relaxed); + let restarts_total = if action == RenderRecoveryAction::Restart { + previous_restarts.saturating_add(1) + } else { + previous_restarts + }; + let record = failure_record(failure, action, restart_attempt, restarts_total); + log_failure(&record, plan, self.inner.policy.max_restarts); + if let Some(store) = self.inner.store.as_ref() + && let Err(error) = store.persist(&record) + { + tracing::warn!(path = %store.path.display(), error = %format!("{error:#}"), "could not persist the render failure record"); + } + // The failure, its counters, and the phase change together under + // the failure lock, which `snapshot` holds while it reads them, so + // no reader sees a counted restart without the failure behind it. + let mut last_failure = self + .inner + .last_failure + .lock() + .unwrap_or_else(PoisonError::into_inner); + *last_failure = Some(record.clone()); + self.inner + .consecutive_failures + .store(consecutive_failures, Ordering::Relaxed); + self.inner + .restarts_total + .store(restarts_total, Ordering::Relaxed); + self.set_phase(phase); + drop(last_failure); + record + } + + /// The record for a terminating failure that could not be logged or + /// persisted, built without touching either. + pub(crate) fn unrecorded_termination(&self, failure: RenderFailure) -> RenderFailureRecord { + let restarts_total = self.inner.restarts_total.load(Ordering::Relaxed); + failure_record( + failure, + RenderRecoveryAction::TerminateProcess, + 0, + restarts_total, + ) + } + + /// Ask the daemon to exit because the render thread cannot recover. + pub(crate) fn request_termination(&self, record: Arc) { + self.set_phase(RenderThreadPhase::Terminating); + self.inner.termination.send_replace(Some(record)); + } + + /// An observer for the process panic hook that persists render thread + /// panics, or `None` when this supervision does not persist. + /// + /// In a `panic = "abort"` build this is the only code that runs between + /// a render panic and the abort, so it persists a panic on the render + /// thread or any of its runtime workers. An unwinding build persists + /// only this supervision's own render thread: the supervisor records + /// that unwind itself right after, while Tokio catches a worker panic + /// and the supervisor only learns of it if the pipeline then fails. + pub(crate) fn panic_observer(&self) -> Option { + let store = self.inner.store.clone()?; + let inner: Weak = Arc::downgrade(&self.inner); + Some(Arc::new(move |report: &PanicReport| { + let Some(inner) = inner.upgrade() else { + return; + }; + let ours = if cfg!(panic = "abort") { + is_render_thread_name(&report.thread_name) + } else { + inner + .render_thread + .try_lock() + .ok() + .and_then(|thread| *thread) + == Some(report.thread_id) + }; + if !ours { + return; + } + let action = if cfg!(panic = "abort") { + RenderRecoveryAction::ProcessAbort + } else { + RenderRecoveryAction::AwaitingSupervisor + }; + let failure = RenderFailure { + kind: RenderFailureKind::Panic, + message: report.payload.clone(), + thread: Some(report.thread_name.clone()), + location: report.location.clone(), + backtrace: Some(report.backtrace_excerpt()), + }; + let restarts_total = inner.restarts_total.load(Ordering::Relaxed); + let record = failure_record(failure, action, 0, restarts_total); + // Nothing useful can be done with an error from inside the + // panic hook; the tracing record and stderr still carry it. + let _ = store.persist(&record); + })) + } + + /// Queue a fault for the render thread to raise on itself. + #[cfg(feature = "persistence-test-hooks")] + #[doc(hidden)] + pub fn inject_fault(&self, fault: InjectedRenderFault) { + self.inner + .injected_faults + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push_back(fault); + self.inner + .injected_fault_pending + .store(true, Ordering::Release); + } + + /// Report this render thread's compositor device lost, the way the + /// device-lost callback raises a real device's loss signal. + #[cfg(feature = "persistence-test-hooks")] + #[doc(hidden)] + pub fn inject_compositor_device_lost(&self, message: &str) { + self.inner.injected_device_loss.raise(format!( + "injected compositor device lost (unknown: {message})" + )); + } + + #[cfg(feature = "persistence-test-hooks")] + pub(crate) fn take_injected_fault(&self) -> Option { + if !self.inner.injected_fault_pending.load(Ordering::Acquire) { + return None; + } + let mut faults = self + .inner + .injected_faults + .lock() + .unwrap_or_else(PoisonError::into_inner); + let fault = faults.pop_front(); + if faults.is_empty() { + self.inner + .injected_fault_pending + .store(false, Ordering::Release); + } + fault + } + + /// Microseconds since this supervision's epoch, offset by one so zero + /// can mean "never". + fn stamp(&self, now: Instant) -> u64 { + u64::try_from(now.saturating_duration_since(self.inner.epoch).as_micros()) + .unwrap_or(u64::MAX - 1) + .saturating_add(1) + } + + fn frame_due_age(&self, now: Instant) -> Option { + let due_since = self + .inner + .last_frame_us + .load(Ordering::Relaxed) + .max(self.inner.idle_until_us.load(Ordering::Relaxed)); + (due_since != 0).then(|| Duration::from_micros(self.stamp(now).saturating_sub(due_since))) + } + + fn age(&self, stamp: &AtomicU64, now: Instant) -> Option { + let stamp = stamp.load(Ordering::Relaxed); + (stamp != 0).then(|| Duration::from_micros(self.stamp(now).saturating_sub(stamp))) + } +} + +fn failure_record( + failure: RenderFailure, + action: RenderRecoveryAction, + restart_attempt: u32, + restarts_total: u64, +) -> RenderFailureRecord { + let now = SystemTime::now(); + RenderFailureRecord { + kind: failure.kind, + message: failure.message, + thread: failure.thread, + location: failure.location, + occurred_at: chrono::DateTime::::from(now) + .to_rfc3339_opts(chrono::SecondsFormat::Millis, true), + occurred_at_unix_ms: now.duration_since(UNIX_EPOCH).map_or(0, |elapsed| { + u64::try_from(elapsed.as_millis()).unwrap_or(u64::MAX) + }), + pid: std::process::id(), + process_instance: PROCESS_INSTANCE.clone(), + daemon_version: env!("CARGO_PKG_VERSION").to_owned(), + restart_attempt, + restarts_total, + action, + backtrace: failure.backtrace, + } +} + +fn log_failure(record: &RenderFailureRecord, plan: RecoveryPlan, max_restarts: u32) { + let backtrace = record.backtrace.as_deref().unwrap_or("unavailable"); + let thread = record.thread.as_deref().unwrap_or("unknown"); + let location = record.location.as_deref().unwrap_or("unknown"); + match plan { + RecoveryPlan::Restart { attempt, backoff } => tracing::error!( + kind = record.kind.as_str(), + message = %record.message, + thread, + location, + restart = attempt, + max_restarts, + backoff_ms = duration_millis(backoff), + backtrace, + "render pipeline failed; restarting it with fresh GPU and render state" + ), + RecoveryPlan::Terminate(TerminationCause::RestartBudgetExhausted { failures }) => { + tracing::error!( + kind = record.kind.as_str(), + message = %record.message, + thread, + location, + failures, + max_restarts, + backtrace, + "render pipeline keeps failing; shutting the daemon down so its service manager restarts it" + ); + } + RecoveryPlan::Terminate(TerminationCause::SupervisorFailed) => tracing::error!( + kind = record.kind.as_str(), + message = %record.message, + thread, + location, + backtrace, + "render supervisor failed; shutting the daemon down so its service manager restarts it" + ), + RecoveryPlan::Terminate(TerminationCause::GpuDeviceLost) => tracing::error!( + kind = record.kind.as_str(), + message = %record.message, + thread, + location, + backtrace, + "compositor GPU device lost; shutting the daemon down so it reopens the GPU in a fresh process" + ), + RecoveryPlan::DaemonStopping => tracing::error!( + kind = record.kind.as_str(), + message = %record.message, + thread, + location, + backtrace, + "render pipeline failed while the daemon was stopping" + ), + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, Instant}; + + use super::{ + RENDER_FAILURE_FILE, RENDER_WEDGED_AFTER, RecoveryPlan, RenderFailure, RenderFailureKind, + RenderFailureStore, RenderRecoveryAction, RenderRestartPolicy, RenderSupervision, + RestartBudget, TerminationCause, failure_record, + }; + + #[test] + fn a_record_from_this_process_is_not_reported_as_an_earlier_run() { + let state_dir = tempfile::tempdir().expect("state tempdir should be created"); + let path = state_dir.path().join(RENDER_FAILURE_FILE); + let store = RenderFailureStore { path: path.clone() }; + let record = failure_record( + RenderFailure::new(RenderFailureKind::Panic, "this process".to_owned()), + RenderRecoveryAction::Restart, + 1, + 1, + ); + + store.persist(&record).expect("record should persist"); + let supervision = RenderSupervision::new(RenderRestartPolicy::default(), Some(path)); + + assert!(supervision.snapshot().last_failure.is_none()); + let leftovers: Vec<_> = std::fs::read_dir(state_dir.path()) + .expect("state dir should list") + .filter_map(Result::ok) + .map(|entry| entry.file_name()) + .collect(); + assert_eq!( + leftovers, + vec![std::ffi::OsString::from(RENDER_FAILURE_FILE)] + ); + } + + #[test] + fn only_a_running_pipeline_that_stopped_iterating_fails_the_watchdog() { + let supervision = RenderSupervision::detached(); + let start = Instant::now(); + let later = start + RENDER_WEDGED_AFTER + Duration::from_secs(1); + assert!(supervision.watchdog_healthy_at(later), "not started yet"); + + supervision.attempt_started(start); + assert!(supervision.watchdog_healthy_at(start + Duration::from_secs(1))); + assert!(!supervision.watchdog_healthy_at(later)); + + supervision.heartbeat(later); + assert!(supervision.watchdog_healthy_at(later + Duration::from_secs(1))); + supervision.set_phase(super::RenderThreadPhase::Restarting); + assert!(supervision.watchdog_healthy_at(later + RENDER_WEDGED_AFTER * 2)); + } + + fn policy() -> RenderRestartPolicy { + RenderRestartPolicy { + max_restarts: 3, + window: Duration::from_secs(60), + initial_backoff: Duration::from_millis(100), + max_backoff: Duration::from_millis(350), + stable_uptime: Duration::from_secs(10), + } + } + + #[test] + fn restarts_back_off_until_the_budget_runs_out() { + let mut budget = RestartBudget::new(policy()); + let start = Instant::now(); + + let plans: Vec<_> = (0..4) + .map(|index| budget.record(start + Duration::from_secs(index), Duration::ZERO)) + .collect(); + + assert_eq!( + plans, + vec![ + RecoveryPlan::Restart { + attempt: 1, + backoff: Duration::from_millis(100) + }, + RecoveryPlan::Restart { + attempt: 2, + backoff: Duration::from_millis(200) + }, + RecoveryPlan::Restart { + attempt: 3, + backoff: Duration::from_millis(350) + }, + RecoveryPlan::Terminate(TerminationCause::RestartBudgetExhausted { failures: 4 }), + ] + ); + } + + #[test] + fn failures_outside_the_window_leave_the_budget() { + let mut budget = RestartBudget::new(policy()); + let start = Instant::now(); + for index in 0..3 { + budget.record(start + Duration::from_secs(index), Duration::ZERO); + } + + let plan = budget.record(start + Duration::from_secs(120), Duration::ZERO); + + assert!(matches!(plan, RecoveryPlan::Restart { attempt: 1, .. })); + } + + #[test] + fn a_stable_run_resets_the_backoff_streak() { + let mut budget = RestartBudget::new(policy()); + let start = Instant::now(); + budget.record(start, Duration::ZERO); + budget.record(start + Duration::from_secs(1), Duration::ZERO); + + let plan = budget.record(start + Duration::from_secs(20), Duration::from_secs(15)); + + assert_eq!( + plan, + RecoveryPlan::Restart { + attempt: 3, + backoff: Duration::from_millis(100) + } + ); + assert_eq!(budget.consecutive(), 1); + } +} diff --git a/crates/hypercolor-daemon/src/render_thread/supervisor.rs b/crates/hypercolor-daemon/src/render_thread/supervisor.rs new file mode 100644 index 000000000..bedeb377b --- /dev/null +++ b/crates/hypercolor-daemon/src/render_thread/supervisor.rs @@ -0,0 +1,475 @@ +//! Body of the `hypercolor-render` thread: start the render runtime and the +//! input publication pump, build the pipeline, and keep a pipeline running. +//! +//! A pipeline that panics, loses its input pump, or loses its compositor +//! GPU device ends its attempt. The supervisor records the failure, and +//! either rebuilds the pipeline in-process with fresh GPU and render state +//! (within the restart budget, after a backoff) or asks the daemon to exit +//! so its service manager starts a fresh process. A panic anywhere else on +//! this thread, in startup or in the supervisor itself, is caught at the +//! top of the thread and also ends in that exit, so the thread never ends +//! without the daemon knowing. +//! +//! What a restart rebuilds: both compositors and every GPU resource on +//! them, the zone runtime with its effect renderers and Servo sessions, the +//! scene snapshot and frame state, and the input pump when it is the part +//! that failed. What it keeps: the compositor GPU device, which stays valid +//! after a panic or a validation error, and the shared daemon state the +//! pipeline reads. A lost device is not rebuilt in-process: Servo's GPU +//! import device is a process-wide once-only binding to the device opened +//! at startup, and the native screen bridges and the interactive preview +//! device came from the same adapter, so only a fresh process can rebind +//! all of them. + +use std::panic::AssertUnwindSafe; +use std::sync::{Arc, mpsc}; +use std::time::{Duration, Instant}; + +use anyhow::{Result, anyhow}; +use arc_swap::ArcSwapOption; +use tokio_util::sync::CancellationToken; +use tracing::{debug, info, warn}; + +use super::gpu_errors::DeviceLossSignal; +use super::input_publication::{InputPublicationMonitor, InputPublicationPump}; +use super::pipeline_driver::run_pipeline; +use super::pipeline_runtime::PipelineRuntime; +#[cfg(feature = "wgpu")] +use super::screen_parity_diagnostics::ScreenParityDiagnosticMailbox; +use super::supervision::{ + RENDER_RUNTIME_THREAD_NAME, RecoveryPlan, RenderFailure, RenderFailureKind, RenderSupervision, + RenderThreadPhase, RestartBudget, TerminationCause, +}; +use super::{InputPublicationDemandHandle, RenderThreadState, panic_payload_message}; +use crate::startup::{StartupProgress, startup_step}; + +/// How recent a runtime worker panic must be to explain a pump failure. +const RECENT_WORKER_PANIC: Duration = Duration::from_secs(10); + +/// Everything the render thread takes ownership of when it starts. +pub(super) struct RenderThreadLaunch { + pub(super) state: RenderThreadState, + pub(super) build_runtime: F, + pub(super) progress: Option, + pub(super) ready: mpsc::SyncSender>, + pub(super) cancel: CancellationToken, + pub(super) input_demands: InputPublicationDemandHandle, + pub(super) input_monitor: Arc>, + pub(super) supervision: RenderSupervision, + #[cfg(feature = "wgpu")] + pub(super) screen_parity_mailbox: ScreenParityDiagnosticMailbox, +} + +/// Run the render thread until shutdown, a clean loop exit, or a failure +/// the supervisor gives up on. +/// +/// Startup failures are reported through `ready` and end the thread with +/// `Ok`, so the spawner sees the error instead of the join result. A panic +/// that escapes everything below is recorded and requests termination. +pub(super) fn run(launch: RenderThreadLaunch) -> Result<()> +where + F: FnOnce() -> Result, +{ + let supervision = launch.supervision.clone(); + supervision.bind_render_thread(); + std::panic::catch_unwind(AssertUnwindSafe(|| run_supervised(launch))).unwrap_or_else( + |payload| { + let failure = RenderFailure::panic( + panic_payload_message(payload.as_ref()), + crate::panic_hook::take_recent_report(std::thread::current().id()), + ); + // Recording logs and writes a file; if that panics too, the + // termination request below must still go out. + let record = std::panic::catch_unwind(AssertUnwindSafe(|| { + supervision.record_failure( + failure.clone(), + RecoveryPlan::Terminate(TerminationCause::SupervisorFailed), + 0, + ) + })) + .unwrap_or_else(|_| supervision.unrecorded_termination(failure)); + let error = anyhow!( + "render thread panicked outside a pipeline attempt: {}", + record.message + ); + supervision.request_termination(Arc::new(record)); + Err(error) + }, + ) +} + +fn run_supervised(launch: RenderThreadLaunch) -> Result<()> +where + F: FnOnce() -> Result, +{ + let RenderThreadLaunch { + state, + build_runtime, + progress, + ready, + cancel, + input_demands, + input_monitor, + supervision, + #[cfg(feature = "wgpu")] + screen_parity_mailbox, + } = launch; + let _scene_transaction_consumer = state.scene_transactions.consumer(); + crate::process::configure_render_thread_priority(); + let runtime = match startup_step(progress.as_ref(), "render runtime", build_runtime) { + Ok(runtime) => runtime, + Err(error) => { + supervision.set_phase(RenderThreadPhase::Stopped); + let _ = ready.send(Err(error)); + return Ok(()); + } + }; + let mut input_pump = match startup_step(progress.as_ref(), "input publication pump", || { + runtime.block_on(start_input_pump(&state, input_demands.clone())) + }) { + Ok(pump) => pump, + Err(error) => { + supervision.set_phase(RenderThreadPhase::Stopped); + let _ = ready.send(Err(error)); + return Ok(()); + } + }; + let pipeline = build_pipeline( + &state, + &input_pump, + &input_demands, + #[cfg(feature = "wgpu")] + &screen_parity_mailbox, + progress.as_ref(), + ); + // Startup reporting ends with the pipeline build. + drop(progress); + let pipeline = match pipeline { + Ok(pipeline) => pipeline, + Err(error) => { + if let Err(shutdown_error) = runtime.block_on(input_pump.shutdown()) { + warn!(%shutdown_error, "input publication cleanup failed after render startup error"); + } + supervision.set_phase(RenderThreadPhase::Stopped); + let _ = ready.send(Err(error)); + return Ok(()); + } + }; + input_monitor.store(Some(Arc::new(input_pump.monitor()))); + if ready.send(Ok(())).is_err() { + supervision.set_phase(RenderThreadPhase::Stopped); + return runtime.block_on(input_pump.shutdown()); + } + + let mut supervisor = Supervisor { + runtime: &runtime, + state: &state, + cancel: &cancel, + input_demands: &input_demands, + input_monitor: &input_monitor, + budget: RestartBudget::new(supervision.policy()), + supervision: &supervision, + device_loss: None, + input_pump, + #[cfg(feature = "wgpu")] + screen_parity_mailbox: &screen_parity_mailbox, + }; + let result = supervisor.supervise(pipeline); + let shutdown_result = runtime.block_on(supervisor.input_pump.shutdown()); + result.and(shutdown_result) +} + +/// How one pipeline attempt ended. +enum AttemptOutcome { + /// The render loop is no longer running or paused. + Exited, + /// The render thread was asked to stop. + Cancelled, + Failed(RenderFailure), +} + +/// What the supervisor does next after a failure. +enum Flow { + RetryAfter(Duration), + Stop(Result<()>), +} + +struct Supervisor<'a> { + runtime: &'a tokio::runtime::Runtime, + state: &'a RenderThreadState, + cancel: &'a CancellationToken, + input_demands: &'a InputPublicationDemandHandle, + input_monitor: &'a Arc>, + supervision: &'a RenderSupervision, + budget: RestartBudget, + /// Loss signal of the GPU device the pipelines draw with. Every rebuild + /// keeps the same device, so a loss stays visible to later attempts. + device_loss: Option, + input_pump: InputPublicationPump, + #[cfg(feature = "wgpu")] + screen_parity_mailbox: &'a ScreenParityDiagnosticMailbox, +} + +impl Supervisor<'_> { + fn supervise(&mut self, first: PipelineRuntime) -> Result<()> { + let mut next = Some(first); + let mut restarted = false; + loop { + let built = match next.take() { + Some(pipeline) => Ok(pipeline), + None => self.rebuild(), + }; + let (failure, uptime) = match built { + Ok(pipeline) => { + self.device_loss = pipeline.render.sparkleflinger.device_loss_signal(); + if restarted { + info!( + restarts_total = self.supervision.snapshot().restarts_total, + "render pipeline restarted with fresh GPU and render state" + ); + } + let started = Instant::now(); + self.supervision.attempt_started(started); + match self.run_attempt(pipeline) { + AttemptOutcome::Exited => { + self.supervision.set_phase(RenderThreadPhase::Stopped); + return Ok(()); + } + AttemptOutcome::Cancelled => { + self.runtime + .block_on(async { self.state.render_loop.write().await.stop() }); + self.supervision.set_phase(RenderThreadPhase::Stopped); + return Ok(()); + } + AttemptOutcome::Failed(failure) => (failure, started.elapsed()), + } + } + Err(failure) => (failure, Duration::ZERO), + }; + match self.handle_failure(failure, uptime) { + Flow::RetryAfter(backoff) => { + if !self.wait_backoff(backoff) { + return Ok(()); + } + restarted = true; + } + Flow::Stop(result) => return result, + } + } + } + + /// Run one pipeline until it exits, fails, or the thread is cancelled. + /// + /// The pipeline future runs on this thread inside `block_on`, so a + /// panic anywhere in a frame unwinds to the `catch_unwind` here. The + /// unwind drops the pipeline and everything it owned. Shared daemon + /// state behind Tokio locks survives intact, since those do not + /// poison. A std lock the panic poisoned can make the rebuilt pipeline + /// (or its rebuild) fail again, because some of its users refuse a + /// poisoned lock; the restart budget then runs out and ends in a + /// process exit, which is the recovery such a failure needs. + fn run_attempt(&self, pipeline: PipelineRuntime) -> AttemptOutcome { + let monitor = self.input_pump.monitor(); + let state = self.state.clone(); + let supervision = self.supervision.clone(); + let cancel = self.cancel.clone(); + let device_loss = self.device_loss_wait(); + let outcome = std::panic::catch_unwind(AssertUnwindSafe(|| { + self.runtime.block_on(async move { + tokio::select! { + () = run_pipeline(state, pipeline, supervision) => AttemptOutcome::Exited, + status = monitor.wait_for_terminal() => AttemptOutcome::Failed( + RenderFailure::new( + RenderFailureKind::InputPumpTerminated, + format!("input publication pump terminated while rendering: {status:?}"), + ) + .with_panic_report(crate::panic_hook::take_recent_report_where(|report| { + report.thread_name.starts_with(RENDER_RUNTIME_THREAD_NAME) + && report + .occurred_at + .elapsed() + .is_ok_and(|age| age < RECENT_WORKER_PANIC) + })), + ), + message = device_loss => AttemptOutcome::Failed( + RenderFailure::new(RenderFailureKind::GpuDeviceLost, message), + ), + () = cancel.cancelled() => AttemptOutcome::Cancelled, + } + }) + })); + outcome.unwrap_or_else(|payload| { + let message = panic_payload_message(payload.as_ref()); + let report = crate::panic_hook::take_recent_report(std::thread::current().id()); + AttemptOutcome::Failed(RenderFailure::panic(message, report)) + }) + } + + /// The compositor device's loss description, once it is lost. + fn device_lost(&self) -> Option { + let lost = self + .device_loss + .as_ref() + .and_then(DeviceLossSignal::lost) + .map(|lost| lost.to_string()); + #[cfg(feature = "persistence-test-hooks")] + let lost = lost.or_else(|| { + self.supervision + .injected_device_loss() + .lost() + .map(|lost| lost.to_string()) + }); + lost + } + + /// Resolves when the compositor device is lost, immediately when it + /// already was. + fn device_loss_wait(&self) -> impl Future + use<> { + let device = self.device_loss.clone(); + #[cfg(feature = "persistence-test-hooks")] + let injected = self.supervision.injected_device_loss(); + async move { + let device = async move { + match device { + Some(device) => device.wait().await, + None => std::future::pending().await, + } + }; + #[cfg(feature = "persistence-test-hooks")] + let device = async move { + tokio::select! { + message = device => message, + message = injected.wait() => message, + } + }; + device.await + } + } + + fn handle_failure(&mut self, failure: RenderFailure, uptime: Duration) -> Flow { + let failure = if failure.kind != RenderFailureKind::GpuDeviceLost + && let Some(lost) = self.device_lost() + { + failure.after_device_loss(&lost) + } else { + failure + }; + let plan = if self.cancel.is_cancelled() { + RecoveryPlan::DaemonStopping + } else if failure.kind == RenderFailureKind::GpuDeviceLost { + RecoveryPlan::Terminate(TerminationCause::GpuDeviceLost) + } else { + self.budget.record(Instant::now(), uptime) + }; + let record = self + .supervision + .record_failure(failure, plan, self.budget.consecutive()); + match plan { + RecoveryPlan::Restart { backoff, .. } => Flow::RetryAfter(backoff), + RecoveryPlan::DaemonStopping => Flow::Stop(Ok(())), + RecoveryPlan::Terminate(_) => { + let error = anyhow!( + "render pipeline cannot recover ({}): {}", + record.kind.as_str(), + record.message + ); + self.supervision.request_termination(Arc::new(record)); + Flow::Stop(Err(error)) + } + } + } + + /// Sleep out a restart backoff. Returns `false` when the thread was + /// cancelled meanwhile. + fn wait_backoff(&self, backoff: Duration) -> bool { + let cancel = self.cancel.clone(); + let restart = self.runtime.block_on(async move { + tokio::select! { + () = tokio::time::sleep(backoff) => true, + () = cancel.cancelled() => false, + } + }); + if !restart { + self.supervision.set_phase(RenderThreadPhase::Stopped); + } + restart + } + + /// Build a replacement pipeline, restarting the input pump first when + /// it was the part that failed. A panic during the rebuild is a failure + /// like any other and counts against the restart budget. + fn rebuild(&mut self) -> Result { + std::panic::catch_unwind(AssertUnwindSafe(|| self.try_rebuild())).unwrap_or_else( + |payload| { + let message = panic_payload_message(payload.as_ref()); + let report = crate::panic_hook::take_recent_report(std::thread::current().id()); + let mut failure = RenderFailure::panic(message, report); + failure.message = format!("while rebuilding the pipeline: {}", failure.message); + Err(failure) + }, + ) + } + + fn try_rebuild(&mut self) -> Result { + if self.input_pump.monitor().status().is_terminal() { + self.restart_input_pump()?; + } + build_pipeline( + self.state, + &self.input_pump, + self.input_demands, + #[cfg(feature = "wgpu")] + self.screen_parity_mailbox, + None, + ) + .map_err(|error| { + RenderFailure::from_error(RenderFailureKind::PipelineRebuildFailed, &error) + }) + } + + fn restart_input_pump(&mut self) -> Result<(), RenderFailure> { + if let Err(error) = self.runtime.block_on(self.input_pump.shutdown()) { + debug!(%error, "failed input publication pump reported its failure again during cleanup"); + } + let pump = self + .runtime + .block_on(start_input_pump(self.state, self.input_demands.clone())) + .map_err(|error| { + RenderFailure::from_error(RenderFailureKind::InputPumpRestartFailed, &error) + })?; + self.input_pump = pump; + self.input_monitor + .store(Some(Arc::new(self.input_pump.monitor()))); + info!("input publication pump restarted"); + Ok(()) + } +} + +async fn start_input_pump( + state: &RenderThreadState, + demands: InputPublicationDemandHandle, +) -> Result { + InputPublicationPump::start( + state.input_manager.clone(), + demands, + Some(state.interaction_routing.device_input().clone()), + ) + .await +} + +fn build_pipeline( + state: &RenderThreadState, + input_pump: &InputPublicationPump, + input_demands: &InputPublicationDemandHandle, + #[cfg(feature = "wgpu")] screen_parity_mailbox: &ScreenParityDiagnosticMailbox, + progress: Option<&StartupProgress>, +) -> Result { + PipelineRuntime::from_state( + state, + input_pump.reader(), + input_demands.clone(), + #[cfg(feature = "wgpu")] + screen_parity_mailbox.clone(), + progress, + ) +} diff --git a/crates/hypercolor-daemon/src/scene_transactions.rs b/crates/hypercolor-daemon/src/scene_transactions.rs index a7a3af801..17649e4c0 100644 --- a/crates/hypercolor-daemon/src/scene_transactions.rs +++ b/crates/hypercolor-daemon/src/scene_transactions.rs @@ -4,7 +4,8 @@ use std::sync::atomic::{AtomicU8, Ordering}; use std::sync::{Arc, Mutex as StdMutex}; use thiserror::Error; -use tokio::sync::{Mutex, OwnedMutexGuard, oneshot}; +use tokio::sync::futures::Notified; +use tokio::sync::{Mutex, Notify, OwnedMutexGuard, oneshot}; use hypercolor_core::spatial::{SpatialEngine, SpatialPlanError}; use hypercolor_types::scene::{SceneId, UnassignedBehavior, Zone}; @@ -320,6 +321,9 @@ struct SceneTransactionQueueState { #[derive(Clone)] pub struct SceneTransactionQueue { inner: Arc>, + /// Wakes a render loop that parked because nothing consumed its frames; + /// a queued transaction has a caller waiting on it. + queued: Arc, scene_activation_lock: Arc>, layout_update_lock: Arc>, } @@ -575,6 +579,7 @@ impl SceneTransactionQueue { pub fn new() -> Self { Self { inner: Arc::default(), + queued: Arc::default(), scene_activation_lock: Arc::default(), layout_update_lock: Arc::default(), } @@ -592,9 +597,20 @@ impl SceneTransactionQueue { return Err(LayoutTransactionRejection::RendererStopped); } state.pending.push_back(transaction); + drop(state); + self.queued.notify_waiters(); Ok(()) } + /// Future that resolves the next time a transaction is queued. + /// + /// Pin it and call [`Notified::enable`] before checking + /// [`Self::has_pending`], so a push that lands between the check and the + /// await still wakes the waiter. + pub(crate) fn queued(&self) -> Notified<'_> { + self.queued.notified() + } + fn submit_layout_preparation( &self, spatial_engine: SpatialEngine, diff --git a/crates/hypercolor-daemon/src/startup/lifecycle.rs b/crates/hypercolor-daemon/src/startup/lifecycle.rs index 53f09703c..7f575e5cc 100644 --- a/crates/hypercolor-daemon/src/startup/lifecycle.rs +++ b/crates/hypercolor-daemon/src/startup/lifecycle.rs @@ -169,8 +169,12 @@ impl DaemonState { face_fps_cap: config.display.effective_face_fps_cap(), }; self.render_thread = Some( - RenderThread::try_spawn_reporting(rt_state, self.startup_progress.clone()) - .context("failed to spawn render thread with resolved compositor mode")?, + RenderThread::try_spawn_reporting( + rt_state, + self.startup_progress.clone(), + self.domains.output.render_supervision().clone(), + ) + .context("failed to spawn render thread with resolved compositor mode")?, ); self.startup_progress .enter(DaemonStartupPhase::StartingServices); diff --git a/crates/hypercolor-daemon/src/startup/services.rs b/crates/hypercolor-daemon/src/startup/services.rs index abf8a1cc6..fd108d93d 100644 --- a/crates/hypercolor-daemon/src/startup/services.rs +++ b/crates/hypercolor-daemon/src/startup/services.rs @@ -820,6 +820,9 @@ impl DaemonState { display_frames: Arc::clone(&display_frames), device_metrics: Arc::clone(&device_metrics), input_manager: input_manager.clone(), + render_supervision: crate::render_thread::RenderSupervision::for_state_dir( + &state_dir, + ), }, |layout| { let discovery_runtime = crate::discovery::DiscoveryRuntime { @@ -1656,6 +1659,7 @@ pub(crate) struct DomainAssemblyResources { pub display_frames: Arc>, pub device_metrics: DeviceMetricsSnapshotStore, pub input_manager: InputManager, + pub render_supervision: crate::render_thread::RenderSupervision, } /// The assembled domain graph and the driver host it was wired around. @@ -1696,6 +1700,7 @@ pub(crate) fn assemble_domains( display_frames, device_metrics, input_manager, + render_supervision, } = resources; let runtime_projection = RuntimeSessionProjection::new( @@ -1739,6 +1744,7 @@ pub(crate) fn assemble_domains( preview_runtime, devices.clone(), start_time, + render_supervision, ); let domains = DomainContexts::assemble( runtime_session, diff --git a/crates/hypercolor-daemon/src/windows_service.rs b/crates/hypercolor-daemon/src/windows_service.rs index 0fff761e4..b203c618e 100644 --- a/crates/hypercolor-daemon/src/windows_service.rs +++ b/crates/hypercolor-daemon/src/windows_service.rs @@ -104,6 +104,20 @@ fn run_service() -> Result<()> { shutdown_rx, configuration.extension_installers, )); + if let Err(error) = &run_result + && daemon::is_render_termination(error) + { + // A reported stop, even with an error code, runs the recovery + // actions only when the service's failure flag is set, and + // services installed before the installer set it do not have it. + // Exiting without reporting a stop is a crash to the service + // manager, which restarts the service either way. + tracing::error!( + error = %format!("{error:#}"), + "exiting the service without reporting a stop so the service manager restarts it" + ); + std::process::exit(daemon::RENDER_FAILURE_EXIT_CODE); + } let exit_code = u32::from(run_result.is_err()); report_status( &status_handle, diff --git a/crates/hypercolor-daemon/tests/output_demand_tests.rs b/crates/hypercolor-daemon/tests/output_demand_tests.rs new file mode 100644 index 000000000..cc2a13d21 --- /dev/null +++ b/crates/hypercolor-daemon/tests/output_demand_tests.rs @@ -0,0 +1,862 @@ +//! Integration tests for output-demand gating. +//! +//! The render loop parks while nothing consumes its frames and resumes at +//! its current tier the moment a consumer appears. These tests drive the real +//! render thread and prove each consumer kind keeps or wakes it, that a +//! capture after a park returns a freshly rendered frame, and that effect +//! time carries across the park without a jump. + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use tokio::sync::{Mutex, RwLock, watch}; + +use hypercolor_core::asset::AssetLibrary; +use hypercolor_core::bus::HypercolorBus; +use hypercolor_core::device::mock::{MockDeviceBackend, MockDeviceConfig}; +use hypercolor_core::device::{BackendManager, DeviceRegistry}; +use hypercolor_core::effect::{EffectRegistry, builtin::register_builtin_effects}; +use hypercolor_core::engine::{FpsTier, RenderLoop}; +use hypercolor_core::input::{InputManager, SourceKind}; +use hypercolor_core::scene::{SceneManager, make_scene}; +use hypercolor_core::spatial::SpatialEngine; +use hypercolor_daemon::SceneTransactionQueue; +use hypercolor_daemon::canvas_capture::{CanvasCaptureRequest, CanvasImageFormat, capture_canvas}; +use hypercolor_daemon::domain::scene::SceneService; +use hypercolor_daemon::domain::spatial::SpatialService; +use hypercolor_daemon::output_demand::{ + OutputConsumers, OutputDemandTelemetry, is_output_backend, is_output_device, + output_device_count, +}; +use hypercolor_daemon::output_power::OutputPowerState; +use hypercolor_daemon::performance::PerformanceTracker; +use hypercolor_daemon::preview_runtime::PreviewRuntime; +use hypercolor_daemon::render_thread::{ + CanvasDims, InputPublicationConsumer, InputPublicationDemand, RenderThread, RenderThreadState, +}; +use hypercolor_daemon::simulators::SimulatedDisplayExt; +use hypercolor_driver_api::DeviceBackend; +use hypercolor_types::api::simulators::SimulatedDisplay; +use hypercolor_types::config::RenderAccelerationMode; +use hypercolor_types::control::ControlValue; +use hypercolor_types::device::{DeviceId, DeviceInfo, DeviceState}; +use hypercolor_types::effect::{EffectId, EffectMetadata}; +use hypercolor_types::event::{FrameData, SceneChangeReason}; +use hypercolor_types::layer::{SceneLayer, SceneLayerId}; +use hypercolor_types::scene::{Zone, ZoneId, ZoneRole}; +use hypercolor_types::spatial::{ + EdgeBehavior, LedTopology, NormalizedPosition, Output, SamplingMode, SpatialLayout, +}; + +/// Deadline for liveness waits. Generous on purpose: these bound hangs, +/// they do not assert latency. +const WAIT_DEADLINE: Duration = Duration::from_secs(10); + +/// Upper bound on how long an event-driven wake may take. The parked +/// safety tick is a full second, so a wake inside this bound came from the +/// demand signal, not the tick. +const WAKE_LATENCY_BOUND: Duration = Duration::from_millis(500); + +/// Window used to prove a running loop renders at its tier rather than +/// trickling. +const RATE_WINDOW: Duration = Duration::from_millis(600); + +/// Frames a 60 fps loop must render inside [`RATE_WINDOW`]. Well below the +/// 36 an unloaded machine renders, so a busy test host does not flake, and +/// far above anything a parked or tick-driven loop produces. +const MIN_FRAMES_IN_RATE_WINDOW: u64 = 15; + +fn test_layout(zones: Vec) -> SpatialLayout { + SpatialLayout { + id: "output-demand".into(), + name: "Output Demand".into(), + description: None, + canvas_width: 320, + canvas_height: 200, + zones, + default_sampling_mode: SamplingMode::Bilinear, + default_edge_behavior: EdgeBehavior::Clamp, + version: 1, + } +} + +fn point_zone(id: &str, device_id: &str, x: f32, y: f32) -> Output { + Output { + id: id.into(), + name: id.into(), + device_id: device_id.into(), + zone_name: None, + position: NormalizedPosition { x, y }, + size: NormalizedPosition { x: 0.2, y: 0.2 }, + rotation: 0.0, + scale: 1.0, + orientation: None, + topology: LedTopology::Point, + led_positions: Vec::new(), + led_mapping: None, + sampling_mode: None, + edge_behavior: None, + shape: None, + shape_preset: None, + display_order: 0, + attachment: None, + brightness: None, + } +} + +fn builtin_effect_registry() -> EffectRegistry { + let mut registry = EffectRegistry::new(Vec::new()); + register_builtin_effects(&mut registry); + registry +} + +fn builtin_effect_metadata(stem: &str) -> EffectMetadata { + builtin_effect_registry() + .iter() + .find_map(|(_, entry)| { + (entry.metadata.source.source_stem() == Some(stem)).then_some(entry.metadata.clone()) + }) + .expect("builtin effect should exist") +} + +fn builtin_effect_id(stem: &str) -> EffectId { + builtin_effect_metadata(stem).id +} + +fn primary_zone( + effect_id: EffectId, + controls: HashMap, + layout: SpatialLayout, +) -> Zone { + Zone { + id: ZoneId::new(), + name: "Primary".into(), + description: None, + layers: vec![SceneLayer::from_effect( + SceneLayerId::new(), + effect_id, + controls, + HashMap::new(), + None, + )], + layout, + brightness: 1.0, + enabled: true, + color: None, + display_target: None, + role: ZoneRole::Primary, + controls_version: 0, + layers_version: 0, + } +} + +fn solid_color_controls(r: u8, g: u8, b: u8) -> HashMap { + HashMap::from([( + "color".into(), + ControlValue::linear_color([ + f32::from(r) / 255.0, + f32::from(g) / 255.0, + f32::from(b) / 255.0, + 1.0, + ]), + )]) +} + +fn test_asset_library() -> Arc> { + let asset_tempdir = tempfile::tempdir().expect("test asset tempdir should be created"); + let asset_dir = asset_tempdir.path().join("assets"); + Arc::new(RwLock::new( + AssetLibrary::open(asset_dir).expect("test asset library should open"), + )) +} + +/// A render state running `effect` on one point zone at the canvas center, +/// with no devices attached and nothing subscribed. +fn make_render_state(effect: &str, controls: HashMap) -> RenderThreadState { + let (_, power_state) = watch::channel(OutputPowerState::default()); + let event_bus = Arc::new(HypercolorBus::new()); + let layout = test_layout(vec![point_zone("zone_main", "mock:main", 0.5, 0.5)]); + let spatial_engine = SpatialEngine::new(layout.clone()); + let mut scene_manager = SceneManager::with_default(); + scene_manager + .upsert_primary_zone(&builtin_effect_metadata(effect), controls, None, layout) + .expect("test render state should seed a default primary zone"); + let scene_manager = SceneService::with_temporary_store(scene_manager, Arc::clone(&event_bus)) + .expect("temporary scene store should open"); + let scene_plan = scene_manager.plan_reader(); + RenderThreadState { + effect_registry: Arc::new(RwLock::new(builtin_effect_registry())), + asset_library: test_asset_library(), + spatial_engine: SpatialService::new(spatial_engine), + backend_manager: Arc::new(Mutex::new(BackendManager::new())), + device_registry: DeviceRegistry::new(), + performance: Arc::new(RwLock::new(PerformanceTracker::default())), + discovery_runtime: None, + event_bus: Arc::clone(&event_bus), + preview_runtime: Arc::new(PreviewRuntime::new(event_bus)), + zone_layout_previews: Arc::new( + hypercolor_daemon::zone_layout_preview::ZoneLayoutPreviewStore::default(), + ), + render_loop: Arc::new(RwLock::new(RenderLoop::new(60))), + scene_manager, + scene_plan, + input_manager: InputManager::new(), + interaction_routing: + hypercolor_daemon::interaction_routing::InteractionRoutingControl::default(), + power_state, + scene_transactions: SceneTransactionQueue::default(), + screen_capture_configured: false, + canvas_dims: CanvasDims::new(320, 200), + render_acceleration_mode: RenderAccelerationMode::Cpu, + #[cfg(feature = "wgpu")] + render_gpu_device: None, + configured_max_fps_tier: FpsTier::Full.into(), + face_fps_cap: 30, + } +} + +async fn start(state: &RenderThreadState) -> RenderThread { + state.render_loop.write().await.start(); + RenderThread::spawn(state.clone()) +} + +async fn stop(state: &RenderThreadState, mut render_thread: RenderThread) { + state.render_loop.write().await.stop(); + render_thread.shutdown().await.expect("shutdown"); +} + +async fn telemetry(state: &RenderThreadState) -> OutputDemandTelemetry { + state.performance.read().await.output_demand_telemetry() +} + +async fn frame_number(state: &RenderThreadState) -> u64 { + state.render_loop.read().await.frame_number() +} + +/// Wait until the loop has parked `parks` times in total and return the +/// render loop frame number it parked at. +async fn wait_until_parked(state: &RenderThreadState, parks: u64) -> u64 { + tokio::time::timeout(WAIT_DEADLINE, async { + loop { + let telemetry = telemetry(state).await; + if telemetry.parked_since.is_some() && telemetry.parks_total >= parks { + return frame_number(state).await; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("render loop should park (park #{parks}) in time")) +} + +/// Wait until the loop renders past `frame`, returning how long it took. +async fn wait_until_rendering_past(state: &RenderThreadState, frame: u64) -> Duration { + let started = Instant::now(); + tokio::time::timeout(WAIT_DEADLINE, async { + while frame_number(state).await <= frame { + tokio::time::sleep(Duration::from_millis(1)).await; + } + }) + .await + .expect("render loop should resume rendering in time"); + started.elapsed() +} + +async fn frames_rendered_in(state: &RenderThreadState, window: Duration) -> u64 { + let before = frame_number(state).await; + tokio::time::sleep(window).await; + frame_number(state).await.saturating_sub(before) +} + +async fn assert_renders_at_full_tier(state: &RenderThreadState, consumer: &str) { + let frames = frames_rendered_in(state, RATE_WINDOW).await; + assert!( + frames >= MIN_FRAMES_IN_RATE_WINDOW, + "{consumer} should keep the loop rendering at its tier, rendered {frames} frames in {RATE_WINDOW:?}" + ); + assert_eq!( + state.render_loop.read().await.stats().tier, + FpsTier::Full, + "{consumer} should not change the FPS tier" + ); +} + +fn mock_device_info(name: &str) -> DeviceInfo { + let backend = MockDeviceBackend::new().with_device(&MockDeviceConfig { + name: name.into(), + led_count: 8, + topology: LedTopology::Point, + id: Some(DeviceId::new()), + }); + backend.device_infos()[0].clone() +} + +async fn connect_device(registry: &DeviceRegistry, info: DeviceInfo) -> DeviceId { + let device_id = registry.add(info).await; + assert!( + registry.set_state(&device_id, DeviceState::Connected).await, + "test device should move to connected" + ); + device_id +} + +fn zone_main_color(frame: &FrameData) -> [u8; 3] { + frame + .zones + .iter() + .find(|zone| zone.zone_id == "zone_main") + .and_then(|zone| zone.colors.first().copied()) + .expect("frame should carry the main zone") +} + +/// HSV hue of an 8-bit color, in degrees. +fn hue_degrees([r, g, b]: [u8; 3]) -> f32 { + let (r, g, b) = (f32::from(r), f32::from(g), f32::from(b)); + let max = r.max(g).max(b); + let delta = max - r.min(g).min(b); + if delta <= f32::EPSILON { + return 0.0; + } + let sector = if (max - r).abs() <= f32::EPSILON { + ((g - b) / delta).rem_euclid(6.0) + } else if (max - g).abs() <= f32::EPSILON { + (b - r) / delta + 2.0 + } else { + (r - g) / delta + 4.0 + }; + sector * 60.0 +} + +fn hue_distance(a: f32, b: f32) -> f32 { + let difference = (a - b).rem_euclid(360.0); + difference.min(360.0 - difference) +} + +#[tokio::test] +async fn unconsumed_render_loop_parks_and_renders_nothing() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + + let parked_at = wait_until_parked(&state, 1).await; + tokio::time::sleep(Duration::from_millis(1_500)).await; + + assert_eq!( + frame_number(&state).await, + parked_at, + "a parked loop must not render, safety ticks included" + ); + let telemetry = telemetry(&state).await; + assert!(telemetry.parked_since.is_some()); + assert_eq!(telemetry.parks_total, 1); + assert_eq!( + state.render_loop.read().await.stats().tier, + FpsTier::Full, + "parking leaves the tier where it was" + ); + assert!( + !telemetry.consumers.any(), + "the gate should have counted no consumers: {:?}", + telemetry.consumers + ); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn connected_device_keeps_the_loop_rendering_without_parking() { + let state = make_render_state("rainbow", HashMap::new()); + connect_device(&state.device_registry, mock_device_info("Desk Strip")).await; + let render_thread = start(&state).await; + + assert_renders_at_full_tier(&state, "a connected device").await; + assert_renders_at_full_tier(&state, "a connected device").await; + assert_eq!( + telemetry(&state).await.parks_total, + 0, + "a rig with a connected device never parks" + ); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn device_connect_wakes_a_parked_loop_and_disconnect_parks_it_again() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + + let device_id = connect_device(&state.device_registry, mock_device_info("Fan Ring")).await; + let latency = wait_until_rendering_past(&state, parked_at).await; + assert!( + latency < WAKE_LATENCY_BOUND, + "device connect should wake the loop at once, took {latency:?}" + ); + assert_renders_at_full_tier(&state, "a newly connected device").await; + assert_eq!( + telemetry(&state).await.last_wake_reason, + Some("output_consumer") + ); + + state + .device_registry + .set_state(&device_id, DeviceState::Reconnecting) + .await; + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn simulated_display_without_a_preview_is_not_output_demand() { + let simulator = SimulatedDisplay { + id: DeviceId::new(), + name: "Virtual LCD".into(), + width: 240, + height: 240, + circular: true, + enabled: true, + } + .device_info(); + assert!(!is_output_device(&simulator, true)); + assert!(is_output_device(&mock_device_info("Strip"), true)); + assert!(!is_output_device(&mock_device_info("Strip"), false)); + + let state = make_render_state("rainbow", HashMap::new()); + connect_device(&state.device_registry, simulator).await; + let render_thread = start(&state).await; + + let parked_at = wait_until_parked(&state, 1).await; + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!(frame_number(&state).await, parked_at); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn canvas_preview_subscription_wakes_a_parked_loop_at_full_tier() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + + let subscribed_at = Instant::now(); + let mut preview = state.preview_runtime.canvas_receiver(); + tokio::time::timeout(WAIT_DEADLINE, preview.changed()) + .await + .expect("a preview subscriber should receive a frame") + .expect("canvas lane should stay open"); + let latency = subscribed_at.elapsed(); + assert!( + latency < WAKE_LATENCY_BOUND, + "a preview subscription should wake the loop at once, took {latency:?}" + ); + assert!( + u64::from(preview.borrow_and_update().frame_number) >= parked_at, + "the preview should receive a frame rendered after the wake" + ); + assert_renders_at_full_tier(&state, "a canvas preview").await; + assert_eq!(telemetry(&state).await.consumers.stream_receivers, 1); + + drop(preview); + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn scene_canvas_receiver_for_a_display_backdrop_wakes_a_parked_loop() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + + let mut backdrop = state.preview_runtime.scene_canvas_receiver(); + let latency = wait_until_rendering_past(&state, parked_at).await; + assert!( + latency < WAKE_LATENCY_BOUND, + "a display backdrop receiver should wake the loop at once, took {latency:?}" + ); + tokio::time::timeout(WAIT_DEADLINE, backdrop.changed()) + .await + .expect("the backdrop should receive a scene canvas") + .expect("scene canvas lane should stay open"); + assert_renders_at_full_tier(&state, "a display backdrop").await; + + drop(backdrop); + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn frame_and_zone_canvas_receivers_and_demand_holds_wake_a_parked_loop() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + + let parked_at = wait_until_parked(&state, 1).await; + let frames = state.event_bus.frame_receiver(); + wait_until_rendering_past(&state, parked_at).await; + assert_renders_at_full_tier(&state, "a frame stream receiver").await; + drop(frames); + + let parked_at = wait_until_parked(&state, 2).await; + let zone_canvas = state + .event_bus + .zone_canvas_receiver(hypercolor_types::scene::ZoneId::new()); + wait_until_rendering_past(&state, parked_at).await; + drop(zone_canvas); + + let parked_at = wait_until_parked(&state, 3).await; + let hold = state.event_bus.hold_output_demand(); + wait_until_rendering_past(&state, parked_at).await; + assert_renders_at_full_tier(&state, "an output demand hold").await; + drop(hold); + wait_until_parked(&state, 4).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn canvas_capture_after_a_park_returns_a_freshly_rendered_frame() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + tokio::time::sleep(Duration::from_millis(300)).await; + + let requested_at = Instant::now(); + let captured = capture_canvas( + &state.preview_runtime, + CanvasCaptureRequest::single(CanvasImageFormat::Png), + ) + .await + .expect("capture should succeed"); + let latency = requested_at.elapsed(); + + let frame = captured.first().expect("capture should return one frame"); + assert!(frame.fresh, "a capture must wake the loop for a new frame"); + // The park closed its frame without rendering, so `parked_at` is the + // number of the first frame after the wake. That frame still composes + // producer output requested before the park; the capture skips it. + assert!( + u64::from(frame.frame_number) > parked_at, + "the capture should skip the wake frame and return one rendered after the request, got frame {} after parking at {parked_at}", + frame.frame_number + ); + assert!( + !state + .preview_runtime + .is_render_resume_frame(frame.frame_number) + ); + assert!( + latency < WAKE_LATENCY_BOUND, + "a capture of a parked loop should land within a frame or two, took {latency:?}" + ); + assert_eq!(frame.source_width, 320); + assert_eq!(frame.source_height, 200); + + // The capture's demand ends with it, so the loop parks again. + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn effect_time_stays_continuous_across_park_and_resume() { + // Rainbow turns its hue 60 degrees per second, about one degree per + // frame at 60 fps. A clock that kept running through a 1.5 second park + // would jump about 90 degrees on resume. + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + wait_until_parked(&state, 1).await; + let (parked_frame, parked_hue) = { + let frame = state.event_bus.frame_lane().borrow(); + (frame.frame_number, hue_degrees(zone_main_color(&frame))) + }; + tokio::time::sleep(Duration::from_millis(1_500)).await; + + let mut frames = state.event_bus.frame_receiver(); + let resumed = tokio::time::timeout(WAIT_DEADLINE, async { + loop { + frames.changed().await.expect("frame lane should stay open"); + let frame = frames.borrow_and_update().clone(); + if frame.frame_number > parked_frame { + break frame; + } + } + }) + .await + .expect("a frame subscriber should wake the loop"); + let resumed_hue = hue_degrees(zone_main_color(&resumed)); + let resume_step = hue_distance(parked_hue, resumed_hue); + assert!( + resume_step < 12.0, + "effect time should step one frame across the park, hue moved {resume_step} degrees ({parked_hue} -> {resumed_hue})" + ); + + tokio::time::sleep(Duration::from_millis(500)).await; + let later_hue = hue_degrees(zone_main_color(&frames.borrow_and_update())); + let progress = hue_distance(resumed_hue, later_hue); + assert!( + progress > 10.0, + "effect time should keep flowing after the resume, hue moved {progress} degrees in 500 ms" + ); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn scene_change_while_parked_renders_the_new_effect_then_parks_again() { + let state = make_render_state("solid_color", solid_color_controls(255, 0, 0)); + let render_thread = start(&state).await; + wait_until_parked(&state, 1).await; + assert_eq!( + zone_main_color(&state.event_bus.frame_lane().borrow()), + [255, 0, 0] + ); + + let layout = state.spatial_engine.snapshot().layout().as_ref().clone(); + let mut mutation = state.scene_manager.begin_mutation().await; + mutation + .upsert_primary_zone( + &builtin_effect_metadata("solid_color"), + solid_color_controls(0, 0, 255), + None, + layout, + hypercolor_types::event::ChangeTrigger::System, + None, + ) + .expect("scene mutation should stage"); + state + .scene_manager + .commit_mutation(mutation) + .await + .expect("scene mutation should commit"); + + // Nothing subscribes, yet the producers settle on the new scene so a + // later consumer sees it at once. + tokio::time::timeout(WAIT_DEADLINE, async { + while zone_main_color(&state.event_bus.frame_lane().borrow()) != [0, 0, 255] { + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("the parked loop should settle the new effect"); + wait_until_parked(&state, 2).await; + assert_eq!( + telemetry(&state).await.last_wake_reason, + Some("producer_state") + ); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn consumer_counts_cover_every_kind_and_skip_simulated_displays() { + let bus = HypercolorBus::new(); + assert!(!OutputConsumers::from_bus(&bus, 0, 0).any()); + assert!(OutputConsumers::from_bus(&bus, 1, 0).any()); + assert!(OutputConsumers::from_bus(&bus, 0, 1).any()); + let canvas = bus.canvas_receiver(); + assert_eq!(OutputConsumers::from_bus(&bus, 0, 0).stream_receivers, 1); + drop(canvas); + let hold = bus.hold_output_demand(); + assert_eq!(OutputConsumers::from_bus(&bus, 0, 0).demand_holds, 1); + drop(hold); + assert!(!OutputConsumers::from_bus(&bus, 0, 0).any()); + + assert!(is_output_backend("wled")); + assert!(!is_output_backend("simulator")); + + let registry = DeviceRegistry::new(); + assert_eq!(output_device_count(®istry).await, 0); + let strip = connect_device(®istry, mock_device_info("Strip")).await; + let _known = registry.add(mock_device_info("Known Only")).await; + let simulator = SimulatedDisplay { + id: DeviceId::new(), + name: "Virtual LCD".into(), + width: 240, + height: 240, + circular: false, + enabled: true, + } + .device_info(); + connect_device(®istry, simulator).await; + assert_eq!( + output_device_count(®istry).await, + 1, + "only the connected physical device counts" + ); + registry.set_state(&strip, DeviceState::Disabled).await; + assert_eq!(output_device_count(®istry).await, 0); +} + +#[tokio::test] +async fn routed_device_missing_from_the_registry_keeps_the_loop_rendering() { + let mut state = make_render_state("rainbow", HashMap::new()); + let device_id = DeviceId::new(); + let backend = Arc::new(MockDeviceBackend::new().with_device(&MockDeviceConfig { + name: "Routed Strip".into(), + led_count: 1, + topology: LedTopology::Point, + id: Some(device_id), + })); + backend + .connect(&device_id) + .await + .expect("mock device should connect"); + let mut manager = BackendManager::new(); + manager.register_backend(backend); + manager.map_device("mock:main", "mock", device_id); + state.backend_manager = Arc::new(Mutex::new(manager)); + let render_thread = start(&state).await; + + assert_renders_at_full_tier(&state, "a routed device").await; + assert_eq!(telemetry(&state).await.parks_total, 0); + assert_eq!(telemetry(&state).await.consumers.output_devices, 1); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn input_event_stream_keeps_the_loop_republishing_host_input() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + + let stream = render_thread.input_publication_demands().register( + InputPublicationConsumer::PassiveStream, + InputPublicationDemand::default().with_source(SourceKind::Interaction, 60), + ); + let latency = wait_until_rendering_past(&state, parked_at).await; + assert!( + latency < WAKE_LATENCY_BOUND, + "an input event stream should wake the loop at once, took {latency:?}" + ); + assert_renders_at_full_tier(&state, "an input event stream").await; + assert_eq!(telemetry(&state).await.consumers.input_event_streams, 1); + + drop(stream); + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn pausing_a_parked_loop_ends_the_park_and_resuming_parks_again() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + wait_until_parked(&state, 1).await; + + state.render_loop.write().await.pause(); + tokio::time::timeout(WAIT_DEADLINE, async { + while telemetry(&state).await.parked_since.is_some() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("a paused loop should stop reporting a park"); + assert_eq!( + telemetry(&state).await.last_wake_reason, + Some("render_loop_state") + ); + assert!(!state.render_loop.read().await.is_parked()); + + state.render_loop.write().await.resume(); + wait_until_parked(&state, 2).await; + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn render_time_holds_still_while_parked() { + let state = make_render_state("rainbow", HashMap::new()); + let render_thread = start(&state).await; + wait_until_parked(&state, 1).await; + + let parked_elapsed = state.render_loop.read().await.elapsed(); + tokio::time::sleep(Duration::from_millis(500)).await; + assert_eq!( + state.render_loop.read().await.elapsed(), + parked_elapsed, + "media layers and time bindings read render time, which must not run on while parked" + ); + + let frames = state.event_bus.frame_receiver(); + wait_until_rendering_past(&state, frame_number(&state).await).await; + tokio::time::sleep(Duration::from_millis(100)).await; + let resumed_elapsed = state.render_loop.read().await.elapsed(); + assert!(resumed_elapsed > parked_elapsed); + assert!( + resumed_elapsed < parked_elapsed + Duration::from_millis(400), + "render time should resume from the park, not jump by it: {parked_elapsed:?} -> {resumed_elapsed:?}" + ); + drop(frames); + + stop(&state, render_thread).await; +} + +#[tokio::test] +async fn scene_transition_finishes_before_the_loop_parks() { + let state = make_render_state("solid_color", solid_color_controls(255, 0, 0)); + let layout = test_layout(vec![point_zone("zone_main", "mock:main", 0.5, 0.5)]); + let solid_id = builtin_effect_id("solid_color"); + let mut scene_a = make_scene("Red"); + scene_a.transition.duration_ms = 0; + scene_a.zones = vec![primary_zone( + solid_id, + solid_color_controls(255, 0, 0), + layout.clone(), + )]; + let mut scene_b = make_scene("Blue"); + scene_b.transition.duration_ms = 600; + scene_b.zones = vec![primary_zone( + solid_id, + solid_color_controls(0, 0, 255), + layout, + )]; + let mut mutation = state.scene_manager.begin_mutation().await; + mutation + .create_scene(scene_a.clone()) + .expect("scene should stage"); + mutation + .create_scene(scene_b.clone()) + .expect("scene should stage"); + mutation + .activate(scene_a.id, None, SceneChangeReason::UserActivate) + .expect("scene should activate"); + state + .scene_manager + .commit_mutation(mutation) + .await + .expect("scenes should commit"); + + let render_thread = start(&state).await; + let parked_at = wait_until_parked(&state, 1).await; + assert_eq!( + zone_main_color(&state.event_bus.frame_lane().borrow()), + [255, 0, 0] + ); + + let mut mutation = state.scene_manager.begin_mutation().await; + mutation + .activate(scene_b.id, None, SceneChangeReason::UserActivate) + .expect("scene should activate"); + state + .scene_manager + .commit_mutation(mutation) + .await + .expect("scene activation should commit"); + + let parked_again_at = wait_until_parked(&state, 2).await; + assert_eq!( + zone_main_color(&state.event_bus.frame_lane().borrow()), + [0, 0, 255], + "the crossfade should finish on the new scene before the loop parks" + ); + assert!( + parked_again_at - parked_at >= 15, + "a 600 ms crossfade renders through to its end, rendered {} frames", + parked_again_at - parked_at + ); + + stop(&state, render_thread).await; +} diff --git a/crates/hypercolor-daemon/tests/panic_hook_tests.rs b/crates/hypercolor-daemon/tests/panic_hook_tests.rs new file mode 100644 index 000000000..749606370 --- /dev/null +++ b/crates/hypercolor-daemon/tests/panic_hook_tests.rs @@ -0,0 +1,127 @@ +//! The process panic hook records every thread's panic with a backtrace, +//! hands it to the registered observers, keeps it for the code that +//! catches the unwind, and still runs the hook that was installed before. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, Once}; + +use hypercolor_daemon::panic_hook::{self, PanicObserver, PanicReport}; + +/// Observers and the hook are process-wide, so tests that panic take turns. +static PANIC_SLOT: Mutex<()> = Mutex::new(()); + +/// Panics the hook that was installed before ours has seen. +static PREVIOUS_HOOK_CALLS: AtomicUsize = AtomicUsize::new(0); + +/// Install a counting hook, then the daemon hook on top of it, once. +fn install_over_a_previous_hook() { + static INSTALL: Once = Once::new(); + INSTALL.call_once(|| { + let default = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + PREVIOUS_HOOK_CALLS.fetch_add(1, Ordering::SeqCst); + default(info); + })); + panic_hook::install(); + }); +} + +fn collecting_observer() -> (PanicObserver, Arc>>) { + let observed: Arc>> = Arc::default(); + let sink = Arc::clone(&observed); + let observer: PanicObserver = Arc::new(move |report: &PanicReport| { + sink.lock() + .expect("observer sink should lock") + .push(report.clone()); + }); + (observer, observed) +} + +fn panic_on_named_thread(name: &str, payload: &'static str) -> Option { + std::thread::Builder::new() + .name(name.to_owned()) + .spawn(move || { + let caught = std::panic::catch_unwind(|| panic!("{payload}")); + assert!(caught.is_err()); + panic_hook::take_recent_report(std::thread::current().id()) + }) + .expect("probe thread should spawn") + .join() + .expect("probe thread caught its own panic") +} + +#[test] +fn hook_reports_a_named_thread_panic_to_observers_the_catcher_and_the_previous_hook() { + let _slot = PANIC_SLOT + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + install_over_a_previous_hook(); + let (observer, observed) = collecting_observer(); + let registration = panic_hook::add_observer(observer); + let previous_calls = PREVIOUS_HOOK_CALLS.load(Ordering::SeqCst); + + let recent = panic_on_named_thread("hypercolor-panic-probe", "probe panic payload") + .expect("the hook should keep the report for the catcher"); + drop(registration); + + assert_eq!(recent.thread_name, "hypercolor-panic-probe"); + assert_eq!(recent.payload, "probe panic payload"); + assert!( + recent + .location + .as_deref() + .is_some_and(|location| location.contains("panic_hook_tests.rs")), + "location should point at the panic site: {:?}", + recent.location + ); + // A build without debuginfo cannot name this test's frames at all, so + // the excerpt can only be held to the frames the backtrace does name. + let excerpt = recent.backtrace_excerpt(); + assert!( + !recent.backtrace.contains("panic_hook_tests") || excerpt.contains("panic_hook_tests"), + "the excerpt dropped the panicking frame:\n{excerpt}\nfull backtrace:\n{}", + recent.backtrace + ); + assert!( + !excerpt.contains("Backtrace::force_capture"), + "the excerpt should skip the capture frames:\n{excerpt}" + ); + assert!( + PREVIOUS_HOOK_CALLS.load(Ordering::SeqCst) > previous_calls, + "the hook installed before the daemon hook should still run" + ); + let observed = observed.lock().expect("observer sink should lock"); + assert!( + observed + .iter() + .any(|report| report.thread_name == "hypercolor-panic-probe" + && report.payload == "probe panic payload"), + "the observer should see the panic before the unwind is caught" + ); +} + +#[test] +fn dropping_a_registration_removes_only_that_observer() { + let _slot = PANIC_SLOT + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + install_over_a_previous_hook(); + let (first, first_seen) = collecting_observer(); + let (second, second_seen) = collecting_observer(); + let first_registration = panic_hook::add_observer(first); + let second_registration = panic_hook::add_observer(second); + + drop(first_registration); + let _ = panic_on_named_thread("hypercolor-observer-probe", "second observer payload"); + drop(second_registration); + + let seen = |reports: &Arc>>| { + reports + .lock() + .expect("observer sink should lock") + .iter() + .any(|report| report.payload == "second observer payload") + }; + assert!(!seen(&first_seen)); + assert!(seen(&second_seen)); +} diff --git a/crates/hypercolor-daemon/tests/render_supervision_tests.rs b/crates/hypercolor-daemon/tests/render_supervision_tests.rs new file mode 100644 index 000000000..5ac926e52 --- /dev/null +++ b/crates/hypercolor-daemon/tests/render_supervision_tests.rs @@ -0,0 +1,104 @@ +//! Render supervision reports the failure an earlier daemon run left in the +//! state directory, and its status projection gates private detail. + +use hypercolor_daemon::render_thread::supervision::{ + RENDER_FAILURE_FILE, RENDER_RUNTIME_THREAD_NAME, RENDER_THREAD_NAME, is_render_thread_name, +}; +use hypercolor_daemon::render_thread::{RenderSupervision, RenderThreadPhase}; + +/// A record as an earlier run wrote it, before records carried a process +/// instance id. +fn earlier_run_record() -> serde_json::Value { + serde_json::json!({ + "kind": "panic", + "message": "frame panicked in the earlier run", + "thread": "hypercolor-render", + "location": "crates/hypercolor-daemon/src/render_thread/frame_executor.rs:120:9", + "occurred_at": "2026-10-19T22:30:00.000Z", + "occurred_at_unix_ms": 1_792_449_000_000_u64, + "pid": 1, + "daemon_version": "0.7.1", + "restart_attempt": 0, + "restarts_total": 0, + "action": "process_abort", + "backtrace": " 4: hypercolor_daemon::render_thread::frame_executor::execute_frame\n", + }) +} + +#[test] +fn a_failure_persisted_by_an_earlier_run_is_reported_with_private_detail_gated() { + let state_dir = tempfile::tempdir().expect("state tempdir should be created"); + let record_path = state_dir.path().join(RENDER_FAILURE_FILE); + std::fs::write( + &record_path, + serde_json::to_vec(&earlier_run_record()).expect("record should serialize"), + ) + .expect("record should be written"); + + let supervision = RenderSupervision::for_state_dir(state_dir.path()); + + assert_eq!( + supervision.failure_record_path(), + Some(record_path.as_path()) + ); + let snapshot = supervision.snapshot(); + assert_eq!(snapshot.phase, RenderThreadPhase::NotStarted); + assert!(!snapshot.last_failure_in_this_process); + let failure = snapshot + .last_failure + .clone() + .expect("the earlier run's failure should be reported"); + assert_eq!( + failure.pid, 1, + "a container's pid 1 must not hide the record" + ); + assert!(failure.process_instance.is_empty()); + + let private = snapshot.status(true); + assert_eq!(private.phase, "not_started"); + assert_eq!(private.restarts_total, 0); + let reported = private + .last_failure + .expect("status should carry the earlier failure"); + assert_eq!(reported.kind, "panic"); + assert_eq!(reported.action, "process_abort"); + assert_eq!(reported.message, "frame panicked in the earlier run"); + assert!(reported.location.is_some()); + assert!(!reported.current_process); + + let public = snapshot + .status(false) + .last_failure + .expect("kind stays public"); + assert_eq!(public.kind, "panic"); + assert!(public.message.is_empty()); + assert!(public.location.is_none()); + assert!(snapshot.status(false).gpu_errors.last_error.is_none()); +} + +#[test] +fn an_unreadable_failure_record_is_ignored() { + let state_dir = tempfile::tempdir().expect("state tempdir should be created"); + std::fs::write(state_dir.path().join(RENDER_FAILURE_FILE), b"not a record") + .expect("record should be written"); + + let staged = state_dir + .path() + .join(".render-failure.left-by-a-killed-run.tmp"); + std::fs::write(&staged, b"half written").expect("staged record should be written"); + + let supervision = RenderSupervision::for_state_dir(state_dir.path()); + + assert!(supervision.snapshot().last_failure.is_none()); + assert!(supervision.termination().is_none()); + assert!(supervision.watchdog_healthy()); + assert!(!staged.exists(), "a stale staged record should be removed"); +} + +#[test] +fn render_thread_names_cover_the_thread_and_its_runtime_workers() { + assert!(is_render_thread_name(RENDER_THREAD_NAME)); + assert!(is_render_thread_name(RENDER_RUNTIME_THREAD_NAME)); + assert!(!is_render_thread_name("hypercolor-daemon-runtime")); + assert!(!is_render_thread_name("servo-worker")); +} diff --git a/crates/hypercolor-daemon/tests/render_thread_tests.rs b/crates/hypercolor-daemon/tests/render_thread_tests.rs index ee3156f6e..301af9927 100644 --- a/crates/hypercolor-daemon/tests/render_thread_tests.rs +++ b/crates/hypercolor-daemon/tests/render_thread_tests.rs @@ -99,15 +99,21 @@ use hypercolor_daemon::SceneTransactionQueue; #[cfg(feature = "wgpu")] use hypercolor_daemon::canvas_capture::{CanvasCaptureRequest, CanvasImageFormat, capture_canvas}; use hypercolor_daemon::discovery::DiscoveryRuntime; +use hypercolor_daemon::domain::diagnostics::render_thread_check; use hypercolor_daemon::logical_devices::LogicalDevice; use hypercolor_daemon::output_power::{OutputPower, OutputPowerState}; use hypercolor_daemon::performance::PerformanceTracker; #[cfg(feature = "wgpu")] use hypercolor_daemon::preview_runtime::ZonePreviewFrameReceiver; use hypercolor_daemon::preview_runtime::{PreviewPixelFormat, PreviewRuntime, PreviewStreamDemand}; +#[cfg(feature = "wgpu")] +use hypercolor_daemon::render_thread::gpu_errors::gpu_error_totals; +use hypercolor_daemon::render_thread::supervision::{InjectedRenderFault, RENDER_FAILURE_FILE}; use hypercolor_daemon::render_thread::{ CanvasDims, InputPublicationConsumer, InputPublicationDemand, - InputPublicationDemandRegistration, RenderThread, RenderThreadState, + InputPublicationDemandRegistration, RenderFailureKind, RenderFailureRecord, + RenderRecoveryAction, RenderRestartPolicy, RenderSupervision, RenderSupervisionSnapshot, + RenderThread, RenderThreadPhase, RenderThreadState, }; // ── Helpers ───────────────────────────────────────────────────────────────── @@ -5871,6 +5877,11 @@ async fn pipeline_keeps_rendering_while_async_write_failure_disconnects() { state.discovery_runtime = Some(discovery_runtime.clone()); let mut event_rx = event_bus.subscribe_all(); + // The disconnect unmaps the only device, which would leave nothing + // consuming frames and park the loop. A frame subscriber keeps output + // demand alive, so the frames below prove the loop is not blocked on + // the disconnect I/O rather than that it parked. + let _frame_consumer = event_bus.frame_receiver(); { let mut rl = state.render_loop.write().await; rl.start(); @@ -7766,3 +7777,552 @@ async fn release_sleep_clears_published_frame_and_canvas_once() { "release sleep should publish a blank canvas instead of the stale preview" ); } + +// ── Render Thread Supervision Tests ───────────────────────────────────────── + +/// Error and warning logs from every thread, captured once per test binary +/// so assertions can prove a failure was logged. +fn captured_logs() -> Arc>> { + static LOGS: std::sync::OnceLock>>> = std::sync::OnceLock::new(); + Arc::clone(LOGS.get_or_init(|| { + let logs: Arc>> = Arc::default(); + let sink = Arc::clone(&logs); + let subscriber = tracing_subscriber::fmt() + .with_ansi(false) + .with_max_level(tracing::Level::WARN) + .with_writer(move || CapturedLogWriter(Arc::clone(&sink))) + .finish(); + tracing::subscriber::set_global_default(subscriber) + .expect("only the supervision tests install a global subscriber"); + logs + })) +} + +struct CapturedLogWriter(Arc>>); + +impl std::io::Write for CapturedLogWriter { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Whether one captured log line contains every needle. +fn logs_contain_line(logs: &Arc>>, needles: &[&str]) -> bool { + let logs = logs + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + String::from_utf8_lossy(&logs) + .lines() + .any(|line| needles.iter().all(|needle| line.contains(needle))) +} + +fn fast_restart_policy(max_restarts: u32) -> RenderRestartPolicy { + RenderRestartPolicy { + max_restarts, + window: Duration::from_secs(60), + initial_backoff: Duration::from_millis(20), + max_backoff: Duration::from_millis(80), + stable_uptime: Duration::from_secs(60), + } +} + +/// A one-strip scene with no output device. Callers hold a frame receiver +/// so the loop has a consumer and keeps rendering instead of parking. +fn solid_strip_state() -> RenderThreadState { + make_render_state( + active_builtin_effect("solid_color", solid_color_controls(0, 255, 0)), + SpatialEngine::new(test_layout(vec![strip_zone("zone_0", "mock:strip", 8)])), + BackendManager::new(), + ) +} + +async fn wait_for_render_frames(state: &RenderThreadState, minimum_frame_number: u64) { + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + loop { + let frame_number = state.render_loop.read().await.frame_number(); + if frame_number >= minimum_frame_number { + return; + } + assert!( + tokio::time::Instant::now() < deadline, + "expected render frame {minimum_frame_number}, still at {frame_number}" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + +async fn wait_for_supervision( + supervision: &RenderSupervision, + what: &str, + predicate: F, +) -> RenderSupervisionSnapshot +where + F: Fn(&RenderSupervisionSnapshot) -> bool, +{ + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + loop { + let snapshot = supervision.snapshot(); + if predicate(&snapshot) { + return snapshot; + } + assert!( + tokio::time::Instant::now() < deadline, + "timed out waiting for {what}: phase={:?}, restarts_total={}, last_failure={:?}", + snapshot.phase, + snapshot.restarts_total, + snapshot.last_failure + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} + +fn read_failure_record(path: &std::path::Path) -> RenderFailureRecord { + serde_json::from_slice(&std::fs::read(path).expect("render failure record should exist")) + .expect("render failure record should parse") +} + +#[tokio::test] +async fn injected_render_panic_is_logged_recorded_and_the_pipeline_resumes() { + hypercolor_daemon::panic_hook::install(); + let logs = captured_logs(); + let state_dir = tempfile::tempdir().expect("state tempdir should be created"); + let record_path = state_dir.path().join(RENDER_FAILURE_FILE); + let supervision = RenderSupervision::new(fast_restart_policy(3), Some(record_path.clone())); + let state = solid_strip_state(); + let _frame_consumer = state.event_bus.frame_receiver(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 3).await; + + supervision.inject_fault(InjectedRenderFault::Panic( + "injected panic for the logged and recorded restart", + )); + + let snapshot = wait_for_supervision(&supervision, "the restarted pipeline", |snapshot| { + snapshot.restarts_total == 1 && snapshot.phase == RenderThreadPhase::Running + }) + .await; + let resumed_from = state.render_loop.read().await.frame_number(); + wait_for_render_frames(&state, resumed_from + 5).await; + + let failure = snapshot + .last_failure + .expect("the restart should keep the failure that caused it"); + assert!(snapshot.last_failure_in_this_process); + assert_eq!(failure.kind, RenderFailureKind::Panic); + assert_eq!( + failure.message, + "injected panic for the logged and recorded restart" + ); + assert_eq!(failure.action, RenderRecoveryAction::Restart); + assert_eq!(failure.thread.as_deref(), Some("hypercolor-render")); + assert!( + failure + .location + .as_deref() + .is_some_and(|location| location.contains("pipeline_driver.rs")), + "{failure:?}" + ); + assert!( + failure + .backtrace + .as_deref() + .is_some_and(|backtrace| backtrace.contains("pipeline_driver")), + "the record should carry the backtrace captured at the panic: {failure:?}" + ); + assert_eq!( + rt.supervision().failure_record_path(), + Some(record_path.as_path()) + ); + let persisted = read_failure_record(&record_path); + assert_eq!(persisted.kind, RenderFailureKind::Panic); + assert_eq!(persisted.action, RenderRecoveryAction::Restart); + assert_eq!(persisted.restarts_total, 1); + assert_eq!(persisted.pid, std::process::id()); + assert!(persisted.backtrace.is_some()); + assert!(logs_contain_line( + &logs, + &[ + "render pipeline failed; restarting it with fresh GPU and render state", + "injected panic for the logged and recorded restart", + ] + )); + assert!(logs_contain_line( + &logs, + &[ + "thread panicked", + "injected panic for the logged and recorded restart" + ] + )); + + state.render_loop.write().await.stop(); + rt.shutdown() + .await + .expect("a recovered render thread should shut down cleanly"); + assert_eq!(supervision.phase(), RenderThreadPhase::Stopped); +} + +#[tokio::test] +async fn repeated_render_panics_exhaust_the_restart_budget_and_request_termination() { + hypercolor_daemon::panic_hook::install(); + let state_dir = tempfile::tempdir().expect("state tempdir should be created"); + let record_path = state_dir.path().join(RENDER_FAILURE_FILE); + let supervision = RenderSupervision::new(fast_restart_policy(2), Some(record_path.clone())); + let state = solid_strip_state(); + let _frame_consumer = state.event_bus.frame_receiver(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 1).await; + + for _ in 0..3 { + supervision.inject_fault(InjectedRenderFault::Panic("injected crash loop panic")); + } + + let record = tokio::time::timeout(WAIT_DEADLINE, supervision.terminated()) + .await + .expect("the third panic should exhaust a two-restart budget"); + assert_eq!(record.kind, RenderFailureKind::Panic); + assert_eq!(record.action, RenderRecoveryAction::TerminateProcess); + assert_eq!(record.restart_attempt, 3); + assert_eq!(record.restarts_total, 2); + assert_eq!(supervision.phase(), RenderThreadPhase::Terminating); + let persisted = read_failure_record(&record_path); + assert_eq!(persisted.action, RenderRecoveryAction::TerminateProcess); + let error = rt + .shutdown() + .await + .expect_err("a render thread that gave up should end with an error"); + assert!(format!("{error:#}").contains("cannot recover"), "{error:#}"); +} + +#[tokio::test] +async fn lost_compositor_device_terminates_without_an_in_process_restart() { + let supervision = RenderSupervision::new(fast_restart_policy(5), None); + let state = solid_strip_state(); + let _frame_consumer = state.event_bus.frame_receiver(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 1).await; + + supervision.inject_compositor_device_lost("injected driver reset"); + + let record = tokio::time::timeout(WAIT_DEADLINE, supervision.terminated()) + .await + .expect("a lost compositor device should end the render thread"); + assert_eq!(record.kind, RenderFailureKind::GpuDeviceLost); + assert_eq!(record.action, RenderRecoveryAction::TerminateProcess); + assert!( + record.message.contains("injected driver reset"), + "{record:?}" + ); + assert_eq!(record.restarts_total, 0); + assert!(rt.shutdown().await.is_err()); +} + +#[cfg(feature = "wgpu")] +#[tokio::test] +async fn injected_uncaptured_wgpu_error_is_logged_and_counted_without_killing_the_render_thread() { + let logs = captured_logs(); + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let mut state = solid_strip_state(); + let _frame_consumer = state.event_bus.frame_receiver(); + state.render_acceleration_mode = RenderAccelerationMode::Gpu; + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised GPU render thread should spawn"); + wait_for_render_frames(&state, 3).await; + let before = gpu_error_totals(); + + supervision.inject_fault(InjectedRenderFault::GpuValidationError); + + let snapshot = wait_for_supervision(&supervision, "the counted validation error", |snapshot| { + snapshot.gpu_errors.validation > before.validation + }) + .await; + let counted_at = state.render_loop.read().await.frame_number(); + wait_for_render_frames(&state, counted_at + 5).await; + + assert_eq!(snapshot.phase, RenderThreadPhase::Running); + assert_eq!(supervision.snapshot().restarts_total, 0); + let last = snapshot + .gpu_errors + .last + .expect("the latest uncaptured error should be kept"); + assert_eq!(last.kind, "validation"); + assert_eq!(last.device, "SparkleFlinger GPU compositor"); + assert!(logs_contain_line( + &logs, + &["uncaptured wgpu error", "injected validation error"] + )); + + state.render_loop.write().await.stop(); + rt.shutdown() + .await + .expect("a render thread that only logged a GPU error should shut down cleanly"); +} + +async fn render_thread_check_now( + state: &RenderThreadState, + supervision: &RenderSupervision, + stall_after: Duration, +) -> hypercolor_types::api::diagnose::DiagnoseCheck { + let loop_state = state.render_loop.read().await.state(); + // Diagnose reads the park from the performance telemetry, not from the + // render loop's flag; the two differ briefly around a wake. + let parked = state + .performance + .read() + .await + .output_demand_telemetry() + .parked_since + .is_some(); + render_thread_check(&supervision.snapshot(), loop_state, parked, stall_after) +} + +async fn wait_for_render_thread_check( + state: &RenderThreadState, + supervision: &RenderSupervision, + stall_after: Duration, + expected: &str, +) -> hypercolor_types::api::diagnose::DiagnoseCheck { + wait_for_render_thread_check_where(state, supervision, stall_after, |check| { + check.status == expected + }) + .await +} + +async fn wait_for_render_thread_check_where( + state: &RenderThreadState, + supervision: &RenderSupervision, + stall_after: Duration, + accept: F, +) -> hypercolor_types::api::diagnose::DiagnoseCheck +where + F: Fn(&hypercolor_types::api::diagnose::DiagnoseCheck) -> bool, +{ + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + loop { + let check = render_thread_check_now(state, supervision, stall_after).await; + if accept(&check) { + return check; + } + assert!( + tokio::time::Instant::now() < deadline, + "render_thread check never reached the expected state: {} ({})", + check.status, + check.detail + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +#[tokio::test] +async fn render_thread_check_fails_while_frames_stall_and_passes_after_recovery() { + let stall_after = Duration::from_millis(300); + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let state = solid_strip_state(); + let _frame_consumer = state.event_bus.frame_receiver(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 3).await; + wait_for_render_thread_check(&state, &supervision, stall_after, "pass").await; + + supervision.inject_fault(InjectedRenderFault::Stall(Duration::from_millis(1_500))); + + let stalled = wait_for_render_thread_check(&state, &supervision, stall_after, "fail").await; + assert!( + stalled.detail.contains("has not completed a frame"), + "{}", + stalled.detail + ); + wait_for_render_thread_check(&state, &supervision, stall_after, "pass").await; + assert_eq!(supervision.snapshot().restarts_total, 0); + + state.render_loop.write().await.stop(); + rt.shutdown().await.expect("shutdown after a stall"); +} + +#[tokio::test] +async fn render_thread_check_fails_while_the_pipeline_restarts_and_passes_after() { + let stall_after = Duration::from_millis(300); + let supervision = RenderSupervision::new( + RenderRestartPolicy { + initial_backoff: Duration::from_millis(600), + max_backoff: Duration::from_millis(600), + ..fast_restart_policy(3) + }, + None, + ); + let state = solid_strip_state(); + let _frame_consumer = state.event_bus.frame_receiver(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 3).await; + + supervision.inject_fault(InjectedRenderFault::Panic( + "injected panic for the restarting check", + )); + + // The panic hook symbolizes a backtrace on the render thread, which can + // outlast the stall threshold, so a stall failure may come first. + wait_for_render_thread_check_where(&state, &supervision, stall_after, |check| { + check.status == "fail" && check.detail.contains("restarting") + }) + .await; + let recovered = wait_for_render_thread_check(&state, &supervision, stall_after, "pass").await; + assert!( + recovered.detail.contains("restarts=1"), + "{}", + recovered.detail + ); + let resumed_from = state.render_loop.read().await.frame_number(); + wait_for_render_frames(&state, resumed_from + 3).await; + + state.render_loop.write().await.stop(); + rt.shutdown() + .await + .expect("shutdown after a recovered restart"); +} + +#[tokio::test] +async fn a_parked_loop_keeps_its_supervisor_heartbeat_fresh() { + let stall_after = Duration::from_millis(2_500); + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let state = solid_strip_state(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + while !state.render_loop.read().await.is_parked() { + assert!( + tokio::time::Instant::now() < deadline, + "a loop with no consumer should park" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + + // Longer than the stall threshold: only the park wait's heartbeat can + // keep the check passing, since no frame completes while parked. + tokio::time::sleep(Duration::from_millis(3_000)).await; + assert!(state.render_loop.read().await.is_parked()); + let check = render_thread_check_now(&state, &supervision, stall_after).await; + assert_eq!(check.status, "pass", "{}", check.detail); + assert!( + check.detail.contains("parked with no output consumers"), + "{}", + check.detail + ); + assert!(supervision.watchdog_healthy()); + + state.render_loop.write().await.stop(); + rt.shutdown().await.expect("shutdown while parked"); +} + +#[tokio::test] +async fn a_new_pipeline_never_inherits_a_park_it_did_not_enter() { + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let state = solid_strip_state(); + let _frame_consumer = state.event_bus.frame_receiver(); + { + let mut render_loop = state.render_loop.write().await; + render_loop.start(); + // A park left behind by a pipeline that died while parked. + render_loop.park(); + } + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + + wait_for_render_frames(&state, 3).await; + assert!( + !state.render_loop.read().await.is_parked(), + "a consumed loop must render unparked so effect time advances" + ); + + state.render_loop.write().await.stop(); + rt.shutdown() + .await + .expect("shutdown after clearing a stale park"); +} + +#[tokio::test] +async fn a_loop_resumed_from_a_long_pause_is_not_reported_as_stalled() { + let stall_after = Duration::from_millis(1_000); + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let state = solid_strip_state(); + let _frame_consumer = state.event_bus.frame_receiver(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + wait_for_render_frames(&state, 3).await; + + state.render_loop.write().await.pause(); + tokio::time::sleep(Duration::from_millis(1_500)).await; + state.render_loop.write().await.resume(); + + // Read before the render thread can notice the resume: the last frame is + // a whole pause old, but frames have only been due since the last poll. + let check = render_thread_check_now(&state, &supervision, stall_after).await; + assert_eq!(check.status, "pass", "{}", check.detail); + let resumed_from = state.render_loop.read().await.frame_number(); + wait_for_render_frames(&state, resumed_from + 3).await; + let check = render_thread_check_now(&state, &supervision, stall_after).await; + assert_eq!(check.status, "pass", "{}", check.detail); + + state.render_loop.write().await.stop(); + rt.shutdown().await.expect("shutdown after a long pause"); +} + +#[tokio::test] +async fn a_loop_woken_from_a_long_park_is_not_reported_as_stalled() { + let stall_after = Duration::from_millis(1_000); + let supervision = RenderSupervision::new(fast_restart_policy(3), None); + let state = solid_strip_state(); + state.render_loop.write().await.start(); + let mut rt = RenderThread::try_spawn_supervised(state.clone(), supervision.clone()) + .expect("supervised render thread should spawn"); + + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + while !state.render_loop.read().await.is_parked() { + assert!( + tokio::time::Instant::now() < deadline, + "a loop with no consumer should park" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } + tokio::time::sleep(Duration::from_millis(1_500)).await; + + let parked_frame = state.render_loop.read().await.frame_number(); + let _frame_consumer = state.event_bus.frame_receiver(); + // Sample across the wake, including the first frame whose gate ends the + // park before that frame completes. + let deadline = tokio::time::Instant::now() + WAIT_DEADLINE; + loop { + let check = render_thread_check_now(&state, &supervision, stall_after).await; + assert_eq!(check.status, "pass", "{}", check.detail); + if state.render_loop.read().await.frame_number() >= parked_frame + 3 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "the woken loop should render" + ); + tokio::task::yield_now().await; + } + + state.render_loop.write().await.stop(); + rt.shutdown().await.expect("shutdown after a long park"); +} diff --git a/crates/hypercolor-linux-session/src/lib.rs b/crates/hypercolor-linux-session/src/lib.rs index 2729baf50..35b2ea6b5 100644 --- a/crates/hypercolor-linux-session/src/lib.rs +++ b/crates/hypercolor-linux-session/src/lib.rs @@ -19,7 +19,7 @@ pub use self::parent_death::arm_parent_death; pub use self::procfs::{process_resident_memory_mb, running_processes}; #[cfg(target_os = "linux")] pub use self::screensaver::ScreensaverMonitor; -pub use self::systemd::{notify_ready, spawn_watchdog}; +pub use self::systemd::{notify_ready, spawn_watchdog, spawn_watchdog_while}; /// Build the Linux session monitors for daemon composition. #[must_use] diff --git a/crates/hypercolor-linux-session/src/systemd.rs b/crates/hypercolor-linux-session/src/systemd.rs index 623c6c49e..5f0cbf287 100644 --- a/crates/hypercolor-linux-session/src/systemd.rs +++ b/crates/hypercolor-linux-session/src/systemd.rs @@ -25,16 +25,50 @@ pub fn notify_ready() { /// /// Off Linux this does nothing. On Linux it must run inside a Tokio runtime. pub fn spawn_watchdog() { + spawn_watchdog_while(|| true); +} + +/// Spawn the periodic watchdog keepalive, sending each keepalive only while +/// `healthy` returns `true`. +/// +/// systemd kills and restarts a service that goes `WatchdogSec` without a +/// keepalive, so withholding them is how a daemon with a wedged subsystem +/// it cannot recover in-process gets a fresh process. `healthy` runs once +/// per keepalive interval on the runtime, so it must not block. +/// +/// Off Linux this does nothing. On Linux it must run inside a Tokio runtime. +pub fn spawn_watchdog_while(healthy: F) +where + F: Fn() -> bool + Send + 'static, +{ #[cfg(target_os = "linux")] { - tokio::spawn(async { + tokio::spawn(async move { let mut interval = tokio::time::interval(WATCHDOG_INTERVAL); + let mut withheld = false; loop { interval.tick().await; + if !healthy() { + if !withheld { + tracing::error!( + "withholding the systemd watchdog keepalive because the daemon \ + reports itself unhealthy; systemd restarts it once the watchdog \ + timeout passes" + ); + withheld = true; + } + continue; + } + if withheld { + tracing::warn!("daemon is healthy again; resuming systemd watchdog keepalives"); + withheld = false; + } if let Err(error) = sd_notify::notify(&[sd_notify::NotifyState::Watchdog]) { tracing::debug!("failed to notify systemd watchdog: {error}"); } } }); } + #[cfg(not(target_os = "linux"))] + let _ = healthy; } diff --git a/crates/hypercolor-linux-session/tests/watchdog_tests.rs b/crates/hypercolor-linux-session/tests/watchdog_tests.rs new file mode 100644 index 000000000..307aa86bf --- /dev/null +++ b/crates/hypercolor-linux-session/tests/watchdog_tests.rs @@ -0,0 +1,18 @@ +//! The systemd watchdog keepalive consults the daemon's health on every +//! interval, starting with the first. + +#[cfg(target_os = "linux")] +#[tokio::test] +async fn watchdog_consults_the_health_check_on_its_first_interval() { + let (checked_tx, mut checked_rx) = tokio::sync::mpsc::unbounded_channel(); + + hypercolor_linux_session::spawn_watchdog_while(move || { + let _ = checked_tx.send(()); + false + }); + + tokio::time::timeout(std::time::Duration::from_secs(5), checked_rx.recv()) + .await + .expect("the first keepalive interval should run the health check") + .expect("the watchdog task should still hold the health check"); +} diff --git a/crates/hypercolor-tui/tests/rest_client_tests.rs b/crates/hypercolor-tui/tests/rest_client_tests.rs index 8670766fd..10b6e0519 100644 --- a/crates/hypercolor-tui/tests/rest_client_tests.rs +++ b/crates/hypercolor-tui/tests/rest_client_tests.rs @@ -208,6 +208,7 @@ async fn get_status_maps_the_system_response_without_an_active_effect_call() { actual_fps: 59.8, consecutive_misses: 0, total_frames: 12_000, + ..RenderLoopStatus::default() }, ..SystemStatus::default() }) diff --git a/crates/hypercolor-types/src/api/diagnose.rs b/crates/hypercolor-types/src/api/diagnose.rs index dccf47d78..f04dc5eb2 100644 --- a/crates/hypercolor-types/src/api/diagnose.rs +++ b/crates/hypercolor-types/src/api/diagnose.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; -use crate::api::system::InputStatus; +use crate::api::system::{InputStatus, RenderSupervisionStatus}; /// Optional body for `POST /api/v1/diagnose`. /// @@ -59,6 +59,14 @@ pub struct DiagnoseSnapshot { pub struct DiagnoseRenderSnapshot { pub latest_frame: Option, pub recent_window: DiagnoseRenderWindowSnapshot, + /// What consumes rendered frames, and whether the loop parked because + /// nothing does. Absent from older daemons. + #[serde(default)] + pub output_demand: crate::api::system::RenderOutputDemandStatus, + /// Supervised render thread health, restart counts, and the last + /// failure. Absent from older daemons. + #[serde(default)] + pub supervision: RenderSupervisionStatus, } #[derive(Debug, Clone, Serialize)] diff --git a/crates/hypercolor-types/src/api/system.rs b/crates/hypercolor-types/src/api/system.rs index c5c809ccd..fc5066fd8 100644 --- a/crates/hypercolor-types/src/api/system.rs +++ b/crates/hypercolor-types/src/api/system.rs @@ -33,6 +33,9 @@ pub struct SystemStatus { pub macos_daemon_ownership: Option, pub compositor_acceleration: RenderAccelerationStatus, pub render_loop: RenderLoopStatus, + /// Supervised render thread health. Absent from older daemons. + #[serde(default)] + pub render_supervision: RenderSupervisionStatus, pub session_performance: SessionPerformanceStatus, pub latest_frame: Option, pub effect_health: EffectHealthStatus, @@ -323,6 +326,118 @@ pub struct RenderLoopStatus { pub actual_fps: f64, pub consecutive_misses: u32, pub total_frames: u64, + /// Whether anything consumes rendered frames, and why the loop parked + /// when nothing does. Absent from older daemons. + #[serde(default)] + pub output_demand: RenderOutputDemandStatus, +} + +/// Output demand: what consumes the render loop's frames. +/// +/// The render loop renders only while something consumes its output: a +/// connected device, a display, a preview or canvas stream, a capture, or +/// an explicit demand hold. With none of those it parks, renders nothing, +/// and resumes at its current tier the moment a consumer appears. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +pub struct RenderOutputDemandStatus { + /// Whether the render loop is parked because nothing consumes its frames. + pub parked: bool, + /// Why the render loop is producing no frames, when it is parked. + /// `no_output_consumers` while parked. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub idle_reason: Option, + /// How long the loop has been parked, in milliseconds. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub parked_ms: Option, + /// Connected devices that receive frames. Simulated displays count only + /// while previewed, through `demand_holds`. + pub output_devices: u32, + /// Live receivers on the frame, canvas, preview, spectrum, and per-zone + /// canvas streams, including in-flight canvas captures. + pub stream_receivers: u32, + /// Stream and diagnostic consumers of host input events, which the + /// render loop republishes. + pub input_event_streams: u32, + /// Explicit demand holds, such as display output for previewed + /// simulated displays. + pub demand_holds: u32, + /// How many times the loop has parked since the daemon started. + pub parks_total: u64, + /// What last woke the loop from a park: `output_consumer`, + /// `scene_transaction`, `producer_state`, or `render_loop_state`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_wake_reason: Option, +} + +/// Health and recovery state of the supervised render thread. +/// +/// The render thread restarts its pipeline in-process after a panic or an +/// error exit, within a bounded budget, and the daemon exits so its +/// service manager restarts it once that budget runs out or the compositor +/// GPU device is lost. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +pub struct RenderSupervisionStatus { + /// `not_started`, `starting`, `running`, `restarting`, `terminating`, + /// or `stopped`. + pub phase: String, + /// In-process pipeline restarts since the daemon started. + pub restarts_total: u64, + /// Failures since the pipeline last ran past its stable uptime. + pub consecutive_failures: u32, + /// Restarts the budget allows within its window. + pub max_restarts: u32, + /// Milliseconds since the render loop last completed a frame. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_frame_age_ms: Option, + /// The latest render failure in this daemon run, or else the one the + /// previous run recorded in the state directory. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_failure: Option, + /// Uncaptured wgpu errors and device losses since the daemon started. + #[serde(default)] + pub gpu_errors: GpuErrorStatus, +} + +/// One render failure, as the supervisor recorded it. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +pub struct RenderFailureStatus { + /// `panic`, `input_pump_terminated`, `input_pump_restart_failed`, + /// `pipeline_rebuild_failed`, or `gpu_device_lost`. + pub kind: String, + /// The panic message or error chain. Empty when the caller may not + /// read private diagnostics. + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thread: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub location: Option, + /// RFC 3339 UTC time of the failure. + pub occurred_at: String, + /// `restart`, `terminate_process`, `daemon_stopping`, `process_abort`, + /// or `awaiting_supervisor`. + pub action: String, + /// Restarts counted in the budget window by this failure. + pub restart_attempt: u32, + /// Whether this daemon run recorded the failure, as opposed to an + /// earlier run whose record survived in the state directory. + pub current_process: bool, +} + +/// Uncaptured wgpu errors and device losses since the daemon started. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] +#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))] +pub struct GpuErrorStatus { + pub validation_errors: u64, + pub out_of_memory_errors: u64, + pub internal_errors: u64, + pub device_lost: u64, + /// `device: kind: message` of the most recent one. Absent when there + /// was none, or when the caller may not read private diagnostics. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub last_error: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)] diff --git a/docs/content/api/rest.md b/docs/content/api/rest.md index 6f8a152f0..e512c5eda 100644 --- a/docs/content/api/rest.md +++ b/docs/content/api/rest.md @@ -238,7 +238,16 @@ keyed daemon receive the public identity without the status block. "target_fps": 60, "capacity_fps": 60.0, "delivered_fps": 59.8, - "actual_fps": 60.0 + "actual_fps": 60.0, + "output_demand": { + "parked": false, + "output_devices": 3, + "stream_receivers": 2, + "input_event_streams": 0, + "demand_holds": 0, + "parks_total": 1, + "last_wake_reason": "output_consumer" + } } } }, @@ -250,6 +259,11 @@ keyed daemon receive the public identity without the status block. } ``` +`render_loop.output_demand` says what consumes rendered frames. With no device, stream, +preview, or capture attached the loop parks: `parked` turns true, `idle_reason` reads +`no_output_consumers`, `parked_ms` counts up, and `delivered_fps` and `actual_fps` read +zero until a consumer appears and the loop resumes at its current tier. + `status.effect_count` reflects whatever the registry holds at request time (native built-ins plus discovered HTML effects); treat it as live, not a fixed product number. diff --git a/docs/content/architecture/render-pipeline.md b/docs/content/architecture/render-pipeline.md index d08e5cdc3..fed9e0744 100644 --- a/docs/content/architecture/render-pipeline.md +++ b/docs/content/architecture/render-pipeline.md @@ -166,6 +166,21 @@ fps_controller.maybe_transition(); `FrameStats` surfaces `tier`, `frame_time`, `headroom`, `ewma_frame_time`, `consecutive_misses`, and `frames_since_tier_change`, all forwarded to the event bus and visible in the dashboard. +## Rendering only what someone receives + +Every frame costs producer, compositor, and GPU work whether or not anything receives it. A daemon with no devices and no open client would otherwise render a WebGL effect at 60 fps for nobody. Before each frame the loop therefore counts the consumers of its output: + +- connected devices, LED or display, that receive frames (a simulated display counts only while someone previews it); +- live receivers on the bus output streams: frame data, spectrum, the canvas and scene canvas previews, screen canvas and screen zones, web viewport, zone previews, and the per-zone canvas streams. WebSocket subscriptions, display backdrops, and canvas captures all hold one; +- WebSocket and diagnostic subscribers to host input events, which the loop republishes; +- explicit output demand holds, taken by anything that reads published frames without a receiver. + +With none of them, the loop **parks**: it renders nothing, withdraws its audio, screen, and interaction capture demand, drops live media streams, and throttles HTML effect pages, which stops the `requestAnimationFrame` loops they schedule for themselves. Page timers keep their own pace, so an effect that animates from `setTimeout` or `setInterval` rather than `requestAnimationFrame` keeps running while parked; the bundled effects all animate from `requestAnimationFrame`. A parked loop sleeps until something can observe a frame. Subscribing to a stream, taking a demand hold, connecting a device, or queuing a layout change wakes it at once, and it resumes at its current tier; any other bus event makes it re-check whether it has work. A one second safety tick re-checks in case a consumer arrived without signalling. No observable cadence changes, because the moment anything can see a frame, it renders at the rate it would have without the park. + +Parking waits for the producers to settle. After the scene, an effect, or the canvas changes, the loop keeps rendering until every producer has shown its new content and any scene transition has finished (bounded at ten seconds, so a longer crossfade pauses partway and finishes when a consumer returns), so an effect applied while nobody watches is already loaded when a client or capture arrives. Time carries across the park: render time stops while parked, so media layers and time bindings continue where they left off, and the first frame after a wake advances effect time by one frame, not by the parked span, with no catch-up frames. A canvas capture that wakes the loop skips the wake frame itself, whose HTML producers still show output requested before the park, and returns the next one. + +`/api/v1/system` reports the state under `render_loop.output_demand` (`parked`, `idle_reason`, `parked_ms`, and the consumer counts the loop last saw), with `delivered_fps` and `actual_fps` at zero while parked. `/api/v1/diagnose` carries the same object under `snapshot.render.output_demand`. Rigs with a connected device never park. + ## Canvas resize and layout updates Two operations trigger mid-run reconfigurations without stalling the render thread: diff --git a/docs/content/troubleshooting/performance.md b/docs/content/troubleshooting/performance.md index f13002af2..b261392ec 100644 --- a/docs/content/troubleshooting/performance.md +++ b/docs/content/troubleshooting/performance.md @@ -41,7 +41,7 @@ Output is a styled table by default. Use `--format plain` for a one-line text su ## Step 2: read the render checks -The `render` category has five checks for FPS and LED output problems. +The `render` category has seven checks for FPS and LED output problems. ### `render_loop` @@ -49,7 +49,7 @@ The `render` category has five checks for FPS and LED output problems. render_loop state=running, tier=60fps (full) ``` -The `tier` field shows where the FPS controller has landed, as its rate and name. Compare it against your configured `daemon.target_fps`: on the default config, `full` (60 FPS) is the healthy steady state. A tier sitting below your configured target means the loop has downshifted under budget pressure. If `state` is anything other than `running`, the render loop has stopped; see [common issues](@/troubleshooting/common-issues.md). +The `tier` field shows where the FPS controller has landed, as its rate and name. Compare it against your configured `daemon.target_fps`: on the default config, `full` (60 FPS) is the healthy steady state. A tier sitting below your configured target means the loop has downshifted under budget pressure. If `state` is anything other than `running`, the render loop is not rendering; see [common issues](@/troubleshooting/common-issues.md). A `paused` loop renders nothing on purpose, so the detail names what paused it. A manual output pause (from `PATCH /api/v1/output`, a pause control, or an integration such as Home Assistant) persists across daemon restarts until output power is set back to running. **Tier reference:** @@ -63,6 +63,30 @@ The `tier` field shows where the FPS controller has landed, as its rate and name The controller downshifts after 2 consecutive budget misses and only upshifts after sustained headroom, so a tier stuck below your target is a real signal. +``` +render_loop state=running, tier=full, parked=no_output_consumers +``` + +`parked` means nothing consumes rendered frames: no connected device, no open preview or stream, no capture. The loop renders nothing until one appears, then resumes at its current tier, so `0 fps` here is expected rather than a stall. `frame_liveness` passes while parked, and `hypercolor status` shows the same state on its render line. `snapshot.render.output_demand` in the JSON report lists the consumer counts the loop last saw and what last woke it. + +### `render_thread` + +``` +render_thread phase=running, last_frame_age_ms=12, restarts=0, last_failure=none +``` + +The render thread supervises its own pipeline. When a frame panics or the pipeline loses its input feed, the daemon logs the failure with a backtrace, rebuilds the pipeline with fresh GPU and render state, and keeps going. This check fails while that restart is in progress, when the thread has gone without completing a frame for 3 seconds while the loop runs, and when the thread is gone while the loop still says it is running. A paused or parked loop passes as long as the thread keeps polling, since no frames are due, and the detail names what paused it. + +`restarts` counts in-process restarts since the daemon started. After five restarts within five minutes, or when the GPU device is lost, the daemon stops recovering in-process and exits with an error so its service manager starts a fresh process. Under systemd, a render thread stuck inside one frame for 20 seconds also stops the service watchdog keepalive, so systemd restarts the daemon. Every failure is also written to `render-failure.json` in the daemon state directory (`~/.local/state/hypercolor/` on Linux), so it survives a log that was lost. The next run reports that record here as `(earlier daemon run)`. + +### `gpu_errors` + +``` +gpu_errors validation=0, out_of_memory=0, internal=0, device_lost=0 +``` + +GPU validation, out-of-memory, and internal errors that nothing handled are counted and logged instead of crashing the thread that raised them. One in the last 10 minutes turns this check into a warning, with the device and the latest message in the detail. The counts since startup always stay in the detail. A lost GPU device fails the check. + ### `frame_liveness` ``` diff --git a/docs/design/30-sparkleflinger-implementation.md b/docs/design/30-sparkleflinger-implementation.md index 0a5602fce..72ab65e49 100644 --- a/docs/design/30-sparkleflinger-implementation.md +++ b/docs/design/30-sparkleflinger-implementation.md @@ -210,6 +210,42 @@ throttle. Both throttle paths bypass composition and the producer stages but still record `LatestFrameMetrics` with pacing-relevant fields populated. +### 3.4.1 Output-demand park + +Before either throttle, right after scene transactions are serviced, the +output gate (`render_thread/output_gate.rs`) asks whether anything consumes +the frame. Consumers are defined once in `output_demand.rs`: renderable +devices in the registry or devices the backend manager routes to +(simulated displays excluded), live receivers on any bus output stream, +stream and diagnostic subscribers to republished host input events, and +explicit `OutputDemandHold`s, which display output takes while it has a +target so previewed simulators count. + +With no consumer the gate parks the loop instead of rendering. It +withdraws the authoritative input capture demand, releases the screen +queue, calls `EffectRenderer::park` on every producer (which throttles +Servo webviews so the page's script thread stops scheduling its own +animation ticks; page timers keep their pace), drops live media streams +on a teardown thread, abandons +the admitted frame without recording it, calls `RenderLoop::park` so +render time stops like it does for a pause, and returns `NextWake::Park`. +The pipeline driver then sleeps on the bus output-demand signal (notified +by lane subscriptions, demand holds, and event publication, and armed only +while the loop waits), the device registry change signal, the input demand +revision, and the scene transaction queue, with a one second safety tick. +On wake it unparks the render loop, rebases the frame clock one target +interval back so effect time steps by one frame, records the wake frame +for capture freshness, and continues at the current tier. + +The gate parks only once the producers are settled: three consecutive +frame starts with no producer reporting `is_warming`, no loading media, +and no scene transition in flight, under an unchanged producer signature +(scene plan generation, zones revision, effect registry generation, zone +layout preview generation, canvas size), bounded at ten seconds, so a +crossfade longer than that pauses partway until a consumer returns. Frames +that consumers drive count toward settling, so losing the last consumer +parks at once when the producers are warm. + ### 3.5 Input sampling If `skip_decision == SkipDecision::None`, `sample_inputs` acquires the diff --git a/python/src/hypercolor/_generated/models/__init__.py b/python/src/hypercolor/_generated/models/__init__.py index d8f852ad0..0ea682853 100644 --- a/python/src/hypercolor/_generated/models/__init__.py +++ b/python/src/hypercolor/_generated/models/__init__.py @@ -418,6 +418,7 @@ from .get_software_conflicts_response_200 import GetSoftwareConflictsResponse200 from .get_system_response_200 import GetSystemResponse200 from .gpu_compositor_probe_status import GpuCompositorProbeStatus +from .gpu_error_status import GpuErrorStatus from .gradient_stop import GradientStop from .health_checks import HealthChecks from .health_response import HealthResponse @@ -561,7 +562,10 @@ from .rejected_control_change_attempted_value import RejectedControlChangeAttemptedValue from .remove_favorite_response_200 import RemoveFavoriteResponse200 from .render_acceleration_status import RenderAccelerationStatus +from .render_failure_status import RenderFailureStatus from .render_loop_status import RenderLoopStatus +from .render_output_demand_status import RenderOutputDemandStatus +from .render_supervision_status import RenderSupervisionStatus from .render_surface_status import RenderSurfaceStatus from .reorder_layers_request import ReorderLayersRequest from .reorder_live_zone_layers_response_200 import ReorderLiveZoneLayersResponse200 @@ -1047,6 +1051,7 @@ "GetSoftwareConflictsResponse200", "GetSystemResponse200", "GpuCompositorProbeStatus", + "GpuErrorStatus", "GradientStop", "HealthChecks", "HealthResponse", @@ -1182,7 +1187,10 @@ "RejectedControlChangeAttemptedValue", "RemoveFavoriteResponse200", "RenderAccelerationStatus", + "RenderFailureStatus", "RenderLoopStatus", + "RenderOutputDemandStatus", + "RenderSupervisionStatus", "RenderSurfaceStatus", "ReorderLayersRequest", "ReorderLiveZoneLayersResponse200", diff --git a/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py b/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py index 01ec30f26..487ec8b55 100644 --- a/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py +++ b/python/src/hypercolor/_generated/models/diagnose_render_snapshot.py @@ -12,6 +12,8 @@ if TYPE_CHECKING: from ..models.diagnose_latest_frame_snapshot import DiagnoseLatestFrameSnapshot from ..models.diagnose_render_window_snapshot import DiagnoseRenderWindowSnapshot + from ..models.render_output_demand_status import RenderOutputDemandStatus + from ..models.render_supervision_status import RenderSupervisionStatus T = TypeVar("T", bound="DiagnoseRenderSnapshot") @@ -23,10 +25,24 @@ class DiagnoseRenderSnapshot: Attributes: recent_window (DiagnoseRenderWindowSnapshot): latest_frame (DiagnoseLatestFrameSnapshot | None | Unset): + output_demand (RenderOutputDemandStatus | Unset): Output demand: what consumes the render loop's frames. + + The render loop renders only while something consumes its output: a + connected device, a display, a preview or canvas stream, a capture, or + an explicit demand hold. With none of those it parks, renders nothing, + and resumes at its current tier the moment a consumer appears. + supervision (RenderSupervisionStatus | Unset): Health and recovery state of the supervised render thread. + + The render thread restarts its pipeline in-process after a panic or an + error exit, within a bounded budget, and the daemon exits so its + service manager restarts it once that budget runs out or the compositor + GPU device is lost. """ recent_window: DiagnoseRenderWindowSnapshot latest_frame: DiagnoseLatestFrameSnapshot | None | Unset = UNSET + output_demand: RenderOutputDemandStatus | Unset = UNSET + supervision: RenderSupervisionStatus | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -44,6 +60,14 @@ def to_dict(self) -> dict[str, Any]: else: latest_frame = self.latest_frame + output_demand: dict[str, Any] | Unset = UNSET + if not isinstance(self.output_demand, Unset): + output_demand = self.output_demand.to_dict() + + supervision: dict[str, Any] | Unset = UNSET + if not isinstance(self.supervision, Unset): + supervision = self.supervision.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -53,6 +77,10 @@ def to_dict(self) -> dict[str, Any]: ) if latest_frame is not UNSET: field_dict["latest_frame"] = latest_frame + if output_demand is not UNSET: + field_dict["output_demand"] = output_demand + if supervision is not UNSET: + field_dict["supervision"] = supervision return field_dict @@ -64,6 +92,12 @@ def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: from ..models.diagnose_render_window_snapshot import ( DiagnoseRenderWindowSnapshot, ) + from ..models.render_output_demand_status import ( + RenderOutputDemandStatus, + ) + from ..models.render_supervision_status import ( + RenderSupervisionStatus, + ) d = dict(src_dict) recent_window = DiagnoseRenderWindowSnapshot.from_dict(d.pop("recent_window")) @@ -87,9 +121,25 @@ def _parse_latest_frame( latest_frame = _parse_latest_frame(d.pop("latest_frame", UNSET)) + _output_demand = d.pop("output_demand", UNSET) + output_demand: RenderOutputDemandStatus | Unset + if isinstance(_output_demand, Unset): + output_demand = UNSET + else: + output_demand = RenderOutputDemandStatus.from_dict(_output_demand) + + _supervision = d.pop("supervision", UNSET) + supervision: RenderSupervisionStatus | Unset + if isinstance(_supervision, Unset): + supervision = UNSET + else: + supervision = RenderSupervisionStatus.from_dict(_supervision) + diagnose_render_snapshot = cls( recent_window=recent_window, latest_frame=latest_frame, + output_demand=output_demand, + supervision=supervision, ) diagnose_render_snapshot.additional_properties = d diff --git a/python/src/hypercolor/_generated/models/gpu_error_status.py b/python/src/hypercolor/_generated/models/gpu_error_status.py new file mode 100644 index 000000000..e87572418 --- /dev/null +++ b/python/src/hypercolor/_generated/models/gpu_error_status.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field +from typing_extensions import Self + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="GpuErrorStatus") + + +@_attrs_define +class GpuErrorStatus: + """Uncaptured wgpu errors and device losses since the daemon started. + + Attributes: + device_lost (int): + internal_errors (int): + out_of_memory_errors (int): + validation_errors (int): + last_error (None | str | Unset): `device: kind: message` of the most recent one. Absent when there + was none, or when the caller may not read private diagnostics. + """ + + device_lost: int + internal_errors: int + out_of_memory_errors: int + validation_errors: int + last_error: None | str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + device_lost = self.device_lost + + internal_errors = self.internal_errors + + out_of_memory_errors = self.out_of_memory_errors + + validation_errors = self.validation_errors + + last_error: None | str | Unset + if isinstance(self.last_error, Unset): + last_error = UNSET + else: + last_error = self.last_error + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "device_lost": device_lost, + "internal_errors": internal_errors, + "out_of_memory_errors": out_of_memory_errors, + "validation_errors": validation_errors, + } + ) + if last_error is not UNSET: + field_dict["last_error"] = last_error + + return field_dict + + @classmethod + def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: + d = dict(src_dict) + device_lost = d.pop("device_lost") + + internal_errors = d.pop("internal_errors") + + out_of_memory_errors = d.pop("out_of_memory_errors") + + validation_errors = d.pop("validation_errors") + + def _parse_last_error(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + last_error = _parse_last_error(d.pop("last_error", UNSET)) + + gpu_error_status = cls( + device_lost=device_lost, + internal_errors=internal_errors, + out_of_memory_errors=out_of_memory_errors, + validation_errors=validation_errors, + last_error=last_error, + ) + + gpu_error_status.additional_properties = d + return gpu_error_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/python/src/hypercolor/_generated/models/render_failure_status.py b/python/src/hypercolor/_generated/models/render_failure_status.py new file mode 100644 index 000000000..0df346890 --- /dev/null +++ b/python/src/hypercolor/_generated/models/render_failure_status.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field +from typing_extensions import Self + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="RenderFailureStatus") + + +@_attrs_define +class RenderFailureStatus: + """One render failure, as the supervisor recorded it. + + Attributes: + action (str): `restart`, `terminate_process`, `daemon_stopping`, `process_abort`, + or `awaiting_supervisor`. + current_process (bool): Whether this daemon run recorded the failure, as opposed to an + earlier run whose record survived in the state directory. + kind (str): `panic`, `input_pump_terminated`, `input_pump_restart_failed`, + `pipeline_rebuild_failed`, or `gpu_device_lost`. + message (str): The panic message or error chain. Empty when the caller may not + read private diagnostics. + occurred_at (str): RFC 3339 UTC time of the failure. + restart_attempt (int): Restarts counted in the budget window by this failure. + location (None | str | Unset): + thread (None | str | Unset): + """ + + action: str + current_process: bool + kind: str + message: str + occurred_at: str + restart_attempt: int + location: None | str | Unset = UNSET + thread: None | str | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + action = self.action + + current_process = self.current_process + + kind = self.kind + + message = self.message + + occurred_at = self.occurred_at + + restart_attempt = self.restart_attempt + + location: None | str | Unset + if isinstance(self.location, Unset): + location = UNSET + else: + location = self.location + + thread: None | str | Unset + if isinstance(self.thread, Unset): + thread = UNSET + else: + thread = self.thread + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "action": action, + "current_process": current_process, + "kind": kind, + "message": message, + "occurred_at": occurred_at, + "restart_attempt": restart_attempt, + } + ) + if location is not UNSET: + field_dict["location"] = location + if thread is not UNSET: + field_dict["thread"] = thread + + return field_dict + + @classmethod + def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: + d = dict(src_dict) + action = d.pop("action") + + current_process = d.pop("current_process") + + kind = d.pop("kind") + + message = d.pop("message") + + occurred_at = d.pop("occurred_at") + + restart_attempt = d.pop("restart_attempt") + + def _parse_location(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + location = _parse_location(d.pop("location", UNSET)) + + def _parse_thread(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + thread = _parse_thread(d.pop("thread", UNSET)) + + render_failure_status = cls( + action=action, + current_process=current_process, + kind=kind, + message=message, + occurred_at=occurred_at, + restart_attempt=restart_attempt, + location=location, + thread=thread, + ) + + render_failure_status.additional_properties = d + return render_failure_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/python/src/hypercolor/_generated/models/render_loop_status.py b/python/src/hypercolor/_generated/models/render_loop_status.py index 00d61a418..68f7f2cfa 100644 --- a/python/src/hypercolor/_generated/models/render_loop_status.py +++ b/python/src/hypercolor/_generated/models/render_loop_status.py @@ -1,12 +1,18 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Any, TypeVar +from typing import TYPE_CHECKING, Any, TypeVar from attrs import define as _attrs_define from attrs import field as _attrs_field from typing_extensions import Self +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.render_output_demand_status import RenderOutputDemandStatus + + T = TypeVar("T", bound="RenderLoopStatus") @@ -23,6 +29,12 @@ class RenderLoopStatus: state (str): target_fps (int): total_frames (int): + output_demand (RenderOutputDemandStatus | Unset): Output demand: what consumes the render loop's frames. + + The render loop renders only while something consumes its output: a + connected device, a display, a preview or canvas stream, a capture, or + an explicit demand hold. With none of those it parks, renders nothing, + and resumes at its current tier the moment a consumer appears. """ actual_fps: float @@ -34,6 +46,7 @@ class RenderLoopStatus: state: str target_fps: int total_frames: int + output_demand: RenderOutputDemandStatus | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -55,6 +68,10 @@ def to_dict(self) -> dict[str, Any]: total_frames = self.total_frames + output_demand: dict[str, Any] | Unset = UNSET + if not isinstance(self.output_demand, Unset): + output_demand = self.output_demand.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -70,11 +87,17 @@ def to_dict(self) -> dict[str, Any]: "total_frames": total_frames, } ) + if output_demand is not UNSET: + field_dict["output_demand"] = output_demand return field_dict @classmethod def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: + from ..models.render_output_demand_status import ( + RenderOutputDemandStatus, + ) + d = dict(src_dict) actual_fps = d.pop("actual_fps") @@ -94,6 +117,13 @@ def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: total_frames = d.pop("total_frames") + _output_demand = d.pop("output_demand", UNSET) + output_demand: RenderOutputDemandStatus | Unset + if isinstance(_output_demand, Unset): + output_demand = UNSET + else: + output_demand = RenderOutputDemandStatus.from_dict(_output_demand) + render_loop_status = cls( actual_fps=actual_fps, capacity_fps=capacity_fps, @@ -104,6 +134,7 @@ def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: state=state, target_fps=target_fps, total_frames=total_frames, + output_demand=output_demand, ) render_loop_status.additional_properties = d diff --git a/python/src/hypercolor/_generated/models/render_output_demand_status.py b/python/src/hypercolor/_generated/models/render_output_demand_status.py new file mode 100644 index 000000000..bccff0573 --- /dev/null +++ b/python/src/hypercolor/_generated/models/render_output_demand_status.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field +from typing_extensions import Self + +from ..types import UNSET, Unset + +T = TypeVar("T", bound="RenderOutputDemandStatus") + + +@_attrs_define +class RenderOutputDemandStatus: + """Output demand: what consumes the render loop's frames. + + The render loop renders only while something consumes its output: a + connected device, a display, a preview or canvas stream, a capture, or + an explicit demand hold. With none of those it parks, renders nothing, + and resumes at its current tier the moment a consumer appears. + + Attributes: + demand_holds (int): Explicit demand holds, such as display output for previewed + simulated displays. + input_event_streams (int): Stream and diagnostic consumers of host input events, which the + render loop republishes. + output_devices (int): Connected devices that receive frames. Simulated displays count only + while previewed, through `demand_holds`. + parked (bool): Whether the render loop is parked because nothing consumes its frames. + parks_total (int): How many times the loop has parked since the daemon started. + stream_receivers (int): Live receivers on the frame, canvas, preview, spectrum, and per-zone + canvas streams, including in-flight canvas captures. + idle_reason (None | str | Unset): Why the render loop is producing no frames, when it is parked. + `no_output_consumers` while parked. + last_wake_reason (None | str | Unset): What last woke the loop from a park: `output_consumer`, + `scene_transaction`, `producer_state`, or `render_loop_state`. + parked_ms (int | None | Unset): How long the loop has been parked, in milliseconds. + """ + + demand_holds: int + input_event_streams: int + output_devices: int + parked: bool + parks_total: int + stream_receivers: int + idle_reason: None | str | Unset = UNSET + last_wake_reason: None | str | Unset = UNSET + parked_ms: int | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + demand_holds = self.demand_holds + + input_event_streams = self.input_event_streams + + output_devices = self.output_devices + + parked = self.parked + + parks_total = self.parks_total + + stream_receivers = self.stream_receivers + + idle_reason: None | str | Unset + if isinstance(self.idle_reason, Unset): + idle_reason = UNSET + else: + idle_reason = self.idle_reason + + last_wake_reason: None | str | Unset + if isinstance(self.last_wake_reason, Unset): + last_wake_reason = UNSET + else: + last_wake_reason = self.last_wake_reason + + parked_ms: int | None | Unset + if isinstance(self.parked_ms, Unset): + parked_ms = UNSET + else: + parked_ms = self.parked_ms + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "demand_holds": demand_holds, + "input_event_streams": input_event_streams, + "output_devices": output_devices, + "parked": parked, + "parks_total": parks_total, + "stream_receivers": stream_receivers, + } + ) + if idle_reason is not UNSET: + field_dict["idle_reason"] = idle_reason + if last_wake_reason is not UNSET: + field_dict["last_wake_reason"] = last_wake_reason + if parked_ms is not UNSET: + field_dict["parked_ms"] = parked_ms + + return field_dict + + @classmethod + def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: + d = dict(src_dict) + demand_holds = d.pop("demand_holds") + + input_event_streams = d.pop("input_event_streams") + + output_devices = d.pop("output_devices") + + parked = d.pop("parked") + + parks_total = d.pop("parks_total") + + stream_receivers = d.pop("stream_receivers") + + def _parse_idle_reason(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + idle_reason = _parse_idle_reason(d.pop("idle_reason", UNSET)) + + def _parse_last_wake_reason(data: object) -> None | str | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(None | str | Unset, data) + + last_wake_reason = _parse_last_wake_reason(d.pop("last_wake_reason", UNSET)) + + def _parse_parked_ms(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + parked_ms = _parse_parked_ms(d.pop("parked_ms", UNSET)) + + render_output_demand_status = cls( + demand_holds=demand_holds, + input_event_streams=input_event_streams, + output_devices=output_devices, + parked=parked, + parks_total=parks_total, + stream_receivers=stream_receivers, + idle_reason=idle_reason, + last_wake_reason=last_wake_reason, + parked_ms=parked_ms, + ) + + render_output_demand_status.additional_properties = d + return render_output_demand_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/python/src/hypercolor/_generated/models/render_supervision_status.py b/python/src/hypercolor/_generated/models/render_supervision_status.py new file mode 100644 index 000000000..ae87e5b8e --- /dev/null +++ b/python/src/hypercolor/_generated/models/render_supervision_status.py @@ -0,0 +1,171 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any, TypeVar, cast + +from attrs import define as _attrs_define +from attrs import field as _attrs_field +from typing_extensions import Self + +from ..types import UNSET, Unset + +if TYPE_CHECKING: + from ..models.gpu_error_status import GpuErrorStatus + from ..models.render_failure_status import RenderFailureStatus + + +T = TypeVar("T", bound="RenderSupervisionStatus") + + +@_attrs_define +class RenderSupervisionStatus: + """Health and recovery state of the supervised render thread. + + The render thread restarts its pipeline in-process after a panic or an + error exit, within a bounded budget, and the daemon exits so its + service manager restarts it once that budget runs out or the compositor + GPU device is lost. + + Attributes: + consecutive_failures (int): Failures since the pipeline last ran past its stable uptime. + max_restarts (int): Restarts the budget allows within its window. + phase (str): `not_started`, `starting`, `running`, `restarting`, `terminating`, + or `stopped`. + restarts_total (int): In-process pipeline restarts since the daemon started. + gpu_errors (GpuErrorStatus | Unset): Uncaptured wgpu errors and device losses since the daemon started. + last_failure (None | RenderFailureStatus | Unset): + last_frame_age_ms (int | None | Unset): Milliseconds since the render loop last completed a frame. + """ + + consecutive_failures: int + max_restarts: int + phase: str + restarts_total: int + gpu_errors: GpuErrorStatus | Unset = UNSET + last_failure: None | RenderFailureStatus | Unset = UNSET + last_frame_age_ms: int | None | Unset = UNSET + additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) + + def to_dict(self) -> dict[str, Any]: + from ..models.render_failure_status import RenderFailureStatus + + consecutive_failures = self.consecutive_failures + + max_restarts = self.max_restarts + + phase = self.phase + + restarts_total = self.restarts_total + + gpu_errors: dict[str, Any] | Unset = UNSET + if not isinstance(self.gpu_errors, Unset): + gpu_errors = self.gpu_errors.to_dict() + + last_failure: dict[str, Any] | None | Unset + if isinstance(self.last_failure, Unset): + last_failure = UNSET + elif isinstance(self.last_failure, RenderFailureStatus): + last_failure = self.last_failure.to_dict() + else: + last_failure = self.last_failure + + last_frame_age_ms: int | None | Unset + if isinstance(self.last_frame_age_ms, Unset): + last_frame_age_ms = UNSET + else: + last_frame_age_ms = self.last_frame_age_ms + + field_dict: dict[str, Any] = {} + field_dict.update(self.additional_properties) + field_dict.update( + { + "consecutive_failures": consecutive_failures, + "max_restarts": max_restarts, + "phase": phase, + "restarts_total": restarts_total, + } + ) + if gpu_errors is not UNSET: + field_dict["gpu_errors"] = gpu_errors + if last_failure is not UNSET: + field_dict["last_failure"] = last_failure + if last_frame_age_ms is not UNSET: + field_dict["last_frame_age_ms"] = last_frame_age_ms + + return field_dict + + @classmethod + def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: + from ..models.gpu_error_status import GpuErrorStatus + from ..models.render_failure_status import RenderFailureStatus + + d = dict(src_dict) + consecutive_failures = d.pop("consecutive_failures") + + max_restarts = d.pop("max_restarts") + + phase = d.pop("phase") + + restarts_total = d.pop("restarts_total") + + _gpu_errors = d.pop("gpu_errors", UNSET) + gpu_errors: GpuErrorStatus | Unset + if isinstance(_gpu_errors, Unset): + gpu_errors = UNSET + else: + gpu_errors = GpuErrorStatus.from_dict(_gpu_errors) + + def _parse_last_failure(data: object) -> None | RenderFailureStatus | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + try: + if not isinstance(data, dict): + raise TypeError() + last_failure_type_0 = RenderFailureStatus.from_dict(data) + + return last_failure_type_0 + except (TypeError, ValueError, AttributeError, KeyError): + pass + return cast(None | RenderFailureStatus | Unset, data) + + last_failure = _parse_last_failure(d.pop("last_failure", UNSET)) + + def _parse_last_frame_age_ms(data: object) -> int | None | Unset: + if data is None: + return data + if isinstance(data, Unset): + return data + return cast(int | None | Unset, data) + + last_frame_age_ms = _parse_last_frame_age_ms(d.pop("last_frame_age_ms", UNSET)) + + render_supervision_status = cls( + consecutive_failures=consecutive_failures, + max_restarts=max_restarts, + phase=phase, + restarts_total=restarts_total, + gpu_errors=gpu_errors, + last_failure=last_failure, + last_frame_age_ms=last_frame_age_ms, + ) + + render_supervision_status.additional_properties = d + return render_supervision_status + + @property + def additional_keys(self) -> list[str]: + return list(self.additional_properties.keys()) + + def __getitem__(self, key: str) -> Any: + return self.additional_properties[key] + + def __setitem__(self, key: str, value: Any) -> None: + self.additional_properties[key] = value + + def __delitem__(self, key: str) -> None: + del self.additional_properties[key] + + def __contains__(self, key: str) -> bool: + return key in self.additional_properties diff --git a/python/src/hypercolor/_generated/models/system_status.py b/python/src/hypercolor/_generated/models/system_status.py index cd869fd28..30445dceb 100644 --- a/python/src/hypercolor/_generated/models/system_status.py +++ b/python/src/hypercolor/_generated/models/system_status.py @@ -17,6 +17,7 @@ from ..models.preview_runtime_status import PreviewRuntimeStatus from ..models.render_acceleration_status import RenderAccelerationStatus from ..models.render_loop_status import RenderLoopStatus + from ..models.render_supervision_status import RenderSupervisionStatus from ..models.screen_capture_capacity_status import ScreenCaptureCapacityStatus from ..models.session_performance_status import SessionPerformanceStatus @@ -65,6 +66,12 @@ class SystemStatus: active_scene (None | str | Unset): latest_frame (LatestFrameStatus | None | Unset): macos_daemon_ownership (MacosDaemonOwnershipStatus | None | Unset): + render_supervision (RenderSupervisionStatus | Unset): Health and recovery state of the supervised render thread. + + The render thread restarts its pipeline in-process after a panic or an + error exit, within a bounded budget, and the daemon exits so its + service manager restarts it once that budget runs out or the compositor + GPU device is lost. """ active_scene_snapshot_locked: bool @@ -93,6 +100,7 @@ class SystemStatus: active_scene: None | str | Unset = UNSET latest_frame: LatestFrameStatus | None | Unset = UNSET macos_daemon_ownership: MacosDaemonOwnershipStatus | None | Unset = UNSET + render_supervision: RenderSupervisionStatus | Unset = UNSET additional_properties: dict[str, Any] = _attrs_field(init=False, factory=dict) def to_dict(self) -> dict[str, Any]: @@ -171,6 +179,10 @@ def to_dict(self) -> dict[str, Any]: else: macos_daemon_ownership = self.macos_daemon_ownership + render_supervision: dict[str, Any] | Unset = UNSET + if not isinstance(self.render_supervision, Unset): + render_supervision = self.render_supervision.to_dict() + field_dict: dict[str, Any] = {} field_dict.update(self.additional_properties) field_dict.update( @@ -207,6 +219,8 @@ def to_dict(self) -> dict[str, Any]: field_dict["latest_frame"] = latest_frame if macos_daemon_ownership is not UNSET: field_dict["macos_daemon_ownership"] = macos_daemon_ownership + if render_supervision is not UNSET: + field_dict["render_supervision"] = render_supervision return field_dict @@ -225,6 +239,9 @@ def from_dict(cls, src_dict: Mapping[str, Any]) -> Self: RenderAccelerationStatus, ) from ..models.render_loop_status import RenderLoopStatus + from ..models.render_supervision_status import ( + RenderSupervisionStatus, + ) from ..models.screen_capture_capacity_status import ( ScreenCaptureCapacityStatus, ) @@ -341,6 +358,13 @@ def _parse_macos_daemon_ownership( d.pop("macos_daemon_ownership", UNSET) ) + _render_supervision = d.pop("render_supervision", UNSET) + render_supervision: RenderSupervisionStatus | Unset + if isinstance(_render_supervision, Unset): + render_supervision = UNSET + else: + render_supervision = RenderSupervisionStatus.from_dict(_render_supervision) + system_status = cls( active_scene_snapshot_locked=active_scene_snapshot_locked, audio_available=audio_available, @@ -368,6 +392,7 @@ def _parse_macos_daemon_ownership( active_scene=active_scene, latest_frame=latest_frame, macos_daemon_ownership=macos_daemon_ownership, + render_supervision=render_supervision, ) system_status.additional_properties = d diff --git a/scripts/install-windows-service.ps1 b/scripts/install-windows-service.ps1 index d266ed7ff..8de20b6fd 100644 --- a/scripts/install-windows-service.ps1 +++ b/scripts/install-windows-service.ps1 @@ -359,6 +359,10 @@ if ($serviceEnvironment.Count -gt 0) { } sc.exe failure $ServiceName reset= 86400 actions= restart/5000/restart/15000/""/60000 | Out-Null +# Without the failure flag the service manager runs those actions only when +# the process crashes. The daemon stops itself with a nonzero exit code when +# its render thread cannot recover, and that stop must restart it too. +sc.exe failureflag $ServiceName 1 | Out-Null Write-Host "Installed $ServiceName" Write-Host " Source: $daemonSourcePath"