fix(studio): survive render panics without freezing the preview

Investigation of an app freeze + OOM while playing dynamic-glass with
the worker pool. Isolated soaks exonerate the native pipeline: 6
workers at 0.5/1.0 scale hold a flat RSS (124/368 MB over 360 heavy
frames), and a 60s full-pipeline soak (moving 30fps playhead, simulated
asset handler, canvas hit pass) stays bounded at 122-482 MB with 3
cache misses in 1200 served frames.

The identified failure mode is a panic cascade: the global cosmic-text
mutexes were locked with .unwrap(), so one Skia panic on any render
thread (caught by the panic fences) poisoned FONT_SYSTEM/SWASH_CACHE
and turned every subsequent text shape on every thread into a panic —
preview permanently stuck, six workers re-rendering panicking frames in
a tight loop.

- cosmic: poison-tolerant font/swash locks + regression test that
  deliberately poisons the mutex and shapes through it
- studio: FailLedger — a frame whose render panics gets
  MAX_RENDER_ATTEMPTS tries (workers and asset handler), then is left
  alone instead of looping
- keep the ignored stress/soak diagnostics for future perf work

Author: LeadcodeDev

crates/rustmotion-core/src/engine/text/cosmic.rs

@@ -96,7 +96,11 @@ pub fn measure_text(text: &str, style: &TextStyle) -> TextMetrics {
             height: metrics_for(style).line_height,
         };
     }
-    let mut fs = font_system().lock().unwrap();
+    // Poison-tolerant: a panic on another render thread (e.g. a Skia panic
+    // caught by a preview worker's panic fence) must not poison text shaping
+    // for every subsequent frame — the FontSystem stays usable, each shape
+    // call builds its own Buffer.
+    let mut fs = font_system().lock().unwrap_or_else(|e| e.into_inner());
     let metrics = metrics_for(style);
     let mut buf = Buffer::new(&mut fs, metrics);
     buf.set_size(style.max_width, None);
@@ -128,15 +132,16 @@ pub fn paint_text(
     if text.is_empty() {
         return;
     }
-    let mut fs = font_system().lock().unwrap();
+    // Poison-tolerant for the same reason as in `measure_text`.
+    let mut fs = font_system().lock().unwrap_or_else(|e| e.into_inner());
     let metrics = metrics_for(style);
     let mut buf = Buffer::new(&mut fs, metrics);
     buf.set_size(style.max_width, None);
     buf.set_wrap(if style.wrap { Wrap::Word } else { Wrap::None });
     buf.set_text(text, &attrs_for(style), Shaping::Advanced, None);
     buf.shape_until_scroll(&mut fs, false);
 
-    let mut sc = swash_cache().lock().unwrap();
+    let mut sc = swash_cache().lock().unwrap_or_else(|e| e.into_inner());
     let ccolor = CColor::rgba(color.r(), color.g(), color.b(), color.a());
 
     for run in buf.layout_runs() {
@@ -264,4 +269,20 @@ mod tests {
         let m = measure_text("the quick brown fox", &style);
         assert!(m.height < 16.0 * 1.2 * 2.0, "should be one line");
     }
+
+    /// A panic on another thread while it holds the font mutex (e.g. a Skia
+    /// panic caught by a preview worker's panic fence) poisons the lock.
+    /// Shaping must survive that — otherwise one panic turns every subsequent
+    /// text render on every thread into a panic cascade.
+    #[test]
+    fn measure_survives_poisoned_font_mutex() {
+        let _ = std::thread::spawn(|| {
+            let _guard = font_system().lock().unwrap_or_else(|e| e.into_inner());
+            panic!("deliberate poison");
+        })
+        .join();
+        assert!(font_system().is_poisoned(), "setup must have poisoned");
+        let m = measure_text("still shaping", &TextStyle::default());
+        assert!(m.width > 0.0, "measure works despite the poisoned mutex");
+    }
 }

crates/rustmotion-studio/src/editor/frames.rs

@@ -185,6 +185,55 @@ mod tests {
         assert_eq!(&jpeg[0..2], &[0xFF, 0xD8], "must be a JPEG");
     }
 
+    /// TEMP diagnostic (not part of CI): replicate the prefetch worker pool on
+    /// the dynamic-glass example and print RSS growth. Run with
+    /// `RM_STRESS_SCALE=0.5 cargo test -p rustmotion-studio --release stress_rss -- --ignored --nocapture`
+    #[test]
+    #[ignore]
+    fn stress_rss_worker_pool() {
+        use std::sync::Arc;
+        let Ok(src) = std::fs::read_to_string("../../examples/dynamic-glass.json") else {
+            eprintln!("skipped: examples/dynamic-glass.json not present");
+            return;
+        };
+        let scenario =
+            Arc::new(rustmotion::loader::load_scenario_from_source(None, Some(&src)).unwrap());
+        let tasks = Arc::new(rustmotion::encode::build_frame_tasks(&scenario));
+        let scale: f32 = std::env::var("RM_STRESS_SCALE")
+            .ok()
+            .and_then(|s| s.parse().ok())
+            .unwrap_or(0.5);
+        let rss_kb = || -> u64 {
+            let out = std::process::Command::new("ps")
+                .args(["-o", "rss=", "-p", &std::process::id().to_string()])
+                .output()
+                .unwrap();
+            String::from_utf8_lossy(&out.stdout).trim().parse().unwrap()
+        };
+        println!("scale={scale} start rss={} MB", rss_kb() / 1024);
+        let workers: Vec<_> = (0..6u32)
+            .map(|w| {
+                let s = scenario.clone();
+                let t = tasks.clone();
+                std::thread::spawn(move || {
+                    for _pass in 0..6 {
+                        for f in (130..190).filter(|f| f % 6 == w) {
+                            let _ = render_frame(&s, &t, f, scale);
+                        }
+                    }
+                })
+            })
+            .collect();
+        while workers.iter().any(|w| !w.is_finished()) {
+            std::thread::sleep(std::time::Duration::from_secs(2));
+            println!("rss={} MB", rss_kb() / 1024);
+        }
+        for w in workers {
+            w.join().unwrap();
+        }
+        println!("end rss={} MB", rss_kb() / 1024);
+    }
+
     #[test]
     fn renders_frame_at_reduced_scale() {
         let scenario = rustmotion::loader::load_scenario_from_source(None, Some(SCENARIO)).unwrap();

crates/rustmotion-studio/src/editor/prefetch.rs

@@ -206,6 +206,39 @@ fn claims() -> &'static Mutex<ClaimSet> {
     SLOT.get_or_init(|| Mutex::new(ClaimSet::default()))
 }
 
+// ── Failure ledger ───────────────────────────────────────────────────────────
+
+/// Attempts a frame gets before workers and the handler stop re-rendering it.
+/// One retry absorbs transient panics; beyond that a panicking frame must not
+/// become an infinite render-panic loop that pegs the workers (the preview
+/// shows the neighboring frame instead).
+pub const MAX_RENDER_ATTEMPTS: u8 = 2;
+
+/// Panicked render attempts per key. Keys carry the generation, so a model
+/// edit naturally retires old entries; the size cap is a runaway backstop.
+#[derive(Default)]
+pub struct FailLedger {
+    map: HashMap<FrameKey, u8>,
+}
+
+impl FailLedger {
+    pub fn record_failure(&mut self, key: FrameKey) {
+        if self.map.len() > 4096 {
+            self.map.clear();
+        }
+        *self.map.entry(key).or_insert(0) += 1;
+    }
+
+    pub fn exhausted(&self, key: &FrameKey) -> bool {
+        self.map.get(key).is_some_and(|&n| n >= MAX_RENDER_ATTEMPTS)
+    }
+}
+
+pub(crate) fn fail_ledger() -> &'static Mutex<FailLedger> {
+    static SLOT: OnceLock<Mutex<FailLedger>> = OnceLock::new();
+    SLOT.get_or_init(|| Mutex::new(FailLedger::default()))
+}
+
 /// Prefetch worker count for a machine with `cores` logical cores: leave two
 /// for the UI/webview and the asset handler, keep at least two workers so a
 /// slow frame never serializes the window, and cap at six (JPEG frames past
@@ -324,6 +357,15 @@ fn prefetch_loop() {
             {
                 continue;
             }
+            // A frame that keeps panicking gets MAX_RENDER_ATTEMPTS, then is
+            // left alone — never an infinite render-panic loop.
+            if fail_ledger()
+                .lock()
+                .unwrap_or_else(|e| e.into_inner())
+                .exhausted(&key)
+            {
+                continue;
+            }
             // Another worker already rendering this frame → take the next one.
             if !claims()
                 .lock()
@@ -344,11 +386,15 @@ fn prefetch_loop() {
                 let rendered = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
                     render_frame(&scenario, &tasks, frame, scale_factor(scale))
                 }));
-                if let Ok(bytes) = rendered {
-                    frame_cache()
+                match rendered {
+                    Ok(bytes) => frame_cache()
+                        .lock()
+                        .unwrap_or_else(|e| e.into_inner())
+                        .insert(key, bytes, gen_b, gen_a, target.current, scale),
+                    Err(_) => fail_ledger()
                         .lock()
                         .unwrap_or_else(|e| e.into_inner())
-                        .insert(key, bytes, gen_b, gen_a, target.current, scale);
+                        .record_failure(key),
                 }
             }
             claims()
@@ -623,6 +669,122 @@ mod tests {
         assert!(claims.try_claim(other_scale), "other scale");
     }
 
+    /// TEMP diagnostic (not CI): full-pipeline soak — real worker pool, a
+    /// simulated 30fps playhead publishing targets, a simulated asset handler
+    /// serving frames, and the canvas hit-map pass. Prints RSS + serve stats.
+    /// `cargo test -p rustmotion-studio --release soak_full -- --ignored --nocapture`
+    #[test]
+    #[ignore]
+    fn soak_full_pipeline_rss() {
+        use crate::editor::frames::frame_hits;
+        use std::sync::atomic::{AtomicU32, Ordering};
+
+        let Ok(src) = std::fs::read_to_string("../../examples/dynamic-glass.json") else {
+            eprintln!("skipped: examples/dynamic-glass.json not present");
+            return;
+        };
+        let scenario =
+            Arc::new(rustmotion::loader::load_scenario_from_source(None, Some(&src)).unwrap());
+        let tasks = Arc::new(rustmotion::encode::build_frame_tasks(&scenario));
+        let total = tasks.len() as u32;
+        let rss_mb = || -> u64 {
+            let out = std::process::Command::new("ps")
+                .args(["-o", "rss=", "-p", &std::process::id().to_string()])
+                .output()
+                .unwrap();
+            String::from_utf8_lossy(&out.stdout)
+                .trim()
+                .parse::<u64>()
+                .unwrap()
+                / 1024
+        };
+
+        ensure_prefetcher();
+        static SERVED: AtomicU32 = AtomicU32::new(0);
+        static MISSES: AtomicU32 = AtomicU32::new(0);
+
+        // Simulated playback + asset handler + canvas hit pass, 30fps for 60s.
+        let s2 = scenario.clone();
+        let t2 = tasks.clone();
+        let sim = std::thread::spawn(move || {
+            let mut current = 0u32;
+            for _tick in 0..(30 * 60) {
+                std::thread::sleep(Duration::from_millis(33));
+                current = (current + 1) % total;
+                *prefetch_slot().lock().unwrap_or_else(|e| e.into_inner()) = PrefetchTarget {
+                    current,
+                    playing: true,
+                    generation: 1,
+                    side: DiffSide::B,
+                    scenario: Some(s2.clone()),
+                    tasks: Some(t2.clone()),
+                    path: None,
+                };
+                let key = FrameKey {
+                    generation: 1,
+                    side: DiffSide::B,
+                    frame: current,
+                    scale_pct: preview_scale_pct(),
+                };
+                let hit = frame_cache()
+                    .lock()
+                    .unwrap_or_else(|e| e.into_inner())
+                    .contains(&key);
+                if !hit {
+                    MISSES.fetch_add(1, Ordering::Relaxed);
+                    let bytes = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                        crate::editor::frames::render_frame(
+                            &s2,
+                            &t2,
+                            current,
+                            scale_factor(key.scale_pct),
+                        )
+                    }));
+                    if let Ok(b) = bytes {
+                        frame_cache()
+                            .lock()
+                            .unwrap_or_else(|e| e.into_inner())
+                            .insert(key, b, Some(1), None, current, key.scale_pct);
+                    }
+                }
+                SERVED.fetch_add(1, Ordering::Relaxed);
+                // The canvas hit-map layout pass runs on every tick in the app.
+                let _ = frame_hits(&s2, &t2, current, "/scenes/0");
+            }
+        });
+
+        while !sim.is_finished() {
+            std::thread::sleep(Duration::from_secs(5));
+            println!(
+                "rss={} MB served={} misses={}",
+                rss_mb(),
+                SERVED.load(Ordering::Relaxed),
+                MISSES.load(Ordering::Relaxed)
+            );
+        }
+        sim.join().unwrap();
+        println!(
+            "end rss={} MB misses={}",
+            rss_mb(),
+            MISSES.load(Ordering::Relaxed)
+        );
+    }
+
+    #[test]
+    fn fail_ledger_exhausts_after_max_attempts() {
+        let mut ledger = FailLedger::default();
+        let k = key(1, DiffSide::B, 7);
+        assert!(!ledger.exhausted(&k));
+        ledger.record_failure(k);
+        assert!(!ledger.exhausted(&k), "one transient failure gets a retry");
+        ledger.record_failure(k);
+        assert!(ledger.exhausted(&k), "gives up after MAX_RENDER_ATTEMPTS");
+        assert!(
+            !ledger.exhausted(&key(2, DiffSide::B, 7)),
+            "a new generation retries the same frame"
+        );
+    }
+
     #[test]
     fn worker_count_leaves_ui_cores_and_stays_bounded() {
         assert_eq!(worker_count(1), 2, "floor: two workers even on tiny CPUs");

crates/rustmotion-studio/src/editor/view.rs

@@ -17,7 +17,7 @@ use super::playback::{
     playback_action, use_hot_reload, use_playback_clock, PlaybackAction, PlaybackBar,
 };
 use super::prefetch::{
-    frame_cache, preview_scale_pct, scale_factor, use_prefetch_publisher, FrameKey,
+    fail_ledger, frame_cache, preview_scale_pct, scale_factor, use_prefetch_publisher, FrameKey,
 };
 use super::topbar::TopBar;
 
@@ -371,10 +371,25 @@ fn serve_or_render(
     {
         return Ok((*bytes).clone());
     }
+    // A frame whose render keeps panicking is not retried on every <img>
+    // request — fail fast so the webview shows the previous frame instead of
+    // hammering a panic loop.
+    if fail_ledger()
+        .lock()
+        .unwrap_or_else(|e| e.into_inner())
+        .exhausted(&key)
+    {
+        return Err(());
+    }
     let rendered = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
         render_frame(scenario, tasks, idx, scale_factor(key.scale_pct))
     }))
-    .map_err(|_| ())?;
+    .map_err(|_| {
+        fail_ledger()
+            .lock()
+            .unwrap_or_else(|e| e.into_inner())
+            .record_failure(key);
+    })?;
     let (gen_b, gen_a) = match key.side {
         DiffSide::B => (gen_b, None),
         DiffSide::A => (None, Some(key.generation)),