fix(vision mixer): keep a source's new shape when it changes size

backend/src/blocks/builtin/vision_mixer/builder/pipeline_gpu.rs

@@ -394,7 +394,9 @@ pub(super) fn build_gpu_pipeline(
             // pixelate, ...). Sits before the tee so the look follows the
             // source everywhere: PGM, PVW, thumbnails and PiPs.
             let fx_look_id = p.id(&format!("fx_look_{}", i));
-            elems.push((fx_look_id.clone(), elements::make_glshader(&fx_look_id)?));
+            let fx_look = elements::make_glshader(&fx_look_id)?;
+            elements::keep_input_size(&fx_look);
+            elems.push((fx_look_id.clone(), fx_look));
             links.push((
                 ElementPadRef::pad(&cc_id, "src"),
                 ElementPadRef::pad(&fx_look_id, "sink"),
@@ -425,7 +427,9 @@ pub(super) fn build_gpu_pipeline(
             // shader transitions. Dist branch only — multiview thumbnails
             // and PVW stay clean during a wipe.
             let fx_take_id = p.id(&format!("fx_take_{}", i));
-            elems.push((fx_take_id.clone(), elements::make_glshader(&fx_take_id)?));
+            let fx_take = elements::make_glshader(&fx_take_id)?;
+            elements::keep_input_size(&fx_take);
+            elems.push((fx_take_id.clone(), fx_take));
             links.push((
                 ElementPadRef::pad(&q_dist_id, "src"),
                 ElementPadRef::pad(&fx_take_id, "sink"),

backend/src/blocks/builtin/vision_mixer/elements.rs

@@ -184,6 +184,56 @@ pub fn make_glshader(name: &str) -> Result<gst::Element, BlockBuildError> {
     Ok(elem)
 }
 
+/// Keep a per-input GL filter at the frame size it is given.
+///
+/// `glshader` can scale, and takes its output size from the first entry of
+/// downstream's caps. A running compositor answers with the size it already
+/// has, so when a source changes size the filter stretches the new frames to
+/// the old size: a camera switched to portrait, or a new publisher on a
+/// reused WHIP slot, keeps the first source's shape, and the mixer never sees
+/// a caps change to aspect-fit. Opening width and height in that answer keeps
+/// the filter at its input size; the compositor pads take any size.
+///
+/// The filter asks its peer pad directly (`gst_pad_query_caps` on the peer),
+/// so the answer can only be rewritten on the peer, which is not known until
+/// the filter's src pad is linked.
+pub fn keep_input_size(filter: &gst::Element) {
+    let Some(src) = filter.static_pad("src") else {
+        return;
+    };
+    src.connect_linked(|_src, peer| {
+        open_caps_answer_size(peer);
+    });
+}
+
+fn open_caps_answer_size(pad: &gst::Pad) {
+    // PULL: runs once the pad has answered a caps query. Never per buffer.
+    pad.add_probe(
+        gst::PadProbeType::QUERY_DOWNSTREAM | gst::PadProbeType::PULL,
+        |_pad, info| {
+            let Some(query) = info.query_mut() else {
+                return gst::PadProbeReturn::Ok;
+            };
+            let gst::QueryViewMut::Caps(q) = query.view_mut() else {
+                return gst::PadProbeReturn::Ok;
+            };
+            let Some(mut result) = q.result_owned() else {
+                return gst::PadProbeReturn::Ok;
+            };
+            for s in result.make_mut().iter_mut() {
+                s.set("width", gst::IntRange::new(1, i32::MAX));
+                s.set("height", gst::IntRange::new(1, i32::MAX));
+                s.remove_field("pixel-aspect-ratio");
+            }
+            if let Some(filter) = q.filter() {
+                result = filter.intersect_with_mode(&result, gst::CapsIntersectMode::First);
+            }
+            q.set_result(&result);
+            gst::PadProbeReturn::Ok
+        },
+    );
+}
+
 /// Create a simple GStreamer element by factory name.
 pub fn make_element(factory: &str, name: &str) -> Result<gst::Element, BlockBuildError> {
     gst::ElementFactory::make(factory)

backend/src/blocks/builtin/vision_mixer/geometry.rs

@@ -14,6 +14,10 @@
 
 use gstreamer as gst;
 use gstreamer::prelude::*;
+use gstreamer_controller::prelude::*;
+use gstreamer_controller::InterpolationControlSource;
+use std::sync::atomic::Ordering;
+use std::time::Duration;
 use strom_types::vision_mixer::{self, aspect_fit_rect, SourceAspects};
 use tracing::debug;
 
@@ -23,12 +27,12 @@ use crate::gst::pipeline::effects::{
 };
 use crate::gst::underlay::UnderlayCtx;
 
-/// Install CAPS event probes on every input video sink pad of the dist and
-/// multiview compositors. Each caps arrival/change triggers a full geometry
-/// refresh from the current overlay state.
+/// Watch the caps of every input video sink pad of the dist and multiview
+/// compositors. Each caps arrival/change triggers a full geometry refresh
+/// from the current overlay state.
 ///
 /// Must run after linking (request pads exist). Elements are captured as
-/// `WeakRef`s in the probe closures — pads own their probes, and a strong
+/// `WeakRef`s in the handlers — pads own their handlers, and a strong
 /// element reference would create a cycle that leaks the pipeline.
 pub fn install_caps_probes(
     block_id: &str,
@@ -55,37 +59,34 @@ pub fn install_caps_probes(
         let block_id = block_id.to_string();
         let mixer_weak = mixer.downgrade();
         let mv_weak = mv_comp.downgrade();
-        pad.add_probe(gst::PadProbeType::EVENT_DOWNSTREAM, move |_pad, info| {
-            if let Some(gst::PadProbeData::Event(ev)) = &info.data {
-                if ev.type_() == gst::EventType::Caps {
-                    // Defer the refresh to the glib main loop instead of
-                    // running it inline: sticky caps are stored on the pad
-                    // only AFTER the probes return, and sibling branches
-                    // negotiate concurrently on their own streaming threads —
-                    // an inline refresh races both, and two probes of the
-                    // same input firing simultaneously can each miss the
-                    // other's caps, leaving that input's thumbnail stretched
-                    // forever. Serialized on the main context, the last
-                    // refresh always sees every negotiated pad.
-                    let block_id = block_id.clone();
-                    let mixer_weak = mixer_weak.clone();
-                    let mv_weak = mv_weak.clone();
-                    gst::glib::idle_add_once(move || {
-                        // Pipeline teardown in progress → nothing to refresh.
-                        let (Some(mixer), Some(mv_comp)) =
-                            (mixer_weak.upgrade(), mv_weak.upgrade())
-                        else {
-                            return;
-                        };
-                        refresh_geometry(&block_id, &mixer, &mv_comp);
-                    });
-                }
+        // `notify::caps` fires once the pad has stored the new caps; a caps
+        // EVENT probe fires before, so a refresh queued from one can read the
+        // old size and leave the rect in the old shape.
+        pad.connect_notify(Some("caps"), move |pad, _| {
+            // Caps are cleared on deactivation; nothing to fit.
+            if pad.current_caps().is_none() {
+                return;
             }
-            gst::PadProbeReturn::Ok
+            // Defer the refresh to the glib main loop instead of running it
+            // inline: sibling branches negotiate concurrently on their own
+            // streaming threads, and two notifies of the same input firing
+            // simultaneously can each miss the other's caps, leaving that
+            // input's thumbnail stretched forever. Serialized on the main
+            // context, the last refresh always sees every negotiated pad.
+            let block_id = block_id.clone();
+            let mixer_weak = mixer_weak.clone();
+            let mv_weak = mv_weak.clone();
+            gst::glib::idle_add_once(move || {
+                // Pipeline teardown in progress → nothing to refresh.
+                let (Some(mixer), Some(mv_comp)) = (mixer_weak.upgrade(), mv_weak.upgrade()) else {
+                    return;
+                };
+                refresh_geometry(&block_id, &mixer, &mv_comp);
+            });
         });
     }
     debug!(
-        "Vision mixer {}: installed {} caps probes for reactive geometry",
+        "Vision mixer {}: watching caps on {} pads for reactive geometry",
         block_id, installed
     );
 }
@@ -122,23 +123,39 @@ fn refresh_geometry(block_id: &str, mixer: &gst::Element, mv_comp: &gst::Element
     // refreshing on each of those events would clear control bindings and
     // kill the very animation producing them. Source dims are read upstream
     // of videocrop, so crop-induced caps events never count as changes.
-    if !state.update_input_dims(&dims) {
+    let dims_changed = state.update_input_dims(&dims);
+    if !dims_changed && !state.layout_refit_pending.load(Ordering::Relaxed) {
         return;
     }
 
+    // A take, FTB or DSK fade animates pad properties through control
+    // bindings, and re-applying a layout wipes them: the animation would
+    // end as a cut. Leave an animating compositor alone and re-fit it once
+    // the animation is over. FTB off re-fits the dist mixer itself.
+    let ftb = state.ftb_active.load(Ordering::Relaxed);
+    let dist_animation = animation_remaining(mixer);
+    let mv_animation = animation_remaining(mv_comp);
+    let dist_busy = ftb || dist_animation.is_some();
+    let mv_busy = mv_animation.is_some();
+    state
+        .layout_refit_pending
+        .store(dist_busy || mv_busy, Ordering::Relaxed);
+    let retry = [dist_animation, mv_animation].into_iter().flatten().max();
+    if let Some(wait) = retry {
+        refit_after(block_id, mixer, mv_comp, wait + REFIT_MARGIN);
+    }
+
     let mut aspects = SourceAspects::new();
     for (i, d) in dims.iter().enumerate() {
         if let Some((w, h)) = d {
             aspects.insert(i, *w as f64 / *h as f64);
         }
     }
 
-    // Dist canvas size from the mixer's negotiated output caps.
-    let (cw, ch) = pad_caps_dims_src(mixer).unwrap_or_else(|| {
-        strom_types::parse_resolution_string(vision_mixer::DEFAULT_PGM_RESOLUTION)
-            .map(|(w, h)| (w as i32, h as i32))
-            .expect("DEFAULT_PGM_RESOLUTION must be valid")
-    });
+    // Dist canvas size from the mixer's negotiated output caps, or the PGM
+    // size it is built to produce: the first refresh can run before the
+    // output has negotiated, and no later one runs unless an input changes.
+    let (cw, ch) = pad_caps_dims_src(mixer).unwrap_or((state.pgm_w as i32, state.pgm_h as i32));
     let fallback = if ch > 0 {
         cw as f64 / ch as f64
     } else {
@@ -155,8 +172,8 @@ fn refresh_geometry(block_id: &str, mixer: &gst::Element, mv_comp: &gst::Element
         scale: state.layout.pvw_rect.w / pgm_w,
     });
 
-    // --- Dist compositor (PGM) — skip while FTB drives the alphas. ---
-    if !state.ftb_active.load(std::sync::atomic::Ordering::Relaxed) {
+    // --- Dist compositor (PGM). ---
+    if !dist_busy {
         if let Some(p) = state.pgm_pip() {
             apply_pip_layout_to_region(
                 mixer,
@@ -201,6 +218,10 @@ fn refresh_geometry(block_id: &str, mixer: &gst::Element, mv_comp: &gst::Element
         pad.set_property("height", h);
     }
 
+    if mv_busy {
+        return;
+    }
+
     // --- Multiview PVW big region. ---
     let r = &state.layout.pvw_rect;
     let pvw_region = (r.x as i32, r.y as i32, r.w as i32, r.h as i32);
@@ -266,6 +287,53 @@ fn refresh_geometry(block_id: &str, mixer: &gst::Element, mv_comp: &gst::Element
     );
 }
 
+/// Pad properties that takes, FTB and layout code animate.
+const ANIMATED_PAD_PROPS: [&str; 6] = ["alpha", "xpos", "ypos", "width", "height", "zorder"];
+
+/// Slack after an animation's last keyframe before re-fitting.
+const REFIT_MARGIN: Duration = Duration::from_millis(50);
+
+/// How long until the last keyframe programmed on any of `compositor`'s sink
+/// pads, or `None` when none lies ahead. Keyframes are in the compositor's
+/// stream time, which is what its position query reports.
+fn animation_remaining(compositor: &gst::Element) -> Option<Duration> {
+    let now = compositor.query_position::<gst::ClockTime>()?;
+    compositor
+        .sink_pads()
+        .iter()
+        .flat_map(|pad| {
+            ANIMATED_PAD_PROPS
+                .iter()
+                .filter_map(move |prop| last_keyframe(pad, prop))
+        })
+        .max()
+        .filter(|end| *end > now)
+        .map(|end| Duration::from_nanos((end - now).nseconds()))
+}
+
+fn last_keyframe(pad: &gst::Pad, prop: &str) -> Option<gst::ClockTime> {
+    let cs = pad
+        .control_binding(prop)?
+        .property::<Option<gst::ControlSource>>("control-source")?
+        .downcast::<InterpolationControlSource>()
+        .ok()?;
+    cs.list_control_points().last().map(|tv| tv.timestamp())
+}
+
+/// Run a geometry refresh after `wait`, on the main loop.
+fn refit_after(block_id: &str, mixer: &gst::Element, mv_comp: &gst::Element, wait: Duration) {
+    let block_id = block_id.to_string();
+    let mixer_weak = mixer.downgrade();
+    let mv_weak = mv_comp.downgrade();
+    gst::glib::timeout_add_once(wait, move || {
+        // Pipeline teardown in progress → nothing to refresh.
+        let (Some(mixer), Some(mv_comp)) = (mixer_weak.upgrade(), mv_weak.upgrade()) else {
+            return;
+        };
+        refresh_geometry(&block_id, &mixer, &mv_comp);
+    });
+}
+
 /// Read `(width, height)` from an element's `src` pad caps.
 fn pad_caps_dims_src(element: &gst::Element) -> Option<(i32, i32)> {
     let pad = element.static_pad("src")?;

backend/src/blocks/builtin/vision_mixer/overlay.rs

@@ -82,6 +82,9 @@ pub struct VisionMixerOverlayState {
     pub num_inputs: usize,
     /// Whether Fade to Black is active.
     pub ftb_active: AtomicBool,
+    /// An input changed size while a take or FTB was animating the layout,
+    /// so the layout still has to be re-fitted once the animation is over.
+    pub layout_refit_pending: AtomicBool,
     /// Multiview overlay alpha (0.0–1.0), stored as f64 bits.
     overlay_alpha: AtomicU64,
     /// DSK enabled states (one per DSK input, max 4).
@@ -199,6 +202,7 @@ impl VisionMixerOverlayState {
             num_pips: pip.num_pips,
             num_inputs,
             ftb_active: AtomicBool::new(false),
+            layout_refit_pending: AtomicBool::new(false),
             overlay_alpha: AtomicU64::new(1.0f64.to_bits()),
             dsk_enabled: (0..num_dsk_inputs)
                 .map(|_| AtomicBool::new(false))