From 7d5e7cfb84fcaf4e5713fcc644a9d9b3a8f49ce0 Mon Sep 17 00:00:00 2001 From: Peter Wagenet Date: Sat, 12 Sep 2026 16:27:13 -0700 Subject: [PATCH 1/5] feat(vision_mixer): composite premultiplied-alpha DSK inputs correctly compositor and glvideomixerelement blend straight alpha, and caps cannot say that a stream is premultiplied. cefsrc paints premultiplied BGRA, so every HTML graphic on a keyed input composites too dark wherever alpha is partial: 50% white over grey 64 reads 96 on both backends instead of 160. Each DSK input gets a dsk_N_alpha_mode property, straight (default) or premultiplied, matching the alpha_mode values used for stinger sources. CPU: a premultiplied input runs through a converter and a new in-place element, stromunpremultiply (a VideoFilter over packed RGBA-family formats, lookup table with a 0/255 fast path), before the existing DSK chain. The converter lets decoded YUV-with-alpha clips reach it. GL: no element. The DSK pad blends source RGB with constant-alpha, and a property binding keeps blend-constant-color-alpha equal to the pad's alpha, so DSK toggles, takes, the property API and fade-to-black's control binding all stay correct. blend-function-src-rgb=one would be right only at full pad alpha. Co-Authored-By: Claude Opus 5 (1M context) --- .../builtin/vision_mixer/builder/mod.rs | 9 + .../vision_mixer/builder/pad_layout.rs | 12 + .../vision_mixer/builder/pipeline_cpu.rs | 45 +- .../vision_mixer/builder/pipeline_gpu.rs | 27 + .../blocks/builtin/vision_mixer/definition.rs | 37 ++ .../blocks/builtin/vision_mixer/properties.rs | 41 +- backend/src/gst/mod.rs | 1 + backend/src/gst/unpremultiply.rs | 257 +++++++++ .../vision_mixer_premultiplied_alpha_test.rs | 516 ++++++++++++++++++ frontend/src/properties.rs | 23 +- types/src/vision_mixer.rs | 67 +++ 11 files changed, 1026 insertions(+), 9 deletions(-) create mode 100644 backend/src/gst/unpremultiply.rs create mode 100644 backend/tests/vision_mixer_premultiplied_alpha_test.rs diff --git a/backend/src/blocks/builtin/vision_mixer/builder/mod.rs b/backend/src/blocks/builtin/vision_mixer/builder/mod.rs index 56cbf574c..ae3da496d 100644 --- a/backend/src/blocks/builtin/vision_mixer/builder/mod.rs +++ b/backend/src/blocks/builtin/vision_mixer/builder/mod.rs @@ -145,6 +145,7 @@ impl BlockBuilder for VisionMixerBuilder { ); let num_dsk_inputs = properties::parse_num_dsk_inputs(props); + let dsk_alpha_modes = properties::parse_dsk_alpha_modes(props, num_dsk_inputs); let pip_bg_inputs: Vec> = (0..num_pips) .map(|i| properties::parse_pip_bg(props, i, num_inputs)) @@ -185,6 +186,7 @@ impl BlockBuilder for VisionMixerBuilder { flow_id: properties::parse_flow_id(props), num_inputs, num_dsk_inputs, + dsk_alpha_modes: &dsk_alpha_modes, num_pips, pgm_input, pvw_input, @@ -225,6 +227,8 @@ pub(super) struct PipelineParams<'a> { pub(super) flow_id: strom_types::FlowId, pub(super) num_inputs: usize, pub(super) num_dsk_inputs: usize, + /// Alpha encoding of each DSK input, `num_dsk_inputs` long. + pub(super) dsk_alpha_modes: &'a [vision_mixer::AlphaMode], pub(super) num_pips: usize, pub(super) pgm_input: usize, pub(super) pvw_input: usize, @@ -266,6 +270,11 @@ impl<'a> PipelineParams<'a> { format!("{}:{}", self.instance_id, name) } + /// Whether DSK input `i` was declared premultiplied. + pub(super) fn dsk_premultiplied(&self, i: usize) -> bool { + self.dsk_alpha_modes.get(i) == Some(&vision_mixer::AlphaMode::Premultiplied) + } + /// Raw video caps with resolution, framerate and an optional pixel format. fn raw_caps(&self, w: u32, h: u32, framerate: (i32, i32), format: Option<&str>) -> gst::Caps { let mut builder = gst::Caps::builder("video/x-raw") diff --git a/backend/src/blocks/builtin/vision_mixer/builder/pad_layout.rs b/backend/src/blocks/builtin/vision_mixer/builder/pad_layout.rs index 11cc8169f..dba3e5478 100644 --- a/backend/src/blocks/builtin/vision_mixer/builder/pad_layout.rs +++ b/backend/src/blocks/builtin/vision_mixer/builder/pad_layout.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use strom_types::{vision_mixer, PropertyValue}; +use super::super::elements::CompositorBackend; use super::super::layout; use super::PipelineParams; @@ -165,6 +166,17 @@ pub(super) fn build_pad_properties( "sizing-policy".to_string(), PropertyValue::String("keep-aspect-ratio".to_string()), ); + if p.backend == CompositorBackend::OpenGL && p.dsk_premultiplied(i) { + // Premultiplied colour must not be scaled by its own alpha again, + // but still has to fade with the pad: blending with the constant + // gives src * pad_alpha + dst * (1 - src_alpha * pad_alpha). + // `one` would be right only at full pad alpha. The constant + // follows `alpha` through a binding set up in `pipeline_gpu`. + props.insert( + "blend-function-src-rgb".to_string(), + PropertyValue::String("constant-alpha".to_string()), + ); + } } // --- Stinger pads: graphic, and on the GPU the matte. Both hidden, full diff --git a/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs b/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs index aeaa434ae..3843a8e8c 100644 --- a/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs +++ b/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs @@ -10,6 +10,7 @@ use super::super::{elements, layout}; use super::{audio_meter, pad_layout, setup_overlay_renderer, PipelineParams}; use crate::blocks::{BlockBuildContext, BlockBuildError, BlockBuildResult}; use crate::gpu; +use crate::gst::unpremultiply; pub(super) fn build_cpu_pipeline( p: &PipelineParams, @@ -162,10 +163,46 @@ pub(super) fn build_cpu_pipeline( elems.push((q_id.clone(), queue)); elems.push((vc_id_dsk.clone(), videoconvert)); - links.push(( - ElementPadRef::pad(&q_id, "src"), - ElementPadRef::pad(&vc_id_dsk, "sink"), - )); + if p.dsk_premultiplied(i) { + // `compositor` blends straight alpha only, so a premultiplied + // source is unpremultiplied first. The element takes packed RGB + // with alpha; the converter in front brings anything else (a + // decoded A420 clip, GPU memory) to that, and passes RGBA-family + // input straight through. + if !unpremultiply::register() { + return Err(BlockBuildError::ElementCreation(format!( + "DSK{} is premultiplied but {} could not be registered", + i + 1, + unpremultiply::ELEMENT_NAME + ))); + } + let vc_pre_id = p.id(&format!("videoconvert_unpremultiply_dsk_{}", i)); + let unpre_id = p.id(&format!("unpremultiply_dsk_{}", i)); + let videoconvert_pre = elements::make_element(vc_factory, &vc_pre_id)?; + gpu::configure_video_convert(&videoconvert_pre); + elems.push((vc_pre_id.clone(), videoconvert_pre)); + elems.push(( + unpre_id.clone(), + elements::make_element(unpremultiply::ELEMENT_NAME, &unpre_id)?, + )); + links.push(( + ElementPadRef::pad(&q_id, "src"), + ElementPadRef::pad(&vc_pre_id, "sink"), + )); + links.push(( + ElementPadRef::pad(&vc_pre_id, "src"), + ElementPadRef::pad(&unpre_id, "sink"), + )); + links.push(( + ElementPadRef::pad(&unpre_id, "src"), + ElementPadRef::pad(&vc_id_dsk, "sink"), + )); + } else { + links.push(( + ElementPadRef::pad(&q_id, "src"), + ElementPadRef::pad(&vc_id_dsk, "sink"), + )); + } // No output_format capsfilter here. A DSK is a keyer: forcing an // alpha-less format (I420/NV12) ahead of the compositor silently diff --git a/backend/src/blocks/builtin/vision_mixer/builder/pipeline_gpu.rs b/backend/src/blocks/builtin/vision_mixer/builder/pipeline_gpu.rs index fb6e3b556..d2b6021d1 100644 --- a/backend/src/blocks/builtin/vision_mixer/builder/pipeline_gpu.rs +++ b/backend/src/blocks/builtin/vision_mixer/builder/pipeline_gpu.rs @@ -665,6 +665,7 @@ pub(super) fn build_gpu_pipeline( let num_inputs = p.num_inputs; let num_pips = p.num_pips; let underlay_state = std::sync::Arc::clone(&overlay_state); + let dist_weak = dist_weak.clone(); ctx.register_element_setup(Box::new(move |_flow_id, _events| { let (Some(mixer), Some(mv_comp)) = (dist_weak.upgrade(), mv_weak.upgrade()) else { return; @@ -680,6 +681,32 @@ pub(super) fn build_gpu_pipeline( })); } + // --- Premultiplied DSK pads: blend constant follows pad alpha --- + // `pad_layout` blends these pads with `constant-alpha`, which is only + // correct while the constant equals the pad's alpha. A property binding + // keeps them equal for every writer: DSK toggles, takes, fade to black + // (whose control binding sets `alpha` per frame), and the property API. + let premultiplied_dsk_pads: Vec = (0..p.num_dsk_inputs) + .filter(|&i| p.dsk_premultiplied(i)) + .map(|i| format!("sink_{}", p.num_inputs + i)) + .collect(); + if !premultiplied_dsk_pads.is_empty() { + ctx.register_element_setup(Box::new(move |_flow_id, _events| { + let Some(mixer) = dist_weak.upgrade() else { + return; + }; + for pad in mixer + .sink_pads() + .into_iter() + .filter(|pad| premultiplied_dsk_pads.iter().any(|n| pad.name() == *n)) + { + pad.bind_property("alpha", &pad, "blend-constant-color-alpha") + .sync_create() + .build(); + } + })); + } + // --- Bus handler that pipes `level` messages into the overlay state --- let bus_message_handler = Some(audio_meter::build_meter_bus_handler( p.instance_id, diff --git a/backend/src/blocks/builtin/vision_mixer/definition.rs b/backend/src/blocks/builtin/vision_mixer/definition.rs index 758079914..d50b77a8e 100644 --- a/backend/src/blocks/builtin/vision_mixer/definition.rs +++ b/backend/src/blocks/builtin/vision_mixer/definition.rs @@ -419,6 +419,43 @@ fn vision_mixer_definition() -> BlockDefinition { }); } + // Per-DSK alpha mode, one property per possible DSK slot; the frontend + // hides slots beyond `num_dsk_inputs` (see `vision_mixer_skip_set`). + for i in 0..MAX_DSK_INPUTS { + let name = dsk_alpha_mode_property(i); + exposed_properties.push(ExposedProperty { + name: name.clone(), + label: format!("DSK{} Alpha", i + 1), + description: format!( + "How DSK{}'s source encodes alpha. HTML sources paint premultiplied; \ + composited as straight they come out too dark wherever alpha is partial", + i + 1 + ), + property_type: PropertyType::Enum { + values: vec![ + EnumValue { + value: AlphaMode::Straight.as_str().to_string(), + label: Some("Straight".to_string()), + }, + EnumValue { + value: AlphaMode::Premultiplied.as_str().to_string(), + label: Some("Premultiplied".to_string()), + }, + ], + }, + default_value: Some(PropertyValue::String( + AlphaMode::default().as_str().to_string(), + )), + mapping: PropertyMapping { + element_id: "_block".to_string(), + property_name: name, + transform: None, + }, + live: false, + persist: None, + }); + } + // Number of PiP tiles { let mut pip_values = vec![EnumValue { diff --git a/backend/src/blocks/builtin/vision_mixer/properties.rs b/backend/src/blocks/builtin/vision_mixer/properties.rs index 66da4cf68..de39298d2 100644 --- a/backend/src/blocks/builtin/vision_mixer/properties.rs +++ b/backend/src/blocks/builtin/vision_mixer/properties.rs @@ -2,8 +2,9 @@ use std::collections::HashMap; use strom_types::vision_mixer::{ - Source, DEFAULT_DSK_INPUTS, DEFAULT_NUM_INPUTS, DEFAULT_NUM_PIPS, DEFAULT_SHOW_VU_METERS, - MAX_DSK_INPUTS, MAX_NUM_INPUTS, MAX_NUM_PIPS, MIN_NUM_INPUTS, + dsk_alpha_mode_property, AlphaMode, Source, DEFAULT_DSK_INPUTS, DEFAULT_NUM_INPUTS, + DEFAULT_NUM_PIPS, DEFAULT_SHOW_VU_METERS, MAX_DSK_INPUTS, MAX_NUM_INPUTS, MAX_NUM_PIPS, + MIN_NUM_INPUTS, }; use strom_types::FlowId; use strom_types::PropertyValue; @@ -37,6 +38,20 @@ pub fn parse_num_dsk_inputs(properties: &HashMap) -> usiz .min(MAX_DSK_INPUTS) } +/// Parse each DSK input's alpha mode. Missing or unrecognised values are +/// straight, which is what the compositors assume. +pub fn parse_dsk_alpha_modes( + properties: &HashMap, + num_dsk_inputs: usize, +) -> Vec { + (0..num_dsk_inputs) + .map(|i| match properties.get(&dsk_alpha_mode_property(i)) { + Some(PropertyValue::String(s)) => s.parse().unwrap_or_default(), + _ => AlphaMode::default(), + }) + .collect() +} + /// Parse the number of PiP tiles from block properties. pub fn parse_num_pips(properties: &HashMap) -> usize { properties @@ -306,6 +321,28 @@ pub fn parse_u64(properties: &HashMap, key: &str, default mod tests { use super::*; + #[test] + fn dsk_alpha_modes_default_to_straight() { + let mut props = HashMap::new(); + props.insert( + "dsk_1_alpha_mode".to_string(), + PropertyValue::String("premultiplied".to_string()), + ); + props.insert( + "dsk_2_alpha_mode".to_string(), + PropertyValue::String("nonsense".to_string()), + ); + assert_eq!( + parse_dsk_alpha_modes(&props, 4), + vec![ + AlphaMode::Straight, + AlphaMode::Premultiplied, + AlphaMode::Straight, + AlphaMode::Straight, + ] + ); + } + #[test] fn parse_framerate_string_valid_fractions() { assert_eq!(parse_framerate_string("25/1"), Some((25, 1))); diff --git a/backend/src/gst/mod.rs b/backend/src/gst/mod.rs index 358152e53..b5317f5df 100644 --- a/backend/src/gst/mod.rs +++ b/backend/src/gst/mod.rs @@ -22,6 +22,7 @@ pub mod thumbnail; pub mod thumbnail_tap; pub mod transitions; pub(crate) mod underlay; +pub mod unpremultiply; pub mod video_adapt; pub mod video_frame; pub mod video_input_bridge; diff --git a/backend/src/gst/unpremultiply.rs b/backend/src/gst/unpremultiply.rs new file mode 100644 index 000000000..488c9c4a2 --- /dev/null +++ b/backend/src/gst/unpremultiply.rs @@ -0,0 +1,257 @@ +//! `stromunpremultiply`: divides colour by alpha, in place. +//! +//! `compositor` blends straight alpha only and has no operator that handles a +//! premultiplied source, so a keyed input declared premultiplied is converted +//! to straight before it reaches the CPU mixer. The GL mixer does not need +//! this element; it corrects the blend on the pad instead. +//! +//! Registration is static and idempotent; see [`register`]. + +use std::sync::{LazyLock, OnceLock}; + +use gstreamer as gst; +use gstreamer::glib; +use gstreamer::prelude::*; +use gstreamer_base as gst_base; +use gstreamer_video as gst_video; +use tracing::warn; + +/// The GStreamer factory name. +pub const ELEMENT_NAME: &str = "stromunpremultiply"; + +glib::wrapper! { + pub struct Unpremultiply(ObjectSubclass) + @extends gst_video::VideoFilter, gst_base::BaseTransform, gst::Element, gst::Object; +} + +/// Register the element with the GStreamer registry, once per process. +/// +/// Returns whether `stromunpremultiply` is available afterwards. Callers must +/// have initialised GStreamer first. +pub fn register() -> bool { + static REGISTERED: OnceLock = OnceLock::new(); + *REGISTERED.get_or_init(|| { + // Rank NONE: built by name for premultiplied keyed inputs, never + // autoplugged. + match gst::Element::register( + None, + ELEMENT_NAME, + gst::Rank::NONE, + Unpremultiply::static_type(), + ) { + Ok(()) => true, + Err(e) => { + warn!("could not register {}: {}", ELEMENT_NAME, e); + false + } + } + }) +} + +/// `LUT[(a << 8) | c]` is `c` unpremultiplied by `a`, rounded and clamped. +/// Rows for alpha 0 and 255 are never read: those pixels are skipped. +static LUT: LazyLock> = LazyLock::new(|| { + let mut lut = vec![0u8; 256 * 256].into_boxed_slice(); + for a in 1..256usize { + for c in 0..256usize { + lut[(a << 8) | c] = ((c * 255 + a / 2) / a).min(255) as u8; + } + } + lut +}); + +/// Unpremultiply one row of packed 4-byte pixels whose alpha is at byte +/// `alpha_at` and whose colour occupies the other three. +fn unpremultiply_row(row: &mut [u8], alpha_at: usize) { + let lut = &**LUT; + for px in row.as_chunks_mut::<4>().0 { + let a = px[alpha_at] as usize; + // Fully transparent and fully opaque pixels are identical in both + // encodings, and they are most of any graphic. + if a == 0 || a == 255 { + continue; + } + let base = a << 8; + for (i, c) in px.iter_mut().enumerate() { + if i != alpha_at { + *c = lut[base | *c as usize]; + } + } + } +} + +mod imp { + use super::*; + use gstreamer::subclass::prelude::*; + use gstreamer_base::subclass::prelude::*; + use gstreamer_video::prelude::*; + use gstreamer_video::subclass::prelude::*; + use gstreamer_video::{VideoFormat, VideoFrameRef}; + + #[derive(Default)] + pub struct Unpremultiply; + + #[glib::object_subclass] + impl ObjectSubclass for Unpremultiply { + const NAME: &'static str = "StromUnpremultiply"; + type Type = super::Unpremultiply; + type ParentType = gst_video::VideoFilter; + } + + impl ObjectImpl for Unpremultiply {} + + impl GstObjectImpl for Unpremultiply {} + + impl ElementImpl for Unpremultiply { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static METADATA: LazyLock = LazyLock::new(|| { + gst::subclass::ElementMetadata::new( + "Unpremultiply alpha", + "Filter/Effect/Video", + "Converts premultiplied-alpha video to straight alpha in place", + "Strom contributors", + ) + }); + Some(&*METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static TEMPLATES: LazyLock> = LazyLock::new(|| { + // 8-bit packed RGB with alpha only. Anything else, including + // decoded YUV-with-alpha clips, needs a converter in front. + let caps = gst_video::VideoCapsBuilder::new() + .format_list([ + VideoFormat::Bgra, + VideoFormat::Rgba, + VideoFormat::Argb, + VideoFormat::Abgr, + ]) + .build(); + vec![ + gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .expect("static sink template"), + gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .expect("static src template"), + ] + }); + TEMPLATES.as_ref() + } + } + + impl BaseTransformImpl for Unpremultiply { + const MODE: gst_base::subclass::BaseTransformMode = + gst_base::subclass::BaseTransformMode::AlwaysInPlace; + const PASSTHROUGH_ON_SAME_CAPS: bool = false; + const TRANSFORM_IP_ON_PASSTHROUGH: bool = false; + } + + impl VideoFilterImpl for Unpremultiply { + fn transform_frame_ip( + &self, + frame: &mut VideoFrameRef<&mut gst::BufferRef>, + ) -> Result { + let alpha_at = match frame.format() { + VideoFormat::Bgra | VideoFormat::Rgba => 3, + VideoFormat::Argb | VideoFormat::Abgr => 0, + _ => return Err(gst::FlowError::NotNegotiated), + }; + let width = frame.width() as usize; + let height = frame.height() as usize; + let stride = frame.plane_stride()[0] as usize; + let data = frame.plane_data_mut(0).map_err(|_| gst::FlowError::Error)?; + for row in data.chunks_mut(stride).take(height) { + unpremultiply_row(&mut row[..width * 4], alpha_at); + } + Ok(gst::FlowSuccess::Ok) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use gstreamer_app as gst_app; + + #[test] + fn lut_matches_the_exact_division() { + for a in 1..255u32 { + // Premultiplied colour never exceeds its alpha. + for c in 0..=a { + let expected = ((c as f64) * 255.0 / a as f64).round() as u32; + let got = LUT[((a as usize) << 8) | c as usize] as u32; + assert!( + got.abs_diff(expected) <= 1, + "a={a} c={c}: got {got}, expected {expected}" + ); + } + } + } + + #[test] + fn row_leaves_alpha_and_the_endpoints_alone() { + // BGRA: half-alpha white, opaque grey, transparent black. + let mut row = [128, 128, 128, 128, 64, 64, 64, 255, 0, 0, 0, 0]; + unpremultiply_row(&mut row, 3); + assert_eq!(row, [255, 255, 255, 128, 64, 64, 64, 255, 0, 0, 0, 0]); + + // ARGB: alpha leads. + let mut row = [64, 32, 16, 64]; + unpremultiply_row(&mut row, 0); + assert_eq!(row, [64, 128, 64, 255]); + } + + /// Runs the registered element on one premultiplied frame per format. + #[test] + fn element_unpremultiplies_every_supported_format() { + gst::init().unwrap(); + assert!(register(), "element registers"); + + for (format, px, expected) in [ + ("BGRA", [128u8, 64, 0, 128], [255u8, 128, 0, 128]), + ("RGBA", [0, 64, 128, 128], [0, 128, 255, 128]), + ("ARGB", [128, 128, 64, 0], [128, 255, 128, 0]), + ("ABGR", [128, 0, 64, 128], [128, 0, 128, 255]), + ] { + let pipeline = gst::parse::launch(&format!( + "appsrc name=src format=time caps=video/x-raw,format={format},width=4,height=2,framerate=30/1 \ + ! {ELEMENT_NAME} ! appsink name=sink sync=false" + )) + .unwrap() + .downcast::() + .unwrap(); + let src = pipeline + .by_name("src") + .unwrap() + .downcast::() + .unwrap(); + let sink = pipeline + .by_name("sink") + .unwrap() + .downcast::() + .unwrap(); + pipeline.set_state(gst::State::Playing).unwrap(); + + let frame: Vec = px.iter().copied().cycle().take(4 * 4 * 2).collect(); + src.push_buffer(gst::Buffer::from_slice(frame)).unwrap(); + let sample = sink + .try_pull_sample(gst::ClockTime::from_seconds(5)) + .unwrap_or_else(|| panic!("{format}: no output")); + let map = sample.buffer().unwrap().map_readable().unwrap(); + for out in map.as_chunks::<4>().0 { + assert_eq!(*out, expected, "{format}"); + } + drop(map); + pipeline.set_state(gst::State::Null).unwrap(); + } + } +} diff --git a/backend/tests/vision_mixer_premultiplied_alpha_test.rs b/backend/tests/vision_mixer_premultiplied_alpha_test.rs new file mode 100644 index 000000000..0c783e04a --- /dev/null +++ b/backend/tests/vision_mixer_premultiplied_alpha_test.rs @@ -0,0 +1,516 @@ +//! Regression test: a DSK input declared premultiplied composites correctly +//! on both vision mixer backends. +//! +//! `compositor` and `glvideomixerelement` blend straight alpha. A premultiplied +//! source (every `cefsrc` HTML graphic) blended as straight is scaled by its +//! alpha twice: 50 % white over grey 64 comes out 96, where the right answer +//! is 160. Grey matters: against black or white, clamping hides the error. +//! +//! CI has no `cefsrc`, so a `videotestsrc` paints the premultiplied encoding +//! of 50 % white, `(128, 128, 128, 128)`, into the real block. + +use gstreamer::prelude::*; +use std::collections::HashMap; +use std::time::{Duration, Instant}; +use strom::blocks::BlockRegistry; +use strom::events::EventBroadcaster; +use strom::gst::pipeline::PipelineManager; +use strom_types::{Flow, PropertyValue as PV}; +use tempfile::NamedTempFile; + +const W: usize = 320; +const H: usize = 180; +const NUM_INPUTS: usize = 2; +/// The DSK pad on the dist mixer: after the video inputs. +const DSK_PAD: &str = "sink_2"; + +/// Premultiplied 50 % white over grey 64 at pad alpha 1. +const CORRECT_FULL: f64 = 160.0; +/// The same at pad alpha 0.5: 255 * 0.25 + 64 * 0.75. +const CORRECT_HALF: f64 = 112.0; + +fn elem(id: &str, ty: &str, props: Vec<(&str, PV)>) -> strom_types::Element { + strom_types::Element { + id: id.to_string(), + element_type: ty.to_string(), + properties: props.into_iter().map(|(k, v)| (k.to_string(), v)).collect(), + position: [0.0, 0.0].into(), + pad_properties: HashMap::new(), + } +} + +fn raw_caps(format: &str) -> PV { + PV::String(format!( + "video/x-raw,format={},width={},height={},framerate=30/1", + format, W, H + )) +} + +/// What feeds DSK 0. +enum DskSource { + /// A live `videotestsrc` painting premultiplied 50 % white as BGRA. + TestPattern, + /// A media player block looping the clip at this path. + MediaPlayer(std::path::PathBuf), +} + +/// Flow: grey 64 on `video_in_0` (the initial program), the premultiplied +/// source on `dsk_in_0`, and program out converted to RGBA into an appsink. +/// `alpha_mode: None` leaves the property unset, as existing flows have it. +fn build_flow(backend: &str, block_id: &str, alpha_mode: Option<&str>, dsk: DskSource) -> Flow { + let mut flow = Flow::new(format!("vm_premul_{}", block_id)); + flow.blocks.push(strom_types::BlockInstance { + id: block_id.to_string(), + block_definition_id: "builtin.vision_mixer".to_string(), + name: None, + properties: { + let mut p = HashMap::new(); + p.insert( + "compositor_preference".to_string(), + PV::String(backend.to_string()), + ); + // The measurement tap is a plain videoconvert, which cannot take + // GL memory. + p.insert("gl_download".to_string(), PV::Bool(backend == "gpu")); + p.insert("num_inputs".to_string(), PV::UInt(NUM_INPUTS as u64)); + p.insert("num_dsk_inputs".to_string(), PV::String("1".to_string())); + if let Some(mode) = alpha_mode { + p.insert("dsk_0_alpha_mode".to_string(), PV::String(mode.to_string())); + } + p.insert( + "pgm_resolution".to_string(), + PV::String(format!("{}x{}", W, H)), + ); + p.insert( + "multiview_resolution".to_string(), + PV::String(format!("{}x{}", W, H)), + ); + p + }, + position: strom_types::block::Position { x: 100.0, y: 100.0 }, + runtime_data: None, + computed_external_pads: None, + }); + + flow.elements.push(elem( + "bg", + "videotestsrc", + vec![ + ("pattern", PV::String("solid-color".into())), + ("foreground-color", PV::UInt(0xff40_4040)), + ("is-live", PV::Bool(true)), + ], + )); + flow.elements.push(elem( + "bgcaps", + "capsfilter", + vec![("caps", raw_caps("BGRA"))], + )); + + let dsk_out = match dsk { + DskSource::TestPattern => { + flow.elements.push(elem( + "gfx", + "videotestsrc", + vec![ + ("pattern", PV::String("solid-color".into())), + // ARGB 0x80808080: every BGRA byte is 128. + ("foreground-color", PV::UInt(0x8080_8080)), + ("is-live", PV::Bool(true)), + ], + )); + flow.elements.push(elem( + "gfxcaps", + "capsfilter", + vec![("caps", raw_caps("BGRA"))], + )); + flow.links.push(strom_types::Link { + from: "gfx:src".to_string(), + to: "gfxcaps:sink".to_string(), + }); + "gfxcaps:src".to_string() + } + DskSource::MediaPlayer(path) => { + flow.blocks.push(strom_types::BlockInstance { + id: "clip".to_string(), + block_definition_id: "builtin.media_player".to_string(), + name: None, + properties: { + let mut p = HashMap::new(); + p.insert("decode".to_string(), PV::Bool(true)); + p.insert("loop_playlist".to_string(), PV::Bool(true)); + p.insert( + "playlist".to_string(), + PV::String(serde_json::to_string(&[path]).unwrap()), + ); + p + }, + position: strom_types::block::Position { x: 0.0, y: 0.0 }, + runtime_data: None, + computed_external_pads: None, + }); + "clip:video_out".to_string() + } + }; + + flow.elements.push(elem("pgmconv", "videoconvert", vec![])); + flow.elements.push(elem( + "pgmcaps", + "capsfilter", + vec![("caps", PV::String("video/x-raw,format=RGBA".into()))], + )); + flow.elements.push(elem( + "pgmsink", + "appsink", + vec![ + ("sync", PV::Bool(false)), + ("max-buffers", PV::UInt(1)), + ("drop", PV::Bool(true)), + ], + )); + + // DSK pads are dynamic (num_dsk_inputs), so the flow needs the computed + // pad set the API normally fills in before validation. + for block in &mut flow.blocks { + if let Some(builder) = strom::blocks::builtin::get_builder(&block.block_definition_id) { + block.computed_external_pads = builder.get_external_pads(&block.properties); + } + } + + for (from, to) in [ + ("bg:src".to_string(), "bgcaps:sink".to_string()), + ("bgcaps:src".to_string(), format!("{}:video_in_0", block_id)), + (dsk_out, format!("{}:dsk_in_0", block_id)), + (format!("{}:pgm_out", block_id), "pgmconv:sink".to_string()), + ("pgmconv:src".to_string(), "pgmcaps:sink".to_string()), + ("pgmcaps:src".to_string(), "pgmsink:sink".to_string()), + ] { + flow.links.push(strom_types::Link { from, to }); + } + flow +} + +/// Mean red channel over the centre quarter of an RGBA frame. The scene is +/// grey, so red stands for all three. +fn centre_red(sample: &gstreamer::Sample) -> f64 { + use gstreamer_video::{VideoFormat, VideoFrameRef, VideoInfo}; + let info = VideoInfo::from_caps(sample.caps().expect("caps")).expect("video info"); + assert_eq!( + info.format(), + VideoFormat::Rgba, + "PGM sample should be RGBA" + ); + let frame = VideoFrameRef::from_buffer_ref_readable(sample.buffer().expect("buffer"), &info) + .expect("map frame"); + let stride = info.stride()[0] as usize; + let data = frame.plane_data(0).expect("plane 0"); + let (mut sum, mut n) = (0u64, 0u64); + for y in (H / 4)..(3 * H / 4) { + for x in (W / 4)..(3 * W / 4) { + sum += data[y * stride + x * 4] as u64; + n += 1; + } + } + sum as f64 / n as f64 +} + +struct Running { + manager: PipelineManager, + block_id: String, + sink: gstreamer_app::AppSink, +} + +impl Running { + fn start(backend: &str, block_id: &str, alpha_mode: Option<&str>, dsk: DskSource) -> Self { + gstreamer::init().unwrap(); + // The CPU builder picks its videoconvert factory from the detected + // GPU capabilities; without this the first lookup panics. + strom::gpu::detect_gpu_capabilities(); + + let temp_file = NamedTempFile::new().unwrap(); + let registry = BlockRegistry::new(temp_file.path()); + let flow = build_flow(backend, block_id, alpha_mode, dsk); + let mut manager = PipelineManager::new( + &flow, + EventBroadcaster::new(10, false, false), + ®istry, + vec![], + "all".to_string(), + None, + std::env::temp_dir(), + std::sync::Arc::new(std::sync::Mutex::new(HashMap::new())), + ) + .unwrap_or_else(|e| panic!("build {} pipeline: {}", backend, e)); + manager + .start() + .unwrap_or_else(|e| panic!("start {} pipeline: {}", backend, e)); + + let sink = manager + .pipeline() + .by_name("pgmsink") + .expect("pgmsink in pipeline") + .downcast::() + .expect("appsink type"); + Running { + manager, + block_id: block_id.to_string(), + sink, + } + } + + fn mixer_id(&self) -> String { + format!("{}:mixer", self.block_id) + } + + /// Pull frames until three in a row read within `tol` of `expected`, or + /// until the deadline. Returns the last reading either way, so a failure + /// reports what was actually on air. + fn settle_on(&self, expected: f64, tol: f64) -> f64 { + let deadline = Instant::now() + Duration::from_secs(20); + let (mut last, mut streak) = (f64::NAN, 0); + while Instant::now() < deadline && streak < 3 { + let Some(sample) = self + .sink + .try_pull_sample(gstreamer::ClockTime::from_mseconds(500)) + else { + continue; + }; + last = centre_red(&sample); + streak = if (last - expected).abs() <= tol { + streak + 1 + } else { + 0 + }; + } + last + } + + fn set_pad_alpha(&self, alpha: f64) { + self.manager + .update_pad_property(&self.mixer_id(), DSK_PAD, "alpha", &PV::Float(alpha)) + .expect("set DSK pad alpha"); + } + + fn stop(mut self) { + self.manager.stop().expect("stop"); + } +} + +fn assert_near(what: &str, got: f64, expected: f64, tol: f64) { + assert!( + (got - expected).abs() <= tol, + "{what}: got {got:.1}, expected {expected:.0} +/- {tol:.0} \ + (premultiplied blended as straight reads 96 at full alpha, 80 at half)" + ); +} + +/// Full and half pad alpha on one running flow, against the correct results +/// `(full, half)` for this source. +fn assert_correct_at_full_and_half_alpha( + backend: &str, + block_id: &str, + dsk: DskSource, + (full_expected, half_expected): (f64, f64), +) { + const TOL: f64 = 3.0; + let run = Running::start(backend, block_id, Some("premultiplied"), dsk); + run.manager + .set_dsk_enabled(block_id, 0, NUM_INPUTS, true) + .expect("enable DSK 0"); + + let full = run.settle_on(full_expected, TOL); + eprintln!("{backend} {block_id}: pad alpha 1.0 -> {full:.1}"); + assert_near("pad alpha 1.0", full, full_expected, TOL); + + // Through the property API, as PATCH on the pad does. + run.set_pad_alpha(0.5); + let half = run.settle_on(half_expected, TOL); + eprintln!("{backend} {block_id}: pad alpha 0.5 -> {half:.1}"); + assert_near("pad alpha 0.5", half, half_expected, TOL); + + run.stop(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cpu_premultiplied_dsk_composites_correctly() { + assert_correct_at_full_and_half_alpha( + "cpu", + "vmp_cpu", + DskSource::TestPattern, + (CORRECT_FULL, CORRECT_HALF), + ); +} + +/// The declaration is what changes the result: the same source with no alpha +/// mode set still composites as straight. Guards the default, so existing +/// flows with straight graphics are not unpremultiplied behind their back. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cpu_default_mode_leaves_the_source_alone() { + let run = Running::start("cpu", "vmp_cpu_default", None, DskSource::TestPattern); + run.manager + .set_dsk_enabled("vmp_cpu_default", 0, NUM_INPUTS, true) + .expect("enable DSK 0"); + let got = run.settle_on(96.0, 3.0); + eprintln!("cpu default mode: pad alpha 1.0 -> {got:.1}"); + assert!( + (got - 96.0).abs() <= 3.0, + "with no alpha mode the source should blend as-is (96), got {got:.1}" + ); + run.stop(); +} + +/// Write a premultiplied 50 % white clip, encoded as A420 so the decoded +/// frames are YUV with alpha and have to be converted before unpremultiplying. +fn write_premultiplied_clip(path: &std::path::Path) -> Result<(), String> { + for factory in ["avenc_ffv1", "matroskamux"] { + if gstreamer::ElementFactory::find(factory).is_none() { + return Err(format!("{factory} missing")); + } + } + let pipeline = gstreamer::parse::launch(&format!( + "videotestsrc num-buffers=30 pattern=solid-color foreground-color=0x80808080 \ + ! video/x-raw,format=BGRA,width={W},height={H},framerate=30/1 \ + ! videoconvert ! video/x-raw,format=A420 ! avenc_ffv1 ! matroskamux \ + ! filesink location={}", + path.display() + )) + .map_err(|e| e.to_string())?; + pipeline + .set_state(gstreamer::State::Playing) + .map_err(|e| e.to_string())?; + let bus = pipeline.bus().expect("bus"); + let msg = bus.timed_pop_filtered( + gstreamer::ClockTime::from_seconds(30), + &[gstreamer::MessageType::Eos, gstreamer::MessageType::Error], + ); + let _ = pipeline.set_state(gstreamer::State::Null); + match msg.as_ref().map(|m| m.view()) { + Some(gstreamer::MessageView::Eos(_)) => Ok(()), + other => Err(format!("clip encode did not finish: {other:?}")), + } +} + +/// A premultiplied clip through the media player: the decoder hands out A420, +/// which the unpremultiply element cannot take directly. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cpu_premultiplied_clip_through_media_player() { + gstreamer::init().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let clip = dir.path().join("premultiplied.mkv"); + // gst-libav and matroska are in CI's package list; failing here rather + // than skipping keeps the test from passing green without running. + write_premultiplied_clip(&clip).expect("write premultiplied clip"); + + // videoconvert's RGB -> A420 -> RGB round trip turns the clip's 128 into + // 126, which unpremultiplies to 251: 157.5 at full alpha, 110.4 at half. + // Blended as straight it would read 95. + assert_correct_at_full_and_half_alpha( + "cpu", + "vmp_clip", + DskSource::MediaPlayer(clip), + (157.5, 110.4), + ); +} + +/// Probe whether this environment can actually render through GL: the plugins +/// being installed is not enough — on headless runners the elements exist but +/// no context can be created. Same probe as `vision_mixer_fx_test`. +fn gl_environment_available() -> bool { + gstreamer::init().unwrap(); + if gstreamer::ElementFactory::find("glvideomixerelement").is_none() + || gstreamer::ElementFactory::find("gltestsrc").is_none() + { + return false; + } + let Ok(pipeline) = gstreamer::parse::launch( + "gltestsrc num-buffers=3 ! video/x-raw(memory:GLMemory),format=RGBA,width=64,height=64,framerate=30/1 ! fakesink sync=false", + ) else { + return false; + }; + let Ok(pipeline) = pipeline.downcast::() else { + return false; + }; + if pipeline.set_state(gstreamer::State::Playing).is_err() { + return false; + } + let bus = pipeline.bus().expect("pipeline has a bus"); + let ok = matches!( + bus.timed_pop_filtered( + gstreamer::ClockTime::from_seconds(20), + &[gstreamer::MessageType::Eos, gstreamer::MessageType::Error], + ), + Some(msg) if matches!(msg.view(), gstreamer::MessageView::Eos(_)) + ); + let _ = pipeline.set_state(gstreamer::State::Null); + ok +} + +/// GL corrects the blend on the pad rather than converting the source, and +/// the correction depends on a blend constant that has to track pad alpha. +/// Half alpha is the case that catches a constant left behind. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn gpu_premultiplied_dsk_composites_correctly() { + if !gl_environment_available() { + eprintln!("SKIP: GL environment unavailable (no context or GL elements missing)"); + return; + } + assert_correct_at_full_and_half_alpha( + "gpu", + "vmp_gpu", + DskSource::TestPattern, + (CORRECT_FULL, CORRECT_HALF), + ); +} + +/// Fade to black drives pad alpha from a control binding, frame by frame, +/// not through `set_property` from Strom code. The blend constant has to +/// follow that too. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn gpu_blend_constant_follows_fade_to_black() { + if !gl_environment_available() { + eprintln!("SKIP: GL environment unavailable (no context or GL elements missing)"); + return; + } + let block_id = "vmp_gpu_ftb"; + let run = Running::start( + "gpu", + block_id, + Some("premultiplied"), + DskSource::TestPattern, + ); + run.manager + .set_dsk_enabled(block_id, 0, NUM_INPUTS, true) + .expect("enable DSK 0"); + let full = run.settle_on(CORRECT_FULL, 3.0); + assert_near("pad alpha 1.0 before the fade", full, CORRECT_FULL, 3.0); + + let pad = run + .manager + .pipeline() + .by_name(&run.mixer_id()) + .expect("mixer") + .sink_pads() + .into_iter() + .find(|p| p.name() == DSK_PAD) + .expect("DSK pad"); + assert_eq!(pad.property::("blend-constant-color-alpha"), 1.0); + + // During the fade the only writer of `alpha` is the control binding. + assert!(run.manager.fade_to_black(block_id, 300).expect("FTB on")); + let deadline = Instant::now() + Duration::from_secs(10); + while pad.property::("alpha") > 0.0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + pad.property::("alpha"), + 0.0, + "fade never reached black" + ); + assert_eq!( + pad.property::("blend-constant-color-alpha"), + 0.0, + "blend constant did not follow the animated pad alpha" + ); + + run.stop(); +} diff --git a/frontend/src/properties.rs b/frontend/src/properties.rs index 621adfee1..ab2ff6c64 100644 --- a/frontend/src/properties.rs +++ b/frontend/src/properties.rs @@ -1463,10 +1463,13 @@ impl PropertyInspector { result } - /// Skip per-input labels for slots beyond the configured `num_inputs` - /// so the property panel shows just the inputs in use. + /// Skip per-input labels and per-DSK alpha modes for slots beyond the + /// configured counts so the property panel shows just the inputs in use. fn vision_mixer_skip_set(block: &BlockInstance) -> std::collections::HashSet { - use strom_types::vision_mixer::{DEFAULT_NUM_INPUTS, MAX_NUM_INPUTS}; + use strom_types::vision_mixer::{ + dsk_alpha_mode_property, DEFAULT_DSK_INPUTS, DEFAULT_NUM_INPUTS, MAX_DSK_INPUTS, + MAX_NUM_INPUTS, + }; let num_inputs = block .properties @@ -1479,10 +1482,24 @@ impl PropertyInspector { }) .unwrap_or(DEFAULT_NUM_INPUTS); + let num_dsk_inputs = block + .properties + .get("num_dsk_inputs") + .and_then(|v| match v { + PropertyValue::String(s) => s.parse().ok(), + PropertyValue::UInt(n) => Some(*n as usize), + PropertyValue::Int(n) => Some(*n as usize), + _ => None, + }) + .unwrap_or(DEFAULT_DSK_INPUTS); + let mut skip = std::collections::HashSet::new(); for i in num_inputs..MAX_NUM_INPUTS { skip.insert(format!("input_{}_label", i)); } + for i in num_dsk_inputs..MAX_DSK_INPUTS { + skip.insert(dsk_alpha_mode_property(i)); + } skip } diff --git a/types/src/vision_mixer.rs b/types/src/vision_mixer.rs index a6478dc9b..a32925933 100644 --- a/types/src/vision_mixer.rs +++ b/types/src/vision_mixer.rs @@ -114,6 +114,63 @@ pub const MAX_DSK_INPUTS: usize = 4; /// Default number of DSK inputs (0 = no DSK). pub const DEFAULT_DSK_INPUTS: usize = 0; +/// How a keyed (DSK) input encodes its alpha. +/// +/// Caps cannot say whether colour is already multiplied by alpha, so it is +/// declared per input. `cefsrc` (HTML graphics) paints premultiplied. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum AlphaMode { + /// Colour is independent of alpha. What the compositors assume. + #[default] + Straight, + /// Colour has already been multiplied by alpha. + Premultiplied, +} + +impl AlphaMode { + /// Property value, as stored on the block. + pub fn as_str(self) -> &'static str { + match self { + AlphaMode::Straight => "straight", + AlphaMode::Premultiplied => "premultiplied", + } + } +} + +impl fmt::Display for AlphaMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Error returned when an [`AlphaMode`] string cannot be parsed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParseAlphaModeError; + +impl fmt::Display for ParseAlphaModeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("expected 'straight' or 'premultiplied'") + } +} + +impl std::error::Error for ParseAlphaModeError {} + +impl FromStr for AlphaMode { + type Err = ParseAlphaModeError; + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "straight" => Ok(AlphaMode::Straight), + "premultiplied" => Ok(AlphaMode::Premultiplied), + _ => Err(ParseAlphaModeError), + } + } +} + +/// Block property declaring DSK input `dsk_index`'s [`AlphaMode`] (0-based). +pub fn dsk_alpha_mode_property(dsk_index: usize) -> String { + format!("dsk_{}_alpha_mode", dsk_index) +} + /// Maximum number of PiP (Picture-in-Picture) tiles rendered virtually in the multiview. /// Each PiP consumes one tile in the multiview thumbnail grid alongside the inputs. pub const MAX_NUM_PIPS: usize = 4; @@ -834,6 +891,16 @@ pub fn compute_pip_overlay_rects( mod tests { use super::*; + #[test] + fn alpha_mode_round_trips_through_its_property_value() { + for mode in [AlphaMode::Straight, AlphaMode::Premultiplied] { + assert_eq!(mode.as_str().parse::(), Ok(mode)); + } + assert_eq!(" Premultiplied ".parse(), Ok(AlphaMode::Premultiplied)); + assert_eq!("".parse::(), Err(ParseAlphaModeError)); + assert_eq!(dsk_alpha_mode_property(2), "dsk_2_alpha_mode"); + } + #[test] fn test_pip_overlay_rects_one_full_aspect() { // 1 source in 1920×1080 with 16:9 aspect → fills the whole container. From 766d329556ead9d192dbbd7b18abac0a4f9b2d28 Mon Sep 17 00:00:00 2001 From: Peter Wagenet Date: Sun, 13 Sep 2026 22:23:20 -0700 Subject: [PATCH 2/5] docs(vision_mixer): say when a DSK input should stay straight cefsrc can now convert to straight alpha itself (centricular/gstcefsrc#113). Declaring such a source premultiplied as well converts twice and composites too bright, so the alpha mode description says to leave it straight. Co-Authored-By: Claude Opus 5 (1M context) --- backend/src/blocks/builtin/vision_mixer/definition.rs | 6 ++++-- types/src/vision_mixer.rs | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/src/blocks/builtin/vision_mixer/definition.rs b/backend/src/blocks/builtin/vision_mixer/definition.rs index d50b77a8e..0481b000d 100644 --- a/backend/src/blocks/builtin/vision_mixer/definition.rs +++ b/backend/src/blocks/builtin/vision_mixer/definition.rs @@ -427,8 +427,10 @@ fn vision_mixer_definition() -> BlockDefinition { name: name.clone(), label: format!("DSK{} Alpha", i + 1), description: format!( - "How DSK{}'s source encodes alpha. HTML sources paint premultiplied; \ - composited as straight they come out too dark wherever alpha is partial", + "How DSK{}'s source encodes alpha. HTML sources (cefsrc) emit premultiplied \ + by default, which composites too dark as straight. Leave straight if the \ + source already converts, e.g. cefsrc with unpremultiply=true; converting \ + twice comes out too bright", i + 1 ), property_type: PropertyType::Enum { diff --git a/types/src/vision_mixer.rs b/types/src/vision_mixer.rs index a32925933..456417cac 100644 --- a/types/src/vision_mixer.rs +++ b/types/src/vision_mixer.rs @@ -117,7 +117,8 @@ pub const DEFAULT_DSK_INPUTS: usize = 0; /// How a keyed (DSK) input encodes its alpha. /// /// Caps cannot say whether colour is already multiplied by alpha, so it is -/// declared per input. `cefsrc` (HTML graphics) paints premultiplied. +/// declared per input. `cefsrc` (HTML graphics) emits premultiplied unless its +/// `unpremultiply` property is set. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum AlphaMode { /// Colour is independent of alpha. What the compositors assume. From ca55b54cd80d6009d61c11800037d52df78afc9b Mon Sep 17 00:00:00 2001 From: Peter Wagenet Date: Tue, 15 Sep 2026 10:21:00 -0700 Subject: [PATCH 3/5] fix(vision_mixer): set the DSK blend constant before alpha on DSK on A premultiplied DSK pad on the GL mixer blends with constant-alpha, and a property binding copies alpha into the blend constant after each write. DSK on sets alpha from a non-streaming thread, so a frame drawn between the two writes sees alpha 1 with the constant still 0 and puts the graphic's silhouette on air in black. Set the constant first. Disabling needs nothing: the mixer skips pads at alpha 0. No test: the window is two consecutive property writes, which a test cannot hit reliably. Co-Authored-By: Claude Opus 5 (1M context) --- backend/src/gst/pipeline/effects/mixer_ops.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/backend/src/gst/pipeline/effects/mixer_ops.rs b/backend/src/gst/pipeline/effects/mixer_ops.rs index a941dcaeb..fd8265d7f 100644 --- a/backend/src/gst/pipeline/effects/mixer_ops.rs +++ b/backend/src/gst/pipeline/effects/mixer_ops.rs @@ -242,6 +242,14 @@ impl PipelineManager { let pad_name = format!("sink_{}", num_inputs + dsk_index); if let Some(pad) = find_pad(mixer, &pad_name) { let alpha = if enabled { 1.0f64 } else { 0.0f64 }; + // A premultiplied DSK pad on the GL mixer blends with a constant + // that a binding copies from `alpha` after the write. Enabling + // sets it first, so no frame is drawn at alpha 1 with the constant + // still 0, which puts the graphic's silhouette on air in black. + // Disabling needs no help: the mixer skips pads at alpha 0. + if enabled && pad.has_property("blend-constant-color-alpha") { + pad.set_property("blend-constant-color-alpha", alpha); + } pad.set_property("alpha", alpha); // Update overlay state for DSK tracking if let Some(state) = crate::blocks::builtin::vision_mixer::overlay::get_overlay_state( From 0dd1d0fcff03fd18a9108314b9d88828446afc19 Mon Sep 17 00:00:00 2001 From: Peter Wagenet Date: Tue, 15 Sep 2026 10:21:17 -0700 Subject: [PATCH 4/5] chore(vision_mixer): warn on bad DSK alpha modes, note the CPU edge - Log an unrecognised or non-string dsk_N_alpha_mode before falling back to straight. Open Live writes the value from code, and a typo otherwise shows only as a graphic that is too dark. - The property description no longer names cefsrc's unpremultiply option, which the pinned build lacks, and says a premultiplied source on the CPU mixer should arrive at program resolution: compositor resizes after the conversion, which darkens edges by about a pixel. - The CPU chain comment no longer claims the converter handles GPU memory; plain videoconvert does not. - The build log line includes each DSK's alpha mode. Co-Authored-By: Claude Opus 5 (1M context) --- .../blocks/builtin/vision_mixer/builder/mod.rs | 4 ++-- .../vision_mixer/builder/pipeline_cpu.rs | 6 +++--- .../blocks/builtin/vision_mixer/definition.rs | 9 +++++---- .../blocks/builtin/vision_mixer/properties.rs | 18 ++++++++++++++---- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/backend/src/blocks/builtin/vision_mixer/builder/mod.rs b/backend/src/blocks/builtin/vision_mixer/builder/mod.rs index ae3da496d..39f3b9642 100644 --- a/backend/src/blocks/builtin/vision_mixer/builder/mod.rs +++ b/backend/src/blocks/builtin/vision_mixer/builder/mod.rs @@ -175,10 +175,10 @@ impl BlockBuilder for VisionMixerBuilder { let backend = super::elements::select_backend(pref)?; info!( - "Building vision mixer: {} inputs, PGM={}x{}@{}/{}, MV={}x{}@{}/{}, backend={:?}, pgm={}, pvw={}", + "Building vision mixer: {} inputs, PGM={}x{}@{}/{}, MV={}x{}@{}/{}, backend={:?}, pgm={}, pvw={}, dsk_alpha={:?}", num_inputs, pgm_w, pgm_h, pgm_framerate.0, pgm_framerate.1, mv_w, mv_h, mv_framerate.0, mv_framerate.1, - backend, pgm_input, pvw_input + backend, pgm_input, pvw_input, dsk_alpha_modes ); let p = PipelineParams { diff --git a/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs b/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs index 3843a8e8c..06fd6596d 100644 --- a/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs +++ b/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs @@ -166,9 +166,9 @@ pub(super) fn build_cpu_pipeline( if p.dsk_premultiplied(i) { // `compositor` blends straight alpha only, so a premultiplied // source is unpremultiplied first. The element takes packed RGB - // with alpha; the converter in front brings anything else (a - // decoded A420 clip, GPU memory) to that, and passes RGBA-family - // input straight through. + // with alpha; the converter in front brings other formats (such as + // a decoded A420 clip) to that, and passes RGBA-family input + // straight through. if !unpremultiply::register() { return Err(BlockBuildError::ElementCreation(format!( "DSK{} is premultiplied but {} could not be registered", diff --git a/backend/src/blocks/builtin/vision_mixer/definition.rs b/backend/src/blocks/builtin/vision_mixer/definition.rs index 0481b000d..9b5c9be0a 100644 --- a/backend/src/blocks/builtin/vision_mixer/definition.rs +++ b/backend/src/blocks/builtin/vision_mixer/definition.rs @@ -427,10 +427,11 @@ fn vision_mixer_definition() -> BlockDefinition { name: name.clone(), label: format!("DSK{} Alpha", i + 1), description: format!( - "How DSK{}'s source encodes alpha. HTML sources (cefsrc) emit premultiplied \ - by default, which composites too dark as straight. Leave straight if the \ - source already converts, e.g. cefsrc with unpremultiply=true; converting \ - twice comes out too bright", + "How DSK{}'s source encodes alpha. HTML sources (cefsrc) emit premultiplied, \ + which composites too dark as straight. Leave straight if the source already \ + outputs straight alpha; converting twice comes out too bright. On the CPU \ + mixer, feed a premultiplied source at program resolution (e.g. through a \ + Video Format block), or its edges come out slightly dark", i + 1 ), property_type: PropertyType::Enum { diff --git a/backend/src/blocks/builtin/vision_mixer/properties.rs b/backend/src/blocks/builtin/vision_mixer/properties.rs index de39298d2..f72c2fc95 100644 --- a/backend/src/blocks/builtin/vision_mixer/properties.rs +++ b/backend/src/blocks/builtin/vision_mixer/properties.rs @@ -8,6 +8,7 @@ use strom_types::vision_mixer::{ }; use strom_types::FlowId; use strom_types::PropertyValue; +use tracing::warn; /// Parse the owning flow id, injected as `_flow_id` by block expansion. /// @@ -39,15 +40,24 @@ pub fn parse_num_dsk_inputs(properties: &HashMap) -> usiz } /// Parse each DSK input's alpha mode. Missing or unrecognised values are -/// straight, which is what the compositors assume. +/// straight, which is what the compositors assume; unrecognised ones are +/// logged, since the only symptom on air is a graphic that is too dark. pub fn parse_dsk_alpha_modes( properties: &HashMap, num_dsk_inputs: usize, ) -> Vec { (0..num_dsk_inputs) - .map(|i| match properties.get(&dsk_alpha_mode_property(i)) { - Some(PropertyValue::String(s)) => s.parse().unwrap_or_default(), - _ => AlphaMode::default(), + .map(|i| { + let name = dsk_alpha_mode_property(i); + let parsed = match properties.get(&name) { + None => return AlphaMode::default(), + Some(PropertyValue::String(s)) => s.parse().map_err(|e| format!("{s:?}: {e}")), + Some(other) => Err(format!("{other:?} is not a string")), + }; + parsed.unwrap_or_else(|e| { + warn!("{name} = {e}; using straight"); + AlphaMode::default() + }) }) .collect() } From 8cc98b3327888d89601a1c9a06392ab13e6910f8 Mon Sep 17 00:00:00 2001 From: Peter Wagenet Date: Fri, 9 Oct 2026 08:57:03 -0700 Subject: [PATCH 5/5] test(vision_mixer): use the shared GL gate for the premultiplied tests Use common::gl_available, which renders through surfaceless EGL on a headless Linux host, and serialise the tests that render through llvmpipe. Co-Authored-By: Claude Sonnet 5.5 --- .../vision_mixer_premultiplied_alpha_test.rs | 65 +++++++++---------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/backend/tests/vision_mixer_premultiplied_alpha_test.rs b/backend/tests/vision_mixer_premultiplied_alpha_test.rs index 0c783e04a..d18eafd43 100644 --- a/backend/tests/vision_mixer_premultiplied_alpha_test.rs +++ b/backend/tests/vision_mixer_premultiplied_alpha_test.rs @@ -9,7 +9,10 @@ //! CI has no `cefsrc`, so a `videotestsrc` paints the premultiplied encoding //! of 50 % white, `(128, 128, 128, 128)`, into the real block. +pub mod common; + use gstreamer::prelude::*; +use serial_test::serial; use std::collections::HashMap; use std::time::{Duration, Instant}; use strom::blocks::BlockRegistry; @@ -214,10 +217,16 @@ fn centre_red(sample: &gstreamer::Sample) -> f64 { sum as f64 / n as f64 } +/// A running flow and the glib main loop it needs: the media player starts +/// its internal pipeline from a bus handler on the default main context when +/// the flow reaches PLAYING, which only runs while a main loop does. The server +/// has one. struct Running { manager: PipelineManager, block_id: String, sink: gstreamer_app::AppSink, + main_loop: gstreamer::glib::MainLoop, + main_loop_thread: std::thread::JoinHandle<()>, } impl Running { @@ -227,6 +236,11 @@ impl Running { // GPU capabilities; without this the first lookup panics. strom::gpu::detect_gpu_capabilities(); + let main_loop = gstreamer::glib::MainLoop::new(None, false); + let main_loop_thread = { + let ml = main_loop.clone(); + std::thread::spawn(move || ml.run()) + }; let temp_file = NamedTempFile::new().unwrap(); let registry = BlockRegistry::new(temp_file.path()); let flow = build_flow(backend, block_id, alpha_mode, dsk); @@ -255,6 +269,8 @@ impl Running { manager, block_id: block_id.to_string(), sink, + main_loop, + main_loop_thread, } } @@ -293,6 +309,8 @@ impl Running { fn stop(mut self) { self.manager.stop().expect("stop"); + self.main_loop.quit(); + self.main_loop_thread.join().expect("main loop thread"); } } @@ -331,7 +349,11 @@ fn assert_correct_at_full_and_half_alpha( run.stop(); } +// Every case here renders a flow and probes program frames; the GPU ones render +// through llvmpipe in CI, which costs whole cores. Serialised so a sibling test +// cannot starve the pipeline under measurement. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(gl)] async fn cpu_premultiplied_dsk_composites_correctly() { assert_correct_at_full_and_half_alpha( "cpu", @@ -345,6 +367,7 @@ async fn cpu_premultiplied_dsk_composites_correctly() { /// mode set still composites as straight. Guards the default, so existing /// flows with straight graphics are not unpremultiplied behind their back. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(gl)] async fn cpu_default_mode_leaves_the_source_alone() { let run = Running::start("cpu", "vmp_cpu_default", None, DskSource::TestPattern); run.manager @@ -393,6 +416,7 @@ fn write_premultiplied_clip(path: &std::path::Path) -> Result<(), String> { /// A premultiplied clip through the media player: the decoder hands out A420, /// which the unpremultiply element cannot take directly. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(gl)] async fn cpu_premultiplied_clip_through_media_player() { gstreamer::init().unwrap(); let dir = tempfile::tempdir().unwrap(); @@ -412,46 +436,15 @@ async fn cpu_premultiplied_clip_through_media_player() { ); } -/// Probe whether this environment can actually render through GL: the plugins -/// being installed is not enough — on headless runners the elements exist but -/// no context can be created. Same probe as `vision_mixer_fx_test`. -fn gl_environment_available() -> bool { - gstreamer::init().unwrap(); - if gstreamer::ElementFactory::find("glvideomixerelement").is_none() - || gstreamer::ElementFactory::find("gltestsrc").is_none() - { - return false; - } - let Ok(pipeline) = gstreamer::parse::launch( - "gltestsrc num-buffers=3 ! video/x-raw(memory:GLMemory),format=RGBA,width=64,height=64,framerate=30/1 ! fakesink sync=false", - ) else { - return false; - }; - let Ok(pipeline) = pipeline.downcast::() else { - return false; - }; - if pipeline.set_state(gstreamer::State::Playing).is_err() { - return false; - } - let bus = pipeline.bus().expect("pipeline has a bus"); - let ok = matches!( - bus.timed_pop_filtered( - gstreamer::ClockTime::from_seconds(20), - &[gstreamer::MessageType::Eos, gstreamer::MessageType::Error], - ), - Some(msg) if matches!(msg.view(), gstreamer::MessageView::Eos(_)) - ); - let _ = pipeline.set_state(gstreamer::State::Null); - ok -} +const GL_ELEMENTS: &[&str] = &["glvideomixerelement", "gltestsrc"]; /// GL corrects the blend on the pad rather than converting the source, and /// the correction depends on a blend constant that has to track pad alpha. /// Half alpha is the case that catches a constant left behind. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(gl)] async fn gpu_premultiplied_dsk_composites_correctly() { - if !gl_environment_available() { - eprintln!("SKIP: GL environment unavailable (no context or GL elements missing)"); + if !common::gl_available(GL_ELEMENTS) { return; } assert_correct_at_full_and_half_alpha( @@ -466,9 +459,9 @@ async fn gpu_premultiplied_dsk_composites_correctly() { /// not through `set_property` from Strom code. The blend constant has to /// follow that too. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(gl)] async fn gpu_blend_constant_follows_fade_to_black() { - if !gl_environment_available() { - eprintln!("SKIP: GL environment unavailable (no context or GL elements missing)"); + if !common::gl_available(GL_ELEMENTS) { return; } let block_id = "vmp_gpu_ftb";