diff --git a/backend/src/blocks/builtin/vision_mixer/builder/mod.rs b/backend/src/blocks/builtin/vision_mixer/builder/mod.rs index 56cbf574c..39f3b9642 100644 --- a/backend/src/blocks/builtin/vision_mixer/builder/mod.rs +++ b/backend/src/blocks/builtin/vision_mixer/builder/mod.rs @@ -145,6 +145,7 @@ impl BlockBuilder for VisionMixerBuilder { ); let num_dsk_inputs = properties::parse_num_dsk_inputs(props); + let dsk_alpha_modes = properties::parse_dsk_alpha_modes(props, num_dsk_inputs); let pip_bg_inputs: Vec> = (0..num_pips) .map(|i| properties::parse_pip_bg(props, i, num_inputs)) @@ -174,10 +175,10 @@ impl BlockBuilder for VisionMixerBuilder { let backend = super::elements::select_backend(pref)?; info!( - "Building vision mixer: {} inputs, PGM={}x{}@{}/{}, MV={}x{}@{}/{}, backend={:?}, pgm={}, pvw={}", + "Building vision mixer: {} inputs, PGM={}x{}@{}/{}, MV={}x{}@{}/{}, backend={:?}, pgm={}, pvw={}, dsk_alpha={:?}", num_inputs, pgm_w, pgm_h, pgm_framerate.0, pgm_framerate.1, mv_w, mv_h, mv_framerate.0, mv_framerate.1, - backend, pgm_input, pvw_input + backend, pgm_input, pvw_input, dsk_alpha_modes ); let p = PipelineParams { @@ -185,6 +186,7 @@ impl BlockBuilder for VisionMixerBuilder { flow_id: properties::parse_flow_id(props), num_inputs, num_dsk_inputs, + dsk_alpha_modes: &dsk_alpha_modes, num_pips, pgm_input, pvw_input, @@ -225,6 +227,8 @@ pub(super) struct PipelineParams<'a> { pub(super) flow_id: strom_types::FlowId, pub(super) num_inputs: usize, pub(super) num_dsk_inputs: usize, + /// Alpha encoding of each DSK input, `num_dsk_inputs` long. + pub(super) dsk_alpha_modes: &'a [vision_mixer::AlphaMode], pub(super) num_pips: usize, pub(super) pgm_input: usize, pub(super) pvw_input: usize, @@ -266,6 +270,11 @@ impl<'a> PipelineParams<'a> { format!("{}:{}", self.instance_id, name) } + /// Whether DSK input `i` was declared premultiplied. + pub(super) fn dsk_premultiplied(&self, i: usize) -> bool { + self.dsk_alpha_modes.get(i) == Some(&vision_mixer::AlphaMode::Premultiplied) + } + /// Raw video caps with resolution, framerate and an optional pixel format. fn raw_caps(&self, w: u32, h: u32, framerate: (i32, i32), format: Option<&str>) -> gst::Caps { let mut builder = gst::Caps::builder("video/x-raw") diff --git a/backend/src/blocks/builtin/vision_mixer/builder/pad_layout.rs b/backend/src/blocks/builtin/vision_mixer/builder/pad_layout.rs index 11cc8169f..dba3e5478 100644 --- a/backend/src/blocks/builtin/vision_mixer/builder/pad_layout.rs +++ b/backend/src/blocks/builtin/vision_mixer/builder/pad_layout.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use strom_types::{vision_mixer, PropertyValue}; +use super::super::elements::CompositorBackend; use super::super::layout; use super::PipelineParams; @@ -165,6 +166,17 @@ pub(super) fn build_pad_properties( "sizing-policy".to_string(), PropertyValue::String("keep-aspect-ratio".to_string()), ); + if p.backend == CompositorBackend::OpenGL && p.dsk_premultiplied(i) { + // Premultiplied colour must not be scaled by its own alpha again, + // but still has to fade with the pad: blending with the constant + // gives src * pad_alpha + dst * (1 - src_alpha * pad_alpha). + // `one` would be right only at full pad alpha. The constant + // follows `alpha` through a binding set up in `pipeline_gpu`. + props.insert( + "blend-function-src-rgb".to_string(), + PropertyValue::String("constant-alpha".to_string()), + ); + } } // --- Stinger pads: graphic, and on the GPU the matte. Both hidden, full diff --git a/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs b/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs index aeaa434ae..06fd6596d 100644 --- a/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs +++ b/backend/src/blocks/builtin/vision_mixer/builder/pipeline_cpu.rs @@ -10,6 +10,7 @@ use super::super::{elements, layout}; use super::{audio_meter, pad_layout, setup_overlay_renderer, PipelineParams}; use crate::blocks::{BlockBuildContext, BlockBuildError, BlockBuildResult}; use crate::gpu; +use crate::gst::unpremultiply; pub(super) fn build_cpu_pipeline( p: &PipelineParams, @@ -162,10 +163,46 @@ pub(super) fn build_cpu_pipeline( elems.push((q_id.clone(), queue)); elems.push((vc_id_dsk.clone(), videoconvert)); - links.push(( - ElementPadRef::pad(&q_id, "src"), - ElementPadRef::pad(&vc_id_dsk, "sink"), - )); + if p.dsk_premultiplied(i) { + // `compositor` blends straight alpha only, so a premultiplied + // source is unpremultiplied first. The element takes packed RGB + // with alpha; the converter in front brings other formats (such as + // a decoded A420 clip) to that, and passes RGBA-family input + // straight through. + if !unpremultiply::register() { + return Err(BlockBuildError::ElementCreation(format!( + "DSK{} is premultiplied but {} could not be registered", + i + 1, + unpremultiply::ELEMENT_NAME + ))); + } + let vc_pre_id = p.id(&format!("videoconvert_unpremultiply_dsk_{}", i)); + let unpre_id = p.id(&format!("unpremultiply_dsk_{}", i)); + let videoconvert_pre = elements::make_element(vc_factory, &vc_pre_id)?; + gpu::configure_video_convert(&videoconvert_pre); + elems.push((vc_pre_id.clone(), videoconvert_pre)); + elems.push(( + unpre_id.clone(), + elements::make_element(unpremultiply::ELEMENT_NAME, &unpre_id)?, + )); + links.push(( + ElementPadRef::pad(&q_id, "src"), + ElementPadRef::pad(&vc_pre_id, "sink"), + )); + links.push(( + ElementPadRef::pad(&vc_pre_id, "src"), + ElementPadRef::pad(&unpre_id, "sink"), + )); + links.push(( + ElementPadRef::pad(&unpre_id, "src"), + ElementPadRef::pad(&vc_id_dsk, "sink"), + )); + } else { + links.push(( + ElementPadRef::pad(&q_id, "src"), + ElementPadRef::pad(&vc_id_dsk, "sink"), + )); + } // No output_format capsfilter here. A DSK is a keyer: forcing an // alpha-less format (I420/NV12) ahead of the compositor silently diff --git a/backend/src/blocks/builtin/vision_mixer/builder/pipeline_gpu.rs b/backend/src/blocks/builtin/vision_mixer/builder/pipeline_gpu.rs index fb6e3b556..d2b6021d1 100644 --- a/backend/src/blocks/builtin/vision_mixer/builder/pipeline_gpu.rs +++ b/backend/src/blocks/builtin/vision_mixer/builder/pipeline_gpu.rs @@ -665,6 +665,7 @@ pub(super) fn build_gpu_pipeline( let num_inputs = p.num_inputs; let num_pips = p.num_pips; let underlay_state = std::sync::Arc::clone(&overlay_state); + let dist_weak = dist_weak.clone(); ctx.register_element_setup(Box::new(move |_flow_id, _events| { let (Some(mixer), Some(mv_comp)) = (dist_weak.upgrade(), mv_weak.upgrade()) else { return; @@ -680,6 +681,32 @@ pub(super) fn build_gpu_pipeline( })); } + // --- Premultiplied DSK pads: blend constant follows pad alpha --- + // `pad_layout` blends these pads with `constant-alpha`, which is only + // correct while the constant equals the pad's alpha. A property binding + // keeps them equal for every writer: DSK toggles, takes, fade to black + // (whose control binding sets `alpha` per frame), and the property API. + let premultiplied_dsk_pads: Vec = (0..p.num_dsk_inputs) + .filter(|&i| p.dsk_premultiplied(i)) + .map(|i| format!("sink_{}", p.num_inputs + i)) + .collect(); + if !premultiplied_dsk_pads.is_empty() { + ctx.register_element_setup(Box::new(move |_flow_id, _events| { + let Some(mixer) = dist_weak.upgrade() else { + return; + }; + for pad in mixer + .sink_pads() + .into_iter() + .filter(|pad| premultiplied_dsk_pads.iter().any(|n| pad.name() == *n)) + { + pad.bind_property("alpha", &pad, "blend-constant-color-alpha") + .sync_create() + .build(); + } + })); + } + // --- Bus handler that pipes `level` messages into the overlay state --- let bus_message_handler = Some(audio_meter::build_meter_bus_handler( p.instance_id, diff --git a/backend/src/blocks/builtin/vision_mixer/definition.rs b/backend/src/blocks/builtin/vision_mixer/definition.rs index 758079914..9b5c9be0a 100644 --- a/backend/src/blocks/builtin/vision_mixer/definition.rs +++ b/backend/src/blocks/builtin/vision_mixer/definition.rs @@ -419,6 +419,46 @@ fn vision_mixer_definition() -> BlockDefinition { }); } + // Per-DSK alpha mode, one property per possible DSK slot; the frontend + // hides slots beyond `num_dsk_inputs` (see `vision_mixer_skip_set`). + for i in 0..MAX_DSK_INPUTS { + let name = dsk_alpha_mode_property(i); + exposed_properties.push(ExposedProperty { + name: name.clone(), + label: format!("DSK{} Alpha", i + 1), + description: format!( + "How DSK{}'s source encodes alpha. HTML sources (cefsrc) emit premultiplied, \ + which composites too dark as straight. Leave straight if the source already \ + outputs straight alpha; converting twice comes out too bright. On the CPU \ + mixer, feed a premultiplied source at program resolution (e.g. through a \ + Video Format block), or its edges come out slightly dark", + i + 1 + ), + property_type: PropertyType::Enum { + values: vec![ + EnumValue { + value: AlphaMode::Straight.as_str().to_string(), + label: Some("Straight".to_string()), + }, + EnumValue { + value: AlphaMode::Premultiplied.as_str().to_string(), + label: Some("Premultiplied".to_string()), + }, + ], + }, + default_value: Some(PropertyValue::String( + AlphaMode::default().as_str().to_string(), + )), + mapping: PropertyMapping { + element_id: "_block".to_string(), + property_name: name, + transform: None, + }, + live: false, + persist: None, + }); + } + // Number of PiP tiles { let mut pip_values = vec![EnumValue { diff --git a/backend/src/blocks/builtin/vision_mixer/properties.rs b/backend/src/blocks/builtin/vision_mixer/properties.rs index 66da4cf68..f72c2fc95 100644 --- a/backend/src/blocks/builtin/vision_mixer/properties.rs +++ b/backend/src/blocks/builtin/vision_mixer/properties.rs @@ -2,11 +2,13 @@ use std::collections::HashMap; use strom_types::vision_mixer::{ - Source, DEFAULT_DSK_INPUTS, DEFAULT_NUM_INPUTS, DEFAULT_NUM_PIPS, DEFAULT_SHOW_VU_METERS, - MAX_DSK_INPUTS, MAX_NUM_INPUTS, MAX_NUM_PIPS, MIN_NUM_INPUTS, + dsk_alpha_mode_property, AlphaMode, Source, DEFAULT_DSK_INPUTS, DEFAULT_NUM_INPUTS, + DEFAULT_NUM_PIPS, DEFAULT_SHOW_VU_METERS, MAX_DSK_INPUTS, MAX_NUM_INPUTS, MAX_NUM_PIPS, + MIN_NUM_INPUTS, }; use strom_types::FlowId; use strom_types::PropertyValue; +use tracing::warn; /// Parse the owning flow id, injected as `_flow_id` by block expansion. /// @@ -37,6 +39,29 @@ pub fn parse_num_dsk_inputs(properties: &HashMap) -> usiz .min(MAX_DSK_INPUTS) } +/// Parse each DSK input's alpha mode. Missing or unrecognised values are +/// straight, which is what the compositors assume; unrecognised ones are +/// logged, since the only symptom on air is a graphic that is too dark. +pub fn parse_dsk_alpha_modes( + properties: &HashMap, + num_dsk_inputs: usize, +) -> Vec { + (0..num_dsk_inputs) + .map(|i| { + let name = dsk_alpha_mode_property(i); + let parsed = match properties.get(&name) { + None => return AlphaMode::default(), + Some(PropertyValue::String(s)) => s.parse().map_err(|e| format!("{s:?}: {e}")), + Some(other) => Err(format!("{other:?} is not a string")), + }; + parsed.unwrap_or_else(|e| { + warn!("{name} = {e}; using straight"); + AlphaMode::default() + }) + }) + .collect() +} + /// Parse the number of PiP tiles from block properties. pub fn parse_num_pips(properties: &HashMap) -> usize { properties @@ -306,6 +331,28 @@ pub fn parse_u64(properties: &HashMap, key: &str, default mod tests { use super::*; + #[test] + fn dsk_alpha_modes_default_to_straight() { + let mut props = HashMap::new(); + props.insert( + "dsk_1_alpha_mode".to_string(), + PropertyValue::String("premultiplied".to_string()), + ); + props.insert( + "dsk_2_alpha_mode".to_string(), + PropertyValue::String("nonsense".to_string()), + ); + assert_eq!( + parse_dsk_alpha_modes(&props, 4), + vec![ + AlphaMode::Straight, + AlphaMode::Premultiplied, + AlphaMode::Straight, + AlphaMode::Straight, + ] + ); + } + #[test] fn parse_framerate_string_valid_fractions() { assert_eq!(parse_framerate_string("25/1"), Some((25, 1))); diff --git a/backend/src/gst/mod.rs b/backend/src/gst/mod.rs index 358152e53..b5317f5df 100644 --- a/backend/src/gst/mod.rs +++ b/backend/src/gst/mod.rs @@ -22,6 +22,7 @@ pub mod thumbnail; pub mod thumbnail_tap; pub mod transitions; pub(crate) mod underlay; +pub mod unpremultiply; pub mod video_adapt; pub mod video_frame; pub mod video_input_bridge; diff --git a/backend/src/gst/pipeline/effects/mixer_ops.rs b/backend/src/gst/pipeline/effects/mixer_ops.rs index a941dcaeb..fd8265d7f 100644 --- a/backend/src/gst/pipeline/effects/mixer_ops.rs +++ b/backend/src/gst/pipeline/effects/mixer_ops.rs @@ -242,6 +242,14 @@ impl PipelineManager { let pad_name = format!("sink_{}", num_inputs + dsk_index); if let Some(pad) = find_pad(mixer, &pad_name) { let alpha = if enabled { 1.0f64 } else { 0.0f64 }; + // A premultiplied DSK pad on the GL mixer blends with a constant + // that a binding copies from `alpha` after the write. Enabling + // sets it first, so no frame is drawn at alpha 1 with the constant + // still 0, which puts the graphic's silhouette on air in black. + // Disabling needs no help: the mixer skips pads at alpha 0. + if enabled && pad.has_property("blend-constant-color-alpha") { + pad.set_property("blend-constant-color-alpha", alpha); + } pad.set_property("alpha", alpha); // Update overlay state for DSK tracking if let Some(state) = crate::blocks::builtin::vision_mixer::overlay::get_overlay_state( diff --git a/backend/src/gst/unpremultiply.rs b/backend/src/gst/unpremultiply.rs new file mode 100644 index 000000000..488c9c4a2 --- /dev/null +++ b/backend/src/gst/unpremultiply.rs @@ -0,0 +1,257 @@ +//! `stromunpremultiply`: divides colour by alpha, in place. +//! +//! `compositor` blends straight alpha only and has no operator that handles a +//! premultiplied source, so a keyed input declared premultiplied is converted +//! to straight before it reaches the CPU mixer. The GL mixer does not need +//! this element; it corrects the blend on the pad instead. +//! +//! Registration is static and idempotent; see [`register`]. + +use std::sync::{LazyLock, OnceLock}; + +use gstreamer as gst; +use gstreamer::glib; +use gstreamer::prelude::*; +use gstreamer_base as gst_base; +use gstreamer_video as gst_video; +use tracing::warn; + +/// The GStreamer factory name. +pub const ELEMENT_NAME: &str = "stromunpremultiply"; + +glib::wrapper! { + pub struct Unpremultiply(ObjectSubclass) + @extends gst_video::VideoFilter, gst_base::BaseTransform, gst::Element, gst::Object; +} + +/// Register the element with the GStreamer registry, once per process. +/// +/// Returns whether `stromunpremultiply` is available afterwards. Callers must +/// have initialised GStreamer first. +pub fn register() -> bool { + static REGISTERED: OnceLock = OnceLock::new(); + *REGISTERED.get_or_init(|| { + // Rank NONE: built by name for premultiplied keyed inputs, never + // autoplugged. + match gst::Element::register( + None, + ELEMENT_NAME, + gst::Rank::NONE, + Unpremultiply::static_type(), + ) { + Ok(()) => true, + Err(e) => { + warn!("could not register {}: {}", ELEMENT_NAME, e); + false + } + } + }) +} + +/// `LUT[(a << 8) | c]` is `c` unpremultiplied by `a`, rounded and clamped. +/// Rows for alpha 0 and 255 are never read: those pixels are skipped. +static LUT: LazyLock> = LazyLock::new(|| { + let mut lut = vec![0u8; 256 * 256].into_boxed_slice(); + for a in 1..256usize { + for c in 0..256usize { + lut[(a << 8) | c] = ((c * 255 + a / 2) / a).min(255) as u8; + } + } + lut +}); + +/// Unpremultiply one row of packed 4-byte pixels whose alpha is at byte +/// `alpha_at` and whose colour occupies the other three. +fn unpremultiply_row(row: &mut [u8], alpha_at: usize) { + let lut = &**LUT; + for px in row.as_chunks_mut::<4>().0 { + let a = px[alpha_at] as usize; + // Fully transparent and fully opaque pixels are identical in both + // encodings, and they are most of any graphic. + if a == 0 || a == 255 { + continue; + } + let base = a << 8; + for (i, c) in px.iter_mut().enumerate() { + if i != alpha_at { + *c = lut[base | *c as usize]; + } + } + } +} + +mod imp { + use super::*; + use gstreamer::subclass::prelude::*; + use gstreamer_base::subclass::prelude::*; + use gstreamer_video::prelude::*; + use gstreamer_video::subclass::prelude::*; + use gstreamer_video::{VideoFormat, VideoFrameRef}; + + #[derive(Default)] + pub struct Unpremultiply; + + #[glib::object_subclass] + impl ObjectSubclass for Unpremultiply { + const NAME: &'static str = "StromUnpremultiply"; + type Type = super::Unpremultiply; + type ParentType = gst_video::VideoFilter; + } + + impl ObjectImpl for Unpremultiply {} + + impl GstObjectImpl for Unpremultiply {} + + impl ElementImpl for Unpremultiply { + fn metadata() -> Option<&'static gst::subclass::ElementMetadata> { + static METADATA: LazyLock = LazyLock::new(|| { + gst::subclass::ElementMetadata::new( + "Unpremultiply alpha", + "Filter/Effect/Video", + "Converts premultiplied-alpha video to straight alpha in place", + "Strom contributors", + ) + }); + Some(&*METADATA) + } + + fn pad_templates() -> &'static [gst::PadTemplate] { + static TEMPLATES: LazyLock> = LazyLock::new(|| { + // 8-bit packed RGB with alpha only. Anything else, including + // decoded YUV-with-alpha clips, needs a converter in front. + let caps = gst_video::VideoCapsBuilder::new() + .format_list([ + VideoFormat::Bgra, + VideoFormat::Rgba, + VideoFormat::Argb, + VideoFormat::Abgr, + ]) + .build(); + vec![ + gst::PadTemplate::new( + "sink", + gst::PadDirection::Sink, + gst::PadPresence::Always, + &caps, + ) + .expect("static sink template"), + gst::PadTemplate::new( + "src", + gst::PadDirection::Src, + gst::PadPresence::Always, + &caps, + ) + .expect("static src template"), + ] + }); + TEMPLATES.as_ref() + } + } + + impl BaseTransformImpl for Unpremultiply { + const MODE: gst_base::subclass::BaseTransformMode = + gst_base::subclass::BaseTransformMode::AlwaysInPlace; + const PASSTHROUGH_ON_SAME_CAPS: bool = false; + const TRANSFORM_IP_ON_PASSTHROUGH: bool = false; + } + + impl VideoFilterImpl for Unpremultiply { + fn transform_frame_ip( + &self, + frame: &mut VideoFrameRef<&mut gst::BufferRef>, + ) -> Result { + let alpha_at = match frame.format() { + VideoFormat::Bgra | VideoFormat::Rgba => 3, + VideoFormat::Argb | VideoFormat::Abgr => 0, + _ => return Err(gst::FlowError::NotNegotiated), + }; + let width = frame.width() as usize; + let height = frame.height() as usize; + let stride = frame.plane_stride()[0] as usize; + let data = frame.plane_data_mut(0).map_err(|_| gst::FlowError::Error)?; + for row in data.chunks_mut(stride).take(height) { + unpremultiply_row(&mut row[..width * 4], alpha_at); + } + Ok(gst::FlowSuccess::Ok) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use gstreamer_app as gst_app; + + #[test] + fn lut_matches_the_exact_division() { + for a in 1..255u32 { + // Premultiplied colour never exceeds its alpha. + for c in 0..=a { + let expected = ((c as f64) * 255.0 / a as f64).round() as u32; + let got = LUT[((a as usize) << 8) | c as usize] as u32; + assert!( + got.abs_diff(expected) <= 1, + "a={a} c={c}: got {got}, expected {expected}" + ); + } + } + } + + #[test] + fn row_leaves_alpha_and_the_endpoints_alone() { + // BGRA: half-alpha white, opaque grey, transparent black. + let mut row = [128, 128, 128, 128, 64, 64, 64, 255, 0, 0, 0, 0]; + unpremultiply_row(&mut row, 3); + assert_eq!(row, [255, 255, 255, 128, 64, 64, 64, 255, 0, 0, 0, 0]); + + // ARGB: alpha leads. + let mut row = [64, 32, 16, 64]; + unpremultiply_row(&mut row, 0); + assert_eq!(row, [64, 128, 64, 255]); + } + + /// Runs the registered element on one premultiplied frame per format. + #[test] + fn element_unpremultiplies_every_supported_format() { + gst::init().unwrap(); + assert!(register(), "element registers"); + + for (format, px, expected) in [ + ("BGRA", [128u8, 64, 0, 128], [255u8, 128, 0, 128]), + ("RGBA", [0, 64, 128, 128], [0, 128, 255, 128]), + ("ARGB", [128, 128, 64, 0], [128, 255, 128, 0]), + ("ABGR", [128, 0, 64, 128], [128, 0, 128, 255]), + ] { + let pipeline = gst::parse::launch(&format!( + "appsrc name=src format=time caps=video/x-raw,format={format},width=4,height=2,framerate=30/1 \ + ! {ELEMENT_NAME} ! appsink name=sink sync=false" + )) + .unwrap() + .downcast::() + .unwrap(); + let src = pipeline + .by_name("src") + .unwrap() + .downcast::() + .unwrap(); + let sink = pipeline + .by_name("sink") + .unwrap() + .downcast::() + .unwrap(); + pipeline.set_state(gst::State::Playing).unwrap(); + + let frame: Vec = px.iter().copied().cycle().take(4 * 4 * 2).collect(); + src.push_buffer(gst::Buffer::from_slice(frame)).unwrap(); + let sample = sink + .try_pull_sample(gst::ClockTime::from_seconds(5)) + .unwrap_or_else(|| panic!("{format}: no output")); + let map = sample.buffer().unwrap().map_readable().unwrap(); + for out in map.as_chunks::<4>().0 { + assert_eq!(*out, expected, "{format}"); + } + drop(map); + pipeline.set_state(gst::State::Null).unwrap(); + } + } +} diff --git a/backend/tests/vision_mixer_premultiplied_alpha_test.rs b/backend/tests/vision_mixer_premultiplied_alpha_test.rs new file mode 100644 index 000000000..d18eafd43 --- /dev/null +++ b/backend/tests/vision_mixer_premultiplied_alpha_test.rs @@ -0,0 +1,509 @@ +//! Regression test: a DSK input declared premultiplied composites correctly +//! on both vision mixer backends. +//! +//! `compositor` and `glvideomixerelement` blend straight alpha. A premultiplied +//! source (every `cefsrc` HTML graphic) blended as straight is scaled by its +//! alpha twice: 50 % white over grey 64 comes out 96, where the right answer +//! is 160. Grey matters: against black or white, clamping hides the error. +//! +//! CI has no `cefsrc`, so a `videotestsrc` paints the premultiplied encoding +//! of 50 % white, `(128, 128, 128, 128)`, into the real block. + +pub mod common; + +use gstreamer::prelude::*; +use serial_test::serial; +use std::collections::HashMap; +use std::time::{Duration, Instant}; +use strom::blocks::BlockRegistry; +use strom::events::EventBroadcaster; +use strom::gst::pipeline::PipelineManager; +use strom_types::{Flow, PropertyValue as PV}; +use tempfile::NamedTempFile; + +const W: usize = 320; +const H: usize = 180; +const NUM_INPUTS: usize = 2; +/// The DSK pad on the dist mixer: after the video inputs. +const DSK_PAD: &str = "sink_2"; + +/// Premultiplied 50 % white over grey 64 at pad alpha 1. +const CORRECT_FULL: f64 = 160.0; +/// The same at pad alpha 0.5: 255 * 0.25 + 64 * 0.75. +const CORRECT_HALF: f64 = 112.0; + +fn elem(id: &str, ty: &str, props: Vec<(&str, PV)>) -> strom_types::Element { + strom_types::Element { + id: id.to_string(), + element_type: ty.to_string(), + properties: props.into_iter().map(|(k, v)| (k.to_string(), v)).collect(), + position: [0.0, 0.0].into(), + pad_properties: HashMap::new(), + } +} + +fn raw_caps(format: &str) -> PV { + PV::String(format!( + "video/x-raw,format={},width={},height={},framerate=30/1", + format, W, H + )) +} + +/// What feeds DSK 0. +enum DskSource { + /// A live `videotestsrc` painting premultiplied 50 % white as BGRA. + TestPattern, + /// A media player block looping the clip at this path. + MediaPlayer(std::path::PathBuf), +} + +/// Flow: grey 64 on `video_in_0` (the initial program), the premultiplied +/// source on `dsk_in_0`, and program out converted to RGBA into an appsink. +/// `alpha_mode: None` leaves the property unset, as existing flows have it. +fn build_flow(backend: &str, block_id: &str, alpha_mode: Option<&str>, dsk: DskSource) -> Flow { + let mut flow = Flow::new(format!("vm_premul_{}", block_id)); + flow.blocks.push(strom_types::BlockInstance { + id: block_id.to_string(), + block_definition_id: "builtin.vision_mixer".to_string(), + name: None, + properties: { + let mut p = HashMap::new(); + p.insert( + "compositor_preference".to_string(), + PV::String(backend.to_string()), + ); + // The measurement tap is a plain videoconvert, which cannot take + // GL memory. + p.insert("gl_download".to_string(), PV::Bool(backend == "gpu")); + p.insert("num_inputs".to_string(), PV::UInt(NUM_INPUTS as u64)); + p.insert("num_dsk_inputs".to_string(), PV::String("1".to_string())); + if let Some(mode) = alpha_mode { + p.insert("dsk_0_alpha_mode".to_string(), PV::String(mode.to_string())); + } + p.insert( + "pgm_resolution".to_string(), + PV::String(format!("{}x{}", W, H)), + ); + p.insert( + "multiview_resolution".to_string(), + PV::String(format!("{}x{}", W, H)), + ); + p + }, + position: strom_types::block::Position { x: 100.0, y: 100.0 }, + runtime_data: None, + computed_external_pads: None, + }); + + flow.elements.push(elem( + "bg", + "videotestsrc", + vec![ + ("pattern", PV::String("solid-color".into())), + ("foreground-color", PV::UInt(0xff40_4040)), + ("is-live", PV::Bool(true)), + ], + )); + flow.elements.push(elem( + "bgcaps", + "capsfilter", + vec![("caps", raw_caps("BGRA"))], + )); + + let dsk_out = match dsk { + DskSource::TestPattern => { + flow.elements.push(elem( + "gfx", + "videotestsrc", + vec![ + ("pattern", PV::String("solid-color".into())), + // ARGB 0x80808080: every BGRA byte is 128. + ("foreground-color", PV::UInt(0x8080_8080)), + ("is-live", PV::Bool(true)), + ], + )); + flow.elements.push(elem( + "gfxcaps", + "capsfilter", + vec![("caps", raw_caps("BGRA"))], + )); + flow.links.push(strom_types::Link { + from: "gfx:src".to_string(), + to: "gfxcaps:sink".to_string(), + }); + "gfxcaps:src".to_string() + } + DskSource::MediaPlayer(path) => { + flow.blocks.push(strom_types::BlockInstance { + id: "clip".to_string(), + block_definition_id: "builtin.media_player".to_string(), + name: None, + properties: { + let mut p = HashMap::new(); + p.insert("decode".to_string(), PV::Bool(true)); + p.insert("loop_playlist".to_string(), PV::Bool(true)); + p.insert( + "playlist".to_string(), + PV::String(serde_json::to_string(&[path]).unwrap()), + ); + p + }, + position: strom_types::block::Position { x: 0.0, y: 0.0 }, + runtime_data: None, + computed_external_pads: None, + }); + "clip:video_out".to_string() + } + }; + + flow.elements.push(elem("pgmconv", "videoconvert", vec![])); + flow.elements.push(elem( + "pgmcaps", + "capsfilter", + vec![("caps", PV::String("video/x-raw,format=RGBA".into()))], + )); + flow.elements.push(elem( + "pgmsink", + "appsink", + vec![ + ("sync", PV::Bool(false)), + ("max-buffers", PV::UInt(1)), + ("drop", PV::Bool(true)), + ], + )); + + // DSK pads are dynamic (num_dsk_inputs), so the flow needs the computed + // pad set the API normally fills in before validation. + for block in &mut flow.blocks { + if let Some(builder) = strom::blocks::builtin::get_builder(&block.block_definition_id) { + block.computed_external_pads = builder.get_external_pads(&block.properties); + } + } + + for (from, to) in [ + ("bg:src".to_string(), "bgcaps:sink".to_string()), + ("bgcaps:src".to_string(), format!("{}:video_in_0", block_id)), + (dsk_out, format!("{}:dsk_in_0", block_id)), + (format!("{}:pgm_out", block_id), "pgmconv:sink".to_string()), + ("pgmconv:src".to_string(), "pgmcaps:sink".to_string()), + ("pgmcaps:src".to_string(), "pgmsink:sink".to_string()), + ] { + flow.links.push(strom_types::Link { from, to }); + } + flow +} + +/// Mean red channel over the centre quarter of an RGBA frame. The scene is +/// grey, so red stands for all three. +fn centre_red(sample: &gstreamer::Sample) -> f64 { + use gstreamer_video::{VideoFormat, VideoFrameRef, VideoInfo}; + let info = VideoInfo::from_caps(sample.caps().expect("caps")).expect("video info"); + assert_eq!( + info.format(), + VideoFormat::Rgba, + "PGM sample should be RGBA" + ); + let frame = VideoFrameRef::from_buffer_ref_readable(sample.buffer().expect("buffer"), &info) + .expect("map frame"); + let stride = info.stride()[0] as usize; + let data = frame.plane_data(0).expect("plane 0"); + let (mut sum, mut n) = (0u64, 0u64); + for y in (H / 4)..(3 * H / 4) { + for x in (W / 4)..(3 * W / 4) { + sum += data[y * stride + x * 4] as u64; + n += 1; + } + } + sum as f64 / n as f64 +} + +/// A running flow and the glib main loop it needs: the media player starts +/// its internal pipeline from a bus handler on the default main context when +/// the flow reaches PLAYING, which only runs while a main loop does. The server +/// has one. +struct Running { + manager: PipelineManager, + block_id: String, + sink: gstreamer_app::AppSink, + main_loop: gstreamer::glib::MainLoop, + main_loop_thread: std::thread::JoinHandle<()>, +} + +impl Running { + fn start(backend: &str, block_id: &str, alpha_mode: Option<&str>, dsk: DskSource) -> Self { + gstreamer::init().unwrap(); + // The CPU builder picks its videoconvert factory from the detected + // GPU capabilities; without this the first lookup panics. + strom::gpu::detect_gpu_capabilities(); + + let main_loop = gstreamer::glib::MainLoop::new(None, false); + let main_loop_thread = { + let ml = main_loop.clone(); + std::thread::spawn(move || ml.run()) + }; + let temp_file = NamedTempFile::new().unwrap(); + let registry = BlockRegistry::new(temp_file.path()); + let flow = build_flow(backend, block_id, alpha_mode, dsk); + let mut manager = PipelineManager::new( + &flow, + EventBroadcaster::new(10, false, false), + ®istry, + vec![], + "all".to_string(), + None, + std::env::temp_dir(), + std::sync::Arc::new(std::sync::Mutex::new(HashMap::new())), + ) + .unwrap_or_else(|e| panic!("build {} pipeline: {}", backend, e)); + manager + .start() + .unwrap_or_else(|e| panic!("start {} pipeline: {}", backend, e)); + + let sink = manager + .pipeline() + .by_name("pgmsink") + .expect("pgmsink in pipeline") + .downcast::() + .expect("appsink type"); + Running { + manager, + block_id: block_id.to_string(), + sink, + main_loop, + main_loop_thread, + } + } + + fn mixer_id(&self) -> String { + format!("{}:mixer", self.block_id) + } + + /// Pull frames until three in a row read within `tol` of `expected`, or + /// until the deadline. Returns the last reading either way, so a failure + /// reports what was actually on air. + fn settle_on(&self, expected: f64, tol: f64) -> f64 { + let deadline = Instant::now() + Duration::from_secs(20); + let (mut last, mut streak) = (f64::NAN, 0); + while Instant::now() < deadline && streak < 3 { + let Some(sample) = self + .sink + .try_pull_sample(gstreamer::ClockTime::from_mseconds(500)) + else { + continue; + }; + last = centre_red(&sample); + streak = if (last - expected).abs() <= tol { + streak + 1 + } else { + 0 + }; + } + last + } + + fn set_pad_alpha(&self, alpha: f64) { + self.manager + .update_pad_property(&self.mixer_id(), DSK_PAD, "alpha", &PV::Float(alpha)) + .expect("set DSK pad alpha"); + } + + fn stop(mut self) { + self.manager.stop().expect("stop"); + self.main_loop.quit(); + self.main_loop_thread.join().expect("main loop thread"); + } +} + +fn assert_near(what: &str, got: f64, expected: f64, tol: f64) { + assert!( + (got - expected).abs() <= tol, + "{what}: got {got:.1}, expected {expected:.0} +/- {tol:.0} \ + (premultiplied blended as straight reads 96 at full alpha, 80 at half)" + ); +} + +/// Full and half pad alpha on one running flow, against the correct results +/// `(full, half)` for this source. +fn assert_correct_at_full_and_half_alpha( + backend: &str, + block_id: &str, + dsk: DskSource, + (full_expected, half_expected): (f64, f64), +) { + const TOL: f64 = 3.0; + let run = Running::start(backend, block_id, Some("premultiplied"), dsk); + run.manager + .set_dsk_enabled(block_id, 0, NUM_INPUTS, true) + .expect("enable DSK 0"); + + let full = run.settle_on(full_expected, TOL); + eprintln!("{backend} {block_id}: pad alpha 1.0 -> {full:.1}"); + assert_near("pad alpha 1.0", full, full_expected, TOL); + + // Through the property API, as PATCH on the pad does. + run.set_pad_alpha(0.5); + let half = run.settle_on(half_expected, TOL); + eprintln!("{backend} {block_id}: pad alpha 0.5 -> {half:.1}"); + assert_near("pad alpha 0.5", half, half_expected, TOL); + + run.stop(); +} + +// Every case here renders a flow and probes program frames; the GPU ones render +// through llvmpipe in CI, which costs whole cores. Serialised so a sibling test +// cannot starve the pipeline under measurement. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(gl)] +async fn cpu_premultiplied_dsk_composites_correctly() { + assert_correct_at_full_and_half_alpha( + "cpu", + "vmp_cpu", + DskSource::TestPattern, + (CORRECT_FULL, CORRECT_HALF), + ); +} + +/// The declaration is what changes the result: the same source with no alpha +/// mode set still composites as straight. Guards the default, so existing +/// flows with straight graphics are not unpremultiplied behind their back. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(gl)] +async fn cpu_default_mode_leaves_the_source_alone() { + let run = Running::start("cpu", "vmp_cpu_default", None, DskSource::TestPattern); + run.manager + .set_dsk_enabled("vmp_cpu_default", 0, NUM_INPUTS, true) + .expect("enable DSK 0"); + let got = run.settle_on(96.0, 3.0); + eprintln!("cpu default mode: pad alpha 1.0 -> {got:.1}"); + assert!( + (got - 96.0).abs() <= 3.0, + "with no alpha mode the source should blend as-is (96), got {got:.1}" + ); + run.stop(); +} + +/// Write a premultiplied 50 % white clip, encoded as A420 so the decoded +/// frames are YUV with alpha and have to be converted before unpremultiplying. +fn write_premultiplied_clip(path: &std::path::Path) -> Result<(), String> { + for factory in ["avenc_ffv1", "matroskamux"] { + if gstreamer::ElementFactory::find(factory).is_none() { + return Err(format!("{factory} missing")); + } + } + let pipeline = gstreamer::parse::launch(&format!( + "videotestsrc num-buffers=30 pattern=solid-color foreground-color=0x80808080 \ + ! video/x-raw,format=BGRA,width={W},height={H},framerate=30/1 \ + ! videoconvert ! video/x-raw,format=A420 ! avenc_ffv1 ! matroskamux \ + ! filesink location={}", + path.display() + )) + .map_err(|e| e.to_string())?; + pipeline + .set_state(gstreamer::State::Playing) + .map_err(|e| e.to_string())?; + let bus = pipeline.bus().expect("bus"); + let msg = bus.timed_pop_filtered( + gstreamer::ClockTime::from_seconds(30), + &[gstreamer::MessageType::Eos, gstreamer::MessageType::Error], + ); + let _ = pipeline.set_state(gstreamer::State::Null); + match msg.as_ref().map(|m| m.view()) { + Some(gstreamer::MessageView::Eos(_)) => Ok(()), + other => Err(format!("clip encode did not finish: {other:?}")), + } +} + +/// A premultiplied clip through the media player: the decoder hands out A420, +/// which the unpremultiply element cannot take directly. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(gl)] +async fn cpu_premultiplied_clip_through_media_player() { + gstreamer::init().unwrap(); + let dir = tempfile::tempdir().unwrap(); + let clip = dir.path().join("premultiplied.mkv"); + // gst-libav and matroska are in CI's package list; failing here rather + // than skipping keeps the test from passing green without running. + write_premultiplied_clip(&clip).expect("write premultiplied clip"); + + // videoconvert's RGB -> A420 -> RGB round trip turns the clip's 128 into + // 126, which unpremultiplies to 251: 157.5 at full alpha, 110.4 at half. + // Blended as straight it would read 95. + assert_correct_at_full_and_half_alpha( + "cpu", + "vmp_clip", + DskSource::MediaPlayer(clip), + (157.5, 110.4), + ); +} + +const GL_ELEMENTS: &[&str] = &["glvideomixerelement", "gltestsrc"]; + +/// GL corrects the blend on the pad rather than converting the source, and +/// the correction depends on a blend constant that has to track pad alpha. +/// Half alpha is the case that catches a constant left behind. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(gl)] +async fn gpu_premultiplied_dsk_composites_correctly() { + if !common::gl_available(GL_ELEMENTS) { + return; + } + assert_correct_at_full_and_half_alpha( + "gpu", + "vmp_gpu", + DskSource::TestPattern, + (CORRECT_FULL, CORRECT_HALF), + ); +} + +/// Fade to black drives pad alpha from a control binding, frame by frame, +/// not through `set_property` from Strom code. The blend constant has to +/// follow that too. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial(gl)] +async fn gpu_blend_constant_follows_fade_to_black() { + if !common::gl_available(GL_ELEMENTS) { + return; + } + let block_id = "vmp_gpu_ftb"; + let run = Running::start( + "gpu", + block_id, + Some("premultiplied"), + DskSource::TestPattern, + ); + run.manager + .set_dsk_enabled(block_id, 0, NUM_INPUTS, true) + .expect("enable DSK 0"); + let full = run.settle_on(CORRECT_FULL, 3.0); + assert_near("pad alpha 1.0 before the fade", full, CORRECT_FULL, 3.0); + + let pad = run + .manager + .pipeline() + .by_name(&run.mixer_id()) + .expect("mixer") + .sink_pads() + .into_iter() + .find(|p| p.name() == DSK_PAD) + .expect("DSK pad"); + assert_eq!(pad.property::("blend-constant-color-alpha"), 1.0); + + // During the fade the only writer of `alpha` is the control binding. + assert!(run.manager.fade_to_black(block_id, 300).expect("FTB on")); + let deadline = Instant::now() + Duration::from_secs(10); + while pad.property::("alpha") > 0.0 && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(20)); + } + assert_eq!( + pad.property::("alpha"), + 0.0, + "fade never reached black" + ); + assert_eq!( + pad.property::("blend-constant-color-alpha"), + 0.0, + "blend constant did not follow the animated pad alpha" + ); + + run.stop(); +} diff --git a/frontend/src/properties.rs b/frontend/src/properties.rs index 621adfee1..ab2ff6c64 100644 --- a/frontend/src/properties.rs +++ b/frontend/src/properties.rs @@ -1463,10 +1463,13 @@ impl PropertyInspector { result } - /// Skip per-input labels for slots beyond the configured `num_inputs` - /// so the property panel shows just the inputs in use. + /// Skip per-input labels and per-DSK alpha modes for slots beyond the + /// configured counts so the property panel shows just the inputs in use. fn vision_mixer_skip_set(block: &BlockInstance) -> std::collections::HashSet { - use strom_types::vision_mixer::{DEFAULT_NUM_INPUTS, MAX_NUM_INPUTS}; + use strom_types::vision_mixer::{ + dsk_alpha_mode_property, DEFAULT_DSK_INPUTS, DEFAULT_NUM_INPUTS, MAX_DSK_INPUTS, + MAX_NUM_INPUTS, + }; let num_inputs = block .properties @@ -1479,10 +1482,24 @@ impl PropertyInspector { }) .unwrap_or(DEFAULT_NUM_INPUTS); + let num_dsk_inputs = block + .properties + .get("num_dsk_inputs") + .and_then(|v| match v { + PropertyValue::String(s) => s.parse().ok(), + PropertyValue::UInt(n) => Some(*n as usize), + PropertyValue::Int(n) => Some(*n as usize), + _ => None, + }) + .unwrap_or(DEFAULT_DSK_INPUTS); + let mut skip = std::collections::HashSet::new(); for i in num_inputs..MAX_NUM_INPUTS { skip.insert(format!("input_{}_label", i)); } + for i in num_dsk_inputs..MAX_DSK_INPUTS { + skip.insert(dsk_alpha_mode_property(i)); + } skip } diff --git a/types/src/vision_mixer.rs b/types/src/vision_mixer.rs index a6478dc9b..456417cac 100644 --- a/types/src/vision_mixer.rs +++ b/types/src/vision_mixer.rs @@ -114,6 +114,64 @@ pub const MAX_DSK_INPUTS: usize = 4; /// Default number of DSK inputs (0 = no DSK). pub const DEFAULT_DSK_INPUTS: usize = 0; +/// How a keyed (DSK) input encodes its alpha. +/// +/// Caps cannot say whether colour is already multiplied by alpha, so it is +/// declared per input. `cefsrc` (HTML graphics) emits premultiplied unless its +/// `unpremultiply` property is set. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum AlphaMode { + /// Colour is independent of alpha. What the compositors assume. + #[default] + Straight, + /// Colour has already been multiplied by alpha. + Premultiplied, +} + +impl AlphaMode { + /// Property value, as stored on the block. + pub fn as_str(self) -> &'static str { + match self { + AlphaMode::Straight => "straight", + AlphaMode::Premultiplied => "premultiplied", + } + } +} + +impl fmt::Display for AlphaMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Error returned when an [`AlphaMode`] string cannot be parsed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParseAlphaModeError; + +impl fmt::Display for ParseAlphaModeError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("expected 'straight' or 'premultiplied'") + } +} + +impl std::error::Error for ParseAlphaModeError {} + +impl FromStr for AlphaMode { + type Err = ParseAlphaModeError; + fn from_str(s: &str) -> Result { + match s.trim().to_ascii_lowercase().as_str() { + "straight" => Ok(AlphaMode::Straight), + "premultiplied" => Ok(AlphaMode::Premultiplied), + _ => Err(ParseAlphaModeError), + } + } +} + +/// Block property declaring DSK input `dsk_index`'s [`AlphaMode`] (0-based). +pub fn dsk_alpha_mode_property(dsk_index: usize) -> String { + format!("dsk_{}_alpha_mode", dsk_index) +} + /// Maximum number of PiP (Picture-in-Picture) tiles rendered virtually in the multiview. /// Each PiP consumes one tile in the multiview thumbnail grid alongside the inputs. pub const MAX_NUM_PIPS: usize = 4; @@ -834,6 +892,16 @@ pub fn compute_pip_overlay_rects( mod tests { use super::*; + #[test] + fn alpha_mode_round_trips_through_its_property_value() { + for mode in [AlphaMode::Straight, AlphaMode::Premultiplied] { + assert_eq!(mode.as_str().parse::(), Ok(mode)); + } + assert_eq!(" Premultiplied ".parse(), Ok(AlphaMode::Premultiplied)); + assert_eq!("".parse::(), Err(ParseAlphaModeError)); + assert_eq!(dsk_alpha_mode_property(2), "dsk_2_alpha_mode"); + } + #[test] fn test_pip_overlay_rects_one_full_aspect() { // 1 source in 1920×1080 with 16:9 aspect → fills the whole container.