Make the IDR interval configurable

InstantReplay.Externals/unienc/crates/unienc_android_mc/src/common.rs

@@ -2,7 +2,7 @@ use bincode::{Decode, Encode};
 use jni::{
     JNIEnv,
     objects::{JObject, JString},
-    sys::{jint, jlong},
+    sys::{jfloat, jint, jlong},
 };
 use std::future::Future;
 use std::pin::Pin;
@@ -496,6 +496,18 @@ pub fn set_format_integer(
     Ok(())
 }
 
+/// Set float parameter on MediaFormat
+pub fn set_format_float(
+    env: &mut JNIEnv,
+    format: &JObject,
+    key: &str,
+    value: jfloat,
+) -> Result<()> {
+    let key_str = to_java_string(env, key)?;
+    bindings::MediaFormat::set_float(env, format, &key_str, value)?;
+    Ok(())
+}
+
 #[derive(Encode, Decode)]
 pub struct CommonEncodedData {
     pub content: CommonEncodedDataContent,

InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs

@@ -1,4 +1,7 @@
-use jni::{JNIEnv, sys::jint};
+use jni::{
+    JNIEnv,
+    sys::{jfloat, jint},
+};
 use std::sync::Arc;
 use std::time::Duration;
 use unienc_common::{
@@ -35,6 +38,7 @@ struct UninitializedState {
     tx: tokio::sync::oneshot::Sender<()>,
     bitrate: u32,
     fps_hint: u32,
+    idr_interval_seconds: f32,
 }
 
 enum MediaCodecVideoEncoderInputProcessor {
@@ -138,6 +142,7 @@ impl<R: unienc_common::Runtime + 'static> MediaCodecVideoEncoder<R> {
                         tx,
                         bitrate: options.bitrate(),
                         fps_hint: options.fps_hint(),
+                        idr_interval_seconds: options.idr_interval_seconds(),
                     },
                 ),
                 runtime: runtime.clone(),
@@ -186,6 +191,7 @@ async fn push_video_impl<R: unienc_common::Runtime + 'static>(
                         this.padded_height,
                         state.bitrate,
                         state.fps_hint,
+                        state.idr_interval_seconds,
                         false, // use_surface = false for buffer mode
                     )?;
                     this.codec.configure(&format)?;
@@ -272,6 +278,7 @@ async fn push_video_impl<R: unienc_common::Runtime + 'static>(
                     this.padded_height,
                     state.bitrate,
                     state.fps_hint,
+                    state.idr_interval_seconds,
                     true, // use_surface = true for hardware buffer mode
                 )?;
                 this.codec.configure(&format)?;
@@ -372,6 +379,7 @@ fn create_video_format_raw(
     padded_height: u32,
     bitrate: u32,
     fps_hint: u32,
+    idr_interval_seconds: f32,
     use_surface: bool,
 ) -> Result<SafeGlobalRef> {
     let mime = to_java_string(env, MIME_TYPE_VIDEO_AVC)?;
@@ -396,7 +404,14 @@ fn create_video_format_raw(
 
     set_format_integer(env, &format_obj, KEY_BITRATE, bitrate as jint)?;
     set_format_integer(env, &format_obj, KEY_FRAME_RATE, fps_hint as jint)?;
-    set_format_integer(env, &format_obj, KEY_I_FRAME_INTERVAL, 1)?;
+    // MediaFormat.KEY_I_FRAME_INTERVAL accepts a float since API 25, and `min_api` is 26, so the
+    // sub-second range is available without a version guard.
+    set_format_float(
+        env,
+        &format_obj,
+        KEY_I_FRAME_INTERVAL,
+        idr_interval_seconds as jfloat,
+    )?;
 
     set_format_integer(env, &format_obj, KEY_PRIORITY, 0)?;
     set_format_integer(env, &format_obj, KEY_OPERATING_RATE, fps_hint as jint)?;

InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs

@@ -14,7 +14,8 @@ use objc2_core_video::{CVPixelBuffer, CVPixelBufferCreateWithBytes, kCVPixelForm
 use objc2_video_toolbox::{
     VTCompressionSession, VTEncodeInfoFlags, VTSessionSetProperty,
     kVTCompressionPropertyKey_AllowFrameReordering, kVTCompressionPropertyKey_AverageBitRate,
-    kVTCompressionPropertyKey_RealTime, kVTInvalidSessionErr,
+    kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, kVTCompressionPropertyKey_RealTime,
+    kVTInvalidSessionErr,
 };
 use tokio::sync::mpsc;
 use unienc_common::{
@@ -44,6 +45,7 @@ pub struct VideoToolboxEncoderInput {
     width: u32,
     height: u32,
     bitrate: u32,
+    idr_interval_seconds: f32,
 }
 
 struct CompressionSession {
@@ -279,7 +281,12 @@ impl EncoderInput for VideoToolboxEncoderInput {
                 // VTCompressionSession turns invalid when the app enters background on iOS; retry
                 // once with a fresh session, re-passing the same reserved permit.
                 retry += 1;
-                match CompressionSession::new(self.width, self.height, self.bitrate) {
+                match CompressionSession::new(
+                    self.width,
+                    self.height,
+                    self.bitrate,
+                    self.idr_interval_seconds,
+                ) {
                     Ok(session) => {
                         self.session = session;
                         continue;
@@ -335,7 +342,7 @@ impl Drop for VideoToolboxEncoderInput {
 }
 
 impl CompressionSession {
-    fn new(width: u32, height: u32, bitrate: u32) -> Result<Self> {
+    fn new(width: u32, height: u32, bitrate: u32, idr_interval_seconds: f32) -> Result<Self> {
         let mut session: *mut VTCompressionSession = std::ptr::null_mut();
 
         unsafe {
@@ -382,6 +389,14 @@ impl CompressionSession {
             )
         }
         .to_result()?;
+        unsafe {
+            VTSessionSetProperty(
+                &session,
+                kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration,
+                Some(&CFNumber::new_f64(idr_interval_seconds as f64)),
+            )
+        }
+        .to_result()?;
 
         Ok(CompressionSession { inner: session })
     }
@@ -393,14 +408,16 @@ impl VideoToolboxEncoder {
         let tx = Box::new(tx);
 
         let (width, height, bitrate) = (options.width(), options.height(), options.bitrate());
+        let idr_interval_seconds = options.idr_interval_seconds();
 
         Ok(VideoToolboxEncoder {
             input: VideoToolboxEncoderInput {
-                session: CompressionSession::new(width, height, bitrate)?,
+                session: CompressionSession::new(width, height, bitrate, idr_interval_seconds)?,
                 tx,
                 width,
                 height,
                 bitrate,
+                idr_interval_seconds,
             },
             output: VideoToolboxEncoderOutput { rx },
         })

InstantReplay.Externals/unienc/crates/unienc_c/src/types.rs

@@ -26,6 +26,9 @@ pub struct VideoEncoderOptionsNative {
     pub height: u32,
     pub fps_hint: u32,
     pub bitrate: u32,
+    /// Maximum interval between IDR (key) frames, in seconds. Required; the managed side rejects
+    /// non-positive and non-finite values before the struct crosses the boundary.
+    pub idr_interval_seconds: f32,
 }
 
 #[repr(C)]
@@ -52,6 +55,10 @@ impl VideoEncoderOptions for VideoEncoderOptionsNative {
     fn bitrate(&self) -> u32 {
         self.bitrate
     }
+
+    fn idr_interval_seconds(&self) -> f32 {
+        self.idr_interval_seconds
+    }
 }
 
 impl AudioEncoderOptions for AudioEncoderOptionsNative {

InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs

@@ -95,6 +95,11 @@ pub trait VideoEncoderOptions: Clone + Copy {
     fn height(&self) -> u32;
     fn fps_hint(&self) -> u32;
     fn bitrate(&self) -> u32;
+
+    /// Maximum interval between IDR (key) frames, in seconds. Must be finite and greater than
+    /// zero; every platform encoder is configured with it, so there is no "leave it to the
+    /// platform" case to represent.
+    fn idr_interval_seconds(&self) -> f32;
 }
 
 pub trait AudioEncoderOptions: Clone + Copy {

InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs

@@ -132,6 +132,8 @@ impl<R: Runtime + 'static> FFmpegVideoEncoder<R> {
         let height = options.height();
         let cfr = options.fps_hint();
 
+        let idr_interval_seconds = options.idr_interval_seconds();
+
         // encode raw BGRA frames into H.264 stream
         let mut spawned = ffmpeg::Builder::new()
             .use_stdin(true)
@@ -166,7 +168,7 @@ impl<R: Runtime + 'static> FFmpegVideoEncoder<R> {
                     "-b:v",
                     &format!("{}", options.bitrate()),
                     "-force_key_frames",
-                    "expr:gte(t,n_forced*1)",
+                    &format!("expr:gte(t,n_forced*{idr_interval_seconds})"),
                 ],
                 ffmpeg::Destination::Stdout,
             )?;

InstantReplay.Externals/unienc/crates/unienc_testkit/src/options.rs

@@ -1,11 +1,16 @@
 use crate::e2e::E2eConfig;
 
+/// Interval the backends applied before it became configurable, kept so that the harness exercises
+/// the same key-frame spacing it always has.
+const DEFAULT_IDR_INTERVAL_SECONDS: f32 = 1.0;
+
 #[derive(Debug, Clone, Copy)]
 pub struct TestVideoOptions {
     pub width: u32,
     pub height: u32,
     pub fps_hint: u32,
     pub bitrate: u32,
+    pub idr_interval_seconds: f32,
 }
 
 impl From<&E2eConfig> for TestVideoOptions {
@@ -15,6 +20,7 @@ impl From<&E2eConfig> for TestVideoOptions {
             height: config.height,
             fps_hint: config.fps,
             bitrate: config.video_bitrate,
+            idr_interval_seconds: DEFAULT_IDR_INTERVAL_SECONDS,
         }
     }
 }
@@ -35,6 +41,10 @@ impl unienc_common::VideoEncoderOptions for TestVideoOptions {
     fn bitrate(&self) -> u32 {
         self.bitrate
     }
+
+    fn idr_interval_seconds(&self) -> f32 {
+        self.idr_interval_seconds
+    }
 }
 
 #[derive(Debug, Clone, Copy)]

InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs

@@ -20,6 +20,7 @@ pub struct WebCodecsVideoEncoderInput<R: Runtime> {
     fps_hint: f64,
     tx: mpsc::Sender<VideoEncodedData>,
     prev_key_timestamp: Option<f64>,
+    idr_interval_seconds: f64,
     runtime: R,
 }
 
@@ -49,6 +50,8 @@ impl<R: Runtime> WebCodecsVideoEncoder<R> {
                 encoder_handle: None,
                 tx,
                 prev_key_timestamp: None,
+                // WebCodecs has no GOP-length knob, so keyframes are requested per frame.
+                idr_interval_seconds: options.idr_interval_seconds() as f64,
                 runtime: runtime.clone(),
             },
             output: WebCodecsVideoEncoderOutput { rx },
@@ -108,17 +111,12 @@ impl<R: Runtime + 'static> EncoderInput for WebCodecsVideoEncoderInput<R> {
             Some(prev) => data.timestamp - prev,
             None => f64::INFINITY,
         };
-        if since_prev_key >= 1.0 {
+        let is_key = since_prev_key >= self.idr_interval_seconds;
+        if is_key {
             self.prev_key_timestamp = Some(data.timestamp);
         }
         encoder_handle
-            .push_video_frame(
-                pixels,
-                frame.width,
-                frame.height,
-                data.timestamp,
-                since_prev_key >= 1.0,
-            )
+            .push_video_frame(pixels, frame.width, frame.height, data.timestamp, is_key)
             .context("Failed to push video frame to WebCodecs EncoderHandle")?;
         Ok(())
     }

InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/audio/mod.rs

@@ -53,6 +53,7 @@ impl MediaFoundationAudioEncoder {
             },
             input_type,
             output_type,
+            &|_| {},
             runtime,
         )?;
 

InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mft.rs

@@ -239,12 +239,16 @@ enum Pipeline {
 }
 
 impl Transform {
+    /// `configure` runs on each candidate MFT after it is activated but before the media types are
+    /// negotiated. It is for optional codec properties: anything it fails to apply must leave the
+    /// transform usable, so it cannot report an error.
     pub fn new(
         category: windows_core::GUID,
         input: MFT_REGISTER_TYPE_INFO,
         output: MFT_REGISTER_TYPE_INFO,
         input_type: IMFMediaType,
         output_type: IMFMediaType,
+        configure: &dyn Fn(&IMFTransform),
         runtime: &impl Runtime,
     ) -> Result<(Self, mpsc::Receiver<UnsafeSend<IMFSample>>)> {
         let mfts = MftIter::new(category, input, output);
@@ -259,7 +263,13 @@ impl Transform {
                 println!("Skipping MFT: {}", Self::get_name(&activate)?);
                 continue;
             }
-            match Self::try_activate(activate, &mut input_type, &mut output_type, runtime) {
+            match Self::try_activate(
+                activate,
+                &mut input_type,
+                &mut output_type,
+                configure,
+                runtime,
+            ) {
                 Ok(r) => {
                     result = Some(r);
                 }
@@ -290,13 +300,16 @@ impl Transform {
         activate: IMFActivate,
         input_type: &mut Option<IMFMediaType>,
         output_type: &mut Option<IMFMediaType>,
+        configure: &dyn Fn(&IMFTransform),
         runtime: &impl Runtime,
     ) -> Result<(Self, mpsc::Receiver<UnsafeSend<IMFSample>>)> {
         println!("Trying MFT: {}", Self::get_name(&activate)?);
 
         let is_async = unsafe { activate.GetUINT32(&MF_TRANSFORM_ASYNC) }.unwrap_or(0) != 0;
         let transform = unsafe { activate.ActivateObject::<IMFTransform>()? };
 
+        configure(&transform);
+
         if is_async {
             let attributes = unsafe { transform.GetAttributes()? };
             unsafe { attributes.SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1)? };

InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs

@@ -6,6 +6,8 @@ use unienc_common::{
     UnsupportedBlitData, VideoEncoderOptions, VideoFrame, VideoSample,
 };
 use windows::Win32::Media::MediaFoundation::*;
+use windows::Win32::System::Variant::{VARIANT, VT_UI4};
+use windows::core::Interface;
 
 use crate::common::*;
 use crate::mft::Transform;
@@ -48,6 +50,29 @@ impl MediaFoundationVideoEncoder {
             output_type
         };
 
+        // Media Foundation expresses the GOP length in frames, so the interval has to be resolved
+        // against the frame rate hint. `CODECAPI_AVEncMPVGOPSize` is optional for an MFT to
+        // implement, so an encoder that rejects it keeps its own GOP length rather than failing to
+        // activate.
+        let gop_size = ((options.idr_interval_seconds() as f64 * options.fps_hint() as f64).round()
+            as u32)
+            .max(1);
+        let configure = |transform: &IMFTransform| {
+            let Ok(codec_api) = transform.cast::<ICodecAPI>() else {
+                println!("MFT does not expose ICodecAPI; leaving the GOP size at its default");
+                return;
+            };
+            let mut value = VARIANT::default();
+            unsafe {
+                let inner = &mut value.Anonymous.Anonymous;
+                inner.vt = VT_UI4;
+                inner.Anonymous.ulVal = gop_size;
+                if let Err(err) = codec_api.SetValue(&CODECAPI_AVEncMPVGOPSize, &value) {
+                    println!("Failed to set CODECAPI_AVEncMPVGOPSize to {gop_size}: {err:?}");
+                }
+            }
+        };
+
         let (transform, output_rx) = Transform::new(
             MFT_CATEGORY_VIDEO_ENCODER,
             MFT_REGISTER_TYPE_INFO {
@@ -60,6 +85,7 @@ impl MediaFoundationVideoEncoder {
             },
             input_type,
             output_type,
+            &configure,
             runtime,
         )?;