From d8551c36df85a91dc0c2d6a482fb4123e658543e Mon Sep 17 00:00:00 2001 From: ruccho Date: Wed, 19 Aug 2026 14:47:48 +0900 Subject: [PATCH 1/4] Make the IDR interval configurable from C# The interval between IDR (key) frames was decided entirely on the native side, and inconsistently so: Android and FFmpeg forced one second, WebCodecs requested a keyframe every second from the frame loop, while VideoToolbox and Media Foundation were left at their platform defaults. Callers that do not rely on the bounded ring buffer, such as UnboundedRecordingSession or UniEnc used directly, pay for that forced interval without needing it. Expose the interval as VideoEncoderOptions.IdrIntervalSeconds, where null means "leave it at the platform encoder's default". It reaches the native options struct as a float whose non-positive values act as the sentinel for the absent case, since Option has no stable repr(C) layout. Each platform applies it through its own knob: - VideoToolbox: kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration - MediaCodec: KEY_I_FRAME_INTERVAL, now set as a float so sub-second intervals are expressible - Media Foundation: CODECAPI_AVEncMPVGOPSize, converted to frames using the frame rate hint, applied best-effort so an encoder that does not implement it still activates - FFmpeg: the -force_key_frames expression, omitted entirely for the default case - WebCodecs: the per-frame keyframe request threshold RealtimeEncodingOptions.Default keeps the previous one-second interval, so existing behaviour is unchanged unless the interval is set explicitly. Co-Authored-By: Claude Opus 5 --- .../crates/unienc/tests/integration_test.rs | 6 +++ .../crates/unienc_android_mc/src/common.rs | 13 ++++++- .../crates/unienc_android_mc/src/video/mod.rs | 24 +++++++++++- .../crates/unienc_apple_vt/src/video/mod.rs | 32 ++++++++++++++-- .../unienc/crates/unienc_c/src/types.rs | 9 +++++ .../unienc/crates/unienc_common/src/lib.rs | 7 ++++ .../crates/unienc_ffmpeg/src/video/mod.rs | 38 ++++++++++--------- .../crates/unienc_webcodecs/src/video/mod.rs | 16 ++++---- .../crates/unienc_windows_mf/src/audio/mod.rs | 1 + .../crates/unienc_windows_mf/src/mft.rs | 15 +++++++- .../crates/unienc_windows_mf/src/video/mod.rs | 28 ++++++++++++++ .../Pipeline/BoundedEncodedFrameBuffer.cs | 8 ++++ .../Pipeline/RealtimeEncodingOptions.cs | 3 +- .../Runtime/Generated/NativeMethods.g.cs | 6 +++ .../UniEnc/Runtime/VideoEncoderOptions.cs | 15 +++++++- 15 files changed, 186 insertions(+), 35 deletions(-) diff --git a/InstantReplay.Externals/unienc/crates/unienc/tests/integration_test.rs b/InstantReplay.Externals/unienc/crates/unienc/tests/integration_test.rs index 6ac88496..37b68b73 100644 --- a/InstantReplay.Externals/unienc/crates/unienc/tests/integration_test.rs +++ b/InstantReplay.Externals/unienc/crates/unienc/tests/integration_test.rs @@ -18,6 +18,7 @@ pub struct VideoEncoderOptions { pub height: u32, pub fps_hint: u32, pub bitrate: u32, + pub idr_interval_seconds: Option, } #[derive(Copy, Clone)] @@ -43,6 +44,10 @@ impl unienc::VideoEncoderOptions for VideoEncoderOptions { fn bitrate(&self) -> u32 { self.bitrate } + + fn idr_interval_seconds(&self) -> Option { + self.idr_interval_seconds + } } impl unienc::AudioEncoderOptions for AudioEncoderOptions { @@ -109,6 +114,7 @@ fn test_e2e() { height: 720, fps_hint: 1, bitrate: 1000000, + idr_interval_seconds: Some(1.0), }, &AudioEncoderOptions { sample_rate: 48000, diff --git a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/common.rs b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/common.rs index 49c7f23b..b2598f53 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/common.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/common.rs @@ -2,7 +2,7 @@ use bincode::{Decode, Encode}; use jni::{ JNIEnv, objects::{JByteArray, JObject, JString, JValue}, - sys::{jboolean, jint, jlong}, + sys::{jboolean, jfloat, jint, jlong}, }; use std::pin::Pin; use std::task::{Context, Poll}; @@ -648,6 +648,17 @@ pub fn set_format_integer(env: &JNIEnv, format: &JObject, key: &str, value: jint ) } +pub fn set_format_float(env: &JNIEnv, format: &JObject, key: &str, value: jfloat) -> Result<()> { + let key_str = to_java_string(env, key)?; + call_void_method( + env, + format, + "setFloat", + "(Ljava/lang/String;F)V", + &[JValue::Object(&key_str), JValue::Float(value)], + ) +} + #[derive(Encode, Decode)] pub struct CommonEncodedData { pub content: CommonEncodedDataContent, diff --git a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs index def6dca6..d12a0d5c 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs @@ -1,4 +1,9 @@ -use jni::{JNIEnv, objects::JValue, signature::ReturnType, sys::jint}; +use jni::{ + JNIEnv, + objects::JValue, + signature::ReturnType, + sys::{jfloat, jint}, +}; use std::sync::Arc; use std::time::Duration; use unienc_common::{ @@ -35,6 +40,7 @@ struct UninitializedState { tx: tokio::sync::oneshot::Sender<()>, bitrate: u32, fps_hint: u32, + idr_interval_seconds: Option, } enum MediaCodecVideoEncoderInputProcessor { @@ -132,6 +138,7 @@ impl MediaCodecVideoEncoder { tx, bitrate: options.bitrate(), fps_hint: options.fps_hint(), + idr_interval_seconds: options.idr_interval_seconds(), }, ), runtime, @@ -179,6 +186,7 @@ async fn push_video_impl( this.padded_height, state.bitrate, state.fps_hint, + state.idr_interval_seconds, false, // use_surface = false for buffer mode )?; this.codec.configure(&format)?; @@ -271,6 +279,7 @@ async fn push_video_impl( this.padded_height, state.bitrate, state.fps_hint, + state.idr_interval_seconds, true, // use_surface = true for hardware buffer mode )?; this.codec.configure(&format)?; @@ -371,6 +380,7 @@ fn create_video_format_raw( padded_height: u32, bitrate: u32, fps_hint: u32, + idr_interval_seconds: Option, use_surface: bool, ) -> Result { let format_class = env.find_class("android/media/MediaFormat")?; @@ -410,7 +420,17 @@ fn create_video_format_raw( set_format_integer(env, &format_obj, KEY_BITRATE, bitrate as jint)?; set_format_integer(env, &format_obj, KEY_FRAME_RATE, fps_hint as jint)?; - set_format_integer(env, &format_obj, KEY_I_FRAME_INTERVAL, 1)?; + // MediaFormat.KEY_I_FRAME_INTERVAL accepts a float since API 25, and `min_api` is 26, so the + // sub-second range is available without a version guard. Left unset when the caller asked for + // the platform default, in which case MediaCodec picks its own interval. + if let Some(idr_interval_seconds) = idr_interval_seconds { + set_format_float( + env, + &format_obj, + KEY_I_FRAME_INTERVAL, + idr_interval_seconds as jfloat, + )?; + } set_format_integer(env, &format_obj, KEY_PRIORITY, 0)?; set_format_integer(env, &format_obj, KEY_OPERATING_RATE, fps_hint as jint)?; diff --git a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs index 116a7e1c..87a61622 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs @@ -14,7 +14,8 @@ use objc2_core_video::{CVPixelBuffer, CVPixelBufferCreateWithBytes, kCVPixelForm use objc2_video_toolbox::{ VTCompressionSession, VTEncodeInfoFlags, VTSessionSetProperty, kVTCompressionPropertyKey_AllowFrameReordering, kVTCompressionPropertyKey_AverageBitRate, - kVTCompressionPropertyKey_RealTime, kVTInvalidSessionErr, + kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, kVTCompressionPropertyKey_RealTime, + kVTInvalidSessionErr, }; use tokio::sync::mpsc; use unienc_common::{ @@ -44,6 +45,7 @@ pub struct VideoToolboxEncoderInput { width: u32, height: u32, bitrate: u32, + idr_interval_seconds: Option, } struct CompressionSession { @@ -279,7 +281,12 @@ impl EncoderInput for VideoToolboxEncoderInput { // VTCompressionSession turns invalid when the app enters background on iOS; retry // once with a fresh session, re-passing the same reserved permit. retry += 1; - match CompressionSession::new(self.width, self.height, self.bitrate) { + match CompressionSession::new( + self.width, + self.height, + self.bitrate, + self.idr_interval_seconds, + ) { Ok(session) => { self.session = session; continue; @@ -335,7 +342,12 @@ impl Drop for VideoToolboxEncoderInput { } impl CompressionSession { - fn new(width: u32, height: u32, bitrate: u32) -> Result { + fn new( + width: u32, + height: u32, + bitrate: u32, + idr_interval_seconds: Option, + ) -> Result { let mut session: *mut VTCompressionSession = std::ptr::null_mut(); unsafe { @@ -382,6 +394,16 @@ impl CompressionSession { ) } .to_result()?; + if let Some(idr_interval_seconds) = idr_interval_seconds { + unsafe { + VTSessionSetProperty( + &session, + kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, + Some(&CFNumber::new_f64(idr_interval_seconds as f64)), + ) + } + .to_result()?; + } Ok(CompressionSession { inner: session }) } @@ -393,14 +415,16 @@ impl VideoToolboxEncoder { let tx = Box::new(tx); let (width, height, bitrate) = (options.width(), options.height(), options.bitrate()); + let idr_interval_seconds = options.idr_interval_seconds(); Ok(VideoToolboxEncoder { input: VideoToolboxEncoderInput { - session: CompressionSession::new(width, height, bitrate)?, + session: CompressionSession::new(width, height, bitrate, idr_interval_seconds)?, tx, width, height, bitrate, + idr_interval_seconds, }, output: VideoToolboxEncoderOutput { rx }, }) diff --git a/InstantReplay.Externals/unienc/crates/unienc_c/src/types.rs b/InstantReplay.Externals/unienc/crates/unienc_c/src/types.rs index 49b1f3db..61d9071e 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_c/src/types.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_c/src/types.rs @@ -26,6 +26,10 @@ pub struct VideoEncoderOptionsNative { pub height: u32, pub fps_hint: u32, pub bitrate: u32, + /// Maximum interval between IDR (key) frames, in seconds. Zero or negative means "leave the + /// interval at the platform encoder's default"; `Option` has no stable `repr(C)` layout, + /// so the absent case is carried by the sentinel instead of a separate discriminant field. + pub idr_interval_seconds: f32, } #[repr(C)] @@ -52,6 +56,11 @@ impl VideoEncoderOptions for VideoEncoderOptionsNative { fn bitrate(&self) -> u32 { self.bitrate } + + fn idr_interval_seconds(&self) -> Option { + (self.idr_interval_seconds > 0.0 && self.idr_interval_seconds.is_finite()) + .then_some(self.idr_interval_seconds) + } } impl AudioEncoderOptions for AudioEncoderOptionsNative { diff --git a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs index 44262da7..966bad60 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs @@ -95,6 +95,13 @@ pub trait VideoEncoderOptions: Clone + Copy { fn height(&self) -> u32; fn fps_hint(&self) -> u32; fn bitrate(&self) -> u32; + + /// Maximum interval between IDR (key) frames, in seconds. `None` leaves the interval at the + /// platform encoder's own default, which differs per platform and may be considerably longer + /// than a second. + fn idr_interval_seconds(&self) -> Option { + None + } } pub trait AudioEncoderOptions: Clone + Copy { diff --git a/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs index 91c2f1a4..fc873b56 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs @@ -131,6 +131,26 @@ impl FFmpegVideoEncoder { let height = options.height(); let cfr = options.fps_hint(); + let mut output_options = vec![ + "-f".to_owned(), + "h264".to_owned(), + "-pix_fmt".to_owned(), + "yuv420p".to_owned(), + "-r".to_owned(), + format!("{cfr}"), + "-c:v".to_owned(), + FFMPEG_CODEC.clone(), + "-b:v".to_owned(), + format!("{}", options.bitrate()), + ]; + + // Without `-force_key_frames` the encoder falls back to its own GOP heuristics, which is + // what the platform-default case asks for. + if let Some(idr_interval_seconds) = options.idr_interval_seconds() { + output_options.push("-force_key_frames".to_owned()); + output_options.push(format!("expr:gte(t,n_forced*{idr_interval_seconds})")); + } + // encode raw BGRA frames into H.264 stream let mut ffmpeg = ffmpeg::Builder::new() .use_stdin(true) @@ -144,23 +164,7 @@ impl FFmpegVideoEncoder { "-framerate", &format!("{cfr}"), ]) - .build( - [ - "-f", - "h264", - "-pix_fmt", - "yuv420p", - "-r", - &format!("{cfr}"), - "-c:v", - &*FFMPEG_CODEC, - "-b:v", - &format!("{}", options.bitrate()), - "-force_key_frames", - "expr:gte(t,n_forced*1)", - ], - ffmpeg::Destination::Stdout, - )?; + .build(output_options, ffmpeg::Destination::Stdout)?; let input = ffmpeg .inputs diff --git a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs index e3a48ff4..7f5e2d88 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs @@ -20,6 +20,7 @@ pub struct WebCodecsVideoEncoderInput { fps_hint: f64, tx: mpsc::Sender, prev_key_timestamp: Option, + idr_interval_seconds: f64, runtime: R, } @@ -49,6 +50,10 @@ impl WebCodecsVideoEncoder { encoder_handle: None, tx, prev_key_timestamp: None, + // WebCodecs has no GOP-length knob; keyframes are requested per frame, so the + // platform-default case has to pick a concrete interval. Keep the previous + // hardcoded one second. + idr_interval_seconds: options.idr_interval_seconds().unwrap_or(1.0) as f64, runtime: runtime.clone(), }, output: WebCodecsVideoEncoderOutput { rx }, @@ -108,17 +113,12 @@ impl EncoderInput for WebCodecsVideoEncoderInput { Some(prev) => data.timestamp - prev, None => f64::INFINITY, }; - if since_prev_key >= 1.0 { + let is_key = since_prev_key >= self.idr_interval_seconds; + if is_key { self.prev_key_timestamp = Some(data.timestamp); } encoder_handle - .push_video_frame( - pixels, - frame.width, - frame.height, - data.timestamp, - since_prev_key >= 1.0, - ) + .push_video_frame(pixels, frame.width, frame.height, data.timestamp, is_key) .context("Failed to push video frame to WebCodecs EncoderHandle")?; Ok(()) } diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/audio/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/audio/mod.rs index 90b54567..bb6be587 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/audio/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/audio/mod.rs @@ -53,6 +53,7 @@ impl MediaFoundationAudioEncoder { }, input_type, output_type, + &|_| {}, runtime, )?; diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mft.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mft.rs index f910182b..c42a5fd1 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mft.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/mft.rs @@ -239,12 +239,16 @@ enum Pipeline { } impl Transform { + /// `configure` runs on each candidate MFT after it is activated but before the media types are + /// negotiated. It is for optional codec properties: anything it fails to apply must leave the + /// transform usable, so it cannot report an error. pub fn new( category: windows_core::GUID, input: MFT_REGISTER_TYPE_INFO, output: MFT_REGISTER_TYPE_INFO, input_type: IMFMediaType, output_type: IMFMediaType, + configure: &dyn Fn(&IMFTransform), runtime: &impl Runtime, ) -> Result<(Self, mpsc::Receiver>)> { let mfts = MftIter::new(category, input, output); @@ -259,7 +263,13 @@ impl Transform { println!("Skipping MFT: {}", Self::get_name(&activate)?); continue; } - match Self::try_activate(activate, &mut input_type, &mut output_type, runtime) { + match Self::try_activate( + activate, + &mut input_type, + &mut output_type, + configure, + runtime, + ) { Ok(r) => { result = Some(r); } @@ -290,6 +300,7 @@ impl Transform { activate: IMFActivate, input_type: &mut Option, output_type: &mut Option, + configure: &dyn Fn(&IMFTransform), runtime: &impl Runtime, ) -> Result<(Self, mpsc::Receiver>)> { println!("Trying MFT: {}", Self::get_name(&activate)?); @@ -297,6 +308,8 @@ impl Transform { let is_async = unsafe { activate.GetUINT32(&MF_TRANSFORM_ASYNC) }.unwrap_or(0) != 0; let transform = unsafe { activate.ActivateObject::()? }; + configure(&transform); + if is_async { let attributes = unsafe { transform.GetAttributes()? }; unsafe { attributes.SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1)? }; diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs index b33708f1..f8aadb44 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs @@ -6,6 +6,8 @@ use unienc_common::{ UnsupportedBlitData, VideoEncoderOptions, VideoFrame, VideoSample, }; use windows::Win32::Media::MediaFoundation::*; +use windows::Win32::System::Variant::{VARIANT, VT_UI4}; +use windows::core::Interface; use crate::common::*; use crate::mft::Transform; @@ -48,6 +50,31 @@ impl MediaFoundationVideoEncoder { output_type }; + // Media Foundation expresses the GOP length in frames, so the interval has to be resolved + // against the frame rate hint. `CODECAPI_AVEncMPVGOPSize` is optional: encoders that do not + // implement it keep their own default rather than failing to activate. + let gop_size = options + .idr_interval_seconds() + .map(|seconds| ((seconds as f64 * options.fps_hint() as f64).round() as u32).max(1)); + let configure = |transform: &IMFTransform| { + let Some(gop_size) = gop_size else { + return; + }; + let Ok(codec_api) = transform.cast::() else { + println!("MFT does not expose ICodecAPI; leaving the GOP size at its default"); + return; + }; + let mut value = VARIANT::default(); + unsafe { + let inner = &mut value.Anonymous.Anonymous; + inner.vt = VT_UI4; + inner.Anonymous.ulVal = gop_size; + if let Err(err) = codec_api.SetValue(&CODECAPI_AVEncMPVGOPSize, &value) { + println!("Failed to set CODECAPI_AVEncMPVGOPSize to {gop_size}: {err:?}"); + } + } + }; + let (transform, output_rx) = Transform::new( MFT_CATEGORY_VIDEO_ENCODER, MFT_REGISTER_TYPE_INFO { @@ -60,6 +87,7 @@ impl MediaFoundationVideoEncoder { }, input_type, output_type, + &configure, runtime, )?; diff --git a/Packages/jp.co.cyberagent.instant-replay/Runtime/Pipeline/BoundedEncodedFrameBuffer.cs b/Packages/jp.co.cyberagent.instant-replay/Runtime/Pipeline/BoundedEncodedFrameBuffer.cs index 6c98ef7f..8737ba40 100644 --- a/Packages/jp.co.cyberagent.instant-replay/Runtime/Pipeline/BoundedEncodedFrameBuffer.cs +++ b/Packages/jp.co.cyberagent.instant-replay/Runtime/Pipeline/BoundedEncodedFrameBuffer.cs @@ -108,6 +108,14 @@ public bool TryAddAudioFrame(EncodedFrame frame) /// /// Gets frames for the specified duration, adjusted to start from a keyframe. /// + /// + /// The segment can only begin at a keyframe, so the IDR interval bounds how precisely the requested + /// duration can be honoured, and no frames at all are returned while the buffer holds no keyframe. Raising + /// beyond the retained duration therefore + /// yields an empty export. Eviction also works one frame at a time rather than one group of pictures at a + /// time, so the frames preceding the oldest surviving keyframe are retained but unusable, and a longer IDR + /// interval leaves a larger share of the memory budget unusable. + /// public void GetFramesForDuration(double? durationSeconds, out ReadOnlyMemory videoFrames, out ReadOnlyMemory audioFrames) { diff --git a/Packages/jp.co.cyberagent.instant-replay/Runtime/Pipeline/RealtimeEncodingOptions.cs b/Packages/jp.co.cyberagent.instant-replay/Runtime/Pipeline/RealtimeEncodingOptions.cs index 9fdbed1b..41b65a4f 100644 --- a/Packages/jp.co.cyberagent.instant-replay/Runtime/Pipeline/RealtimeEncodingOptions.cs +++ b/Packages/jp.co.cyberagent.instant-replay/Runtime/Pipeline/RealtimeEncodingOptions.cs @@ -64,7 +64,8 @@ public int AudioInputQueueSize Width = 1280, Height = 720, FpsHint = 30, - Bitrate = 2500000 // 2.5 Mbps + Bitrate = 2500000, // 2.5 Mbps + IdrIntervalSeconds = 1f }, AudioOptions = new AudioEncoderOptions { diff --git a/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/Generated/NativeMethods.g.cs b/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/Generated/NativeMethods.g.cs index 3d87b392..e8d5be13 100644 --- a/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/Generated/NativeMethods.g.cs +++ b/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/Generated/NativeMethods.g.cs @@ -144,6 +144,12 @@ internal unsafe partial struct VideoEncoderOptionsNative public uint height; public uint fps_hint; public uint bitrate; + /// + /// Maximum interval between IDR (key) frames, in seconds. Zero or negative means "leave the + /// interval at the platform encoder's default"; `Option<f32>` has no stable `repr(C)` layout, + /// so the absent case is carried by the sentinel instead of a separate discriminant field. + /// + public float idr_interval_seconds; } [StructLayout(LayoutKind.Sequential)] diff --git a/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs b/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs index b63587c4..09e1b390 100644 --- a/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs +++ b/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs @@ -28,6 +28,12 @@ public struct VideoEncoderOptions /// public uint Bitrate { get; set; } + /// + /// Maximum interval between IDR (key) frames, in seconds. null leaves the interval at the platform + /// encoder's own default, which differs per platform and may be considerably longer than a second. + /// + public float? IdrIntervalSeconds { get; set; } + /// /// Validates the options and throws if invalid. /// @@ -41,6 +47,11 @@ internal void Validate() if (Bitrate == 0) throw new ArgumentException("Bitrate must be greater than 0"); + + if (IdrIntervalSeconds is { } idrIntervalSeconds && + (idrIntervalSeconds <= 0f || float.IsNaN(idrIntervalSeconds) || + float.IsInfinity(idrIntervalSeconds))) + throw new ArgumentException("IDR interval must be a finite value greater than 0"); } /// @@ -54,7 +65,9 @@ internal VideoEncoderOptionsNative ToNative() width = Width, height = Height, fps_hint = FpsHint, - bitrate = Bitrate + bitrate = Bitrate, + // 0 is the sentinel the native side reads as "use the platform default". + idr_interval_seconds = IdrIntervalSeconds ?? 0f }; } } From c6b1242d28a21344204e65f1ec9b25d5230ff6c7 Mon Sep 17 00:00:00 2001 From: ruccho Date: Wed, 19 Aug 2026 15:09:57 +0900 Subject: [PATCH 2/4] Make the IDR interval mandatory at the UniEnc layer Modelling the interval as optional pushed a "leave it to the platform" case down to every encoder, and that case had no good answer: MediaCodec documents KEY_I_FRAME_INTERVAL as required for video encoders, and WebCodecs has no GOP-length knob at all, so both had to invent a value anyway. Requiring the interval removes the branch instead of choosing arbitrarily on each platform. VideoEncoderOptions.IdrIntervalSeconds becomes a plain float, validated alongside Width, Height and Bitrate, and every platform now applies it unconditionally. On the Rust side the trait method returns f32 with no default implementation, so a missing value is a compile error rather than a silent fallback. The native struct keeps its f32 field, now without the sentinel interpretation, so its layout is unchanged. InstantReplay keeps supplying 1 second: RealtimeEncodingOptions.Default, UniEncTranscoder, the PersistentRecorder sample and the UniEnc example all set it explicitly. BREAKING CHANGE: code that constructs UniEnc.VideoEncoderOptions directly must now set IdrIntervalSeconds. The struct's zero default is rejected by Validate(), so leaving it unset throws ArgumentException. Callers going through RealtimeEncodingOptions.Default are unaffected. Co-Authored-By: Claude Opus 5 --- .../src/UniEnc.Example/Program.cs | 3 +- .../crates/unienc/tests/integration_test.rs | 6 +-- .../crates/unienc_android_mc/src/video/mod.rs | 21 +++++----- .../crates/unienc_apple_vt/src/video/mod.rs | 25 +++++------- .../unienc/crates/unienc_c/src/types.rs | 10 ++--- .../unienc/crates/unienc_common/src/lib.rs | 10 ++--- .../crates/unienc_ffmpeg/src/video/mod.rs | 38 +++++++++---------- .../crates/unienc_webcodecs/src/video/mod.rs | 6 +-- .../crates/unienc_windows_mf/src/video/mod.rs | 14 +++---- .../Runtime/Legacy/UniEncTranscoder.cs | 3 +- .../User Interfaces/PersistentRecorder.cs | 3 +- .../Runtime/Generated/NativeMethods.g.cs | 5 +-- .../UniEnc/Runtime/VideoEncoderOptions.cs | 14 +++---- 13 files changed, 69 insertions(+), 89 deletions(-) diff --git a/InstantReplay.Externals/src/UniEnc.Example/Program.cs b/InstantReplay.Externals/src/UniEnc.Example/Program.cs index aaa53797..5d8eea6e 100644 --- a/InstantReplay.Externals/src/UniEnc.Example/Program.cs +++ b/InstantReplay.Externals/src/UniEnc.Example/Program.cs @@ -14,7 +14,8 @@ Width = width, Height = height, FpsHint = framerate, - Bitrate = 2500000 // 2.5Mbps + Bitrate = 2500000, // 2.5Mbps + IdrIntervalSeconds = 1f }, new AudioEncoderOptions { diff --git a/InstantReplay.Externals/unienc/crates/unienc/tests/integration_test.rs b/InstantReplay.Externals/unienc/crates/unienc/tests/integration_test.rs index 37b68b73..5fc5d7f6 100644 --- a/InstantReplay.Externals/unienc/crates/unienc/tests/integration_test.rs +++ b/InstantReplay.Externals/unienc/crates/unienc/tests/integration_test.rs @@ -18,7 +18,7 @@ pub struct VideoEncoderOptions { pub height: u32, pub fps_hint: u32, pub bitrate: u32, - pub idr_interval_seconds: Option, + pub idr_interval_seconds: f32, } #[derive(Copy, Clone)] @@ -45,7 +45,7 @@ impl unienc::VideoEncoderOptions for VideoEncoderOptions { self.bitrate } - fn idr_interval_seconds(&self) -> Option { + fn idr_interval_seconds(&self) -> f32 { self.idr_interval_seconds } } @@ -114,7 +114,7 @@ fn test_e2e() { height: 720, fps_hint: 1, bitrate: 1000000, - idr_interval_seconds: Some(1.0), + idr_interval_seconds: 1.0, }, &AudioEncoderOptions { sample_rate: 48000, diff --git a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs index d12a0d5c..e39a01e9 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_android_mc/src/video/mod.rs @@ -40,7 +40,7 @@ struct UninitializedState { tx: tokio::sync::oneshot::Sender<()>, bitrate: u32, fps_hint: u32, - idr_interval_seconds: Option, + idr_interval_seconds: f32, } enum MediaCodecVideoEncoderInputProcessor { @@ -380,7 +380,7 @@ fn create_video_format_raw( padded_height: u32, bitrate: u32, fps_hint: u32, - idr_interval_seconds: Option, + idr_interval_seconds: f32, use_surface: bool, ) -> Result { let format_class = env.find_class("android/media/MediaFormat")?; @@ -421,16 +421,13 @@ fn create_video_format_raw( set_format_integer(env, &format_obj, KEY_BITRATE, bitrate as jint)?; set_format_integer(env, &format_obj, KEY_FRAME_RATE, fps_hint as jint)?; // MediaFormat.KEY_I_FRAME_INTERVAL accepts a float since API 25, and `min_api` is 26, so the - // sub-second range is available without a version guard. Left unset when the caller asked for - // the platform default, in which case MediaCodec picks its own interval. - if let Some(idr_interval_seconds) = idr_interval_seconds { - set_format_float( - env, - &format_obj, - KEY_I_FRAME_INTERVAL, - idr_interval_seconds as jfloat, - )?; - } + // sub-second range is available without a version guard. + set_format_float( + env, + &format_obj, + KEY_I_FRAME_INTERVAL, + idr_interval_seconds as jfloat, + )?; set_format_integer(env, &format_obj, KEY_PRIORITY, 0)?; set_format_integer(env, &format_obj, KEY_OPERATING_RATE, fps_hint as jint)?; diff --git a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs index 87a61622..d214e406 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_apple_vt/src/video/mod.rs @@ -45,7 +45,7 @@ pub struct VideoToolboxEncoderInput { width: u32, height: u32, bitrate: u32, - idr_interval_seconds: Option, + idr_interval_seconds: f32, } struct CompressionSession { @@ -342,12 +342,7 @@ impl Drop for VideoToolboxEncoderInput { } impl CompressionSession { - fn new( - width: u32, - height: u32, - bitrate: u32, - idr_interval_seconds: Option, - ) -> Result { + fn new(width: u32, height: u32, bitrate: u32, idr_interval_seconds: f32) -> Result { let mut session: *mut VTCompressionSession = std::ptr::null_mut(); unsafe { @@ -394,16 +389,14 @@ impl CompressionSession { ) } .to_result()?; - if let Some(idr_interval_seconds) = idr_interval_seconds { - unsafe { - VTSessionSetProperty( - &session, - kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, - Some(&CFNumber::new_f64(idr_interval_seconds as f64)), - ) - } - .to_result()?; + unsafe { + VTSessionSetProperty( + &session, + kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, + Some(&CFNumber::new_f64(idr_interval_seconds as f64)), + ) } + .to_result()?; Ok(CompressionSession { inner: session }) } diff --git a/InstantReplay.Externals/unienc/crates/unienc_c/src/types.rs b/InstantReplay.Externals/unienc/crates/unienc_c/src/types.rs index 61d9071e..1e78dd77 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_c/src/types.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_c/src/types.rs @@ -26,9 +26,8 @@ pub struct VideoEncoderOptionsNative { pub height: u32, pub fps_hint: u32, pub bitrate: u32, - /// Maximum interval between IDR (key) frames, in seconds. Zero or negative means "leave the - /// interval at the platform encoder's default"; `Option` has no stable `repr(C)` layout, - /// so the absent case is carried by the sentinel instead of a separate discriminant field. + /// Maximum interval between IDR (key) frames, in seconds. Required; the managed side rejects + /// non-positive and non-finite values before the struct crosses the boundary. pub idr_interval_seconds: f32, } @@ -57,9 +56,8 @@ impl VideoEncoderOptions for VideoEncoderOptionsNative { self.bitrate } - fn idr_interval_seconds(&self) -> Option { - (self.idr_interval_seconds > 0.0 && self.idr_interval_seconds.is_finite()) - .then_some(self.idr_interval_seconds) + fn idr_interval_seconds(&self) -> f32 { + self.idr_interval_seconds } } diff --git a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs index 966bad60..ef056e7f 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_common/src/lib.rs @@ -96,12 +96,10 @@ pub trait VideoEncoderOptions: Clone + Copy { fn fps_hint(&self) -> u32; fn bitrate(&self) -> u32; - /// Maximum interval between IDR (key) frames, in seconds. `None` leaves the interval at the - /// platform encoder's own default, which differs per platform and may be considerably longer - /// than a second. - fn idr_interval_seconds(&self) -> Option { - None - } + /// Maximum interval between IDR (key) frames, in seconds. Must be finite and greater than + /// zero; every platform encoder is configured with it, so there is no "leave it to the + /// platform" case to represent. + fn idr_interval_seconds(&self) -> f32; } pub trait AudioEncoderOptions: Clone + Copy { diff --git a/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs index fc873b56..abd76952 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_ffmpeg/src/video/mod.rs @@ -131,25 +131,7 @@ impl FFmpegVideoEncoder { let height = options.height(); let cfr = options.fps_hint(); - let mut output_options = vec![ - "-f".to_owned(), - "h264".to_owned(), - "-pix_fmt".to_owned(), - "yuv420p".to_owned(), - "-r".to_owned(), - format!("{cfr}"), - "-c:v".to_owned(), - FFMPEG_CODEC.clone(), - "-b:v".to_owned(), - format!("{}", options.bitrate()), - ]; - - // Without `-force_key_frames` the encoder falls back to its own GOP heuristics, which is - // what the platform-default case asks for. - if let Some(idr_interval_seconds) = options.idr_interval_seconds() { - output_options.push("-force_key_frames".to_owned()); - output_options.push(format!("expr:gte(t,n_forced*{idr_interval_seconds})")); - } + let idr_interval_seconds = options.idr_interval_seconds(); // encode raw BGRA frames into H.264 stream let mut ffmpeg = ffmpeg::Builder::new() @@ -164,7 +146,23 @@ impl FFmpegVideoEncoder { "-framerate", &format!("{cfr}"), ]) - .build(output_options, ffmpeg::Destination::Stdout)?; + .build( + [ + "-f", + "h264", + "-pix_fmt", + "yuv420p", + "-r", + &format!("{cfr}"), + "-c:v", + &*FFMPEG_CODEC, + "-b:v", + &format!("{}", options.bitrate()), + "-force_key_frames", + &format!("expr:gte(t,n_forced*{idr_interval_seconds})"), + ], + ffmpeg::Destination::Stdout, + )?; let input = ffmpeg .inputs diff --git a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs index 7f5e2d88..fe1bf2f8 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_webcodecs/src/video/mod.rs @@ -50,10 +50,8 @@ impl WebCodecsVideoEncoder { encoder_handle: None, tx, prev_key_timestamp: None, - // WebCodecs has no GOP-length knob; keyframes are requested per frame, so the - // platform-default case has to pick a concrete interval. Keep the previous - // hardcoded one second. - idr_interval_seconds: options.idr_interval_seconds().unwrap_or(1.0) as f64, + // WebCodecs has no GOP-length knob, so keyframes are requested per frame. + idr_interval_seconds: options.idr_interval_seconds() as f64, runtime: runtime.clone(), }, output: WebCodecsVideoEncoderOutput { rx }, diff --git a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs index f8aadb44..c36373d8 100644 --- a/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs +++ b/InstantReplay.Externals/unienc/crates/unienc_windows_mf/src/video/mod.rs @@ -51,15 +51,13 @@ impl MediaFoundationVideoEncoder { }; // Media Foundation expresses the GOP length in frames, so the interval has to be resolved - // against the frame rate hint. `CODECAPI_AVEncMPVGOPSize` is optional: encoders that do not - // implement it keep their own default rather than failing to activate. - let gop_size = options - .idr_interval_seconds() - .map(|seconds| ((seconds as f64 * options.fps_hint() as f64).round() as u32).max(1)); + // against the frame rate hint. `CODECAPI_AVEncMPVGOPSize` is optional for an MFT to + // implement, so an encoder that rejects it keeps its own GOP length rather than failing to + // activate. + let gop_size = ((options.idr_interval_seconds() as f64 * options.fps_hint() as f64).round() + as u32) + .max(1); let configure = |transform: &IMFTransform| { - let Some(gop_size) = gop_size else { - return; - }; let Ok(codec_api) = transform.cast::() else { println!("MFT does not expose ICodecAPI; leaving the GOP size at its default"); return; diff --git a/Packages/jp.co.cyberagent.instant-replay/Runtime/Legacy/UniEncTranscoder.cs b/Packages/jp.co.cyberagent.instant-replay/Runtime/Legacy/UniEncTranscoder.cs index a7f98949..38981378 100644 --- a/Packages/jp.co.cyberagent.instant-replay/Runtime/Legacy/UniEncTranscoder.cs +++ b/Packages/jp.co.cyberagent.instant-replay/Runtime/Legacy/UniEncTranscoder.cs @@ -41,7 +41,8 @@ public UniEncTranscoder(int width, int height, int sampleRate, int channels, str Height = checked((uint)height), Bitrate = (uint)Mathf.Min(width * height * 30 * 0.2f - 25000, width * height * 30 * 0.1f + 1000), - FpsHint = 30 + FpsHint = 30, + IdrIntervalSeconds = 1f }, new AudioEncoderOptions { diff --git a/Packages/jp.co.cyberagent.instant-replay/Samples~/User Interfaces/PersistentRecorder.cs b/Packages/jp.co.cyberagent.instant-replay/Samples~/User Interfaces/PersistentRecorder.cs index a4237e85..675bcc3b 100644 --- a/Packages/jp.co.cyberagent.instant-replay/Samples~/User Interfaces/PersistentRecorder.cs +++ b/Packages/jp.co.cyberagent.instant-replay/Samples~/User Interfaces/PersistentRecorder.cs @@ -88,7 +88,8 @@ public void NewSession(bool allowStopCurrentSession = true) // and a lower bound width * height * LowerBitPerPixel + LowerBitPerPixelBias ), - FpsHint = (uint)fixedFrameRate + FpsHint = (uint)fixedFrameRate, + IdrIntervalSeconds = 1f }, AudioOptions = new AudioEncoderOptions { diff --git a/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/Generated/NativeMethods.g.cs b/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/Generated/NativeMethods.g.cs index e8d5be13..7e5f5295 100644 --- a/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/Generated/NativeMethods.g.cs +++ b/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/Generated/NativeMethods.g.cs @@ -145,9 +145,8 @@ internal unsafe partial struct VideoEncoderOptionsNative public uint fps_hint; public uint bitrate; /// - /// Maximum interval between IDR (key) frames, in seconds. Zero or negative means "leave the - /// interval at the platform encoder's default"; `Option<f32>` has no stable `repr(C)` layout, - /// so the absent case is carried by the sentinel instead of a separate discriminant field. + /// Maximum interval between IDR (key) frames, in seconds. Required; the managed side rejects + /// non-positive and non-finite values before the struct crosses the boundary. /// public float idr_interval_seconds; } diff --git a/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs b/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs index 09e1b390..e165c4c7 100644 --- a/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs +++ b/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs @@ -29,10 +29,10 @@ public struct VideoEncoderOptions public uint Bitrate { get; set; } /// - /// Maximum interval between IDR (key) frames, in seconds. null leaves the interval at the platform - /// encoder's own default, which differs per platform and may be considerably longer than a second. + /// Maximum interval between IDR (key) frames, in seconds. Required, and must be finite and greater than 0. + /// Every platform encoder is configured with this value, so there is no "leave it to the platform" case. /// - public float? IdrIntervalSeconds { get; set; } + public float IdrIntervalSeconds { get; set; } /// /// Validates the options and throws if invalid. @@ -48,9 +48,8 @@ internal void Validate() if (Bitrate == 0) throw new ArgumentException("Bitrate must be greater than 0"); - if (IdrIntervalSeconds is { } idrIntervalSeconds && - (idrIntervalSeconds <= 0f || float.IsNaN(idrIntervalSeconds) || - float.IsInfinity(idrIntervalSeconds))) + if (IdrIntervalSeconds <= 0f || float.IsNaN(IdrIntervalSeconds) || + float.IsInfinity(IdrIntervalSeconds)) throw new ArgumentException("IDR interval must be a finite value greater than 0"); } @@ -66,8 +65,7 @@ internal VideoEncoderOptionsNative ToNative() height = Height, fps_hint = FpsHint, bitrate = Bitrate, - // 0 is the sentinel the native side reads as "use the platform default". - idr_interval_seconds = IdrIntervalSeconds ?? 0f + idr_interval_seconds = IdrIntervalSeconds }; } } From 7e64401ca89cdf0a00d84b6755eda2e6a5b0c371 Mon Sep 17 00:00:00 2001 From: ruccho Date: Wed, 19 Aug 2026 15:33:48 +0900 Subject: [PATCH 3/4] Fall back to a UniEnc-level default IDR interval instead of requiring one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requiring the interval made the property a breaking addition: the struct's zero default is not a valid interval, so any existing code constructing VideoEncoderOptions directly would start throwing from Validate(). Make IdrIntervalSeconds nullable again, but give null a definite meaning. It selects VideoEncoderOptions.DefaultIdrIntervalSeconds — one second, the interval every platform used before this branch — rather than deferring to the platform encoder, whose own default differs per platform and can be far longer. The native side keeps receiving a concrete f32, so no platform code changes and the struct layout is untouched. With the default covering them, UniEncTranscoder, the PersistentRecorder sample and the UniEnc example no longer need to set the interval, so those three files revert to their original contents. RealtimeEncodingOptions.Default keeps setting it explicitly, since one second is InstantReplay's own choice rather than something to inherit silently. Co-Authored-By: Claude Opus 5 --- .../src/UniEnc.Example/Program.cs | 3 +-- .../Runtime/Legacy/UniEncTranscoder.cs | 3 +-- .../User Interfaces/PersistentRecorder.cs | 3 +-- .../UniEnc/Runtime/VideoEncoderOptions.cs | 19 +++++++++++++------ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/InstantReplay.Externals/src/UniEnc.Example/Program.cs b/InstantReplay.Externals/src/UniEnc.Example/Program.cs index 5d8eea6e..aaa53797 100644 --- a/InstantReplay.Externals/src/UniEnc.Example/Program.cs +++ b/InstantReplay.Externals/src/UniEnc.Example/Program.cs @@ -14,8 +14,7 @@ Width = width, Height = height, FpsHint = framerate, - Bitrate = 2500000, // 2.5Mbps - IdrIntervalSeconds = 1f + Bitrate = 2500000 // 2.5Mbps }, new AudioEncoderOptions { diff --git a/Packages/jp.co.cyberagent.instant-replay/Runtime/Legacy/UniEncTranscoder.cs b/Packages/jp.co.cyberagent.instant-replay/Runtime/Legacy/UniEncTranscoder.cs index 38981378..a7f98949 100644 --- a/Packages/jp.co.cyberagent.instant-replay/Runtime/Legacy/UniEncTranscoder.cs +++ b/Packages/jp.co.cyberagent.instant-replay/Runtime/Legacy/UniEncTranscoder.cs @@ -41,8 +41,7 @@ public UniEncTranscoder(int width, int height, int sampleRate, int channels, str Height = checked((uint)height), Bitrate = (uint)Mathf.Min(width * height * 30 * 0.2f - 25000, width * height * 30 * 0.1f + 1000), - FpsHint = 30, - IdrIntervalSeconds = 1f + FpsHint = 30 }, new AudioEncoderOptions { diff --git a/Packages/jp.co.cyberagent.instant-replay/Samples~/User Interfaces/PersistentRecorder.cs b/Packages/jp.co.cyberagent.instant-replay/Samples~/User Interfaces/PersistentRecorder.cs index 675bcc3b..a4237e85 100644 --- a/Packages/jp.co.cyberagent.instant-replay/Samples~/User Interfaces/PersistentRecorder.cs +++ b/Packages/jp.co.cyberagent.instant-replay/Samples~/User Interfaces/PersistentRecorder.cs @@ -88,8 +88,7 @@ public void NewSession(bool allowStopCurrentSession = true) // and a lower bound width * height * LowerBitPerPixel + LowerBitPerPixelBias ), - FpsHint = (uint)fixedFrameRate, - IdrIntervalSeconds = 1f + FpsHint = (uint)fixedFrameRate }, AudioOptions = new AudioEncoderOptions { diff --git a/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs b/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs index e165c4c7..bf3b2759 100644 --- a/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs +++ b/Packages/jp.co.cyberagent.instant-replay/UniEnc/Runtime/VideoEncoderOptions.cs @@ -8,6 +8,11 @@ namespace UniEnc /// public struct VideoEncoderOptions { + /// + /// Interval between IDR (key) frames applied when is not set, in seconds. + /// + public const float DefaultIdrIntervalSeconds = 1f; + /// /// Width of the video in pixels. /// @@ -29,10 +34,11 @@ public struct VideoEncoderOptions public uint Bitrate { get; set; } /// - /// Maximum interval between IDR (key) frames, in seconds. Required, and must be finite and greater than 0. - /// Every platform encoder is configured with this value, so there is no "leave it to the platform" case. + /// Maximum interval between IDR (key) frames, in seconds. Must be finite and greater than 0 when set. + /// null applies rather than deferring to the platform + /// encoder, whose own default differs per platform and may be considerably longer. /// - public float IdrIntervalSeconds { get; set; } + public float? IdrIntervalSeconds { get; set; } /// /// Validates the options and throws if invalid. @@ -48,8 +54,9 @@ internal void Validate() if (Bitrate == 0) throw new ArgumentException("Bitrate must be greater than 0"); - if (IdrIntervalSeconds <= 0f || float.IsNaN(IdrIntervalSeconds) || - float.IsInfinity(IdrIntervalSeconds)) + if (IdrIntervalSeconds is { } idrIntervalSeconds && + (idrIntervalSeconds <= 0f || float.IsNaN(idrIntervalSeconds) || + float.IsInfinity(idrIntervalSeconds))) throw new ArgumentException("IDR interval must be a finite value greater than 0"); } @@ -65,7 +72,7 @@ internal VideoEncoderOptionsNative ToNative() height = Height, fps_hint = FpsHint, bitrate = Bitrate, - idr_interval_seconds = IdrIntervalSeconds + idr_interval_seconds = IdrIntervalSeconds ?? DefaultIdrIntervalSeconds }; } } From f1236978038c105b98921e2c8c6cb6b19b3e40e3 Mon Sep 17 00:00:00 2001 From: ruccho Date: Wed, 19 Aug 2026 17:11:59 +0900 Subject: [PATCH 4/4] Document IdrIntervalSeconds in the README Adds the new option to the settings sample and explains what null means: the UniEnc default of one second, not the platform encoder's own default. Also notes the interaction with the export path, since an export can only begin at a key frame and an interval longer than the retained duration leaves the buffer with none. Co-Authored-By: Claude Opus 5 --- README.ja.md | 5 ++++- README.md | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/README.ja.md b/README.ja.md index c86e7b54..aef040fc 100644 --- a/README.ja.md +++ b/README.ja.md @@ -151,6 +151,8 @@ File.Move(outputPath, Path.Combine(Application.persistentDataPath, Path.GetFileN 実行時に使用されるメモリとしては、上記のエンコード済みのデータを保持するバッファに加え、エンコード前の生のフレームや音声サンプルがいくつか保持されます。これはエンコーダーが非同期的に動作する関係で、あるフレームをエンコードしている間に次のフレームを受け取るためです。`VideoInputQueueSize` と `AudioInputQueueSizeSeconds` でそれぞれのキューのサイズを指定できるほか、`MaxNumberOfRawFrameBuffers` (オプション) で圧縮前のフレームを保持するバッファの最大数を指定できます。この値を小さくすることでメモリ使用量を削減できる場合がありますが、フレームドロップの可能性が高まります。 +`IdrIntervalSeconds` は IDR (キー) フレームの最大間隔を秒で指定します。書き出しはキーフレームからしか開始できないため、この値は `StopAndExportAsync` が指定した長さにどれだけ近づけられるかの粒度を決めます。`null` を指定した場合は、プラットフォームのエンコーダー自身の既定値ではなく `VideoEncoderOptions.DefaultIdrIntervalSeconds` (1 秒) が適用されます。エンコーダーの既定値はプラットフォームごとに異なり、著しく長い場合があるためです。間隔を長くするとキーフレームに費やされるビットレートが減りますが、保持している時間より長い間隔を指定するとバッファ内にキーフレームが存在しなくなり、書き出しが何も生成しない場合があります。 + ```csharp // デフォルト設定 var options = new RealtimeEncodingOptions @@ -160,7 +162,8 @@ var options = new RealtimeEncodingOptions Width = 1280, Height = 720, FpsHint = 30, - Bitrate = 2500000 // 2.5 Mbps + Bitrate = 2500000, // 2.5 Mbps + IdrIntervalSeconds = 1f // IDR (キー) フレームの最大間隔 (秒)。null の場合は既定値の 1 秒が適用されます。 }, AudioOptions = new AudioEncoderOptions { diff --git a/README.md b/README.md index a510159c..5a08e4fa 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,8 @@ Recording uses memory in two places: buffers for compressed output, and buffers `VideoInputQueueSize`, `AudioInputQueueSizeSeconds`, and `MaxNumberOfRawFrameBuffers` (optional) control how many raw frames and audio samples are queued for encoding. These queues are needed because the encoder runs asynchronously, receiving the next frame while encoding the current one. Reducing these values decreases memory usage but may increase the likelihood of dropped frames. +`IdrIntervalSeconds` sets the maximum interval between IDR (key) frames. An export can only begin at a key frame, so this bounds how closely `StopAndExportAsync` can match the requested duration. `null` applies `VideoEncoderOptions.DefaultIdrIntervalSeconds`, which is one second, rather than deferring to the platform encoder, whose own default differs per platform and may be considerably longer. A longer interval spends less of the bitrate on key frames, but an interval longer than the retained duration can leave the buffer with no key frame to start from, in which case the export produces nothing. + ```csharp // Default settings var options = new RealtimeEncodingOptions @@ -164,7 +166,8 @@ var options = new RealtimeEncodingOptions Width = 1280, Height = 720, FpsHint = 30, - Bitrate = 2500000 // 2.5 Mbps + Bitrate = 2500000, // 2.5 Mbps + IdrIntervalSeconds = 1f // Max interval between IDR (key) frames in seconds. null applies the default of 1 second. }, AudioOptions = new AudioEncoderOptions {