From 4af540e90b779d8b091490c2ff63623e099e8f7e Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 18 Oct 2026 08:18:15 +0900 Subject: [PATCH 1/5] fix(recording): heal Studio display fps that finalization replaced with the average rate --- crates/enc-ffmpeg/src/remux.rs | 184 ++++++++++++++++++++++++++++- crates/recording/src/track_heal.rs | 168 +++++++++++++++++++++++++- 2 files changed, 347 insertions(+), 5 deletions(-) diff --git a/crates/enc-ffmpeg/src/remux.rs b/crates/enc-ffmpeg/src/remux.rs index 984dc93e324..a2d58da8651 100644 --- a/crates/enc-ffmpeg/src/remux.rs +++ b/crates/enc-ffmpeg/src/remux.rs @@ -850,6 +850,105 @@ fn probe_video_pts_ladder_inner(path: &Path, nominal_fps: u32) -> Option Option { + suppress_ffmpeg_logs(); + let result = get_video_capture_fps_inner(path); + restore_ffmpeg_logs(); + result +} + +fn get_video_capture_fps_inner(path: &Path) -> Option { + const MAX_PACKETS: usize = 20_000; + + let mut ictx = avformat::input(path).ok()?; + let (stream_index, time_base) = { + let stream = ictx.streams().best(ffmpeg::media::Type::Video)?; + let tb = stream.time_base(); + if tb.numerator() <= 0 || tb.denominator() <= 0 { + return None; + } + ( + stream.index(), + f64::from(tb.numerator()) / f64::from(tb.denominator()), + ) + }; + let mut timestamps = Vec::new(); + for (stream, packet) in ictx.packets() { + if stream.index() != stream_index { + continue; + } + if let Some(pts) = packet.pts().or(packet.dts()) { + timestamps.push(pts as f64 * time_base); + } + if timestamps.len() >= MAX_PACKETS { + break; + } + } + capture_fps_from_timestamps(timestamps) +} + +/// The most common gap between frames, in 5% bins, refined by the median of +/// the gaps within 20% of it: frames land one capture interval apart +/// whenever the screen changes, whatever the holds between bursts. Arrival +/// jitter leaves the median a frame or two per second off, so a rate within +/// 5% of a standard capture rate is that rate. +pub fn capture_fps_from_timestamps(mut timestamps: Vec) -> Option { + const MIN_GAPS: usize = 10; + const BIN_RATIO: f64 = 1.05; + const CLUSTER_TOLERANCE: f64 = 0.2; + const STANDARD_RATES: [f64; 14] = [ + 24.0, 25.0, 30.0, 48.0, 50.0, 60.0, 72.0, 75.0, 90.0, 100.0, 120.0, 144.0, 165.0, 240.0, + ]; + const SNAP_TOLERANCE: f64 = 0.05; + + timestamps.sort_by(f64::total_cmp); + let gaps: Vec = timestamps + .windows(2) + .map(|pair| pair[1] - pair[0]) + .filter(|gap| *gap > 1e-4) + .collect(); + if gaps.len() < MIN_GAPS { + return None; + } + let bin_of = |gap: f64| (gap.ln() / BIN_RATIO.ln()).round() as i64; + let mut counts = std::collections::BTreeMap::::new(); + for gap in &gaps { + *counts.entry(bin_of(*gap)).or_default() += 1; + } + // Ties go to the shorter gap: a rate above the capture rate costs + // nothing, one below it drops frames. + let (bin, _) = counts + .iter() + .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))?; + // A bin is narrower than the jitter, so the most common one can sit to + // one side of the interval: recentre on the cluster's median. + let mut interval = BIN_RATIO.powi(*bin as i32); + for _ in 0..3 { + let mut cluster: Vec = gaps + .iter() + .copied() + .filter(|gap| (gap / interval - 1.0).abs() <= CLUSTER_TOLERANCE) + .collect(); + if cluster.len() < MIN_GAPS { + return None; + } + cluster.sort_by(f64::total_cmp); + interval = cluster[cluster.len() / 2]; + } + let measured = 1.0 / interval; + let fps = STANDARD_RATES + .into_iter() + .filter(|rate| (measured / rate - 1.0).abs() <= SNAP_TOLERANCE) + .min_by(|a, b| (measured - a).abs().total_cmp(&(measured - b).abs())) + .unwrap_or(measured.round()); + (1.0..=240.0).contains(&fps).then_some(fps as u32) +} + pub fn probe_m4s_can_decode_with_init( init_path: &Path, segment_path: &Path, @@ -1226,10 +1325,11 @@ fn merge_video_audio_inner( #[cfg(test)] mod tests { use super::{ - build_seek_probe_positions, concat_fragment_entry, concatenate_audio_to_ogg, - concatenate_m4s_segments_with_init, concatenate_m4s_segments_with_init_validated, + build_seek_probe_positions, capture_fps_from_timestamps, concat_fragment_entry, + concatenate_audio_to_ogg, concatenate_m4s_segments_with_init, + concatenate_m4s_segments_with_init_validated, concatenate_m4s_segments_with_init_validated_with_reader, concatenate_video_fragments, - probe_video_pts_ladder, write_concat_list, + get_video_capture_fps, get_video_fps, probe_video_pts_ladder, write_concat_list, }; use std::{ path::{Path, PathBuf}, @@ -1701,6 +1801,84 @@ mod tests { dir.join("ladder_probe_test.mp4") } + /// A 60 fps capture of a mostly static screen: a burst of `burst` + /// frames on the 60 fps grid, then a hold of `hold` intervals, repeated. + fn bursty_timestamps(bursts: u64, burst: u64, hold: u64) -> Vec { + (0..bursts) + .flat_map(|b| (0..burst).map(move |i| b * (burst + hold) + i)) + .map(|slot| Duration::from_nanos(slot * 1_000_000_000 / 60)) + .collect() + } + + fn secs(timestamps: &[Duration]) -> Vec { + timestamps.iter().map(Duration::as_secs_f64).collect() + } + + #[test] + fn capture_fps_of_a_bursty_capture_is_its_grid_not_its_average() { + let timestamps = secs(&bursty_timestamps(20, 6, 30)); + let span = timestamps.last().unwrap() - timestamps[0]; + assert!(timestamps.len() as f64 / span < 12.0); + assert_eq!(capture_fps_from_timestamps(timestamps), Some(60)); + } + + /// Up to `max` seconds either side of a frame's slot, uniformly. + fn jitter(index: u64, max: f64) -> f64 { + let mut x = index.wrapping_mul(0x9E37_79B9_7F4A_7C15) | 1; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + (x >> 11) as f64 / (1u64 << 53) as f64 * 2.0 * max - max + } + + #[test] + fn capture_fps_absorbs_arrival_jitter() { + for (fps, max) in [(30, 0.003), (60, 0.002), (120, 0.001)] { + let timestamps = (0..600) + .map(|i| i as f64 / f64::from(fps) + jitter(i, max)) + .collect(); + assert_eq!(capture_fps_from_timestamps(timestamps), Some(fps)); + } + let bursts = bursty_timestamps(20, 6, 30) + .iter() + .enumerate() + .map(|(i, t)| t.as_secs_f64() + jitter(i as u64, 0.0015)) + .collect(); + assert_eq!(capture_fps_from_timestamps(bursts), Some(60)); + } + + #[test] + fn capture_fps_keeps_a_non_standard_rate() { + let timestamps = (0..300).map(|i| f64::from(i) / 37.0).collect(); + assert_eq!(capture_fps_from_timestamps(timestamps), Some(37)); + } + + #[test] + fn capture_fps_of_a_slow_capture_on_a_fine_grid_is_the_capture_rate() { + // 30 fps frames stamped on a 120 Hz display's vsync grid, with holds: + // r_frame_rate reads 120 here. + let timestamps = (0..400u32) + .map(|i| f64::from(i * 4 + (i / 20) * 40) / 120.0) + .collect(); + assert_eq!(capture_fps_from_timestamps(timestamps), Some(30)); + } + + #[test] + fn capture_fps_needs_enough_frames() { + assert_eq!( + capture_fps_from_timestamps((0..5).map(|i| f64::from(i) / 60.0).collect()), + None + ); + } + + #[test] + fn capture_fps_reads_an_encoded_variable_rate_track() { + let dir = tempfile::tempdir().unwrap(); + let path = encode_test_mp4(dir.path(), &bursty_timestamps(20, 6, 30)); + assert!(get_video_fps(&path).unwrap() < 12); + assert_eq!(get_video_capture_fps(&path), Some(60)); + } + #[test] fn ladder_probe_detects_synthetic_cfr_stamping() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/recording/src/track_heal.rs b/crates/recording/src/track_heal.rs index 9ebfba3046f..638ce019a14 100644 --- a/crates/recording/src/track_heal.rs +++ b/crates/recording/src/track_heal.rs @@ -21,7 +21,8 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; use cap_enc_ffmpeg::remux::{ - get_media_duration, get_video_fps, probe_video_pts_ladder, rescale_video_timestamps, + get_media_duration, get_video_capture_fps, get_video_fps, probe_video_pts_ladder, + rescale_video_timestamps, }; use cap_project::{ProjectConfiguration, RecordingMeta, RecordingMetaInner, StudioRecordingMeta}; use tracing::{info, warn}; @@ -39,9 +40,54 @@ pub fn heal_stretched_tracks(project_path: &Path) -> anyhow::Result { return Ok(false); }; + let fps_healed = heal_average_display_fps(project_path)?; let display_healed = heal_stretched_display(project_path)?; let camera_healed = heal_stretched_camera(project_path)?; - Ok(display_healed || camera_healed) + Ok(fps_healed || display_healed || camera_healed) +} + +/// Finalizing a fragmented Studio recording stored the display track's +/// average frame rate as its fps. Decoders place frames on that rate, so a +/// mostly static screen, averaging well under its capture rate, played and +/// exported with frames missing. A track whose fps is still its average +/// rate gets the rate its frames were captured at; the media is untouched, +/// and a healed fps no longer matches the average, so this runs once. +pub fn heal_average_display_fps(project_path: &Path) -> anyhow::Result { + let mut meta = RecordingMeta::load_for_project(project_path) + .map_err(|e| anyhow::anyhow!("failed to load recording meta: {e}"))?; + let RecordingMetaInner::Studio(studio_meta) = &mut meta.inner else { + return Ok(false); + }; + let StudioRecordingMeta::MultipleSegments { inner } = studio_meta.as_mut() else { + return Ok(false); + }; + + let mut healed_any = false; + for (segment_index, segment) in inner.segments.iter_mut().enumerate() { + let display_path = segment.display.path.to_path(project_path); + let stored = segment.display.fps; + if get_video_fps(&display_path) != Some(stored) { + continue; + } + let Some(capture) = get_video_capture_fps(&display_path) else { + continue; + }; + if capture <= stored { + continue; + } + info!( + segment_index, + stored, capture, "Display fps was the track's average rate, restoring its capture rate" + ); + segment.display.fps = capture; + healed_any = true; + } + + if healed_any { + meta.save_for_project() + .map_err(|e| anyhow::anyhow!("failed to save healed recording meta: {e:?}"))?; + } + Ok(healed_any) } struct HealInFlightGuard(PathBuf); @@ -681,6 +727,124 @@ fn evaluate_stretch(camera_secs: f64, references: &[(&str, f64)]) -> Option u32, + ) -> tempfile::TempDir { + use cap_enc_ffmpeg::{h264::H264Encoder, mp4::MP4File}; + + ffmpeg::init().unwrap(); + let directory = tempfile::tempdir().unwrap(); + let project = directory.path(); + let display = project.join("content/segments/segment-0/display.mp4"); + std::fs::create_dir_all(display.parent().unwrap()).unwrap(); + let info = cap_media_info::VideoInfo { + pixel_format: cap_media_info::Pixel::NV12, + width: 320, + height: 240, + time_base: ffmpeg::Rational(1, 1_000_000), + frame_rate: ffmpeg::Rational(60, 1), + }; + let mut file = MP4File::init( + "display", + display.clone(), + false, + |o| H264Encoder::builder(info).build(o), + |_| None, + ) + .unwrap(); + for (index, timestamp) in timestamps.iter().enumerate() { + let mut frame = ffmpeg::frame::Video::new(ffmpeg::format::Pixel::NV12, 320, 240); + frame.data_mut(0).fill(32 + index as u8); + frame.data_mut(1).fill(128); + file.queue_video_frame(frame, *timestamp).unwrap(); + } + file.finish().unwrap(); + RecordingMeta { + platform: None, + project_path: project.to_path_buf(), + pretty_name: "fps heal fixture".into(), + sharing: None, + upload: None, + inner: RecordingMetaInner::Studio(Box::new(StudioRecordingMeta::MultipleSegments { + inner: MultipleSegments { + segments: vec![MultipleSegment { + display: VideoMeta { + path: "content/segments/segment-0/display.mp4".into(), + fps: fps(&display), + start_time: None, + device_id: None, + }, + camera: None, + mic: None, + system_audio: None, + cursor: None, + keyboard: None, + display_notch: None, + }], + cursors: Cursors::default(), + status: None, + }, + })), + } + .save_for_project() + .unwrap(); + directory + } + + fn stored_display_fps(project: &Path) -> u32 { + let meta = RecordingMeta::load_for_project(project).unwrap(); + let RecordingMetaInner::Studio(studio) = meta.inner else { + panic!("not a studio recording"); + }; + studio.max_fps() + } + + /// 60 fps frames in bursts of six between half-second holds. + fn static_screen_capture() -> Vec { + (0..20u64) + .flat_map(|burst| (0..6).map(move |i| burst * 36 + i)) + .map(|slot| Duration::from_nanos(slot * 1_000_000_000 / 60)) + .collect() + } + + #[test] + fn an_average_display_fps_is_healed_to_the_capture_rate() { + let project = display_project(&static_screen_capture(), |display| { + get_video_fps(display).unwrap() + }); + let average = stored_display_fps(project.path()); + assert!(average < 12, "fixture averages {average} fps"); + + assert!(heal_average_display_fps(project.path()).unwrap()); + assert_eq!(stored_display_fps(project.path()), 60); + assert!(!heal_average_display_fps(project.path()).unwrap()); + assert_eq!(stored_display_fps(project.path()), 60); + } + + #[test] + fn a_recorded_display_fps_is_left_alone() { + let project = display_project(&static_screen_capture(), |_| 60); + assert!(!heal_average_display_fps(project.path()).unwrap()); + assert_eq!(stored_display_fps(project.path()), 60); + } + + #[test] + fn a_steady_capture_at_its_average_is_left_alone() { + let timestamps: Vec = (0..90u64) + .map(|i| Duration::from_nanos(i * 1_000_000_000 / 30)) + .collect(); + let project = display_project(×tamps, |display| get_video_fps(display).unwrap()); + assert_eq!(stored_display_fps(project.path()), 30); + assert!(!heal_average_display_fps(project.path()).unwrap()); + assert_eq!(stored_display_fps(project.path()), 30); + } + const USER_REPORT_CAMERA_SECS: f64 = 13.666; fn user_report_references() -> Vec<(&'static str, f64)> { From c2237fb286cd9ed4c22f33c26ebbc983e164f1e2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 18 Oct 2026 08:18:15 +0900 Subject: [PATCH 2/5] fix(gpui): run the recording track heals before the editor opens a project --- apps/desktop-gpui/src/editor_window.rs | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/apps/desktop-gpui/src/editor_window.rs b/apps/desktop-gpui/src/editor_window.rs index 63c57330865..6cec16f0d2e 100644 --- a/apps/desktop-gpui/src/editor_window.rs +++ b/apps/desktop-gpui/src/editor_window.rs @@ -362,6 +362,15 @@ pub fn preflight(path: &std::path::Path) -> Result { return Err(format!("Video path {} not found!", path.display())); } + // Repairs tracks older recorders stamped at the wrong rate, and display + // fps that finalization replaced with the average rate, before anything + // reads them. Non-fatal: the editor opens the files as they are. + match cap_recording::track_heal::heal_stretched_tracks(path) { + Ok(true) => tracing::info!(?path, "Healed recording tracks"), + Ok(false) => {} + Err(error) => tracing::warn!(?path, "Track heal check failed: {error:#}"), + } + let meta = RecordingMeta::load_for_project(path) .map_err(|error| format!("Failed to load recording meta: {error}"))?; From 7441f0da1c385dc3266fe33855083160b0235f0b Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 18 Oct 2026 08:55:12 +0900 Subject: [PATCH 3/5] fix(enc-ffmpeg): capture rate is the shortest frame gap that recurs in runs, not the most common one --- crates/enc-ffmpeg/src/remux.rs | 78 ++++++++++++++++++++++++++-------- 1 file changed, 61 insertions(+), 17 deletions(-) diff --git a/crates/enc-ffmpeg/src/remux.rs b/crates/enc-ffmpeg/src/remux.rs index a2d58da8651..6abd21abb8b 100644 --- a/crates/enc-ffmpeg/src/remux.rs +++ b/crates/enc-ffmpeg/src/remux.rs @@ -892,11 +892,15 @@ fn get_video_capture_fps_inner(path: &Path) -> Option { capture_fps_from_timestamps(timestamps) } -/// The most common gap between frames, in 5% bins, refined by the median of -/// the gaps within 20% of it: frames land one capture interval apart -/// whenever the screen changes, whatever the holds between bursts. Arrival -/// jitter leaves the median a frame or two per second off, so a rate within -/// 5% of a standard capture rate is that rate. +/// The shortest gap between frames that recurs in runs, in 5% bins, refined +/// by the median of the gaps within 20% of it. Frames land one capture +/// interval apart whenever the screen changes, whatever the holds between +/// them, and content that changes at a lower rate for most of a recording +/// still has its faster stretches captured at the full rate. A late frame +/// leaves one short gap between two long ones, so only gaps beside another +/// gap of the same length count. Arrival jitter leaves the median a frame or +/// two per second off, so a rate within 5% of a standard capture rate is +/// that rate. pub fn capture_fps_from_timestamps(mut timestamps: Vec) -> Option { const MIN_GAPS: usize = 10; const BIN_RATIO: f64 = 1.05; @@ -915,19 +919,29 @@ pub fn capture_fps_from_timestamps(mut timestamps: Vec) -> Option { if gaps.len() < MIN_GAPS { return None; } - let bin_of = |gap: f64| (gap.ln() / BIN_RATIO.ln()).round() as i64; - let mut counts = std::collections::BTreeMap::::new(); - for gap in &gaps { - *counts.entry(bin_of(*gap)).or_default() += 1; - } - // Ties go to the shorter gap: a rate above the capture rate costs - // nothing, one below it drops frames. - let (bin, _) = counts + let within = |center: f64| { + gaps.iter() + .map(|gap| (gap / center - 1.0).abs() <= CLUSTER_TOLERANCE) + .collect::>() + }; + let in_runs = |within: &[bool]| { + (0..within.len()) + .filter(|&i| { + within[i] + && ((i > 0 && within[i - 1]) || within.get(i + 1).copied().unwrap_or(false)) + }) + .count() + }; + let bins: std::collections::BTreeSet = gaps .iter() - .max_by(|a, b| a.1.cmp(b.1).then(b.0.cmp(a.0)))?; - // A bin is narrower than the jitter, so the most common one can sit to - // one side of the interval: recentre on the cluster's median. - let mut interval = BIN_RATIO.powi(*bin as i32); + .map(|gap| (gap.ln() / BIN_RATIO.ln()).round() as i64) + .collect(); + // A bin is narrower than the jitter, so the first one with support can + // sit to one side of the interval: recentre on the cluster's median. + let mut interval = bins + .into_iter() + .map(|bin| BIN_RATIO.powi(bin as i32)) + .find(|center| in_runs(&within(*center)) >= MIN_GAPS)?; for _ in 0..3 { let mut cluster: Vec = gaps .iter() @@ -1847,6 +1861,36 @@ mod tests { assert_eq!(capture_fps_from_timestamps(bursts), Some(60)); } + #[test] + fn capture_fps_of_mostly_slower_content_is_its_fastest_run() { + // A 60 fps capture of content that changes 30 times a second for + // most of it, with idle holds and one 12-frame scroll at 60 fps. + let mut timestamps = Vec::new(); + let mut t = 0.0; + for burst in 0..200 { + for i in 0..10 { + t += 1.0 / 30.0; + timestamps.push(t + jitter(burst * 10 + i, 0.0015)); + } + t += 0.5; + } + for i in 0..12 { + t += 1.0 / 60.0; + timestamps.push(t + jitter(5000 + i, 0.001)); + } + assert_eq!(capture_fps_from_timestamps(timestamps), Some(60)); + } + + #[test] + fn capture_fps_ignores_the_short_gap_after_a_late_frame() { + // 30 fps with every seventh frame 12 ms late: each late frame leaves + // one 21 ms gap between a 45 ms one and a 33 ms one. + let timestamps = (0..600u64) + .map(|i| i as f64 / 30.0 + if i % 7 == 0 { 0.012 } else { 0.0 } + jitter(i, 0.001)) + .collect(); + assert_eq!(capture_fps_from_timestamps(timestamps), Some(30)); + } + #[test] fn capture_fps_keeps_a_non_standard_rate() { let timestamps = (0..300).map(|i| f64::from(i) / 37.0).collect(); From a782444a8e48ad6e49bebf71421931b958ed6f2e Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 18 Oct 2026 09:02:23 +0900 Subject: [PATCH 4/5] fix(enc-ffmpeg): a frame gap between two holds counts toward the capture rate --- crates/enc-ffmpeg/src/remux.rs | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/crates/enc-ffmpeg/src/remux.rs b/crates/enc-ffmpeg/src/remux.rs index 6abd21abb8b..b9fc9481962 100644 --- a/crates/enc-ffmpeg/src/remux.rs +++ b/crates/enc-ffmpeg/src/remux.rs @@ -897,8 +897,8 @@ fn get_video_capture_fps_inner(path: &Path) -> Option { /// interval apart whenever the screen changes, whatever the holds between /// them, and content that changes at a lower rate for most of a recording /// still has its faster stretches captured at the full rate. A late frame -/// leaves one short gap between two long ones, so only gaps beside another -/// gap of the same length count. Arrival jitter leaves the median a frame or +/// leaves one short gap between two slightly longer ones, so only gaps in +/// runs, or between holds, count. Arrival jitter leaves the median a frame or /// two per second off, so a rate within 5% of a standard capture rate is /// that rate. pub fn capture_fps_from_timestamps(mut timestamps: Vec) -> Option { @@ -909,6 +909,7 @@ pub fn capture_fps_from_timestamps(mut timestamps: Vec) -> Option { 24.0, 25.0, 30.0, 48.0, 50.0, 60.0, 72.0, 75.0, 90.0, 100.0, 120.0, 144.0, 165.0, 240.0, ]; const SNAP_TOLERANCE: f64 = 0.05; + const HOLD_RATIO: f64 = 3.0; timestamps.sort_by(f64::total_cmp); let gaps: Vec = timestamps @@ -924,11 +925,20 @@ pub fn capture_fps_from_timestamps(mut timestamps: Vec) -> Option { .map(|gap| (gap / center - 1.0).abs() <= CLUSTER_TOLERANCE) .collect::>() }; + // A gap counts beside another of its length (a run of frames) or between + // two holds at least HOLD_RATIO times as long (a change captured as a + // pair). A late frame's short gap sits between gaps under twice its own. let in_runs = |within: &[bool]| { + let held = |i: usize, j: Option| { + j.and_then(|j| gaps.get(j)) + .is_some_and(|gap| *gap >= gaps[i] * HOLD_RATIO) + }; (0..within.len()) .filter(|&i| { within[i] - && ((i > 0 && within[i - 1]) || within.get(i + 1).copied().unwrap_or(false)) + && ((i > 0 && within[i - 1]) + || within.get(i + 1).copied().unwrap_or(false) + || (held(i, i.checked_sub(1)) && held(i, Some(i + 1)))) }) .count() }; @@ -1891,6 +1901,16 @@ mod tests { assert_eq!(capture_fps_from_timestamps(timestamps), Some(30)); } + #[test] + fn capture_fps_of_changes_captured_in_pairs_is_the_pair_gap() { + // A 60 fps capture of a screen that changes once a second, each + // change captured as two frames. + let timestamps = (0..40u64) + .flat_map(|second| [0.0, 1.0 / 60.0].map(|offset| second as f64 + offset)) + .collect(); + assert_eq!(capture_fps_from_timestamps(timestamps), Some(60)); + } + #[test] fn capture_fps_keeps_a_non_standard_rate() { let timestamps = (0..300).map(|i| f64::from(i) / 37.0).collect(); From 763b344e09e87300dd3d0703759cde3b01c5cf17 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 18 Oct 2026 09:27:55 +0900 Subject: [PATCH 5/5] fix(enc-ffmpeg): a hold beside a short frame gap lasts at least 100 ms, so a late frame is not one --- crates/enc-ffmpeg/src/remux.rs | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/crates/enc-ffmpeg/src/remux.rs b/crates/enc-ffmpeg/src/remux.rs index b9fc9481962..ef5d4885a19 100644 --- a/crates/enc-ffmpeg/src/remux.rs +++ b/crates/enc-ffmpeg/src/remux.rs @@ -910,6 +910,7 @@ pub fn capture_fps_from_timestamps(mut timestamps: Vec) -> Option { ]; const SNAP_TOLERANCE: f64 = 0.05; const HOLD_RATIO: f64 = 3.0; + const HOLD_SECS: f64 = 0.1; timestamps.sort_by(f64::total_cmp); let gaps: Vec = timestamps @@ -926,12 +927,13 @@ pub fn capture_fps_from_timestamps(mut timestamps: Vec) -> Option { .collect::>() }; // A gap counts beside another of its length (a run of frames) or between - // two holds at least HOLD_RATIO times as long (a change captured as a - // pair). A late frame's short gap sits between gaps under twice its own. + // two holds (a change captured as a pair): gaps of at least HOLD_SECS and + // HOLD_RATIO times its length. A late frame's short gap always has one + // neighbour one capture interval long, under HOLD_SECS at 10 fps and up. let in_runs = |within: &[bool]| { let held = |i: usize, j: Option| { j.and_then(|j| gaps.get(j)) - .is_some_and(|gap| *gap >= gaps[i] * HOLD_RATIO) + .is_some_and(|gap| *gap >= HOLD_SECS && *gap >= gaps[i] * HOLD_RATIO) }; (0..within.len()) .filter(|&i| { @@ -1893,12 +1895,19 @@ mod tests { #[test] fn capture_fps_ignores_the_short_gap_after_a_late_frame() { - // 30 fps with every seventh frame 12 ms late: each late frame leaves - // one 21 ms gap between a 45 ms one and a 33 ms one. - let timestamps = (0..600u64) - .map(|i| i as f64 / 30.0 + if i % 7 == 0 { 0.012 } else { 0.0 } + jitter(i, 0.001)) - .collect(); - assert_eq!(capture_fps_from_timestamps(timestamps), Some(30)); + // 30 fps with every seventh frame late: 12 ms leaves a 21 ms gap + // between a 45 ms one and a 33 ms one, 25 ms an 8 ms gap between a + // 58 ms one and a 33 ms one. + for late in [0.012, 0.025, 0.03] { + let timestamps = (0..600u64) + .map(|i| i as f64 / 30.0 + if i % 7 == 0 { late } else { 0.0 } + jitter(i, 0.001)) + .collect(); + assert_eq!( + capture_fps_from_timestamps(timestamps), + Some(30), + "{late} s late" + ); + } } #[test]