Record: camera fixes, mirroring, voice clean-up

README.md

@@ -293,7 +293,8 @@ yourself and your screen. Nothing extra to install.
   live in the tab, sounds with a level meter. Splice keeps its own windows
   out of the shot.
 - **Scene:** Screen, Bubble (the camera in a round or square corner, small,
-  medium or large), Side by side, or Camera.
+  medium or large), Side by side, or Camera, the camera mirrored if you like.
+  The file stays as the camera saw it; the mirror is part of the edit.
 - **A file per source**, like OBS's Source Record: every take writes
   `screen.mp4`, `camera.mp4`, `microphone.m4a` and `system.m4a` into its own
   folder under `~/Movies/Splice/Recordings/<project>/`. All start at the same
@@ -307,6 +308,10 @@ yourself and your screen. Nothing extra to install.
   or 1080p. Files are written in one-second pieces, so a crash keeps what
   was recorded.
 - **Pause** and resume within a take. Clicks are noted, to zoom in on later.
+- **Voice clean-up**, like OBS's mic filters: rumble cut, steady noise (fans,
+  hiss, hum) lowered, and the level evened out. Switch it on for the mic in
+  Record, or for any clip under Sound in the inspector; the file stays as
+  recorded.
 - **Each project keeps its own setup.** "Make this my default" sets the one
   new projects start with.
 - **Permissions:** Screen Recording, Camera and Microphone, asked for the

crates/splice-app/capture/main.swift

@@ -143,9 +143,11 @@ struct Config: Decodable {
     /// The longest side of preview frames; 0 sends none.
     var preview: Int = 480
     var previewFps: Int = 12
+    /// The camera's preview flipped left to right (its file never is).
+    var mirror: Bool = false
 
     enum CodingKeys: String, CodingKey {
-        case screen, camera, microphone, codec, quality, fps, cursor, exclude, preview
+        case screen, camera, microphone, codec, quality, fps, cursor, exclude, preview, mirror
         case systemAudio = "system_audio"
         case maxHeight = "max_height"
         case previewFps = "preview_fps"
@@ -165,6 +167,7 @@ struct Config: Decodable {
         exclude = try c.decodeIfPresent(Int32.self, forKey: .exclude)
         preview = try c.decodeIfPresent(Int.self, forKey: .preview) ?? 480
         previewFps = try c.decodeIfPresent(Int.self, forKey: .previewFps) ?? 12
+        mirror = try c.decodeIfPresent(Bool.self, forKey: .mirror) ?? false
     }
 }
 
@@ -224,6 +227,7 @@ final class Sink {
     let input: AVAssetWriterInput
     var appended = 0
     var dropped = 0
+    var last = CMTime.invalid
 
     init(feed: Feed, url: URL, sample: CMSampleBuffer, config: Config) throws {
         self.feed = feed
@@ -276,7 +280,13 @@ final class Sink {
 
     func append(_ buffer: CMSampleBuffer) {
         guard writer.status == .writing else { return }
+        // A frame hard on the heels of the one before (the held frame put at
+        // the start, or at a resume) would last a few milliseconds, which
+        // some readers drop altogether; the earlier one stands in for it.
+        let time = CMSampleBufferGetPresentationTimeStamp(buffer)
+        if feed.isVideo, last.isValid, CMTimeSubtract(time, last).seconds < 0.01 { return }
         if input.isReadyForMoreMediaData, input.append(buffer) {
+            last = time
             appended += 1
         } else {
             dropped += 1
@@ -467,8 +477,14 @@ final class Capture: NSObject, SCStreamOutput, SCStreamDelegate,
         }
         if let best, (try? device.lockForConfiguration()) != nil {
             device.activeFormat = best
+            // Cameras such as the C920 offer fixed rates like 30.00003 that
+            // refuse a duration of exactly 1/30; use the range's own then.
             if let range = best.videoSupportedFrameRateRanges.first(where: { $0.maxFrameRate >= fps - 0.5 }) {
-                let frame = CMTime(value: 1000, timescale: CMTimeScale(min(fps, range.maxFrameRate) * 1000))
+                var frame = range.minFrameDuration
+                if fps < range.maxFrameRate - 0.5 {
+                    let wanted = CMTime(value: 1000, timescale: CMTimeScale(fps * 1000))
+                    frame = CMTimeMaximum(CMTimeMinimum(wanted, range.maxFrameDuration), range.minFrameDuration)
+                }
                 device.activeVideoMinFrameDuration = frame
                 device.activeVideoMaxFrameDuration = frame
             }
@@ -695,7 +711,9 @@ final class Capture: NSObject, SCStreamOutput, SCStreamDelegate,
         let (w, h) = (image.extent.width, image.extent.height)
         let k = min(1, CGFloat(config.preview) / max(w, h))
         let (pw, ph) = (max(2, Int(w * k)), max(2, Int(h * k)))
-        let small = image.transformed(by: CGAffineTransform(scaleX: CGFloat(pw) / w, y: CGFloat(ph) / h))
+        let flip: CGFloat = feed == .camera && config.mirror ? -1 : 1
+        let small = image.transformed(by: CGAffineTransform(scaleX: flip * CGFloat(pw) / w, y: CGFloat(ph) / h)
+            .translatedBy(x: flip < 0 ? -w : 0, y: 0))
         var body = Data(count: 9 + pw * ph * 4)
         body.withUnsafeMutableBytes { raw in
             let p = raw.baseAddress!

crates/splice-app/src/capture.rs

@@ -161,6 +161,7 @@ pub fn config(studio: &Studio, devices: &Devices, preview: u32) -> Result<Value>
         "cursor": out.cursor,
         "exclude": std::process::id(),
         "preview": preview,
+        "mirror": studio.scene.mirror,
     }))
 }
 
@@ -181,10 +182,28 @@ pub enum Feed {
 
 enum Message {
     Event(Value),
-    Frame(Feed, Arc<RenderImage>),
+    /// A preview picture, its full size, and the part of it kept.
+    Frame(Feed, Arc<RenderImage>, (u32, u32), Option<Part>),
     Gone,
 }
 
+/// Part of a picture: left, top, right and bottom, as fractions of it.
+pub type Part = [f32; 4];
+
+/// The pixels of `part` of a BGRA picture, and their size.
+fn cut(bgra: &[u8], (w, h): (u32, u32), part: Part) -> (Vec<u8>, (u32, u32)) {
+    let edge = |f: f32, n: u32| ((f.clamp(0.0, 1.0) * n as f32).round() as u32).min(n);
+    let (l, t) = (edge(part[0], w).min(w - 1), edge(part[1], h).min(h - 1));
+    let (r, b) = (edge(part[2], w).clamp(l + 1, w), edge(part[3], h).clamp(t + 1, h));
+    let (cw, ch) = (r - l, b - t);
+    let mut out = Vec::with_capacity((cw * ch * 4) as usize);
+    for y in t..t + ch {
+        let row = (y * w + l) as usize * 4;
+        out.extend_from_slice(&bgra[row..row + cw as usize * 4]);
+    }
+    (out, (cw, ch))
+}
+
 /// A recording under way.
 pub struct Rec {
     pub dir: PathBuf,
@@ -206,6 +225,12 @@ pub struct Live {
     /// The instructions it was started with.
     pub config: Value,
     pub frames: HashMap<Feed, Arc<RenderImage>>,
+    /// Each picture's full size, and the part of it its frame holds.
+    pub natural: HashMap<Feed, (u32, u32)>,
+    pub parts: HashMap<Feed, Option<Part>>,
+    /// The part of the camera's picture the scene shows; its preview frames
+    /// arrive cut to it, so a bubble can be drawn round.
+    pub camera_part: Arc<std::sync::Mutex<Option<Part>>>,
     /// The loudest each sound has been lately, 0 to 1.
     pub levels: HashMap<String, f32>,
     /// Sources that have sent something.
@@ -242,6 +267,8 @@ impl Live {
         writeln!(stdin, "{config}")?;
         let mut stdout = child.stdout.take().expect("piped");
         let (tx, messages) = channel();
+        let camera_part = Arc::new(std::sync::Mutex::new(None));
+        let wanted = camera_part.clone();
         std::thread::Builder::new()
             .name("splice-capture".into())
             .spawn(move || {
@@ -261,12 +288,20 @@ impl Live {
                             let feed = if body[0] == 0 { Feed::Screen } else { Feed::Camera };
                             let w = u32::from_le_bytes([body[1], body[2], body[3], body[4]]);
                             let h = u32::from_le_bytes([body[5], body[6], body[7], body[8]]);
-                            if body.len() != 9 + (w * h * 4) as usize {
+                            if w == 0 || h == 0 || body.len() != 9 + (w * h * 4) as usize {
                                 continue;
                             }
                             body.drain(..9);
-                            match ffmpeg::render_image(w, h, body) {
-                                Some(image) => Message::Frame(feed, image),
+                            let part = match feed {
+                                Feed::Camera => *wanted.lock().unwrap(),
+                                Feed::Screen => None,
+                            };
+                            let (pixels, size) = match part {
+                                Some(part) => cut(&body, (w, h), part),
+                                None => (body, (w, h)),
+                            };
+                            match ffmpeg::render_image(size.0, size.1, pixels) {
+                                Some(image) => Message::Frame(feed, image, (w, h), part),
                                 None => continue,
                             }
                         }
@@ -284,6 +319,9 @@ impl Live {
             messages,
             config,
             frames: HashMap::new(),
+            natural: HashMap::new(),
+            parts: HashMap::new(),
+            camera_part,
             levels: HashMap::new(),
             live: false,
             recording: None,
@@ -349,8 +387,10 @@ impl Live {
         while let Ok(message) = self.messages.try_recv() {
             heard = true;
             match message {
-                Message::Frame(feed, image) => {
+                Message::Frame(feed, image, natural, part) => {
                     self.live = true;
+                    self.natural.insert(feed, natural);
+                    self.parts.insert(feed, part);
                     if let Some(old) = self.frames.insert(feed, image) {
                         retire.push(old);
                     }
@@ -530,3 +570,20 @@ pub fn take_dir(project: &Path) -> PathBuf {
     }
     dir
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn a_part_of_a_picture_keeps_its_pixels() {
+        // 4×2, each pixel's first byte its index.
+        let bgra: Vec<u8> = (0..8u8).flat_map(|i| [i, 0, 0, 255]).collect();
+        let (pixels, size) = cut(&bgra, (4, 2), [0.25, 0.5, 0.75, 1.0]);
+        assert_eq!(size, (2, 1));
+        assert_eq!(pixels.chunks(4).map(|p| p[0]).collect::<Vec<_>>(), vec![5, 6]);
+        // Nothing left still leaves a pixel, inside the picture.
+        let (pixels, size) = cut(&bgra, (4, 2), [1.0, 1.0, 1.0, 1.0]);
+        assert_eq!((size, pixels[0]), ((1, 1), 7));
+    }
+}

crates/splice-app/src/ctl.rs

@@ -32,6 +32,8 @@ Editing (each is one undo step; clips and media are named by id, media also by n
   delete --clip C [--ripple]               remove clips; --ripple closes the gap
   duplicate --clip C                       copy clips to just after themselves
   volume --clip C --value V                1 is as recorded, 0 silent, up to 4
+  clean --clip C [--noise F] [--even/--no-even] | --off
+                                 clean up a voice: rumble cut, steady noise down (0 to 1, 0.5), level evened out
   track --track N [--mute|--unmute] [--hide|--show]
   canvas [--aspect 16:9|9:16|1:1|4:5] [--size WxH] [--fps N] [--letterbox 2.39|1.85|2|none]
   undo | redo
@@ -56,6 +58,7 @@ Titles, colours and looks
           --key green|blue|#rrggbb|none [--similarity F] [--blend F] [--despill/--no-despill]
           --mask ellipse|rect|none [--mask-x F --mask-y F --mask-width F --mask-height F]
                 [--feather F] [--invert/--no-invert]   (mask numbers are fractions of the clip's frame)
+          --mirror/--no-mirror (flip left to right, as a camera shows you to yourself)
           --crop-left F --crop-right F --crop-top F --crop-bottom F | --crop none
                 (cut edges away, fractions of the picture; it keeps its size and what remains stays put)
           --grade none|warm|cool|vivid|muted|faded|film|teal-orange|mono|noir   (a ready-made look)
@@ -201,7 +204,8 @@ Recording (each source to its own file; the take is laid out as the scene says)
       --screen display:N|window:APP|none   --region X,Y,W,H (points, on that display)
       --camera NAME|none   --microphone NAME|default|none   --system | --no-system (the Mac's sound)
       --layout screen|bubble|side|camera   --corner tl|tr|bl|br   --size 0.3 (bubble height)
-      --shape circle|square   --split 0.32 (camera's share, side by side)
+      --shape circle|square   --split 0.32 (camera's share, side by side)   --mirror/--no-mirror
+      --clean/--no-clean (takes' mic clips come in with the voice cleaned up)
       --codec h264|hevc   --quality good|high|best   --fps 30|60   --height full|4k|1440|1080
       --cursor | --no-cursor   --clicks | --no-clicks (note clicks, to zoom in on later)
   record start [SETUP] [--for S] [--beat B | --at T [--track N] | --into media]
@@ -480,7 +484,7 @@ fn build(args: &[String]) -> Result<Value, String> {
             | "--crop-left" | "--crop-top" | "--crop-right" | "--crop-bottom" | "--exposure" | "--contrast"
             | "--saturation" | "--warmth" | "--tint" | "--rate" | "--blur" | "--vignette" | "--grain" | "--shake"
             | "--glitch" | "--backdrop-blur" | "--backdrop-dim" | "--redact-x" | "--redact-y" | "--redact-width"
-            | "--redact-height" => {
+            | "--redact-height" | "--noise" => {
                 let raw = next(arg)?;
                 let v = match raw.parse::<f64>() {
                     Ok(v) => json!(v),
@@ -525,10 +529,11 @@ fn build(args: &[String]) -> Result<Value, String> {
                 let v: f64 = next(arg)?.parse().map_err(|_| "--for expects seconds".to_string())?;
                 request.insert("for".into(), json!(v));
             }
-            "--mic" | "--clicks" | "--system" | "--cursor" | "--default" => {
+            "--mic" | "--clicks" | "--system" | "--cursor" | "--default" | "--mirror" | "--even" | "--clean"
+            | "--off" => {
                 request.insert(arg[2..].into(), json!(true));
             }
-            "--no-clicks" | "--no-system" | "--no-cursor" => {
+            "--no-clicks" | "--no-system" | "--no-cursor" | "--no-mirror" | "--no-even" | "--no-clean" => {
                 request.insert(arg[5..].into(), json!(false));
             }
             "--screen" | "--camera" | "--microphone" | "--layout" | "--corner" | "--shape" | "--codec"
@@ -613,8 +618,8 @@ fn build(args: &[String]) -> Result<Value, String> {
     }
     let cmd = match command.as_str() {
         "status" | "read" | "versions" | "add" | "move" | "trim" | "split" | "delete" | "duplicate" | "volume"
-        | "track" | "canvas" | "undo" | "redo" | "play" | "pause" | "select" | "save" | "say" | "wait" | "text"
-        | "color" | "style" | "look" | "fonts" | "transition" | "speed" | "freeze" | "keyframe" | "leak"
+        | "clean" | "track" | "canvas" | "undo" | "redo" | "play" | "pause" | "select" | "save" | "say" | "wait"
+        | "text" | "color" | "style" | "look" | "fonts" | "transition" | "speed" | "freeze" | "keyframe" | "leak"
         | "captions" | "zoom" => command.as_str(),
         "restore" => {
             if let Some(id) = words.first().and_then(|w| w.parse::<u64>().ok()) {
@@ -854,6 +859,15 @@ mod tests {
             json!({"cmd": "record", "sub": "setup", "screen": "window:Safari", "camera": "C920", "system": false,
                    "layout": "bubble", "size": 0.25, "fps": 60, "height": "1440", "default": true})
         );
+        let r = build(&args("record setup --mirror --clean")).unwrap();
+        assert_eq!(
+            r,
+            json!({"cmd": "record", "sub": "setup", "mirror": true, "clean": true})
+        );
+        let r = build(&args("clean --clip 9 --noise 0.6 --no-even")).unwrap();
+        assert_eq!(r, json!({"cmd": "clean", "clips": [9], "noise": 0.6, "even": false}));
+        let r = build(&args("look --clip 3 --mirror")).unwrap();
+        assert_eq!(r, json!({"cmd": "look", "clips": [3], "mirror": true}));
     }
 
     #[test]

crates/splice-app/src/editor.rs

@@ -181,12 +181,13 @@ struct Status {
 /// Names the retimed sound of a clip: what it plays, how fast, which way.
 fn retimed_key(clip: &splice_core::model::Clip, media: &Media) -> String {
     format!(
-        "{}:{}:{}:{}:{}",
+        "{}:{}:{}:{}:{}:{:?}",
         media.hash,
         clip.source_in,
         clip.span(),
         clip.speed,
-        clip.reverse
+        clip.reverse,
+        clip.clean
     )
 }
 
@@ -797,10 +798,10 @@ impl Editor {
                 let fades = |f: Option<(Time, Time)>| f.map(|(a, z)| (audio.frames(a), audio.frames(z)));
                 let (fade_in, fade_out) = (fades(sound.fade_in), fades(sound.fade_out));
                 let volume = clip.volume;
-                // A clip played faster, slower or backwards sounds as export
-                // makes it, once that is decoded; until then, its own sound
-                // retimed on the fly.
-                if clip.is_retimed() {
+                // A clip played faster, slower or backwards, or cleaned up,
+                // sounds as export makes it, once that is decoded; until
+                // then, its own sound retimed on the fly.
+                if clip.has_own_sound() {
                     let (start, length) = (audio.frames(clip.start), audio.frames(clip.duration));
                     if let Some(Some(pcm)) = retimed.get(&retimed_key(clip, media)) {
                         return Some(Voice {
@@ -849,12 +850,12 @@ impl Editor {
     }
 
     /// Decodes, in the background, the sound of every clip played at
-    /// another speed or backwards, as export will make it.
+    /// another speed or backwards, or cleaned up, as export will make it.
     fn prepare_retimed_sound(&mut self) {
         let rate = self.audio.rate;
         let mut wanted = Vec::new();
         for (clip, media) in self.doc.project.audible_clips() {
-            if !clip.is_retimed() || clip.frozen {
+            if !clip.has_own_sound() || clip.frozen {
                 continue;
             }
             let key = retimed_key(clip, media);
@@ -866,21 +867,22 @@ impl Editor {
                 path.to_path_buf(),
                 clip.source_in,
                 clip.span(),
+                clip.clean.map(|c| c.filters()).unwrap_or_default(),
                 clip.speed,
                 clip.reverse,
             ));
         }
         let mut cache = self.retimed_sound.lock().unwrap();
         let keep: std::collections::HashSet<&String> = wanted.iter().map(|w| &w.0).collect();
         cache.retain(|k, _| keep.contains(k));
-        for (key, path, from, span, speed, reverse) in wanted {
+        for (key, path, from, span, clean, speed, reverse) in wanted {
             if cache.contains_key(&key) {
                 continue;
             }
             cache.insert(key.clone(), None);
             let cache = self.retimed_sound.clone();
             std::thread::spawn(move || {
-                if let Ok(pcm) = ffmpeg::decode_audio_retimed(&path, rate, from, span, speed, reverse)
+                if let Ok(pcm) = ffmpeg::decode_audio_retimed(&path, rate, from, span, clean, speed, reverse)
                     && let Some(slot) = cache.lock().unwrap().get_mut(&key)
                 {
                     *slot = Some(Arc::new(pcm));