From 9fefe6d998aa2e041f67d1b7bf50734a6a8f19f7 Mon Sep 17 00:00:00 2001 From: Calvin Grunewald Date: Thu, 29 Oct 2026 09:32:06 -0700 Subject: [PATCH 1/3] Record: cameras with fixed frame rates, and stills of variable-rate video The C920 offers 30.00003 fps and refuses a frame duration of exactly 1/30, which crashed the recorder whenever a camera was chosen; use the range's own duration. A still (ctl frame, the viewer's freeze) dropped any video frame lasting under half a frame, which hid the first second of every take: the fps filter now passes the last frame, and the recorder no longer writes a frame a few milliseconds after the one it holds at the start or a resume. `record setup` also brings back sources that stopped. Co-Authored-By: Claude Opus 5.5 --- crates/splice-app/capture/main.swift | 15 ++++++++++++++- crates/splice-app/src/editor/record.rs | 3 ++- crates/splice-core/src/export.rs | 4 +++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/crates/splice-app/capture/main.swift b/crates/splice-app/capture/main.swift index 114ab60..5119f19 100644 --- a/crates/splice-app/capture/main.swift +++ b/crates/splice-app/capture/main.swift @@ -224,6 +224,7 @@ final class Sink { let input: AVAssetWriterInput var appended = 0 var dropped = 0 + var last = CMTime.invalid init(feed: Feed, url: URL, sample: CMSampleBuffer, config: Config) throws { self.feed = feed @@ -276,7 +277,13 @@ final class Sink { func append(_ buffer: CMSampleBuffer) { guard writer.status == .writing else { return } + // A frame hard on the heels of the one before (the held frame put at + // the start, or at a resume) would last a few milliseconds, which + // some readers drop altogether; the earlier one stands in for it. + let time = CMSampleBufferGetPresentationTimeStamp(buffer) + if feed.isVideo, last.isValid, CMTimeSubtract(time, last).seconds < 0.01 { return } if input.isReadyForMoreMediaData, input.append(buffer) { + last = time appended += 1 } else { dropped += 1 @@ -467,8 +474,14 @@ final class Capture: NSObject, SCStreamOutput, SCStreamDelegate, } if let best, (try? device.lockForConfiguration()) != nil { device.activeFormat = best + // Cameras such as the C920 offer fixed rates like 30.00003 that + // refuse a duration of exactly 1/30; use the range's own then. if let range = best.videoSupportedFrameRateRanges.first(where: { $0.maxFrameRate >= fps - 0.5 }) { - let frame = CMTime(value: 1000, timescale: CMTimeScale(min(fps, range.maxFrameRate) * 1000)) + var frame = range.minFrameDuration + if fps < range.maxFrameRate - 0.5 { + let wanted = CMTime(value: 1000, timescale: CMTimeScale(fps * 1000)) + frame = CMTimeMaximum(CMTimeMinimum(wanted, range.maxFrameDuration), range.minFrameDuration) + } device.activeVideoMinFrameDuration = frame device.activeVideoMaxFrameDuration = frame } diff --git a/crates/splice-app/src/editor/record.rs b/crates/splice-app/src/editor/record.rs index c65bf31..2a4c24e 100644 --- a/crates/splice-app/src/editor/record.rs +++ b/crates/splice-app/src/editor/record.rs @@ -1521,8 +1521,9 @@ impl Editor { } if studio != before || self.doc.project.studio.is_none() && request.get("default").is_some() { self.edit(cx, |p| p.studio = Some(studio)); - self.relive(cx); } + // Also brings back sources that stopped. + self.relive(cx); Ok(()) } } diff --git a/crates/splice-core/src/export.rs b/crates/splice-core/src/export.rs index fc03323..c26a9ea 100644 --- a/crates/splice-core/src/export.rs +++ b/crates/splice-core/src/export.rs @@ -1081,7 +1081,7 @@ fn clip_frames( None => format!(",tpad=stop_mode=clone:stop_duration={len},trim=duration={len},setpts=PTS-STARTPTS"), }; filters.push(format!( - "[{i}:v]reverse,trim=end_frame=1,setpts=PTS-STARTPTS,fps={fps}{hold}[{label}]" + "[{i}:v]reverse,trim=end_frame=1,setpts=PTS-STARTPTS,fps={fps}:eof_action=pass{hold}[{label}]" )); return Ok(Frames { label, @@ -1459,6 +1459,8 @@ mod tests { assert_eq!(plan.args[seek + 1], "0.000000"); assert_eq!(plan.args[seek + 3], "0.283333", "up to a frame past 0.25s"); assert!(graph.contains("reverse,trim=end_frame=1")); + // A frame lasting a few milliseconds (variable frame rate) still shows. + assert!(graph.contains(":eof_action=pass")); assert!( graph.contains("colorchannelmixer=aa=0.5000") && !graph.contains("fade="), "the fade is half done: {graph}" From 39f25b23b96daa031bea2a0f83a71d26aa43884b Mon Sep 17 00:00:00 2001 From: Calvin Grunewald Date: Thu, 29 Oct 2026 09:38:25 -0700 Subject: [PATCH 2/3] Mirror: flip any picture left to right, and the camera in Record Look.mirror flips the picture before its crop, mask and hidden box, in the viewer and in export alike (one hflip in the shared filters). The inspector has it under Crop and mirror; `ctl look --mirror`. Record's scene has a Mirror switch: the preview flips at once and a take's camera clip comes in mirrored, while camera.mp4 stays as the camera saw it. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- crates/splice-app/capture/main.swift | 9 +++- crates/splice-app/src/capture.rs | 1 + crates/splice-app/src/ctl.rs | 7 +-- crates/splice-app/src/editor/agent.rs | 5 ++- crates/splice-app/src/editor/inspector.rs | 54 +++++++++++++---------- crates/splice-app/src/editor/record.rs | 29 ++++++++---- crates/splice-core/src/look.rs | 29 +++++++++++- crates/splice-core/src/studio.rs | 21 ++++++++- skill/splice/SKILL.md | 2 +- 10 files changed, 119 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 0b20160..3aaff20 100644 --- a/README.md +++ b/README.md @@ -293,7 +293,8 @@ yourself and your screen. Nothing extra to install. live in the tab, sounds with a level meter. Splice keeps its own windows out of the shot. - **Scene:** Screen, Bubble (the camera in a round or square corner, small, - medium or large), Side by side, or Camera. + medium or large), Side by side, or Camera, the camera mirrored if you like. + The file stays as the camera saw it; the mirror is part of the edit. - **A file per source**, like OBS's Source Record: every take writes `screen.mp4`, `camera.mp4`, `microphone.m4a` and `system.m4a` into its own folder under `~/Movies/Splice/Recordings//`. All start at the same diff --git a/crates/splice-app/capture/main.swift b/crates/splice-app/capture/main.swift index 5119f19..aa26caf 100644 --- a/crates/splice-app/capture/main.swift +++ b/crates/splice-app/capture/main.swift @@ -143,9 +143,11 @@ struct Config: Decodable { /// The longest side of preview frames; 0 sends none. var preview: Int = 480 var previewFps: Int = 12 + /// The camera's preview flipped left to right (its file never is). + var mirror: Bool = false enum CodingKeys: String, CodingKey { - case screen, camera, microphone, codec, quality, fps, cursor, exclude, preview + case screen, camera, microphone, codec, quality, fps, cursor, exclude, preview, mirror case systemAudio = "system_audio" case maxHeight = "max_height" case previewFps = "preview_fps" @@ -165,6 +167,7 @@ struct Config: Decodable { exclude = try c.decodeIfPresent(Int32.self, forKey: .exclude) preview = try c.decodeIfPresent(Int.self, forKey: .preview) ?? 480 previewFps = try c.decodeIfPresent(Int.self, forKey: .previewFps) ?? 12 + mirror = try c.decodeIfPresent(Bool.self, forKey: .mirror) ?? false } } @@ -708,7 +711,9 @@ final class Capture: NSObject, SCStreamOutput, SCStreamDelegate, let (w, h) = (image.extent.width, image.extent.height) let k = min(1, CGFloat(config.preview) / max(w, h)) let (pw, ph) = (max(2, Int(w * k)), max(2, Int(h * k))) - let small = image.transformed(by: CGAffineTransform(scaleX: CGFloat(pw) / w, y: CGFloat(ph) / h)) + let flip: CGFloat = feed == .camera && config.mirror ? -1 : 1 + let small = image.transformed(by: CGAffineTransform(scaleX: flip * CGFloat(pw) / w, y: CGFloat(ph) / h) + .translatedBy(x: flip < 0 ? -w : 0, y: 0)) var body = Data(count: 9 + pw * ph * 4) body.withUnsafeMutableBytes { raw in let p = raw.baseAddress! diff --git a/crates/splice-app/src/capture.rs b/crates/splice-app/src/capture.rs index 3a36c70..c440fd0 100644 --- a/crates/splice-app/src/capture.rs +++ b/crates/splice-app/src/capture.rs @@ -161,6 +161,7 @@ pub fn config(studio: &Studio, devices: &Devices, preview: u32) -> Result "cursor": out.cursor, "exclude": std::process::id(), "preview": preview, + "mirror": studio.scene.mirror, })) } diff --git a/crates/splice-app/src/ctl.rs b/crates/splice-app/src/ctl.rs index 1c3f5ca..e092e86 100644 --- a/crates/splice-app/src/ctl.rs +++ b/crates/splice-app/src/ctl.rs @@ -56,6 +56,7 @@ Titles, colours and looks --key green|blue|#rrggbb|none [--similarity F] [--blend F] [--despill/--no-despill] --mask ellipse|rect|none [--mask-x F --mask-y F --mask-width F --mask-height F] [--feather F] [--invert/--no-invert] (mask numbers are fractions of the clip's frame) + --mirror/--no-mirror (flip left to right, as a camera shows you to yourself) --crop-left F --crop-right F --crop-top F --crop-bottom F | --crop none (cut edges away, fractions of the picture; it keeps its size and what remains stays put) --grade none|warm|cool|vivid|muted|faded|film|teal-orange|mono|noir (a ready-made look) @@ -201,7 +202,7 @@ Recording (each source to its own file; the take is laid out as the scene says) --screen display:N|window:APP|none --region X,Y,W,H (points, on that display) --camera NAME|none --microphone NAME|default|none --system | --no-system (the Mac's sound) --layout screen|bubble|side|camera --corner tl|tr|bl|br --size 0.3 (bubble height) - --shape circle|square --split 0.32 (camera's share, side by side) + --shape circle|square --split 0.32 (camera's share, side by side) --mirror/--no-mirror --codec h264|hevc --quality good|high|best --fps 30|60 --height full|4k|1440|1080 --cursor | --no-cursor --clicks | --no-clicks (note clicks, to zoom in on later) record start [SETUP] [--for S] [--beat B | --at T [--track N] | --into media] @@ -525,10 +526,10 @@ fn build(args: &[String]) -> Result { let v: f64 = next(arg)?.parse().map_err(|_| "--for expects seconds".to_string())?; request.insert("for".into(), json!(v)); } - "--mic" | "--clicks" | "--system" | "--cursor" | "--default" => { + "--mic" | "--clicks" | "--system" | "--cursor" | "--default" | "--mirror" => { request.insert(arg[2..].into(), json!(true)); } - "--no-clicks" | "--no-system" | "--no-cursor" => { + "--no-clicks" | "--no-system" | "--no-cursor" | "--no-mirror" => { request.insert(arg[5..].into(), json!(false)); } "--screen" | "--camera" | "--microphone" | "--layout" | "--corner" | "--shape" | "--codec" diff --git a/crates/splice-app/src/editor/agent.rs b/crates/splice-app/src/editor/agent.rs index 6efc14d..0f35122 100644 --- a/crates/splice-app/src/editor/agent.rs +++ b/crates/splice-app/src/editor/agent.rs @@ -108,6 +108,7 @@ fn look_json(look: &splice_core::look::Look) -> Value { "effects": (!look.fx.is_none()).then_some(look.fx), "backdrop": look.backdrop, "redact": look.redact, + "mirror": look.mirror, "grade": (!look.grade.is_none()).then(|| json!({ "look": look.grade.preset.name(), "exposure": look.grade.exposure, @@ -497,13 +498,14 @@ impl Editor { return Some(fail("a green-screen key is for video and pictures, not text or colour")); } let pictures_only = look.backdrop != clip.look.backdrop + || look.mirror != clip.look.mirror || look.redact != clip.look.redact || look.fx.blur != clip.look.fx.blur || look.fx.vignette != clip.look.fx.vignette || look.fx.grain != clip.look.fx.grain; if pictures_only && clip.generator.is_some() { return Some(fail( - "blur, vignette, grain, --backdrop and --redact are for video and pictures; a title or colour \ + "blur, vignette, grain, --mirror, --backdrop and --redact are for video and pictures; a title or colour \ takes --shake, --glitch and the rest", )); } @@ -1189,6 +1191,7 @@ fn apply_look(look: &mut splice_core::look::Look, request: &Value, at: Time) -> fx.shake = number("shake").unwrap_or(fx.shake); fx.glitch = number("glitch").unwrap_or(fx.glitch); *fx = fx.clamped(); + look.mirror = flag("mirror").unwrap_or(look.mirror); match text("backdrop") { Some("on" | "yes" | "blur") => look.backdrop = Some(look.backdrop.unwrap_or_default()), Some("off" | "no" | "none") => look.backdrop = None, diff --git a/crates/splice-app/src/editor/inspector.rs b/crates/splice-app/src/editor/inspector.rs index 78c55c3..caef986 100644 --- a/crates/splice-app/src/editor/inspector.rs +++ b/crates/splice-app/src/editor/inspector.rs @@ -2208,35 +2208,43 @@ impl Editor { /// Cutting edges off a picture: four sliders, and a way back. fn crop_section(&mut self, id: ClipId, clip: &Clip, cx: &mut Context) -> impl IntoElement + use<> { let crop = clip.look.crop.filter(|c| !c.is_none()); - let summary = match crop { - None => "Off".to_string(), - Some(c) => { + let mirror = clip.look.mirror; + let summary = match (crop, mirror) { + (None, false) => "Off".to_string(), + (None, true) => "Mirrored".to_string(), + (Some(c), _) => { let (w, h) = (1.0 - c.left - c.right, 1.0 - c.top - c.bottom); - format!("{:.0}% × {:.0}% left", w * 100.0, h * 100.0) + let flip = if mirror { ", mirrored" } else { "" }; + format!("{:.0}% × {:.0}% left{flip}", w * 100.0, h * 100.0) } }; - let (section, open) = self.fold(id, "Crop", crop.is_some(), summary, cx); + let (section, open) = self.fold(id, "Crop and mirror", crop.is_some() || mirror, summary, cx); section.when(open, |d| { - d.child(self.slider(id, clip, Field::CropLeft, cx)) - .child(self.slider(id, clip, Field::CropRight, cx)) - .child(self.slider(id, clip, Field::CropTop, cx)) - .child(self.slider(id, clip, Field::CropBottom, cx)) - .child( + d.child( + self.toggle("mirror", "Mirror, left to right", mirror, cx, move |this, cx| { + this.change(id, cx, |c| c.look.mirror = !c.look.mirror) + }), + ) + .child(self.slider(id, clip, Field::CropLeft, cx)) + .child(self.slider(id, clip, Field::CropRight, cx)) + .child(self.slider(id, clip, Field::CropTop, cx)) + .child(self.slider(id, clip, Field::CropBottom, cx)) + .child( + div() + .text_size(px(10.5)) + .line_height(px(15.0)) + .text_color(Theme::overlay()) + .child("The picture keeps its size; what remains stays where it was."), + ) + .when(crop.is_some(), |d| { + d.child( div() - .text_size(px(10.5)) - .line_height(px(15.0)) - .text_color(Theme::overlay()) - .child("The picture keeps its size; what remains stays where it was."), + .flex() + .child(self.chip("crop-reset", "Reset crop", false, cx, move |this, cx| { + this.change(id, cx, |c| c.look.crop = None); + })), ) - .when(crop.is_some(), |d| { - d.child( - div() - .flex() - .child(self.chip("crop-reset", "Reset crop", false, cx, move |this, cx| { - this.change(id, cx, |c| c.look.crop = None); - })), - ) - }) + }) }) } diff --git a/crates/splice-app/src/editor/record.rs b/crates/splice-app/src/editor/record.rs index 2a4c24e..c38011a 100644 --- a/crates/splice-app/src/editor/record.rs +++ b/crates/splice-app/src/editor/record.rs @@ -870,20 +870,30 @@ impl Editor { }) .child(record), ) - .when(scene.layout == Layout::Bubble, |d| { + .when(studio.camera.is_some() && scene.layout != Layout::Screen, |d| { d.child( div() .flex() .flex_wrap() .items_center() .gap(px(6.0)) - .child(label("Corner")) - .children(corners) - .child(div().w(px(10.0))) - .child(label("Size")) - .children(sizes) - .child(div().w(px(10.0))) - .children(shapes), + .when(scene.layout == Layout::Bubble, |d| { + d.child(label("Corner")) + .children(corners) + .child(div().w(px(10.0))) + .child(label("Size")) + .children(sizes) + .child(div().w(px(10.0))) + .children(shapes) + .child(div().w(px(10.0))) + }) + .child( + chip("camera-mirror", "Mirror", scene.mirror) + .tooltip(tip("Flip the camera left to right, as a mirror shows you", None)) + .on_click(cx.listener(|this, _, _, cx| { + this.set_studio(cx, |s| s.scene.mirror = !s.scene.mirror) + })), + ), ) }) } @@ -1483,6 +1493,9 @@ impl Editor { if let Some(split) = request.get("split").and_then(Value::as_f64) { studio.scene.split = (split as f32).clamp(0.15, 0.6); } + if let Some(mirror) = request.get("mirror").and_then(Value::as_bool) { + studio.scene.mirror = mirror; + } if let Some(shape) = text("shape") { studio.scene.shape = match shape.as_str() { "circle" | "round" => Bubble::Circle, diff --git a/crates/splice-core/src/look.rs b/crates/splice-core/src/look.rs index 435760f..dd7383b 100644 --- a/crates/splice-core/src/look.rs +++ b/crates/splice-core/src/look.rs @@ -48,6 +48,10 @@ pub struct Look { /// Part of the picture hidden: a face, a screen, a number plate. #[serde(skip_serializing_if = "Option::is_none")] pub redact: Option, + /// Flipped left to right, as a camera shows you to yourself. Crop, mask + /// and hidden box are then measured on the flipped picture. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub mirror: bool, } /// Effects on a clip's picture, each 0 (off) to 1 (strongest). @@ -223,6 +227,7 @@ impl Default for Look { fx: Effects::default(), backdrop: None, redact: None, + mirror: false, } } } @@ -1231,12 +1236,13 @@ impl Look { || self.fx.glitch > 0.0 } - /// The same look with no crop or hidden box: for frames those were + /// The same look with no flip, crop or hidden box: for frames those were /// already applied to. pub fn clone_without_hiding(&self) -> Look { Look { crop: None, redact: None, + mirror: false, ..self.clone() } } @@ -1266,6 +1272,9 @@ impl Look { pub fn hiding_filters(&self, size: (u32, u32), prefix: &str) -> (Vec, (u32, u32)) { let mut filters = Vec::new(); let mut size = size; + if self.mirror { + filters.push("hflip".into()); + } // Cut first, so a key and a mask work on what remains. if let Some(crop) = self.crop.filter(|c| !c.is_none()) { filters.push(crop.ffmpeg()); @@ -1511,4 +1520,22 @@ mod tests { assert!(f.starts_with("chromakey=0x00ff00:") && f.contains("despill=type=green") && f.contains("geq=")); assert!(Look::default().pixel_filters((640, 360), 1.0, "t").is_none()); } + + #[test] + fn a_mirror_flips_before_the_crop_and_mask() { + let look = Look { + mirror: true, + crop: Some(Crop { + left: 0.25, + ..Crop::default() + }), + mask: Some(Mask::default()), + ..Look::default() + }; + let f = look.pixel_filters((640, 360), 1.0, "t").unwrap(); + assert!(f.starts_with("hflip,crop="), "{f}"); + assert!(!look.clone_without_hiding().mirror); + let json = serde_json::to_string(&Look::default()).unwrap(); + assert!(!json.contains("mirror"), "unflipped looks stay as they were saved"); + } } diff --git a/crates/splice-core/src/studio.rs b/crates/splice-core/src/studio.rs index a5f12d8..16a408b 100644 --- a/crates/splice-core/src/studio.rs +++ b/crates/splice-core/src/studio.rs @@ -142,6 +142,8 @@ pub struct Scene { pub shape: Bubble, /// Side by side: the camera's share of the width. pub split: f32, + /// The camera flipped left to right, as a mirror shows you. + pub mirror: bool, } impl Default for Scene { @@ -152,6 +154,7 @@ impl Default for Scene { size: 0.3, shape: Bubble::Circle, split: 0.32, + mirror: false, } } } @@ -264,7 +267,7 @@ impl Studio { (_, None, Some(_)) => Layout::Camera, (layout, _, _) => layout, }; - match (layout, screen, camera) { + let mut placed = match (layout, screen, camera) { (Layout::Screen, Some(_), _) => Placed { screen: Some(Look::default()), camera: None, @@ -344,7 +347,11 @@ impl Studio { } } _ => Placed::default(), + }; + if let Some(camera) = &mut placed.camera { + camera.mirror = scene.mirror; } + placed } } @@ -426,6 +433,18 @@ mod tests { .camera .unwrap(); assert!(left.x < 0.0 && left.y < 0.0); + assert!(!cam.mirror && !left.mirror); + } + + #[test] + fn a_mirrored_camera_is_flipped_in_every_layout() { + for layout in Layout::ALL { + let mut s = studio(layout); + s.scene.mirror = true; + let placed = s.looks(CANVAS, SCREEN, CAMERA); + assert!(placed.camera.is_none_or(|c| c.mirror), "{layout:?}"); + assert!(placed.screen.is_none_or(|c| !c.mirror), "{layout:?}"); + } } #[test] diff --git a/skill/splice/SKILL.md b/skill/splice/SKILL.md index ab47e49..c5a9ff7 100644 --- a/skill/splice/SKILL.md +++ b/skill/splice/SKILL.md @@ -482,7 +482,7 @@ and lands on the timeline as layers placed by the scene. splice ctl record sources # displays, windows, cameras, microphones, permissions splice ctl record setup # this project's setup splice ctl record setup --screen window:Safari --camera C920 --microphone default --system -splice ctl record setup --layout bubble --corner br --size 0.3 --shape circle +splice ctl record setup --layout bubble --corner br --size 0.3 --shape circle --mirror splice ctl record setup --codec hevc --quality high --fps 60 --height 1440 --default splice ctl record start [--for 20] [--beat 27 | --at 30 | --into media] splice ctl record pause | resume From 50752b01ad74446c11ebaec22b5ecbbf968659d1 Mon Sep 17 00:00:00 2001 From: Calvin Grunewald Date: Thu, 29 Oct 2026 09:54:57 -0700 Subject: [PATCH 3/3] Voice clean-up, and a round bubble in Record's preview A clip's sound can be cleaned up as a voice, like OBS's mic filters: rumble cut below 80 Hz, steady noise lowered (afftdn, its floor rising with the strength, so a webcam's room noise drops about 5 dB at the default), and the level evened out (dynaudnorm, which lifts quiet words without raising the noise in pauses). One list of filters serves export and the viewer, which decodes a cleaned clip's sound as it does a retimed one. Inspector: Sound > Clean up the voice; ctl: `clean --clip C [--noise F] [--even]`; Record: a switch under the mic (`record setup --clean`) so takes' mic clips come in cleaned, the file left as recorded. The live preview drew the camera bubble square: gpui clips to rectangles. Camera preview frames now arrive cut to the part the scene shows, and are drawn round themselves. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 + crates/splice-app/src/capture.rs | 66 ++++++++++++- crates/splice-app/src/ctl.rs | 23 ++++- crates/splice-app/src/editor.rs | 22 +++-- crates/splice-app/src/editor/agent.rs | 38 ++++++++ crates/splice-app/src/editor/inspector.rs | 51 ++++++++++ crates/splice-app/src/editor/record.rs | 111 +++++++++++++++++----- crates/splice-app/src/ffmpeg.rs | 8 +- crates/splice-core/src/export.rs | 9 +- crates/splice-core/src/lib.rs | 1 + crates/splice-core/src/model.rs | 9 ++ crates/splice-core/src/studio.rs | 3 + crates/splice-core/src/voice.rs | 77 +++++++++++++++ skill/splice/SKILL.md | 2 + 14 files changed, 377 insertions(+), 47 deletions(-) create mode 100644 crates/splice-core/src/voice.rs diff --git a/README.md b/README.md index 3aaff20..638fa1a 100644 --- a/README.md +++ b/README.md @@ -308,6 +308,10 @@ yourself and your screen. Nothing extra to install. or 1080p. Files are written in one-second pieces, so a crash keeps what was recorded. - **Pause** and resume within a take. Clicks are noted, to zoom in on later. +- **Voice clean-up**, like OBS's mic filters: rumble cut, steady noise (fans, + hiss, hum) lowered, and the level evened out. Switch it on for the mic in + Record, or for any clip under Sound in the inspector; the file stays as + recorded. - **Each project keeps its own setup.** "Make this my default" sets the one new projects start with. - **Permissions:** Screen Recording, Camera and Microphone, asked for the diff --git a/crates/splice-app/src/capture.rs b/crates/splice-app/src/capture.rs index c440fd0..4903ae9 100644 --- a/crates/splice-app/src/capture.rs +++ b/crates/splice-app/src/capture.rs @@ -182,10 +182,28 @@ pub enum Feed { enum Message { Event(Value), - Frame(Feed, Arc), + /// A preview picture, its full size, and the part of it kept. + Frame(Feed, Arc, (u32, u32), Option), Gone, } +/// Part of a picture: left, top, right and bottom, as fractions of it. +pub type Part = [f32; 4]; + +/// The pixels of `part` of a BGRA picture, and their size. +fn cut(bgra: &[u8], (w, h): (u32, u32), part: Part) -> (Vec, (u32, u32)) { + let edge = |f: f32, n: u32| ((f.clamp(0.0, 1.0) * n as f32).round() as u32).min(n); + let (l, t) = (edge(part[0], w).min(w - 1), edge(part[1], h).min(h - 1)); + let (r, b) = (edge(part[2], w).clamp(l + 1, w), edge(part[3], h).clamp(t + 1, h)); + let (cw, ch) = (r - l, b - t); + let mut out = Vec::with_capacity((cw * ch * 4) as usize); + for y in t..t + ch { + let row = (y * w + l) as usize * 4; + out.extend_from_slice(&bgra[row..row + cw as usize * 4]); + } + (out, (cw, ch)) +} + /// A recording under way. pub struct Rec { pub dir: PathBuf, @@ -207,6 +225,12 @@ pub struct Live { /// The instructions it was started with. pub config: Value, pub frames: HashMap>, + /// Each picture's full size, and the part of it its frame holds. + pub natural: HashMap, + pub parts: HashMap>, + /// The part of the camera's picture the scene shows; its preview frames + /// arrive cut to it, so a bubble can be drawn round. + pub camera_part: Arc>>, /// The loudest each sound has been lately, 0 to 1. pub levels: HashMap, /// Sources that have sent something. @@ -243,6 +267,8 @@ impl Live { writeln!(stdin, "{config}")?; let mut stdout = child.stdout.take().expect("piped"); let (tx, messages) = channel(); + let camera_part = Arc::new(std::sync::Mutex::new(None)); + let wanted = camera_part.clone(); std::thread::Builder::new() .name("splice-capture".into()) .spawn(move || { @@ -262,12 +288,20 @@ impl Live { let feed = if body[0] == 0 { Feed::Screen } else { Feed::Camera }; let w = u32::from_le_bytes([body[1], body[2], body[3], body[4]]); let h = u32::from_le_bytes([body[5], body[6], body[7], body[8]]); - if body.len() != 9 + (w * h * 4) as usize { + if w == 0 || h == 0 || body.len() != 9 + (w * h * 4) as usize { continue; } body.drain(..9); - match ffmpeg::render_image(w, h, body) { - Some(image) => Message::Frame(feed, image), + let part = match feed { + Feed::Camera => *wanted.lock().unwrap(), + Feed::Screen => None, + }; + let (pixels, size) = match part { + Some(part) => cut(&body, (w, h), part), + None => (body, (w, h)), + }; + match ffmpeg::render_image(size.0, size.1, pixels) { + Some(image) => Message::Frame(feed, image, (w, h), part), None => continue, } } @@ -285,6 +319,9 @@ impl Live { messages, config, frames: HashMap::new(), + natural: HashMap::new(), + parts: HashMap::new(), + camera_part, levels: HashMap::new(), live: false, recording: None, @@ -350,8 +387,10 @@ impl Live { while let Ok(message) = self.messages.try_recv() { heard = true; match message { - Message::Frame(feed, image) => { + Message::Frame(feed, image, natural, part) => { self.live = true; + self.natural.insert(feed, natural); + self.parts.insert(feed, part); if let Some(old) = self.frames.insert(feed, image) { retire.push(old); } @@ -531,3 +570,20 @@ pub fn take_dir(project: &Path) -> PathBuf { } dir } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_part_of_a_picture_keeps_its_pixels() { + // 4×2, each pixel's first byte its index. + let bgra: Vec = (0..8u8).flat_map(|i| [i, 0, 0, 255]).collect(); + let (pixels, size) = cut(&bgra, (4, 2), [0.25, 0.5, 0.75, 1.0]); + assert_eq!(size, (2, 1)); + assert_eq!(pixels.chunks(4).map(|p| p[0]).collect::>(), vec![5, 6]); + // Nothing left still leaves a pixel, inside the picture. + let (pixels, size) = cut(&bgra, (4, 2), [1.0, 1.0, 1.0, 1.0]); + assert_eq!((size, pixels[0]), ((1, 1), 7)); + } +} diff --git a/crates/splice-app/src/ctl.rs b/crates/splice-app/src/ctl.rs index e092e86..024114f 100644 --- a/crates/splice-app/src/ctl.rs +++ b/crates/splice-app/src/ctl.rs @@ -32,6 +32,8 @@ Editing (each is one undo step; clips and media are named by id, media also by n delete --clip C [--ripple] remove clips; --ripple closes the gap duplicate --clip C copy clips to just after themselves volume --clip C --value V 1 is as recorded, 0 silent, up to 4 + clean --clip C [--noise F] [--even/--no-even] | --off + clean up a voice: rumble cut, steady noise down (0 to 1, 0.5), level evened out track --track N [--mute|--unmute] [--hide|--show] canvas [--aspect 16:9|9:16|1:1|4:5] [--size WxH] [--fps N] [--letterbox 2.39|1.85|2|none] undo | redo @@ -203,6 +205,7 @@ Recording (each source to its own file; the take is laid out as the scene says) --camera NAME|none --microphone NAME|default|none --system | --no-system (the Mac's sound) --layout screen|bubble|side|camera --corner tl|tr|bl|br --size 0.3 (bubble height) --shape circle|square --split 0.32 (camera's share, side by side) --mirror/--no-mirror + --clean/--no-clean (takes' mic clips come in with the voice cleaned up) --codec h264|hevc --quality good|high|best --fps 30|60 --height full|4k|1440|1080 --cursor | --no-cursor --clicks | --no-clicks (note clicks, to zoom in on later) record start [SETUP] [--for S] [--beat B | --at T [--track N] | --into media] @@ -481,7 +484,7 @@ fn build(args: &[String]) -> Result { | "--crop-left" | "--crop-top" | "--crop-right" | "--crop-bottom" | "--exposure" | "--contrast" | "--saturation" | "--warmth" | "--tint" | "--rate" | "--blur" | "--vignette" | "--grain" | "--shake" | "--glitch" | "--backdrop-blur" | "--backdrop-dim" | "--redact-x" | "--redact-y" | "--redact-width" - | "--redact-height" => { + | "--redact-height" | "--noise" => { let raw = next(arg)?; let v = match raw.parse::() { Ok(v) => json!(v), @@ -526,10 +529,11 @@ fn build(args: &[String]) -> Result { let v: f64 = next(arg)?.parse().map_err(|_| "--for expects seconds".to_string())?; request.insert("for".into(), json!(v)); } - "--mic" | "--clicks" | "--system" | "--cursor" | "--default" | "--mirror" => { + "--mic" | "--clicks" | "--system" | "--cursor" | "--default" | "--mirror" | "--even" | "--clean" + | "--off" => { request.insert(arg[2..].into(), json!(true)); } - "--no-clicks" | "--no-system" | "--no-cursor" | "--no-mirror" => { + "--no-clicks" | "--no-system" | "--no-cursor" | "--no-mirror" | "--no-even" | "--no-clean" => { request.insert(arg[5..].into(), json!(false)); } "--screen" | "--camera" | "--microphone" | "--layout" | "--corner" | "--shape" | "--codec" @@ -614,8 +618,8 @@ fn build(args: &[String]) -> Result { } let cmd = match command.as_str() { "status" | "read" | "versions" | "add" | "move" | "trim" | "split" | "delete" | "duplicate" | "volume" - | "track" | "canvas" | "undo" | "redo" | "play" | "pause" | "select" | "save" | "say" | "wait" | "text" - | "color" | "style" | "look" | "fonts" | "transition" | "speed" | "freeze" | "keyframe" | "leak" + | "clean" | "track" | "canvas" | "undo" | "redo" | "play" | "pause" | "select" | "save" | "say" | "wait" + | "text" | "color" | "style" | "look" | "fonts" | "transition" | "speed" | "freeze" | "keyframe" | "leak" | "captions" | "zoom" => command.as_str(), "restore" => { if let Some(id) = words.first().and_then(|w| w.parse::().ok()) { @@ -855,6 +859,15 @@ mod tests { json!({"cmd": "record", "sub": "setup", "screen": "window:Safari", "camera": "C920", "system": false, "layout": "bubble", "size": 0.25, "fps": 60, "height": "1440", "default": true}) ); + let r = build(&args("record setup --mirror --clean")).unwrap(); + assert_eq!( + r, + json!({"cmd": "record", "sub": "setup", "mirror": true, "clean": true}) + ); + let r = build(&args("clean --clip 9 --noise 0.6 --no-even")).unwrap(); + assert_eq!(r, json!({"cmd": "clean", "clips": [9], "noise": 0.6, "even": false})); + let r = build(&args("look --clip 3 --mirror")).unwrap(); + assert_eq!(r, json!({"cmd": "look", "clips": [3], "mirror": true})); } #[test] diff --git a/crates/splice-app/src/editor.rs b/crates/splice-app/src/editor.rs index 77084e5..7571308 100644 --- a/crates/splice-app/src/editor.rs +++ b/crates/splice-app/src/editor.rs @@ -181,12 +181,13 @@ struct Status { /// Names the retimed sound of a clip: what it plays, how fast, which way. fn retimed_key(clip: &splice_core::model::Clip, media: &Media) -> String { format!( - "{}:{}:{}:{}:{}", + "{}:{}:{}:{}:{}:{:?}", media.hash, clip.source_in, clip.span(), clip.speed, - clip.reverse + clip.reverse, + clip.clean ) } @@ -797,10 +798,10 @@ impl Editor { let fades = |f: Option<(Time, Time)>| f.map(|(a, z)| (audio.frames(a), audio.frames(z))); let (fade_in, fade_out) = (fades(sound.fade_in), fades(sound.fade_out)); let volume = clip.volume; - // A clip played faster, slower or backwards sounds as export - // makes it, once that is decoded; until then, its own sound - // retimed on the fly. - if clip.is_retimed() { + // A clip played faster, slower or backwards, or cleaned up, + // sounds as export makes it, once that is decoded; until + // then, its own sound retimed on the fly. + if clip.has_own_sound() { let (start, length) = (audio.frames(clip.start), audio.frames(clip.duration)); if let Some(Some(pcm)) = retimed.get(&retimed_key(clip, media)) { return Some(Voice { @@ -849,12 +850,12 @@ impl Editor { } /// Decodes, in the background, the sound of every clip played at - /// another speed or backwards, as export will make it. + /// another speed or backwards, or cleaned up, as export will make it. fn prepare_retimed_sound(&mut self) { let rate = self.audio.rate; let mut wanted = Vec::new(); for (clip, media) in self.doc.project.audible_clips() { - if !clip.is_retimed() || clip.frozen { + if !clip.has_own_sound() || clip.frozen { continue; } let key = retimed_key(clip, media); @@ -866,6 +867,7 @@ impl Editor { path.to_path_buf(), clip.source_in, clip.span(), + clip.clean.map(|c| c.filters()).unwrap_or_default(), clip.speed, clip.reverse, )); @@ -873,14 +875,14 @@ impl Editor { let mut cache = self.retimed_sound.lock().unwrap(); let keep: std::collections::HashSet<&String> = wanted.iter().map(|w| &w.0).collect(); cache.retain(|k, _| keep.contains(k)); - for (key, path, from, span, speed, reverse) in wanted { + for (key, path, from, span, clean, speed, reverse) in wanted { if cache.contains_key(&key) { continue; } cache.insert(key.clone(), None); let cache = self.retimed_sound.clone(); std::thread::spawn(move || { - if let Ok(pcm) = ffmpeg::decode_audio_retimed(&path, rate, from, span, speed, reverse) + if let Ok(pcm) = ffmpeg::decode_audio_retimed(&path, rate, from, span, clean, speed, reverse) && let Some(slot) = cache.lock().unwrap().get_mut(&key) { *slot = Some(Arc::new(pcm)); diff --git a/crates/splice-app/src/editor/agent.rs b/crates/splice-app/src/editor/agent.rs index 0f35122..8dcba36 100644 --- a/crates/splice-app/src/editor/agent.rs +++ b/crates/splice-app/src/editor/agent.rs @@ -360,6 +360,41 @@ impl Editor { self.edit(cx, |p| p.set_volume(&ids, volume as f32)); self.edited() } + "clean" => { + let ids = match self.clips(request) { + Ok(ids) => ids, + Err(e) => return Some(fail(e)), + }; + let project = &self.doc.project; + if let Some(id) = ids.iter().find(|id| { + project + .clip(**id) + .is_none_or(|c| c.generator.is_some() || project.media(c.media).is_none_or(|m| !m.has_audio)) + }) { + return Some(fail(format!("clip {} has no sound to clean", id.0))); + } + let off = flag("off") == Some(true); + let noise = request + .get("noise") + .and_then(Value::as_f64) + .map(|v| v.clamp(0.0, 1.0) as f32); + let even = flag("even"); + self.edit(cx, |p| { + for id in &ids { + if let Some(c) = p.clip_mut(*id) { + c.clean = if off { + None + } else { + let mut clean = c.clean.unwrap_or_default(); + clean.noise = noise.unwrap_or(clean.noise); + clean.even = even.unwrap_or(clean.even); + Some(clean) + }; + } + } + }); + self.edited() + } "track" => { let Some(track) = self.track(request).filter(|t| *t < self.doc.project.tracks.len()) else { return Some(fail(format!( @@ -901,6 +936,9 @@ impl Editor { value["media"] = json!(clip.media.0); value["source_in"] = json!(s(clip.source_in)); value["volume"] = json!(clip.volume); + if let Some(clean) = clip.clean { + value["clean"] = json!(clean); + } } Some(Generator::Text(title)) => { let h = project.settings.height as f32; diff --git a/crates/splice-app/src/editor/inspector.rs b/crates/splice-app/src/editor/inspector.rs index caef986..304936c 100644 --- a/crates/splice-app/src/editor/inspector.rs +++ b/crates/splice-app/src/editor/inspector.rs @@ -64,6 +64,7 @@ pub enum Field { RedactWidth, RedactHeight, TextOver, + CleanNoise, } impl Field { @@ -106,6 +107,7 @@ impl Field { | Field::RedactY => (0.0, 1.0), Field::RedactWidth | Field::RedactHeight => (0.02, 1.0), Field::TextOver => (0.1, 5.0), + Field::CleanNoise => (0.0, 1.0), } } @@ -133,6 +135,7 @@ impl Field { Field::RedactWidth => Redact::default().width, Field::RedactHeight => Redact::default().height, Field::TextOver => 1.0, + Field::CleanNoise => splice_core::voice::Clean::default().noise, } } @@ -173,6 +176,7 @@ impl Field { Field::RedactWidth => "Width", Field::RedactHeight => "Height", Field::TextOver => "Over", + Field::CleanNoise => "Noise", } } @@ -252,6 +256,7 @@ impl Field { Some(Generator::Text(t)) => t.animate.map_or(1.0, |m| secs(m.over) as f32), _ => 1.0, }, + Field::CleanNoise => clip.clean.unwrap_or_default().noise, } } @@ -303,6 +308,11 @@ impl Field { Field::Warmth => look.grade.temperature = v, Field::Tint => look.grade.tint = v, Field::Volume => clip.volume = v, + Field::CleanNoise => { + if let Some(c) = &mut clip.clean { + c.noise = v; + } + } Field::TextSize => { if let Some(Generator::Text(t)) = &mut clip.generator { t.size = v; @@ -1061,6 +1071,9 @@ impl Editor { sections.push( self.section("Sound") .child(self.slider(id, &clip, Field::Volume, cx)) + .when(clip.generator.is_none(), |d| { + d.child(self.clean_controls(id, &clip, cx)) + }) .when(speech, |d| { d.child( div().flex().child( @@ -1832,6 +1845,44 @@ impl Editor { }) } + /// Cleaning up a voice: a switch, how hard the noise is pushed down, and + /// whether the level is evened out. + fn clean_controls(&mut self, id: ClipId, clip: &Clip, cx: &mut Context) -> impl IntoElement + use<> { + let clean = clip.clean; + div() + .flex() + .flex_col() + .child(self.toggle( + "clean-voice", + "Clean up the voice", + clean.is_some(), + cx, + move |this, cx| { + this.change(id, cx, |c| { + c.clean = match c.clean { + Some(_) => None, + None => Some(splice_core::voice::Clean::default()), + }; + }); + }, + )) + .when_some(clean, |d, clean| { + d.child(self.slider(id, clip, Field::CleanNoise, cx)).child(self.toggle( + "clean-even", + "Even out the level", + clean.even, + cx, + move |this, cx| { + this.change(id, cx, |c| { + if let Some(c) = &mut c.clean { + c.even = !c.even; + } + }); + }, + )) + }) + } + /// Hiding part of the picture: a face, a screen, a number plate. fn redact_section(&mut self, id: ClipId, clip: &Clip, cx: &mut Context) -> impl IntoElement + use<> { let redact = clip.look.redact; diff --git a/crates/splice-app/src/editor/record.rs b/crates/splice-app/src/editor/record.rs index c38011a..95b1e25 100644 --- a/crates/splice-app/src/editor/record.rs +++ b/crates/splice-app/src/editor/record.rs @@ -509,6 +509,12 @@ impl Editor { placed.push(clip); } } + if studio.clean + && let Some(mic) = clips.get("microphone") + && let Some(c) = project.clip_mut(*mic) + { + c.clean = Some(splice_core::voice::Clean::default()); + } // The first pictures in a project set its shape: place them by it. let canvas = (project.settings.width, project.settings.height); let shown = studio.looks(canvas, size(project, "screen"), size(project, "camera")); @@ -667,22 +673,23 @@ impl Editor { (f32::from(area.size.width) - w) / 2.0, (f32::from(area.size.height) - h) / 2.0, ); - let frames = self.rec.live.as_ref().map(|l| l.frames.clone()).unwrap_or_default(); - let dims = |f: Feed| { - frames.get(&f).map(|i| { - let s = i.size(0); - (s.width.0 as u32, s.height.0 as u32) - }) - }; + let live = self.rec.live.as_ref(); + let frames = live.map(|l| l.frames.clone()).unwrap_or_default(); + let dims = |f: Feed| live.and_then(|l| l.natural.get(&f).copied()); let canvas_px = (settings.width, settings.height); let shown = studio.looks(canvas_px, dims(Feed::Screen), dims(Feed::Camera)); + // Camera frames come cut to what the scene shows of them. + if let Some(live) = live { + *live.camera_part.lock().unwrap() = shown.camera.as_ref().map(|l| shown_part(l).0); + } let mut layers: Vec = Vec::new(); for (feed, look) in [(Feed::Screen, shown.screen), (Feed::Camera, shown.camera)] { let (Some(look), Some(image)) = (look, frames.get(&feed)) else { continue; }; let Some(natural) = dims(feed) else { continue }; - layers.push(picture(image.clone(), natural, &look, (w, h))); + let part = live.and_then(|l| l.parts.get(&feed).copied().flatten()); + layers.push(picture(image.clone(), natural, part, &look, (w, h))); } let failed = self.rec.problem.is_some(); let status = if let Some(problem) = &self.rec.problem { @@ -1100,7 +1107,33 @@ impl Editor { .flex_col() .child(mic_row) .when(studio.microphone.is_some(), |d| { - d.child(meter(levels.get("microphone").copied())) + d.child(meter(levels.get("microphone").copied())).child( + div() + .id("mic-clean") + .h(px(28.0)) + .px(px(8.0)) + .ml(px(23.0)) + .rounded(px(7.0)) + .flex() + .items_center() + .gap(px(9.0)) + .cursor_pointer() + .hover(|d| d.bg(Theme::surface0())) + .child( + div() + .flex_1() + .text_size(px(11.5)) + .text_color(Theme::subtext()) + .child("Clean up the voice"), + ) + .child(switch(studio.clean)) + .tooltip(tip( + "Takes come in with rumble and steady noise lowered and the level evened out; \ + microphone.m4a stays as recorded", + None, + )) + .on_click(cx.listener(|this, _, _, cx| this.set_studio(cx, |s| s.clean = !s.clean))), + ) }) .into_any_element(), ); @@ -1493,6 +1526,9 @@ impl Editor { if let Some(split) = request.get("split").and_then(Value::as_f64) { studio.scene.split = (split as f32).clamp(0.15, 0.6); } + if let Some(clean) = flag("clean") { + studio.clean = clean; + } if let Some(mirror) = request.get("mirror").and_then(Value::as_bool) { studio.scene.mirror = mirror; } @@ -1588,28 +1624,57 @@ fn take_name(path: &Path) -> String { } /// A live picture where `look` puts it on a stage of `stage` points. -fn picture(image: Arc, natural: (u32, u32), look: &Look, stage: (f32, f32)) -> AnyElement { - let (sw, sh) = stage; - let (nw, nh) = (natural.0.max(1) as f32, natural.1.max(1) as f32); - let fit = (sw / nw).min(sh / nh) * look.scale; - let (pw, ph) = (nw * fit, nh * fit); - let (cx, cy) = (sw / 2.0 + look.x * sw, sh / 2.0 + look.y * sh); - let (pl, pt) = (cx - pw / 2.0, cy - ph / 2.0); - // What shows of it: all, a mask's shape, or what a crop leaves. - let (mut l, mut t, mut r, mut b) = (0.0, 0.0, 1.0, 1.0); - let mut round = false; +/// What shows of a picture: all of it, a mask's box, or what a crop +/// leaves (as a `Part`), and whether it is round. +fn shown_part(look: &Look) -> (capture::Part, bool) { if let Some(m) = look.mask { - (l, t, r, b) = ( + let part = [ m.x - m.width / 2.0, m.y - m.height / 2.0, m.x + m.width / 2.0, m.y + m.height / 2.0, - ); - round = m.shape == splice_core::look::Shape::Ellipse; + ]; + ( + part.map(|v| v.clamp(0.0, 1.0)), + m.shape == splice_core::look::Shape::Ellipse, + ) } else if let Some(c) = look.crop { - (l, t, r, b) = (c.left, c.top, 1.0 - c.right, 1.0 - c.bottom); + ([c.left, c.top, 1.0 - c.right, 1.0 - c.bottom], false) + } else { + ([0.0, 0.0, 1.0, 1.0], false) } +} + +/// A live picture placed by `look` on a stage of `stage` points. `part` is +/// the part of the picture the frame holds, when it was cut to fit. +fn picture( + image: Arc, + natural: (u32, u32), + part: Option, + look: &Look, + stage: (f32, f32), +) -> AnyElement { + let (sw, sh) = stage; + let (nw, nh) = (natural.0.max(1) as f32, natural.1.max(1) as f32); + let fit = (sw / nw).min(sh / nh) * look.scale; + let (pw, ph) = (nw * fit, nh * fit); + let (cx, cy) = (sw / 2.0 + look.x * sw, sh / 2.0 + look.y * sh); + let (pl, pt) = (cx - pw / 2.0, cy - ph / 2.0); + let ([l, t, r, b], round) = shown_part(look); let (vl, vt, vw, vh) = (pl + l * pw, pt + t * ph, (r - l) * pw, (b - t) * ph); + // A frame holding just what shows is drawn as it is, round if need be: + // clipping a larger one to a circle is beyond gpui. + if part.is_some_and(|p| p.iter().zip([l, t, r, b]).all(|(a, b)| (a - b).abs() < 1e-3)) { + return img(image) + .absolute() + .left(px(vl)) + .top(px(vt)) + .w(px(vw)) + .h(px(vh)) + .when(round, |d| d.rounded_full()) + .object_fit(ObjectFit::Fill) + .into_any_element(); + } div() .absolute() .left(px(vl)) diff --git a/crates/splice-app/src/ffmpeg.rs b/crates/splice-app/src/ffmpeg.rs index 314b2f7..187c60c 100644 --- a/crates/splice-app/src/ffmpeg.rs +++ b/crates/splice-app/src/ffmpeg.rs @@ -321,17 +321,19 @@ pub fn decode_audio(path: &Path, rate: u32) -> Result { Ok(Pcm { rate, samples }) } -/// The sound of `length` of `path` from `from`, played at `speed` (its -/// pitch kept) and backwards when `reverse`: what a retimed clip sounds like. +/// The sound of `length` of `path` from `from`, run through `first` (a +/// voice's clean-up), played at `speed` (its pitch kept) and backwards when +/// `reverse`: what a retimed or cleaned clip sounds like. pub fn decode_audio_retimed( path: &Path, rate: u32, from: Time, length: Time, + first: Vec, speed: f64, reverse: bool, ) -> Result { - let mut filters = Vec::new(); + let mut filters = first; if reverse { filters.push("areverse".to_string()); } diff --git a/crates/splice-core/src/export.rs b/crates/splice-core/src/export.rs index c26a9ea..fae93b8 100644 --- a/crates/splice-core/src/export.rs +++ b/crates/splice-core/src/export.rs @@ -1259,6 +1259,10 @@ fn clip_sound( } }; let mut chain = vec![format!("{trim}aresample=48000"), "asetpts=PTS-STARTPTS".to_string()]; + // Cleaned as recorded, before any change of speed. + if let Some(clean) = clip.clean { + chain.extend(clean.filters()); + } if clip.reverse { chain.push("areverse".into()); } @@ -1374,12 +1378,15 @@ mod tests { content: None, clicks: Vec::new(), }); - p.append_clip(v); + let clip = p.append_clip(v).unwrap(); + p.clip_mut(clip).unwrap().clean = Some(crate::voice::Clean::default()); p.insert_clip(image, 0, SECOND); let plan = plan(&p, &Fake, Path::new("/o.mp4"), Quality::Draft).unwrap(); let graph = graph(&plan); assert_eq!(graph.matches("overlay=").count(), 2); assert!(graph.contains("amix=inputs=2")); + let cleaned = graph.find("highpass=f=80,afftdn=").expect("the voice is cleaned"); + assert!(cleaned < graph.find("volume=").unwrap()); assert!( graph.contains("scale=1080:1080"), "the square image fits the 1920x1080 canvas: {graph}" diff --git a/crates/splice-core/src/lib.rs b/crates/splice-core/src/lib.rs index 21917d5..9591b1f 100644 --- a/crates/splice-core/src/lib.rs +++ b/crates/splice-core/src/lib.rs @@ -20,4 +20,5 @@ pub mod symbol; pub mod textmotion; pub mod time; pub mod transition; +pub mod voice; pub mod zoom; diff --git a/crates/splice-core/src/model.rs b/crates/splice-core/src/model.rs index 77d204e..4a1b501 100644 --- a/crates/splice-core/src/model.rs +++ b/crates/splice-core/src/model.rs @@ -201,6 +201,9 @@ pub struct Clip { /// The way in from the clip before it (or from what is beneath). #[serde(default, skip_serializing_if = "Option::is_none")] pub transition: Option, + /// Its sound cleaned up as a voice: rumble, steady noise, uneven level. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub clean: Option, } fn is_default(look: &Look) -> bool { @@ -341,6 +344,7 @@ impl Clip { reverse: false, frozen: false, transition: None, + clean: None, } } @@ -389,6 +393,11 @@ impl Clip { pub fn is_retimed(&self) -> bool { self.speed != 1.0 || self.reverse || self.frozen } + + /// Whether its sound is not simply its media's: retimed or cleaned up. + pub fn has_own_sound(&self) -> bool { + self.is_retimed() || self.clean.is_some() + } } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/crates/splice-core/src/studio.rs b/crates/splice-core/src/studio.rs index 16a408b..a35d420 100644 --- a/crates/splice-core/src/studio.rs +++ b/crates/splice-core/src/studio.rs @@ -18,6 +18,9 @@ pub struct Studio { pub microphone: Option, /// The sound the Mac plays (not Splice's own). pub system_audio: bool, + /// Takes' microphone clips come in with their voice cleaned up (the + /// file stays as recorded). + pub clean: bool, pub scene: Scene, pub output: Output, } diff --git a/crates/splice-core/src/voice.rs b/crates/splice-core/src/voice.rs new file mode 100644 index 0000000..c15c5ef --- /dev/null +++ b/crates/splice-core/src/voice.rs @@ -0,0 +1,77 @@ +//! Cleaning up a recorded voice: rumble cut, steady background noise (fans, +//! hiss, hum) pushed down, and quiet and loud passages brought closer +//! together. Written once as ffmpeg filters, which export and the viewer's +//! decoder both run, so what you hear is what you get. + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Clean { + /// How hard steady noise is pushed down, 0 (not at all) to 1. + pub noise: f32, + /// The level evened out: quiet words lifted, loud ones held back. + pub even: bool, +} + +impl Default for Clean { + fn default() -> Clean { + Clean { noise: 0.5, even: true } + } +} + +impl Clean { + /// The filters, in order, for a chain of sound at any rate. + pub fn filters(&self) -> Vec { + // Below 80 Hz is rumble, desk knocks and breath on the mic, not voice. + let mut filters = vec!["highpass=f=80".to_string()]; + let noise = if self.noise.is_finite() { + self.noise.clamp(0.0, 1.0) + } else { + 0.0 + }; + if noise > 0.0 { + // Lowered by up to 36 dB, the noise followed as it changes. A + // stronger setting also counts more as noise (a higher floor): at + // 0.5, a webcam's room noise drops about 5 dB and the voice hardly. + filters.push(format!( + "afftdn=nr={:.1}:nf={:.1}:tn=1", + 12.0 + 24.0 * noise, + -45.0 + 25.0 * noise + )); + } + if self.even { + // Each quarter second brought towards peaks of −6 dB, by at most + // 6×, the gain smoothed over a few seconds: quiet words come up, + // loud ones go down, and a pause keeps the gain of the words + // around it rather than having its noise raised to meet them. + filters.push("dynaudnorm=f=250:g=15:m=6:p=0.5".into()); + } + filters + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stronger_cleaning_takes_more_and_evening_is_its_own_choice() { + let gentle = Clean { + noise: 0.2, + even: false, + } + .filters(); + assert_eq!(gentle.len(), 2); + assert!(gentle[0].starts_with("highpass") && gentle[1].starts_with("afftdn=nr=16.8:nf=-40.0")); + let hard = Clean { noise: 1.0, even: true }.filters(); + // ffmpeg takes a floor no higher than −20 dB. + assert!(hard[1].starts_with("afftdn=nr=36.0:nf=-20.0")); + assert!(hard[2].starts_with("dynaudnorm")); + // No noise reduction at 0, and nonsense is no reduction either. + for noise in [0.0, f32::NAN] { + let f = Clean { noise, even: false }.filters(); + assert_eq!(f, vec!["highpass=f=80".to_string()]); + } + } +} diff --git a/skill/splice/SKILL.md b/skill/splice/SKILL.md index c5a9ff7..5efe293 100644 --- a/skill/splice/SKILL.md +++ b/skill/splice/SKILL.md @@ -93,6 +93,7 @@ splice ctl split --at 8.5 # every clip under 8.5s; --c splice ctl delete --clip 4,5 --ripple # --ripple closes the gap splice ctl duplicate --clip 3 splice ctl volume --clip 9 --value 0.4 # 1 as recorded, 0 silent +splice ctl clean --clip 9 --noise 0.6 --even # clean up a voice (--off to undo) splice ctl track --track 2 --mute --hide # --unmute, --show splice ctl canvas --aspect 9:16 # or --size 1080x1920, --fps 30 splice ctl undo @@ -483,6 +484,7 @@ splice ctl record sources # displays, windows, cameras, microph splice ctl record setup # this project's setup splice ctl record setup --screen window:Safari --camera C920 --microphone default --system splice ctl record setup --layout bubble --corner br --size 0.3 --shape circle --mirror +splice ctl record setup --clean # mic clips come in with the voice cleaned up splice ctl record setup --codec hevc --quality high --fps 60 --height 1440 --default splice ctl record start [--for 20] [--beat 27 | --at 30 | --into media] splice ctl record pause | resume