diff --git a/README.md b/README.md index 0b20160..638fa1a 100644 --- a/README.md +++ b/README.md @@ -293,7 +293,8 @@ yourself and your screen. Nothing extra to install. live in the tab, sounds with a level meter. Splice keeps its own windows out of the shot. - **Scene:** Screen, Bubble (the camera in a round or square corner, small, - medium or large), Side by side, or Camera. + medium or large), Side by side, or Camera, the camera mirrored if you like. + The file stays as the camera saw it; the mirror is part of the edit. - **A file per source**, like OBS's Source Record: every take writes `screen.mp4`, `camera.mp4`, `microphone.m4a` and `system.m4a` into its own folder under `~/Movies/Splice/Recordings//`. All start at the same @@ -307,6 +308,10 @@ yourself and your screen. Nothing extra to install. or 1080p. Files are written in one-second pieces, so a crash keeps what was recorded. - **Pause** and resume within a take. Clicks are noted, to zoom in on later. +- **Voice clean-up**, like OBS's mic filters: rumble cut, steady noise (fans, + hiss, hum) lowered, and the level evened out. Switch it on for the mic in + Record, or for any clip under Sound in the inspector; the file stays as + recorded. - **Each project keeps its own setup.** "Make this my default" sets the one new projects start with. - **Permissions:** Screen Recording, Camera and Microphone, asked for the diff --git a/crates/splice-app/capture/main.swift b/crates/splice-app/capture/main.swift index 114ab60..aa26caf 100644 --- a/crates/splice-app/capture/main.swift +++ b/crates/splice-app/capture/main.swift @@ -143,9 +143,11 @@ struct Config: Decodable { /// The longest side of preview frames; 0 sends none. var preview: Int = 480 var previewFps: Int = 12 + /// The camera's preview flipped left to right (its file never is). + var mirror: Bool = false enum CodingKeys: String, CodingKey { - case screen, camera, microphone, codec, quality, fps, cursor, exclude, preview + case screen, camera, microphone, codec, quality, fps, cursor, exclude, preview, mirror case systemAudio = "system_audio" case maxHeight = "max_height" case previewFps = "preview_fps" @@ -165,6 +167,7 @@ struct Config: Decodable { exclude = try c.decodeIfPresent(Int32.self, forKey: .exclude) preview = try c.decodeIfPresent(Int.self, forKey: .preview) ?? 480 previewFps = try c.decodeIfPresent(Int.self, forKey: .previewFps) ?? 12 + mirror = try c.decodeIfPresent(Bool.self, forKey: .mirror) ?? false } } @@ -224,6 +227,7 @@ final class Sink { let input: AVAssetWriterInput var appended = 0 var dropped = 0 + var last = CMTime.invalid init(feed: Feed, url: URL, sample: CMSampleBuffer, config: Config) throws { self.feed = feed @@ -276,7 +280,13 @@ final class Sink { func append(_ buffer: CMSampleBuffer) { guard writer.status == .writing else { return } + // A frame hard on the heels of the one before (the held frame put at + // the start, or at a resume) would last a few milliseconds, which + // some readers drop altogether; the earlier one stands in for it. + let time = CMSampleBufferGetPresentationTimeStamp(buffer) + if feed.isVideo, last.isValid, CMTimeSubtract(time, last).seconds < 0.01 { return } if input.isReadyForMoreMediaData, input.append(buffer) { + last = time appended += 1 } else { dropped += 1 @@ -467,8 +477,14 @@ final class Capture: NSObject, SCStreamOutput, SCStreamDelegate, } if let best, (try? device.lockForConfiguration()) != nil { device.activeFormat = best + // Cameras such as the C920 offer fixed rates like 30.00003 that + // refuse a duration of exactly 1/30; use the range's own then. if let range = best.videoSupportedFrameRateRanges.first(where: { $0.maxFrameRate >= fps - 0.5 }) { - let frame = CMTime(value: 1000, timescale: CMTimeScale(min(fps, range.maxFrameRate) * 1000)) + var frame = range.minFrameDuration + if fps < range.maxFrameRate - 0.5 { + let wanted = CMTime(value: 1000, timescale: CMTimeScale(fps * 1000)) + frame = CMTimeMaximum(CMTimeMinimum(wanted, range.maxFrameDuration), range.minFrameDuration) + } device.activeVideoMinFrameDuration = frame device.activeVideoMaxFrameDuration = frame } @@ -695,7 +711,9 @@ final class Capture: NSObject, SCStreamOutput, SCStreamDelegate, let (w, h) = (image.extent.width, image.extent.height) let k = min(1, CGFloat(config.preview) / max(w, h)) let (pw, ph) = (max(2, Int(w * k)), max(2, Int(h * k))) - let small = image.transformed(by: CGAffineTransform(scaleX: CGFloat(pw) / w, y: CGFloat(ph) / h)) + let flip: CGFloat = feed == .camera && config.mirror ? -1 : 1 + let small = image.transformed(by: CGAffineTransform(scaleX: flip * CGFloat(pw) / w, y: CGFloat(ph) / h) + .translatedBy(x: flip < 0 ? -w : 0, y: 0)) var body = Data(count: 9 + pw * ph * 4) body.withUnsafeMutableBytes { raw in let p = raw.baseAddress! diff --git a/crates/splice-app/src/capture.rs b/crates/splice-app/src/capture.rs index 3a36c70..4903ae9 100644 --- a/crates/splice-app/src/capture.rs +++ b/crates/splice-app/src/capture.rs @@ -161,6 +161,7 @@ pub fn config(studio: &Studio, devices: &Devices, preview: u32) -> Result "cursor": out.cursor, "exclude": std::process::id(), "preview": preview, + "mirror": studio.scene.mirror, })) } @@ -181,10 +182,28 @@ pub enum Feed { enum Message { Event(Value), - Frame(Feed, Arc), + /// A preview picture, its full size, and the part of it kept. + Frame(Feed, Arc, (u32, u32), Option), Gone, } +/// Part of a picture: left, top, right and bottom, as fractions of it. +pub type Part = [f32; 4]; + +/// The pixels of `part` of a BGRA picture, and their size. +fn cut(bgra: &[u8], (w, h): (u32, u32), part: Part) -> (Vec, (u32, u32)) { + let edge = |f: f32, n: u32| ((f.clamp(0.0, 1.0) * n as f32).round() as u32).min(n); + let (l, t) = (edge(part[0], w).min(w - 1), edge(part[1], h).min(h - 1)); + let (r, b) = (edge(part[2], w).clamp(l + 1, w), edge(part[3], h).clamp(t + 1, h)); + let (cw, ch) = (r - l, b - t); + let mut out = Vec::with_capacity((cw * ch * 4) as usize); + for y in t..t + ch { + let row = (y * w + l) as usize * 4; + out.extend_from_slice(&bgra[row..row + cw as usize * 4]); + } + (out, (cw, ch)) +} + /// A recording under way. pub struct Rec { pub dir: PathBuf, @@ -206,6 +225,12 @@ pub struct Live { /// The instructions it was started with. pub config: Value, pub frames: HashMap>, + /// Each picture's full size, and the part of it its frame holds. + pub natural: HashMap, + pub parts: HashMap>, + /// The part of the camera's picture the scene shows; its preview frames + /// arrive cut to it, so a bubble can be drawn round. + pub camera_part: Arc>>, /// The loudest each sound has been lately, 0 to 1. pub levels: HashMap, /// Sources that have sent something. @@ -242,6 +267,8 @@ impl Live { writeln!(stdin, "{config}")?; let mut stdout = child.stdout.take().expect("piped"); let (tx, messages) = channel(); + let camera_part = Arc::new(std::sync::Mutex::new(None)); + let wanted = camera_part.clone(); std::thread::Builder::new() .name("splice-capture".into()) .spawn(move || { @@ -261,12 +288,20 @@ impl Live { let feed = if body[0] == 0 { Feed::Screen } else { Feed::Camera }; let w = u32::from_le_bytes([body[1], body[2], body[3], body[4]]); let h = u32::from_le_bytes([body[5], body[6], body[7], body[8]]); - if body.len() != 9 + (w * h * 4) as usize { + if w == 0 || h == 0 || body.len() != 9 + (w * h * 4) as usize { continue; } body.drain(..9); - match ffmpeg::render_image(w, h, body) { - Some(image) => Message::Frame(feed, image), + let part = match feed { + Feed::Camera => *wanted.lock().unwrap(), + Feed::Screen => None, + }; + let (pixels, size) = match part { + Some(part) => cut(&body, (w, h), part), + None => (body, (w, h)), + }; + match ffmpeg::render_image(size.0, size.1, pixels) { + Some(image) => Message::Frame(feed, image, (w, h), part), None => continue, } } @@ -284,6 +319,9 @@ impl Live { messages, config, frames: HashMap::new(), + natural: HashMap::new(), + parts: HashMap::new(), + camera_part, levels: HashMap::new(), live: false, recording: None, @@ -349,8 +387,10 @@ impl Live { while let Ok(message) = self.messages.try_recv() { heard = true; match message { - Message::Frame(feed, image) => { + Message::Frame(feed, image, natural, part) => { self.live = true; + self.natural.insert(feed, natural); + self.parts.insert(feed, part); if let Some(old) = self.frames.insert(feed, image) { retire.push(old); } @@ -530,3 +570,20 @@ pub fn take_dir(project: &Path) -> PathBuf { } dir } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_part_of_a_picture_keeps_its_pixels() { + // 4×2, each pixel's first byte its index. + let bgra: Vec = (0..8u8).flat_map(|i| [i, 0, 0, 255]).collect(); + let (pixels, size) = cut(&bgra, (4, 2), [0.25, 0.5, 0.75, 1.0]); + assert_eq!(size, (2, 1)); + assert_eq!(pixels.chunks(4).map(|p| p[0]).collect::>(), vec![5, 6]); + // Nothing left still leaves a pixel, inside the picture. + let (pixels, size) = cut(&bgra, (4, 2), [1.0, 1.0, 1.0, 1.0]); + assert_eq!((size, pixels[0]), ((1, 1), 7)); + } +} diff --git a/crates/splice-app/src/ctl.rs b/crates/splice-app/src/ctl.rs index 1c3f5ca..024114f 100644 --- a/crates/splice-app/src/ctl.rs +++ b/crates/splice-app/src/ctl.rs @@ -32,6 +32,8 @@ Editing (each is one undo step; clips and media are named by id, media also by n delete --clip C [--ripple] remove clips; --ripple closes the gap duplicate --clip C copy clips to just after themselves volume --clip C --value V 1 is as recorded, 0 silent, up to 4 + clean --clip C [--noise F] [--even/--no-even] | --off + clean up a voice: rumble cut, steady noise down (0 to 1, 0.5), level evened out track --track N [--mute|--unmute] [--hide|--show] canvas [--aspect 16:9|9:16|1:1|4:5] [--size WxH] [--fps N] [--letterbox 2.39|1.85|2|none] undo | redo @@ -56,6 +58,7 @@ Titles, colours and looks --key green|blue|#rrggbb|none [--similarity F] [--blend F] [--despill/--no-despill] --mask ellipse|rect|none [--mask-x F --mask-y F --mask-width F --mask-height F] [--feather F] [--invert/--no-invert] (mask numbers are fractions of the clip's frame) + --mirror/--no-mirror (flip left to right, as a camera shows you to yourself) --crop-left F --crop-right F --crop-top F --crop-bottom F | --crop none (cut edges away, fractions of the picture; it keeps its size and what remains stays put) --grade none|warm|cool|vivid|muted|faded|film|teal-orange|mono|noir (a ready-made look) @@ -201,7 +204,8 @@ Recording (each source to its own file; the take is laid out as the scene says) --screen display:N|window:APP|none --region X,Y,W,H (points, on that display) --camera NAME|none --microphone NAME|default|none --system | --no-system (the Mac's sound) --layout screen|bubble|side|camera --corner tl|tr|bl|br --size 0.3 (bubble height) - --shape circle|square --split 0.32 (camera's share, side by side) + --shape circle|square --split 0.32 (camera's share, side by side) --mirror/--no-mirror + --clean/--no-clean (takes' mic clips come in with the voice cleaned up) --codec h264|hevc --quality good|high|best --fps 30|60 --height full|4k|1440|1080 --cursor | --no-cursor --clicks | --no-clicks (note clicks, to zoom in on later) record start [SETUP] [--for S] [--beat B | --at T [--track N] | --into media] @@ -480,7 +484,7 @@ fn build(args: &[String]) -> Result { | "--crop-left" | "--crop-top" | "--crop-right" | "--crop-bottom" | "--exposure" | "--contrast" | "--saturation" | "--warmth" | "--tint" | "--rate" | "--blur" | "--vignette" | "--grain" | "--shake" | "--glitch" | "--backdrop-blur" | "--backdrop-dim" | "--redact-x" | "--redact-y" | "--redact-width" - | "--redact-height" => { + | "--redact-height" | "--noise" => { let raw = next(arg)?; let v = match raw.parse::() { Ok(v) => json!(v), @@ -525,10 +529,11 @@ fn build(args: &[String]) -> Result { let v: f64 = next(arg)?.parse().map_err(|_| "--for expects seconds".to_string())?; request.insert("for".into(), json!(v)); } - "--mic" | "--clicks" | "--system" | "--cursor" | "--default" => { + "--mic" | "--clicks" | "--system" | "--cursor" | "--default" | "--mirror" | "--even" | "--clean" + | "--off" => { request.insert(arg[2..].into(), json!(true)); } - "--no-clicks" | "--no-system" | "--no-cursor" => { + "--no-clicks" | "--no-system" | "--no-cursor" | "--no-mirror" | "--no-even" | "--no-clean" => { request.insert(arg[5..].into(), json!(false)); } "--screen" | "--camera" | "--microphone" | "--layout" | "--corner" | "--shape" | "--codec" @@ -613,8 +618,8 @@ fn build(args: &[String]) -> Result { } let cmd = match command.as_str() { "status" | "read" | "versions" | "add" | "move" | "trim" | "split" | "delete" | "duplicate" | "volume" - | "track" | "canvas" | "undo" | "redo" | "play" | "pause" | "select" | "save" | "say" | "wait" | "text" - | "color" | "style" | "look" | "fonts" | "transition" | "speed" | "freeze" | "keyframe" | "leak" + | "clean" | "track" | "canvas" | "undo" | "redo" | "play" | "pause" | "select" | "save" | "say" | "wait" + | "text" | "color" | "style" | "look" | "fonts" | "transition" | "speed" | "freeze" | "keyframe" | "leak" | "captions" | "zoom" => command.as_str(), "restore" => { if let Some(id) = words.first().and_then(|w| w.parse::().ok()) { @@ -854,6 +859,15 @@ mod tests { json!({"cmd": "record", "sub": "setup", "screen": "window:Safari", "camera": "C920", "system": false, "layout": "bubble", "size": 0.25, "fps": 60, "height": "1440", "default": true}) ); + let r = build(&args("record setup --mirror --clean")).unwrap(); + assert_eq!( + r, + json!({"cmd": "record", "sub": "setup", "mirror": true, "clean": true}) + ); + let r = build(&args("clean --clip 9 --noise 0.6 --no-even")).unwrap(); + assert_eq!(r, json!({"cmd": "clean", "clips": [9], "noise": 0.6, "even": false})); + let r = build(&args("look --clip 3 --mirror")).unwrap(); + assert_eq!(r, json!({"cmd": "look", "clips": [3], "mirror": true})); } #[test] diff --git a/crates/splice-app/src/editor.rs b/crates/splice-app/src/editor.rs index 77084e5..7571308 100644 --- a/crates/splice-app/src/editor.rs +++ b/crates/splice-app/src/editor.rs @@ -181,12 +181,13 @@ struct Status { /// Names the retimed sound of a clip: what it plays, how fast, which way. fn retimed_key(clip: &splice_core::model::Clip, media: &Media) -> String { format!( - "{}:{}:{}:{}:{}", + "{}:{}:{}:{}:{}:{:?}", media.hash, clip.source_in, clip.span(), clip.speed, - clip.reverse + clip.reverse, + clip.clean ) } @@ -797,10 +798,10 @@ impl Editor { let fades = |f: Option<(Time, Time)>| f.map(|(a, z)| (audio.frames(a), audio.frames(z))); let (fade_in, fade_out) = (fades(sound.fade_in), fades(sound.fade_out)); let volume = clip.volume; - // A clip played faster, slower or backwards sounds as export - // makes it, once that is decoded; until then, its own sound - // retimed on the fly. - if clip.is_retimed() { + // A clip played faster, slower or backwards, or cleaned up, + // sounds as export makes it, once that is decoded; until + // then, its own sound retimed on the fly. + if clip.has_own_sound() { let (start, length) = (audio.frames(clip.start), audio.frames(clip.duration)); if let Some(Some(pcm)) = retimed.get(&retimed_key(clip, media)) { return Some(Voice { @@ -849,12 +850,12 @@ impl Editor { } /// Decodes, in the background, the sound of every clip played at - /// another speed or backwards, as export will make it. + /// another speed or backwards, or cleaned up, as export will make it. fn prepare_retimed_sound(&mut self) { let rate = self.audio.rate; let mut wanted = Vec::new(); for (clip, media) in self.doc.project.audible_clips() { - if !clip.is_retimed() || clip.frozen { + if !clip.has_own_sound() || clip.frozen { continue; } let key = retimed_key(clip, media); @@ -866,6 +867,7 @@ impl Editor { path.to_path_buf(), clip.source_in, clip.span(), + clip.clean.map(|c| c.filters()).unwrap_or_default(), clip.speed, clip.reverse, )); @@ -873,14 +875,14 @@ impl Editor { let mut cache = self.retimed_sound.lock().unwrap(); let keep: std::collections::HashSet<&String> = wanted.iter().map(|w| &w.0).collect(); cache.retain(|k, _| keep.contains(k)); - for (key, path, from, span, speed, reverse) in wanted { + for (key, path, from, span, clean, speed, reverse) in wanted { if cache.contains_key(&key) { continue; } cache.insert(key.clone(), None); let cache = self.retimed_sound.clone(); std::thread::spawn(move || { - if let Ok(pcm) = ffmpeg::decode_audio_retimed(&path, rate, from, span, speed, reverse) + if let Ok(pcm) = ffmpeg::decode_audio_retimed(&path, rate, from, span, clean, speed, reverse) && let Some(slot) = cache.lock().unwrap().get_mut(&key) { *slot = Some(Arc::new(pcm)); diff --git a/crates/splice-app/src/editor/agent.rs b/crates/splice-app/src/editor/agent.rs index 6efc14d..8dcba36 100644 --- a/crates/splice-app/src/editor/agent.rs +++ b/crates/splice-app/src/editor/agent.rs @@ -108,6 +108,7 @@ fn look_json(look: &splice_core::look::Look) -> Value { "effects": (!look.fx.is_none()).then_some(look.fx), "backdrop": look.backdrop, "redact": look.redact, + "mirror": look.mirror, "grade": (!look.grade.is_none()).then(|| json!({ "look": look.grade.preset.name(), "exposure": look.grade.exposure, @@ -359,6 +360,41 @@ impl Editor { self.edit(cx, |p| p.set_volume(&ids, volume as f32)); self.edited() } + "clean" => { + let ids = match self.clips(request) { + Ok(ids) => ids, + Err(e) => return Some(fail(e)), + }; + let project = &self.doc.project; + if let Some(id) = ids.iter().find(|id| { + project + .clip(**id) + .is_none_or(|c| c.generator.is_some() || project.media(c.media).is_none_or(|m| !m.has_audio)) + }) { + return Some(fail(format!("clip {} has no sound to clean", id.0))); + } + let off = flag("off") == Some(true); + let noise = request + .get("noise") + .and_then(Value::as_f64) + .map(|v| v.clamp(0.0, 1.0) as f32); + let even = flag("even"); + self.edit(cx, |p| { + for id in &ids { + if let Some(c) = p.clip_mut(*id) { + c.clean = if off { + None + } else { + let mut clean = c.clean.unwrap_or_default(); + clean.noise = noise.unwrap_or(clean.noise); + clean.even = even.unwrap_or(clean.even); + Some(clean) + }; + } + } + }); + self.edited() + } "track" => { let Some(track) = self.track(request).filter(|t| *t < self.doc.project.tracks.len()) else { return Some(fail(format!( @@ -497,13 +533,14 @@ impl Editor { return Some(fail("a green-screen key is for video and pictures, not text or colour")); } let pictures_only = look.backdrop != clip.look.backdrop + || look.mirror != clip.look.mirror || look.redact != clip.look.redact || look.fx.blur != clip.look.fx.blur || look.fx.vignette != clip.look.fx.vignette || look.fx.grain != clip.look.fx.grain; if pictures_only && clip.generator.is_some() { return Some(fail( - "blur, vignette, grain, --backdrop and --redact are for video and pictures; a title or colour \ + "blur, vignette, grain, --mirror, --backdrop and --redact are for video and pictures; a title or colour \ takes --shake, --glitch and the rest", )); } @@ -899,6 +936,9 @@ impl Editor { value["media"] = json!(clip.media.0); value["source_in"] = json!(s(clip.source_in)); value["volume"] = json!(clip.volume); + if let Some(clean) = clip.clean { + value["clean"] = json!(clean); + } } Some(Generator::Text(title)) => { let h = project.settings.height as f32; @@ -1189,6 +1229,7 @@ fn apply_look(look: &mut splice_core::look::Look, request: &Value, at: Time) -> fx.shake = number("shake").unwrap_or(fx.shake); fx.glitch = number("glitch").unwrap_or(fx.glitch); *fx = fx.clamped(); + look.mirror = flag("mirror").unwrap_or(look.mirror); match text("backdrop") { Some("on" | "yes" | "blur") => look.backdrop = Some(look.backdrop.unwrap_or_default()), Some("off" | "no" | "none") => look.backdrop = None, diff --git a/crates/splice-app/src/editor/inspector.rs b/crates/splice-app/src/editor/inspector.rs index 78c55c3..304936c 100644 --- a/crates/splice-app/src/editor/inspector.rs +++ b/crates/splice-app/src/editor/inspector.rs @@ -64,6 +64,7 @@ pub enum Field { RedactWidth, RedactHeight, TextOver, + CleanNoise, } impl Field { @@ -106,6 +107,7 @@ impl Field { | Field::RedactY => (0.0, 1.0), Field::RedactWidth | Field::RedactHeight => (0.02, 1.0), Field::TextOver => (0.1, 5.0), + Field::CleanNoise => (0.0, 1.0), } } @@ -133,6 +135,7 @@ impl Field { Field::RedactWidth => Redact::default().width, Field::RedactHeight => Redact::default().height, Field::TextOver => 1.0, + Field::CleanNoise => splice_core::voice::Clean::default().noise, } } @@ -173,6 +176,7 @@ impl Field { Field::RedactWidth => "Width", Field::RedactHeight => "Height", Field::TextOver => "Over", + Field::CleanNoise => "Noise", } } @@ -252,6 +256,7 @@ impl Field { Some(Generator::Text(t)) => t.animate.map_or(1.0, |m| secs(m.over) as f32), _ => 1.0, }, + Field::CleanNoise => clip.clean.unwrap_or_default().noise, } } @@ -303,6 +308,11 @@ impl Field { Field::Warmth => look.grade.temperature = v, Field::Tint => look.grade.tint = v, Field::Volume => clip.volume = v, + Field::CleanNoise => { + if let Some(c) = &mut clip.clean { + c.noise = v; + } + } Field::TextSize => { if let Some(Generator::Text(t)) = &mut clip.generator { t.size = v; @@ -1061,6 +1071,9 @@ impl Editor { sections.push( self.section("Sound") .child(self.slider(id, &clip, Field::Volume, cx)) + .when(clip.generator.is_none(), |d| { + d.child(self.clean_controls(id, &clip, cx)) + }) .when(speech, |d| { d.child( div().flex().child( @@ -1832,6 +1845,44 @@ impl Editor { }) } + /// Cleaning up a voice: a switch, how hard the noise is pushed down, and + /// whether the level is evened out. + fn clean_controls(&mut self, id: ClipId, clip: &Clip, cx: &mut Context) -> impl IntoElement + use<> { + let clean = clip.clean; + div() + .flex() + .flex_col() + .child(self.toggle( + "clean-voice", + "Clean up the voice", + clean.is_some(), + cx, + move |this, cx| { + this.change(id, cx, |c| { + c.clean = match c.clean { + Some(_) => None, + None => Some(splice_core::voice::Clean::default()), + }; + }); + }, + )) + .when_some(clean, |d, clean| { + d.child(self.slider(id, clip, Field::CleanNoise, cx)).child(self.toggle( + "clean-even", + "Even out the level", + clean.even, + cx, + move |this, cx| { + this.change(id, cx, |c| { + if let Some(c) = &mut c.clean { + c.even = !c.even; + } + }); + }, + )) + }) + } + /// Hiding part of the picture: a face, a screen, a number plate. fn redact_section(&mut self, id: ClipId, clip: &Clip, cx: &mut Context) -> impl IntoElement + use<> { let redact = clip.look.redact; @@ -2208,35 +2259,43 @@ impl Editor { /// Cutting edges off a picture: four sliders, and a way back. fn crop_section(&mut self, id: ClipId, clip: &Clip, cx: &mut Context) -> impl IntoElement + use<> { let crop = clip.look.crop.filter(|c| !c.is_none()); - let summary = match crop { - None => "Off".to_string(), - Some(c) => { + let mirror = clip.look.mirror; + let summary = match (crop, mirror) { + (None, false) => "Off".to_string(), + (None, true) => "Mirrored".to_string(), + (Some(c), _) => { let (w, h) = (1.0 - c.left - c.right, 1.0 - c.top - c.bottom); - format!("{:.0}% × {:.0}% left", w * 100.0, h * 100.0) + let flip = if mirror { ", mirrored" } else { "" }; + format!("{:.0}% × {:.0}% left{flip}", w * 100.0, h * 100.0) } }; - let (section, open) = self.fold(id, "Crop", crop.is_some(), summary, cx); + let (section, open) = self.fold(id, "Crop and mirror", crop.is_some() || mirror, summary, cx); section.when(open, |d| { - d.child(self.slider(id, clip, Field::CropLeft, cx)) - .child(self.slider(id, clip, Field::CropRight, cx)) - .child(self.slider(id, clip, Field::CropTop, cx)) - .child(self.slider(id, clip, Field::CropBottom, cx)) - .child( + d.child( + self.toggle("mirror", "Mirror, left to right", mirror, cx, move |this, cx| { + this.change(id, cx, |c| c.look.mirror = !c.look.mirror) + }), + ) + .child(self.slider(id, clip, Field::CropLeft, cx)) + .child(self.slider(id, clip, Field::CropRight, cx)) + .child(self.slider(id, clip, Field::CropTop, cx)) + .child(self.slider(id, clip, Field::CropBottom, cx)) + .child( + div() + .text_size(px(10.5)) + .line_height(px(15.0)) + .text_color(Theme::overlay()) + .child("The picture keeps its size; what remains stays where it was."), + ) + .when(crop.is_some(), |d| { + d.child( div() - .text_size(px(10.5)) - .line_height(px(15.0)) - .text_color(Theme::overlay()) - .child("The picture keeps its size; what remains stays where it was."), + .flex() + .child(self.chip("crop-reset", "Reset crop", false, cx, move |this, cx| { + this.change(id, cx, |c| c.look.crop = None); + })), ) - .when(crop.is_some(), |d| { - d.child( - div() - .flex() - .child(self.chip("crop-reset", "Reset crop", false, cx, move |this, cx| { - this.change(id, cx, |c| c.look.crop = None); - })), - ) - }) + }) }) } diff --git a/crates/splice-app/src/editor/record.rs b/crates/splice-app/src/editor/record.rs index c65bf31..95b1e25 100644 --- a/crates/splice-app/src/editor/record.rs +++ b/crates/splice-app/src/editor/record.rs @@ -509,6 +509,12 @@ impl Editor { placed.push(clip); } } + if studio.clean + && let Some(mic) = clips.get("microphone") + && let Some(c) = project.clip_mut(*mic) + { + c.clean = Some(splice_core::voice::Clean::default()); + } // The first pictures in a project set its shape: place them by it. let canvas = (project.settings.width, project.settings.height); let shown = studio.looks(canvas, size(project, "screen"), size(project, "camera")); @@ -667,22 +673,23 @@ impl Editor { (f32::from(area.size.width) - w) / 2.0, (f32::from(area.size.height) - h) / 2.0, ); - let frames = self.rec.live.as_ref().map(|l| l.frames.clone()).unwrap_or_default(); - let dims = |f: Feed| { - frames.get(&f).map(|i| { - let s = i.size(0); - (s.width.0 as u32, s.height.0 as u32) - }) - }; + let live = self.rec.live.as_ref(); + let frames = live.map(|l| l.frames.clone()).unwrap_or_default(); + let dims = |f: Feed| live.and_then(|l| l.natural.get(&f).copied()); let canvas_px = (settings.width, settings.height); let shown = studio.looks(canvas_px, dims(Feed::Screen), dims(Feed::Camera)); + // Camera frames come cut to what the scene shows of them. + if let Some(live) = live { + *live.camera_part.lock().unwrap() = shown.camera.as_ref().map(|l| shown_part(l).0); + } let mut layers: Vec = Vec::new(); for (feed, look) in [(Feed::Screen, shown.screen), (Feed::Camera, shown.camera)] { let (Some(look), Some(image)) = (look, frames.get(&feed)) else { continue; }; let Some(natural) = dims(feed) else { continue }; - layers.push(picture(image.clone(), natural, &look, (w, h))); + let part = live.and_then(|l| l.parts.get(&feed).copied().flatten()); + layers.push(picture(image.clone(), natural, part, &look, (w, h))); } let failed = self.rec.problem.is_some(); let status = if let Some(problem) = &self.rec.problem { @@ -870,20 +877,30 @@ impl Editor { }) .child(record), ) - .when(scene.layout == Layout::Bubble, |d| { + .when(studio.camera.is_some() && scene.layout != Layout::Screen, |d| { d.child( div() .flex() .flex_wrap() .items_center() .gap(px(6.0)) - .child(label("Corner")) - .children(corners) - .child(div().w(px(10.0))) - .child(label("Size")) - .children(sizes) - .child(div().w(px(10.0))) - .children(shapes), + .when(scene.layout == Layout::Bubble, |d| { + d.child(label("Corner")) + .children(corners) + .child(div().w(px(10.0))) + .child(label("Size")) + .children(sizes) + .child(div().w(px(10.0))) + .children(shapes) + .child(div().w(px(10.0))) + }) + .child( + chip("camera-mirror", "Mirror", scene.mirror) + .tooltip(tip("Flip the camera left to right, as a mirror shows you", None)) + .on_click(cx.listener(|this, _, _, cx| { + this.set_studio(cx, |s| s.scene.mirror = !s.scene.mirror) + })), + ), ) }) } @@ -1090,7 +1107,33 @@ impl Editor { .flex_col() .child(mic_row) .when(studio.microphone.is_some(), |d| { - d.child(meter(levels.get("microphone").copied())) + d.child(meter(levels.get("microphone").copied())).child( + div() + .id("mic-clean") + .h(px(28.0)) + .px(px(8.0)) + .ml(px(23.0)) + .rounded(px(7.0)) + .flex() + .items_center() + .gap(px(9.0)) + .cursor_pointer() + .hover(|d| d.bg(Theme::surface0())) + .child( + div() + .flex_1() + .text_size(px(11.5)) + .text_color(Theme::subtext()) + .child("Clean up the voice"), + ) + .child(switch(studio.clean)) + .tooltip(tip( + "Takes come in with rumble and steady noise lowered and the level evened out; \ + microphone.m4a stays as recorded", + None, + )) + .on_click(cx.listener(|this, _, _, cx| this.set_studio(cx, |s| s.clean = !s.clean))), + ) }) .into_any_element(), ); @@ -1483,6 +1526,12 @@ impl Editor { if let Some(split) = request.get("split").and_then(Value::as_f64) { studio.scene.split = (split as f32).clamp(0.15, 0.6); } + if let Some(clean) = flag("clean") { + studio.clean = clean; + } + if let Some(mirror) = request.get("mirror").and_then(Value::as_bool) { + studio.scene.mirror = mirror; + } if let Some(shape) = text("shape") { studio.scene.shape = match shape.as_str() { "circle" | "round" => Bubble::Circle, @@ -1521,8 +1570,9 @@ impl Editor { } if studio != before || self.doc.project.studio.is_none() && request.get("default").is_some() { self.edit(cx, |p| p.studio = Some(studio)); - self.relive(cx); } + // Also brings back sources that stopped. + self.relive(cx); Ok(()) } } @@ -1574,28 +1624,57 @@ fn take_name(path: &Path) -> String { } /// A live picture where `look` puts it on a stage of `stage` points. -fn picture(image: Arc, natural: (u32, u32), look: &Look, stage: (f32, f32)) -> AnyElement { - let (sw, sh) = stage; - let (nw, nh) = (natural.0.max(1) as f32, natural.1.max(1) as f32); - let fit = (sw / nw).min(sh / nh) * look.scale; - let (pw, ph) = (nw * fit, nh * fit); - let (cx, cy) = (sw / 2.0 + look.x * sw, sh / 2.0 + look.y * sh); - let (pl, pt) = (cx - pw / 2.0, cy - ph / 2.0); - // What shows of it: all, a mask's shape, or what a crop leaves. - let (mut l, mut t, mut r, mut b) = (0.0, 0.0, 1.0, 1.0); - let mut round = false; +/// What shows of a picture: all of it, a mask's box, or what a crop +/// leaves (as a `Part`), and whether it is round. +fn shown_part(look: &Look) -> (capture::Part, bool) { if let Some(m) = look.mask { - (l, t, r, b) = ( + let part = [ m.x - m.width / 2.0, m.y - m.height / 2.0, m.x + m.width / 2.0, m.y + m.height / 2.0, - ); - round = m.shape == splice_core::look::Shape::Ellipse; + ]; + ( + part.map(|v| v.clamp(0.0, 1.0)), + m.shape == splice_core::look::Shape::Ellipse, + ) } else if let Some(c) = look.crop { - (l, t, r, b) = (c.left, c.top, 1.0 - c.right, 1.0 - c.bottom); + ([c.left, c.top, 1.0 - c.right, 1.0 - c.bottom], false) + } else { + ([0.0, 0.0, 1.0, 1.0], false) } +} + +/// A live picture placed by `look` on a stage of `stage` points. `part` is +/// the part of the picture the frame holds, when it was cut to fit. +fn picture( + image: Arc, + natural: (u32, u32), + part: Option, + look: &Look, + stage: (f32, f32), +) -> AnyElement { + let (sw, sh) = stage; + let (nw, nh) = (natural.0.max(1) as f32, natural.1.max(1) as f32); + let fit = (sw / nw).min(sh / nh) * look.scale; + let (pw, ph) = (nw * fit, nh * fit); + let (cx, cy) = (sw / 2.0 + look.x * sw, sh / 2.0 + look.y * sh); + let (pl, pt) = (cx - pw / 2.0, cy - ph / 2.0); + let ([l, t, r, b], round) = shown_part(look); let (vl, vt, vw, vh) = (pl + l * pw, pt + t * ph, (r - l) * pw, (b - t) * ph); + // A frame holding just what shows is drawn as it is, round if need be: + // clipping a larger one to a circle is beyond gpui. + if part.is_some_and(|p| p.iter().zip([l, t, r, b]).all(|(a, b)| (a - b).abs() < 1e-3)) { + return img(image) + .absolute() + .left(px(vl)) + .top(px(vt)) + .w(px(vw)) + .h(px(vh)) + .when(round, |d| d.rounded_full()) + .object_fit(ObjectFit::Fill) + .into_any_element(); + } div() .absolute() .left(px(vl)) diff --git a/crates/splice-app/src/ffmpeg.rs b/crates/splice-app/src/ffmpeg.rs index 314b2f7..187c60c 100644 --- a/crates/splice-app/src/ffmpeg.rs +++ b/crates/splice-app/src/ffmpeg.rs @@ -321,17 +321,19 @@ pub fn decode_audio(path: &Path, rate: u32) -> Result { Ok(Pcm { rate, samples }) } -/// The sound of `length` of `path` from `from`, played at `speed` (its -/// pitch kept) and backwards when `reverse`: what a retimed clip sounds like. +/// The sound of `length` of `path` from `from`, run through `first` (a +/// voice's clean-up), played at `speed` (its pitch kept) and backwards when +/// `reverse`: what a retimed or cleaned clip sounds like. pub fn decode_audio_retimed( path: &Path, rate: u32, from: Time, length: Time, + first: Vec, speed: f64, reverse: bool, ) -> Result { - let mut filters = Vec::new(); + let mut filters = first; if reverse { filters.push("areverse".to_string()); } diff --git a/crates/splice-core/src/export.rs b/crates/splice-core/src/export.rs index fc03323..fae93b8 100644 --- a/crates/splice-core/src/export.rs +++ b/crates/splice-core/src/export.rs @@ -1081,7 +1081,7 @@ fn clip_frames( None => format!(",tpad=stop_mode=clone:stop_duration={len},trim=duration={len},setpts=PTS-STARTPTS"), }; filters.push(format!( - "[{i}:v]reverse,trim=end_frame=1,setpts=PTS-STARTPTS,fps={fps}{hold}[{label}]" + "[{i}:v]reverse,trim=end_frame=1,setpts=PTS-STARTPTS,fps={fps}:eof_action=pass{hold}[{label}]" )); return Ok(Frames { label, @@ -1259,6 +1259,10 @@ fn clip_sound( } }; let mut chain = vec![format!("{trim}aresample=48000"), "asetpts=PTS-STARTPTS".to_string()]; + // Cleaned as recorded, before any change of speed. + if let Some(clean) = clip.clean { + chain.extend(clean.filters()); + } if clip.reverse { chain.push("areverse".into()); } @@ -1374,12 +1378,15 @@ mod tests { content: None, clicks: Vec::new(), }); - p.append_clip(v); + let clip = p.append_clip(v).unwrap(); + p.clip_mut(clip).unwrap().clean = Some(crate::voice::Clean::default()); p.insert_clip(image, 0, SECOND); let plan = plan(&p, &Fake, Path::new("/o.mp4"), Quality::Draft).unwrap(); let graph = graph(&plan); assert_eq!(graph.matches("overlay=").count(), 2); assert!(graph.contains("amix=inputs=2")); + let cleaned = graph.find("highpass=f=80,afftdn=").expect("the voice is cleaned"); + assert!(cleaned < graph.find("volume=").unwrap()); assert!( graph.contains("scale=1080:1080"), "the square image fits the 1920x1080 canvas: {graph}" @@ -1459,6 +1466,8 @@ mod tests { assert_eq!(plan.args[seek + 1], "0.000000"); assert_eq!(plan.args[seek + 3], "0.283333", "up to a frame past 0.25s"); assert!(graph.contains("reverse,trim=end_frame=1")); + // A frame lasting a few milliseconds (variable frame rate) still shows. + assert!(graph.contains(":eof_action=pass")); assert!( graph.contains("colorchannelmixer=aa=0.5000") && !graph.contains("fade="), "the fade is half done: {graph}" diff --git a/crates/splice-core/src/lib.rs b/crates/splice-core/src/lib.rs index 21917d5..9591b1f 100644 --- a/crates/splice-core/src/lib.rs +++ b/crates/splice-core/src/lib.rs @@ -20,4 +20,5 @@ pub mod symbol; pub mod textmotion; pub mod time; pub mod transition; +pub mod voice; pub mod zoom; diff --git a/crates/splice-core/src/look.rs b/crates/splice-core/src/look.rs index 435760f..dd7383b 100644 --- a/crates/splice-core/src/look.rs +++ b/crates/splice-core/src/look.rs @@ -48,6 +48,10 @@ pub struct Look { /// Part of the picture hidden: a face, a screen, a number plate. #[serde(skip_serializing_if = "Option::is_none")] pub redact: Option, + /// Flipped left to right, as a camera shows you to yourself. Crop, mask + /// and hidden box are then measured on the flipped picture. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub mirror: bool, } /// Effects on a clip's picture, each 0 (off) to 1 (strongest). @@ -223,6 +227,7 @@ impl Default for Look { fx: Effects::default(), backdrop: None, redact: None, + mirror: false, } } } @@ -1231,12 +1236,13 @@ impl Look { || self.fx.glitch > 0.0 } - /// The same look with no crop or hidden box: for frames those were + /// The same look with no flip, crop or hidden box: for frames those were /// already applied to. pub fn clone_without_hiding(&self) -> Look { Look { crop: None, redact: None, + mirror: false, ..self.clone() } } @@ -1266,6 +1272,9 @@ impl Look { pub fn hiding_filters(&self, size: (u32, u32), prefix: &str) -> (Vec, (u32, u32)) { let mut filters = Vec::new(); let mut size = size; + if self.mirror { + filters.push("hflip".into()); + } // Cut first, so a key and a mask work on what remains. if let Some(crop) = self.crop.filter(|c| !c.is_none()) { filters.push(crop.ffmpeg()); @@ -1511,4 +1520,22 @@ mod tests { assert!(f.starts_with("chromakey=0x00ff00:") && f.contains("despill=type=green") && f.contains("geq=")); assert!(Look::default().pixel_filters((640, 360), 1.0, "t").is_none()); } + + #[test] + fn a_mirror_flips_before_the_crop_and_mask() { + let look = Look { + mirror: true, + crop: Some(Crop { + left: 0.25, + ..Crop::default() + }), + mask: Some(Mask::default()), + ..Look::default() + }; + let f = look.pixel_filters((640, 360), 1.0, "t").unwrap(); + assert!(f.starts_with("hflip,crop="), "{f}"); + assert!(!look.clone_without_hiding().mirror); + let json = serde_json::to_string(&Look::default()).unwrap(); + assert!(!json.contains("mirror"), "unflipped looks stay as they were saved"); + } } diff --git a/crates/splice-core/src/model.rs b/crates/splice-core/src/model.rs index 77d204e..4a1b501 100644 --- a/crates/splice-core/src/model.rs +++ b/crates/splice-core/src/model.rs @@ -201,6 +201,9 @@ pub struct Clip { /// The way in from the clip before it (or from what is beneath). #[serde(default, skip_serializing_if = "Option::is_none")] pub transition: Option, + /// Its sound cleaned up as a voice: rumble, steady noise, uneven level. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub clean: Option, } fn is_default(look: &Look) -> bool { @@ -341,6 +344,7 @@ impl Clip { reverse: false, frozen: false, transition: None, + clean: None, } } @@ -389,6 +393,11 @@ impl Clip { pub fn is_retimed(&self) -> bool { self.speed != 1.0 || self.reverse || self.frozen } + + /// Whether its sound is not simply its media's: retimed or cleaned up. + pub fn has_own_sound(&self) -> bool { + self.is_retimed() || self.clean.is_some() + } } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/crates/splice-core/src/studio.rs b/crates/splice-core/src/studio.rs index a5f12d8..a35d420 100644 --- a/crates/splice-core/src/studio.rs +++ b/crates/splice-core/src/studio.rs @@ -18,6 +18,9 @@ pub struct Studio { pub microphone: Option, /// The sound the Mac plays (not Splice's own). pub system_audio: bool, + /// Takes' microphone clips come in with their voice cleaned up (the + /// file stays as recorded). + pub clean: bool, pub scene: Scene, pub output: Output, } @@ -142,6 +145,8 @@ pub struct Scene { pub shape: Bubble, /// Side by side: the camera's share of the width. pub split: f32, + /// The camera flipped left to right, as a mirror shows you. + pub mirror: bool, } impl Default for Scene { @@ -152,6 +157,7 @@ impl Default for Scene { size: 0.3, shape: Bubble::Circle, split: 0.32, + mirror: false, } } } @@ -264,7 +270,7 @@ impl Studio { (_, None, Some(_)) => Layout::Camera, (layout, _, _) => layout, }; - match (layout, screen, camera) { + let mut placed = match (layout, screen, camera) { (Layout::Screen, Some(_), _) => Placed { screen: Some(Look::default()), camera: None, @@ -344,7 +350,11 @@ impl Studio { } } _ => Placed::default(), + }; + if let Some(camera) = &mut placed.camera { + camera.mirror = scene.mirror; } + placed } } @@ -426,6 +436,18 @@ mod tests { .camera .unwrap(); assert!(left.x < 0.0 && left.y < 0.0); + assert!(!cam.mirror && !left.mirror); + } + + #[test] + fn a_mirrored_camera_is_flipped_in_every_layout() { + for layout in Layout::ALL { + let mut s = studio(layout); + s.scene.mirror = true; + let placed = s.looks(CANVAS, SCREEN, CAMERA); + assert!(placed.camera.is_none_or(|c| c.mirror), "{layout:?}"); + assert!(placed.screen.is_none_or(|c| !c.mirror), "{layout:?}"); + } } #[test] diff --git a/crates/splice-core/src/voice.rs b/crates/splice-core/src/voice.rs new file mode 100644 index 0000000..c15c5ef --- /dev/null +++ b/crates/splice-core/src/voice.rs @@ -0,0 +1,77 @@ +//! Cleaning up a recorded voice: rumble cut, steady background noise (fans, +//! hiss, hum) pushed down, and quiet and loud passages brought closer +//! together. Written once as ffmpeg filters, which export and the viewer's +//! decoder both run, so what you hear is what you get. + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct Clean { + /// How hard steady noise is pushed down, 0 (not at all) to 1. + pub noise: f32, + /// The level evened out: quiet words lifted, loud ones held back. + pub even: bool, +} + +impl Default for Clean { + fn default() -> Clean { + Clean { noise: 0.5, even: true } + } +} + +impl Clean { + /// The filters, in order, for a chain of sound at any rate. + pub fn filters(&self) -> Vec { + // Below 80 Hz is rumble, desk knocks and breath on the mic, not voice. + let mut filters = vec!["highpass=f=80".to_string()]; + let noise = if self.noise.is_finite() { + self.noise.clamp(0.0, 1.0) + } else { + 0.0 + }; + if noise > 0.0 { + // Lowered by up to 36 dB, the noise followed as it changes. A + // stronger setting also counts more as noise (a higher floor): at + // 0.5, a webcam's room noise drops about 5 dB and the voice hardly. + filters.push(format!( + "afftdn=nr={:.1}:nf={:.1}:tn=1", + 12.0 + 24.0 * noise, + -45.0 + 25.0 * noise + )); + } + if self.even { + // Each quarter second brought towards peaks of −6 dB, by at most + // 6×, the gain smoothed over a few seconds: quiet words come up, + // loud ones go down, and a pause keeps the gain of the words + // around it rather than having its noise raised to meet them. + filters.push("dynaudnorm=f=250:g=15:m=6:p=0.5".into()); + } + filters + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stronger_cleaning_takes_more_and_evening_is_its_own_choice() { + let gentle = Clean { + noise: 0.2, + even: false, + } + .filters(); + assert_eq!(gentle.len(), 2); + assert!(gentle[0].starts_with("highpass") && gentle[1].starts_with("afftdn=nr=16.8:nf=-40.0")); + let hard = Clean { noise: 1.0, even: true }.filters(); + // ffmpeg takes a floor no higher than −20 dB. + assert!(hard[1].starts_with("afftdn=nr=36.0:nf=-20.0")); + assert!(hard[2].starts_with("dynaudnorm")); + // No noise reduction at 0, and nonsense is no reduction either. + for noise in [0.0, f32::NAN] { + let f = Clean { noise, even: false }.filters(); + assert_eq!(f, vec!["highpass=f=80".to_string()]); + } + } +} diff --git a/skill/splice/SKILL.md b/skill/splice/SKILL.md index ab47e49..5efe293 100644 --- a/skill/splice/SKILL.md +++ b/skill/splice/SKILL.md @@ -93,6 +93,7 @@ splice ctl split --at 8.5 # every clip under 8.5s; --c splice ctl delete --clip 4,5 --ripple # --ripple closes the gap splice ctl duplicate --clip 3 splice ctl volume --clip 9 --value 0.4 # 1 as recorded, 0 silent +splice ctl clean --clip 9 --noise 0.6 --even # clean up a voice (--off to undo) splice ctl track --track 2 --mute --hide # --unmute, --show splice ctl canvas --aspect 9:16 # or --size 1080x1920, --fps 30 splice ctl undo @@ -482,7 +483,8 @@ and lands on the timeline as layers placed by the scene. splice ctl record sources # displays, windows, cameras, microphones, permissions splice ctl record setup # this project's setup splice ctl record setup --screen window:Safari --camera C920 --microphone default --system -splice ctl record setup --layout bubble --corner br --size 0.3 --shape circle +splice ctl record setup --layout bubble --corner br --size 0.3 --shape circle --mirror +splice ctl record setup --clean # mic clips come in with the voice cleaned up splice ctl record setup --codec hevc --quality high --fps 60 --height 1440 --default splice ctl record start [--for 20] [--beat 27 | --at 30 | --into media] splice ctl record pause | resume