diff --git a/apps/web/src/app/api/convert-video/route.ts b/apps/web/src/app/api/convert-video/route.ts new file mode 100644 index 000000000..17b909d45 --- /dev/null +++ b/apps/web/src/app/api/convert-video/route.ts @@ -0,0 +1,144 @@ +import { type NextRequest, NextResponse } from "next/server"; +import { writeFile, readFile, rm, mkdtemp } from "fs/promises"; +import { spawn } from "child_process"; +import path from "path"; +import os from "os"; + +const GENERIC_CONVERSION_ERROR = "Video conversion failed."; + +function sanitizeContentDispositionName({ + rawName, +}: { + rawName: string; +}): string { + // Strip characters that can break the Content-Disposition header (quotes, + // backslashes, CR/LF, control bytes) and replace non-ASCII so the + // "filename=" parameter stays unambiguous across clients. + const stripped = rawName + // eslint-disable-next-line no-control-regex + .replace(/[\\"\r\n\x00-\x1f]/g, "_") + .replace(/[^\x20-\x7e]/g, "_") + .slice(0, 200); + return stripped.length > 0 ? stripped : "video"; +} + +export async function POST(request: NextRequest) { + let workDir: string | null = null; + + try { + const formData = await request.formData(); + const file = formData.get("file") as File | null; + + if (!file || !(file instanceof File)) { + return NextResponse.json( + { error: "No file provided" }, + { status: 400 }, + ); + } + + workDir = await mkdtemp(path.join(os.tmpdir(), "opencut-convert-")); + const ext = path.extname(file.name) || ".mp4"; + const inputPath = path.join(workDir, `input${ext}`); + const outputPath = path.join(workDir, "output.mp4"); + + const buffer = Buffer.from(await file.arrayBuffer()); + await writeFile(inputPath, buffer); + + await runFfmpeg({ + inputPath, + outputPath, + }); + + const convertedBuffer = await readFile(outputPath); + + const safeName = sanitizeContentDispositionName({ + rawName: path.basename(file.name, ext), + }); + + return new Response(new Uint8Array(convertedBuffer), { + status: 200, + headers: { + "Content-Type": "video/mp4", + "Content-Disposition": `attachment; filename="${safeName}.mp4"`, + }, + }); + } catch (error) { + console.error("Video conversion error:", error); + return NextResponse.json( + { error: GENERIC_CONVERSION_ERROR }, + { status: 500 }, + ); + } finally { + if (workDir) { + await rm(workDir, { recursive: true, force: true }).catch(() => {}); + } + } +} + +function runFfmpeg({ + inputPath, + outputPath, +}: { + inputPath: string; + outputPath: string; +}): Promise { + return new Promise((resolve, reject) => { + const proc = spawn("ffmpeg", [ + "-i", + inputPath, + "-c:v", + "libx264", + "-profile:v", + "baseline", + "-level", + "3.1", + "-preset", + "veryfast", + "-crf", + "23", + "-pix_fmt", + "yuv420p", + "-g", + "60", + "-keyint_min", + "60", + "-sc_threshold", + "0", + "-movflags", + "+faststart", + "-tag:v", + "avc1", + "-c:a", + "aac", + "-ac", + "2", + "-ar", + "48000", + "-b:a", + "128k", + "-y", + outputPath, + ]); + + let stderr = ""; + proc.stderr.on("data", (data) => { + stderr += data.toString(); + }); + + proc.on("close", (code) => { + if (code === 0) { + resolve(); + } else { + reject( + new Error( + `ffmpeg exited with code ${code}. stderr: ${stderr.slice(-500)}`, + ), + ); + } + }); + + proc.on("error", (err) => { + reject(err); + }); + }); +} diff --git a/apps/web/src/components/editor/export-button.tsx b/apps/web/src/components/editor/export-button.tsx index e738fb68b..6524aca4b 100644 --- a/apps/web/src/components/editor/export-button.tsx +++ b/apps/web/src/components/editor/export-button.tsx @@ -1,6 +1,6 @@ "use client"; -import { useState } from "react"; +import { useMemo, useState } from "react"; import { TransitionTopIcon } from "@hugeicons/core-free-icons"; import { HugeiconsIcon } from "@hugeicons/react"; import { @@ -34,6 +34,9 @@ import { } from "@/components/section"; import { useEditor } from "@/editor/use-editor"; import { DEFAULT_EXPORT_OPTIONS } from "@/export/defaults"; +import { isWebCodecsExportSupported } from "@/services/renderer/media-recorder-support"; +import { convertVideoToH264 } from "@/media/ffmpeg-convert"; +import { toast } from "sonner"; function isExportFormat(value: string): value is ExportFormat { return EXPORT_FORMAT_VALUES.some((formatValue) => formatValue === value); @@ -111,6 +114,20 @@ function ExportPopover({ DEFAULT_EXPORT_OPTIONS.includeAudio ?? true, ); + // When the browser lacks WebCodecs encoders, the SceneExporter falls back to + // a MediaRecorder-based path that always produces WebM (VP8/VP9 + Opus). + // QuickTime and Safari can't play WebM natively, so we offer a one-click + // server-side transcode to H.264 MP4 when the user originally asked for MP4. + const webCodecsExportSupported = useMemo( + () => isWebCodecsExportSupported(), + [], + ); + const willFallbackToWebM = !webCodecsExportSupported; + const shouldOfferServerTranscode = willFallbackToWebM && format === "mp4"; + const [transcodeToMp4OnServer, setTranscodeToMp4OnServer] = + useState(true); + const [isTranscoding, setIsTranscoding] = useState(false); + const handleExport = async () => { if (!activeProject) return; @@ -129,10 +146,39 @@ function ExportPopover({ } if (result.success && result.buffer) { + let outBuffer = result.buffer; + let outFormat: ExportFormat = format; + + if (shouldOfferServerTranscode) { + if (transcodeToMp4OnServer) { + setIsTranscoding(true); + try { + const webmFile = new File([outBuffer], "export.webm", { + type: "video/webm", + }); + const mp4File = await convertVideoToH264({ file: webmFile }); + outBuffer = await mp4File.arrayBuffer(); + } catch (error) { + const message = + error instanceof Error ? error.message : "Unknown error"; + toast.error("Server transcode failed", { + description: `${message}. Saving WebM instead.`, + }); + outFormat = "webm"; + } finally { + setIsTranscoding(false); + } + } else { + // User opted out of server transcode: keep WebM extension/mime so + // the file matches its actual container. + outFormat = "webm"; + } + } + downloadBuffer({ - buffer: result.buffer, - filename: `${activeProject.metadata.name}${getExportFileExtension({ format })}`, - mimeType: getExportMimeType({ format }), + buffer: outBuffer, + filename: `${activeProject.metadata.name}${getExportFileExtension({ format: outFormat })}`, + mimeType: getExportMimeType({ format: outFormat }), }); editor.project.clearExportState(); @@ -250,6 +296,39 @@ function ExportPopover({ + + {shouldOfferServerTranscode && ( +
+ + Browser compatibility + + +

+ This browser can't encode H.264 directly, so the + export will produce a WebM file. WebM plays in Chrome, + Firefox and VLC, but not natively in QuickTime or + Safari. +

+
+ + setTranscodeToMp4OnServer(!!checked) + } + /> + +
+ +
+ )}
@@ -282,6 +361,18 @@ function ExportPopover({
)} + + {isTranscoding && !isExporting && ( +
+

+ Transcoding to MP4 on server… +

+

+ Uploading the WebM file and re-encoding via ffmpeg. This + runs once per export and is independent of preview playback. +

+
+ )} )} diff --git a/apps/web/src/core/managers/audio-manager.ts b/apps/web/src/core/managers/audio-manager.ts index d0716aadc..780421a20 100644 --- a/apps/web/src/core/managers/audio-manager.ts +++ b/apps/web/src/core/managers/audio-manager.ts @@ -20,6 +20,10 @@ import { Input, type WrappedAudioBuffer, } from "mediabunny"; +import { + type AudioBufferSinkLike, + FallbackAudioBufferSink, +} from "@/services/audio-cache/fallback-audio-buffer-sink"; export class AudioManager { private audioContext: AudioContext | null = null; @@ -30,7 +34,7 @@ export class AudioManager { private lookaheadSeconds = 2; private scheduleIntervalMs = 500; private clips: AudioClipSource[] = []; - private sinks = new Map(); + private sinks = new Map(); private inputs = new Map(); private activeClipIds = new Set(); private clipIterators = new Map< @@ -267,7 +271,42 @@ export class AudioManager { const iterator = sink.buffers(sourceStartTime); this.clipIterators.set(clip.id, iterator); - let consecutiveDroppedBufferCount = 0; + + try { + await this.consumeClipIterator({ + clip, + clipEnd, + iterator, + audioContext, + sessionId, + initialDroppedCount: 0, + }); + } catch (error) { + console.warn( + `[AudioManager] clip iterator failed for ${clip.id}; aborting playback for this clip.`, + error, + ); + this.clipIterators.delete(clip.id); + this.activeClipIds.delete(clip.id); + } + } + + private async consumeClipIterator({ + clip, + clipEnd, + iterator, + audioContext, + sessionId, + initialDroppedCount, + }: { + clip: AudioClipSource; + clipEnd: number; + iterator: AsyncGenerator; + audioContext: AudioContext; + sessionId: number; + initialDroppedCount: number; + }): Promise { + let consecutiveDroppedBufferCount = initialDroppedCount; for await (const { buffer, timestamp } of iterator) { if (!this.editor.playback.getIsPlaying()) return; @@ -605,7 +644,18 @@ export class AudioManager { return null; } - const sink = new AudioBufferSink(audioTrack); + const canDecode = await safeCanDecode({ audioTrack }); + const sink: AudioBufferSinkLike = canDecode + ? new AudioBufferSink(audioTrack) + : await FallbackAudioBufferSink.create({ + file: clip.file, + audioContext, + }); + if (!canDecode) { + console.warn( + "[AudioManager] WebCodecs unavailable; decoding clip audio via decodeAudioData.", + ); + } const chunks: AudioBuffer[] = []; let totalSamples = 0; @@ -673,12 +723,13 @@ export class AudioManager { clip, }: { clip: AudioClipSource; - }): Promise { + }): Promise { const existingSink = this.sinks.get(clip.sourceKey); if (existingSink) return existingSink; + let input: Input | null = null; try { - const input = new Input({ + input = new Input({ source: new BlobSource(clip.file), formats: ALL_FORMATS, }); @@ -688,13 +739,47 @@ export class AudioManager { return null; } + const canDecode = await safeCanDecode({ audioTrack }); + if (!canDecode) { + input.dispose(); + input = null; + const audioContext = this.ensureAudioContext(); + if (!audioContext) { + return null; + } + const fallback = await FallbackAudioBufferSink.create({ + file: clip.file, + audioContext, + }); + this.sinks.set(clip.sourceKey, fallback); + console.warn( + "[AudioManager] WebCodecs unavailable; using decodeAudioData fallback for audio clip.", + ); + return fallback; + } + const sink = new AudioBufferSink(audioTrack); this.inputs.set(clip.sourceKey, input); this.sinks.set(clip.sourceKey, sink); return sink; } catch (error) { + if (input) { + input.dispose(); + } console.warn("Failed to initialize audio sink:", error); return null; } } } + +async function safeCanDecode({ + audioTrack, +}: { + audioTrack: { canDecode: () => Promise }; +}): Promise { + try { + return await audioTrack.canDecode(); + } catch { + return false; + } +} diff --git a/apps/web/src/media/audio.ts b/apps/web/src/media/audio.ts index 7abaf90e7..29dd3cce4 100644 --- a/apps/web/src/media/audio.ts +++ b/apps/web/src/media/audio.ts @@ -20,6 +20,10 @@ import { canTrackHaveAudio } from "@/timeline"; import { mediaSupportsAudio } from "@/media/media-utils"; import { getSourceTimeAtClipTime, renderRetimedBuffer } from "@/retime"; import { Input, ALL_FORMATS, BlobSource, AudioBufferSink } from "mediabunny"; +import { + type AudioBufferSinkLike, + FallbackAudioBufferSink, +} from "@/services/audio-cache/fallback-audio-buffer-sink"; import { TICKS_PER_SECOND } from "@/wasm"; import { computeRmsBuckets, @@ -271,7 +275,23 @@ async function resolveAudioBufferForAsset({ const audioTrack = await input.getPrimaryAudioTrack(); if (!audioTrack) return null; - const sink = new AudioBufferSink(audioTrack); + let audioTrackCanDecode = false; + try { + audioTrackCanDecode = await audioTrack.canDecode(); + } catch { + audioTrackCanDecode = false; + } + const sink: AudioBufferSinkLike = audioTrackCanDecode + ? new AudioBufferSink(audioTrack) + : await FallbackAudioBufferSink.create({ + file: asset.file, + audioContext, + }); + if (!audioTrackCanDecode) { + console.warn( + "[audio] WebCodecs unavailable; decoding asset via decodeAudioData fallback.", + ); + } const targetSampleRate = audioContext.sampleRate; const chunks: AudioBuffer[] = []; diff --git a/apps/web/src/media/ffmpeg-convert.ts b/apps/web/src/media/ffmpeg-convert.ts new file mode 100644 index 000000000..0bc76c038 --- /dev/null +++ b/apps/web/src/media/ffmpeg-convert.ts @@ -0,0 +1,22 @@ +export async function convertVideoToH264({ + file, +}: { + file: File; +}): Promise { + const formData = new FormData(); + formData.append("file", file); + + const response = await fetch("/api/convert-video", { + method: "POST", + body: formData, + }); + + if (!response.ok) { + const errorData = await response.json().catch(() => ({})); + throw new Error(errorData.error || `Conversion failed: ${response.status}`); + } + + const blob = await response.blob(); + const baseName = file.name.replace(/\.[^.]+$/, ""); + return new File([blob], `${baseName}.mp4`, { type: "video/mp4" }); +} diff --git a/apps/web/src/media/processing.ts b/apps/web/src/media/processing.ts index 34ccadbc2..a2107b0bc 100644 --- a/apps/web/src/media/processing.ts +++ b/apps/web/src/media/processing.ts @@ -6,6 +6,7 @@ import type { MediaAsset } from "@/media/types"; import { readVideoFile } from "./mediabunny"; import type { VideoFileData } from "./mediabunny"; import { renderThumbnailDataUrl } from "./thumbnail"; +import { convertVideoToH264 } from "./ffmpeg-convert"; export interface ProcessedMediaAsset extends Omit {} @@ -132,8 +133,49 @@ export async function processMediaAssets({ width = result.width; height = result.height; } else if (fileType === "video") { + let currentFile = file; + let currentUrl = url; try { - const videoData = await readVideoFile({ file }); + let videoData = await readVideoFile({ file: currentFile }); + + if (!videoData.canDecode) { + toast.info(`Converting ${file.name} to a compatible format…`); + const convertedFile = await convertVideoToH264({ + file: currentFile, + }); + + const convertedStorageCheck = await storageService.canStoreFile({ + size: convertedFile.size, + }); + if (!convertedStorageCheck.canStore) { + throw new Error( + getStorageLimitDescription({ + fileSize: convertedFile.size, + availableBytes: convertedStorageCheck.availableBytes, + }), + ); + } + + URL.revokeObjectURL(currentUrl); + currentFile = convertedFile; + currentUrl = URL.createObjectURL(currentFile); + videoData = await readVideoFile({ file: currentFile }); + + if (!videoData.canDecode) { + // Conversion produced a file the browser still can't decode + // via mediabunny/WebCodecs. The HTMLVideoElement fallback in + // VideoCache will handle preview rendering, so warn but + // continue importing. + toast.warning(`${file.name} converted, preview may be limited`, { + description: getUnsupportedVideoDescription({ + codec: videoData.codec, + }), + }); + } else { + toast.success(`${file.name} converted successfully`); + } + } + duration = videoData.duration; width = videoData.width; height = videoData.height; @@ -143,13 +185,27 @@ export async function processMediaAssets({ hasAudio = videoData.hasAudio; thumbnailUrl = videoData.thumbnailUrl ?? undefined; - if (!videoData.canDecode) { - toast.error(`Can't preview ${file.name}`, { - description: getUnsupportedVideoDescription({ - codec: videoData.codec, - }), - }); + processedAssets.push({ + name: currentFile.name, + type: fileType, + file: currentFile, + url: currentUrl, + thumbnailUrl, + duration, + width, + height, + fps, + hasAudio, + }); + + await new Promise((resolve) => setTimeout(resolve, 0)); + + completed += 1; + if (onProgress) { + const percent = Math.round((completed / total) * 100); + onProgress({ progress: percent }); } + continue; } catch (error) { const message = error instanceof Error @@ -159,6 +215,8 @@ export async function processMediaAssets({ toast.error(`Couldn't process ${file.name}`, { description: message, }); + URL.revokeObjectURL(currentUrl); + continue; } } else if (fileType === "audio") { duration = await getMediaDuration({ file }); diff --git a/apps/web/src/services/audio-cache/__tests__/fallback-audio-buffer-sink.test.ts b/apps/web/src/services/audio-cache/__tests__/fallback-audio-buffer-sink.test.ts new file mode 100644 index 000000000..6f1815145 --- /dev/null +++ b/apps/web/src/services/audio-cache/__tests__/fallback-audio-buffer-sink.test.ts @@ -0,0 +1,308 @@ +import { + afterEach, + beforeEach, + describe, + expect, + test, +} from "bun:test"; +import { + FallbackAudioBufferSink, + isWebAudioDecoderAvailable, + sliceAudioBuffer, +} from "../fallback-audio-buffer-sink"; + +type AudioBufferLike = { + length: number; + sampleRate: number; + numberOfChannels: number; + duration: number; + getChannelData: (channel: number) => Float32Array; + copyToChannel: ( + source: Float32Array, + channel: number, + startInChannel?: number, + ) => void; +}; + +function createTestAudioBuffer({ + durationSeconds = 1, + sampleRate = 8, + channelCount = 2, +}: { + durationSeconds?: number; + sampleRate?: number; + channelCount?: number; +} = {}): AudioBufferLike { + const length = Math.round(durationSeconds * sampleRate); + const channels = Array.from( + { length: channelCount }, + (_, channelIndex) => + new Float32Array( + Array.from({ length }, (_, i) => channelIndex + 1 + i * 0.01), + ), + ); + return { + length, + sampleRate, + numberOfChannels: channelCount, + duration: durationSeconds, + getChannelData(channel: number) { + return channels[channel]; + }, + // eslint-disable-next-line opencut/prefer-object-params -- mirrors the AudioBuffer.copyToChannel Web API signature so this fake behaves like the real one + copyToChannel( + source: Float32Array, + channel: number, + startInChannel = 0, + ) { + channels[channel].set(source, startInChannel); + }, + }; +} + +function installFakeAudioBuffer(): { restore: () => void } { + const root = globalThis as { AudioBuffer?: unknown }; + const original = root.AudioBuffer; + class FakeAudioBuffer implements AudioBufferLike { + length: number; + sampleRate: number; + numberOfChannels: number; + duration: number; + private channels: Float32Array[]; + + constructor(opts: { + length: number; + numberOfChannels: number; + sampleRate: number; + }) { + this.length = opts.length; + this.sampleRate = opts.sampleRate; + this.numberOfChannels = opts.numberOfChannels; + this.duration = opts.length / opts.sampleRate; + this.channels = Array.from( + { length: opts.numberOfChannels }, + () => new Float32Array(opts.length), + ); + } + + getChannelData(channel: number): Float32Array { + return this.channels[channel]; + } + + // eslint-disable-next-line opencut/prefer-object-params -- mirrors AudioBuffer.copyToChannel Web API signature + copyToChannel( + source: Float32Array, + channel: number, + startInChannel = 0, + ): void { + this.channels[channel].set(source, startInChannel); + } + } + root.AudioBuffer = FakeAudioBuffer; + return { + restore() { + if (original === undefined) { + delete root.AudioBuffer; + } else { + root.AudioBuffer = original; + } + }, + }; +} + +describe("isWebAudioDecoderAvailable", () => { + const root = globalThis as { AudioDecoder?: unknown }; + let original: unknown; + + beforeEach(() => { + original = root.AudioDecoder; + }); + + afterEach(() => { + if (original === undefined) { + delete root.AudioDecoder; + } else { + root.AudioDecoder = original; + } + }); + + test("returns false when AudioDecoder undefined", () => { + delete root.AudioDecoder; + expect(isWebAudioDecoderAvailable()).toBe(false); + }); + + test("returns true when AudioDecoder defined", () => { + root.AudioDecoder = class {}; + expect(isWebAudioDecoderAvailable()).toBe(true); + }); +}); + +describe("sliceAudioBuffer", () => { + let fakeAudioBuffer: ReturnType; + + beforeEach(() => { + fakeAudioBuffer = installFakeAudioBuffer(); + }); + + afterEach(() => { + fakeAudioBuffer.restore(); + }); + + test("slices to the requested range", () => { + const buffer = createTestAudioBuffer({ + durationSeconds: 1, + sampleRate: 8, + channelCount: 1, + }); + const sliced = sliceAudioBuffer({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + buffer: buffer as unknown as AudioBuffer, + start: 0.25, + end: 0.75, + }); + expect(sliced.length).toBe(4); + expect(sliced.numberOfChannels).toBe(1); + expect(sliced.sampleRate).toBe(8); + const channel = sliced.getChannelData(0); + expect(channel[0]).toBeCloseTo(1 + 0.02, 5); + expect(channel[3]).toBeCloseTo(1 + 0.05, 5); + }); + + test("clamps out-of-range start/end", () => { + const buffer = createTestAudioBuffer({ + durationSeconds: 0.5, + sampleRate: 8, + channelCount: 1, + }); + const sliced = sliceAudioBuffer({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + buffer: buffer as unknown as AudioBuffer, + start: -1, + end: 5, + }); + expect(sliced.length).toBe(buffer.length); + }); + + test("preserves channel count", () => { + const buffer = createTestAudioBuffer({ + durationSeconds: 1, + sampleRate: 8, + channelCount: 2, + }); + const sliced = sliceAudioBuffer({ + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + buffer: buffer as unknown as AudioBuffer, + start: 0, + end: 0.5, + }); + expect(sliced.numberOfChannels).toBe(2); + }); +}); + +describe("FallbackAudioBufferSink.buffers", () => { + let fakeAudioBuffer: ReturnType; + + beforeEach(() => { + fakeAudioBuffer = installFakeAudioBuffer(); + }); + + afterEach(() => { + fakeAudioBuffer.restore(); + }); + + test("yields a single buffer covering the full range by default", async () => { + const decoded = createTestAudioBuffer({ + durationSeconds: 1, + sampleRate: 8, + channelCount: 1, + }); + const sink = new FallbackAudioBufferSink( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + decoded as unknown as AudioBuffer, + ); + const yielded = []; + for await (const wrapped of sink.buffers()) { + yielded.push(wrapped); + } + expect(yielded).toHaveLength(1); + expect(yielded[0].timestamp).toBe(0); + expect(yielded[0].duration).toBeCloseTo(1, 5); + }); + + test("yields a slice when startTimestamp/endTimestamp provided", async () => { + const decoded = createTestAudioBuffer({ + durationSeconds: 2, + sampleRate: 8, + channelCount: 1, + }); + const sink = new FallbackAudioBufferSink( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + decoded as unknown as AudioBuffer, + ); + const yielded = []; + for await (const wrapped of sink.buffers(0.5, 1.5)) { + yielded.push(wrapped); + } + expect(yielded).toHaveLength(1); + expect(yielded[0].timestamp).toBe(0.5); + expect(yielded[0].duration).toBeCloseTo(1, 5); + }); + + test("treats NaN/Infinity startTimestamp as 0", async () => { + const decoded = createTestAudioBuffer({ + durationSeconds: 1, + sampleRate: 8, + channelCount: 1, + }); + const sink = new FallbackAudioBufferSink( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + decoded as unknown as AudioBuffer, + ); + const yielded = []; + for await (const wrapped of sink.buffers(Number.NaN)) { + yielded.push(wrapped); + } + expect(yielded).toHaveLength(1); + expect(yielded[0].timestamp).toBe(0); + expect(yielded[0].duration).toBeCloseTo(1, 5); + }); + + test("yields nothing when start >= end (or beyond duration)", async () => { + const decoded = createTestAudioBuffer({ + durationSeconds: 1, + sampleRate: 8, + channelCount: 1, + }); + const sink = new FallbackAudioBufferSink( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + decoded as unknown as AudioBuffer, + ); + + const yielded = []; + for await (const wrapped of sink.buffers(2)) { + yielded.push(wrapped); + } + expect(yielded).toHaveLength(0); + + const yielded2 = []; + for await (const wrapped of sink.buffers(0.4, 0.4)) { + yielded2.push(wrapped); + } + expect(yielded2).toHaveLength(0); + }); + + test("exposes sampleRate/numberOfChannels/duration getters", () => { + const decoded = createTestAudioBuffer({ + durationSeconds: 3, + sampleRate: 16, + channelCount: 2, + }); + const sink = new FallbackAudioBufferSink( + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- the fake duck-typed AudioBuffer is sufficient for this unit test + decoded as unknown as AudioBuffer, + ); + expect(sink.sampleRate).toBe(16); + expect(sink.numberOfChannels).toBe(2); + expect(sink.duration).toBe(3); + }); +}); diff --git a/apps/web/src/services/audio-cache/fallback-audio-buffer-sink.ts b/apps/web/src/services/audio-cache/fallback-audio-buffer-sink.ts new file mode 100644 index 000000000..931a2b8d6 --- /dev/null +++ b/apps/web/src/services/audio-cache/fallback-audio-buffer-sink.ts @@ -0,0 +1,135 @@ +import type { WrappedAudioBuffer } from "mediabunny"; + +export interface AudioBufferSinkLike { + buffers( + startTimestamp?: number, + endTimestamp?: number, + ): AsyncGenerator; +} + +export function isWebAudioDecoderAvailable(): boolean { + return ( + typeof (globalThis as { AudioDecoder?: unknown }).AudioDecoder !== + "undefined" + ); +} + +/** + * AudioBufferSink implementation backed by AudioContext.decodeAudioData. + * + * Used when mediabunny's AudioSampleSink cannot decode the track because the + * environment lacks the WebCodecs AudioDecoder API. AudioContext.decodeAudioData + * uses the system's native audio decoders, mirroring the strategy used by + * HTMLVideoElementSink for video. The entire compressed audio is decoded + * upfront and exposed through a buffers() iterator that yields a single + * AudioBuffer slice covering the requested range. + */ +export class FallbackAudioBufferSink implements AudioBufferSinkLike { + constructor(private readonly decoded: AudioBuffer) {} + + static async create({ + file, + audioContext, + }: { + file: File; + audioContext: BaseAudioContext; + }): Promise { + const arrayBuffer = await file.arrayBuffer(); + const decoded = await decodeArrayBuffer({ arrayBuffer, audioContext }); + return new FallbackAudioBufferSink(decoded); + } + + get sampleRate(): number { + return this.decoded.sampleRate; + } + + get numberOfChannels(): number { + return this.decoded.numberOfChannels; + } + + get duration(): number { + return this.decoded.duration; + } + + // eslint-disable-next-line opencut/prefer-object-params -- mirrors mediabunny's AudioBufferSink.buffers(start, end) so consumers can swap sinks transparently + async *buffers( + startTimestamp = 0, + endTimestamp?: number, + ): AsyncGenerator { + const totalDuration = this.decoded.duration; + const safeStart = Number.isFinite(startTimestamp) ? startTimestamp : 0; + const start = Math.max(0, Math.min(safeStart, totalDuration)); + const end = + typeof endTimestamp === "number" && Number.isFinite(endTimestamp) + ? Math.min(endTimestamp, totalDuration) + : totalDuration; + if (start >= end) return; + + const sliced = sliceAudioBuffer({ + buffer: this.decoded, + start, + end, + }); + + yield { + buffer: sliced, + timestamp: start, + duration: sliced.duration, + }; + } +} + +function decodeArrayBuffer({ + arrayBuffer, + audioContext, +}: { + arrayBuffer: ArrayBuffer; + audioContext: BaseAudioContext; +}): Promise { + return new Promise((resolve, reject) => { + audioContext.decodeAudioData( + arrayBuffer, + (buffer) => resolve(buffer), + (err) => + reject( + err instanceof Error + ? err + : new Error(`decodeAudioData failed: ${String(err)}`), + ), + ); + }); +} + +export function sliceAudioBuffer({ + buffer, + start, + end, +}: { + buffer: AudioBuffer; + start: number; + end: number; +}): AudioBuffer { + const sampleRate = buffer.sampleRate; + const startSample = Math.max( + 0, + Math.min(buffer.length, Math.floor(start * sampleRate)), + ); + const endSample = Math.max( + startSample, + Math.min(buffer.length, Math.ceil(end * sampleRate)), + ); + const length = Math.max(1, endSample - startSample); + + const target = new AudioBuffer({ + length, + numberOfChannels: Math.max(1, buffer.numberOfChannels), + sampleRate, + }); + for (let channel = 0; channel < buffer.numberOfChannels; channel++) { + const channelView = buffer + .getChannelData(channel) + .subarray(startSample, endSample); + target.copyToChannel(channelView, channel, 0); + } + return target; +} diff --git a/apps/web/src/services/renderer/__tests__/media-recorder-exporter.test.ts b/apps/web/src/services/renderer/__tests__/media-recorder-exporter.test.ts new file mode 100644 index 000000000..fb6847d05 --- /dev/null +++ b/apps/web/src/services/renderer/__tests__/media-recorder-exporter.test.ts @@ -0,0 +1,96 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { + detectMediaRecorderSupport, + isWebCodecsExportSupported, +} from "../media-recorder-support"; + +describe("isWebCodecsExportSupported", () => { + const root = globalThis as { + VideoEncoder?: unknown; + AudioEncoder?: unknown; + }; + let originalVideoEncoder: unknown; + let originalAudioEncoder: unknown; + + beforeEach(() => { + originalVideoEncoder = root.VideoEncoder; + originalAudioEncoder = root.AudioEncoder; + }); + + afterEach(() => { + if (originalVideoEncoder === undefined) delete root.VideoEncoder; + else root.VideoEncoder = originalVideoEncoder; + if (originalAudioEncoder === undefined) delete root.AudioEncoder; + else root.AudioEncoder = originalAudioEncoder; + }); + + test("false when either WebCodecs encoder is missing", () => { + delete root.VideoEncoder; + delete root.AudioEncoder; + expect(isWebCodecsExportSupported()).toBe(false); + + root.VideoEncoder = class {}; + delete root.AudioEncoder; + expect(isWebCodecsExportSupported()).toBe(false); + + delete root.VideoEncoder; + root.AudioEncoder = class {}; + expect(isWebCodecsExportSupported()).toBe(false); + }); + + test("true only when both encoders are defined", () => { + root.VideoEncoder = class {}; + root.AudioEncoder = class {}; + expect(isWebCodecsExportSupported()).toBe(true); + }); +}); + +describe("detectMediaRecorderSupport", () => { + const root = globalThis as { + MediaRecorder?: unknown; + }; + let originalMediaRecorder: unknown; + + beforeEach(() => { + originalMediaRecorder = root.MediaRecorder; + }); + + afterEach(() => { + if (originalMediaRecorder === undefined) delete root.MediaRecorder; + else root.MediaRecorder = originalMediaRecorder; + }); + + test("unsupported when MediaRecorder is undefined", () => { + delete root.MediaRecorder; + expect(detectMediaRecorderSupport()).toEqual({ + supported: false, + mimeType: null, + }); + }); + + test("returns the first supported mime type", () => { + class FakeMediaRecorder { + static isTypeSupported(mimeType: string): boolean { + return mimeType === "video/webm;codecs=vp8,opus"; + } + } + root.MediaRecorder = FakeMediaRecorder; + expect(detectMediaRecorderSupport()).toEqual({ + supported: true, + mimeType: "video/webm;codecs=vp8,opus", + }); + }); + + test("unsupported when no candidate mime type matches", () => { + class FakeMediaRecorder { + static isTypeSupported(): boolean { + return false; + } + } + root.MediaRecorder = FakeMediaRecorder; + expect(detectMediaRecorderSupport()).toEqual({ + supported: false, + mimeType: null, + }); + }); +}); diff --git a/apps/web/src/services/renderer/media-recorder-exporter.ts b/apps/web/src/services/renderer/media-recorder-exporter.ts new file mode 100644 index 000000000..d62c9450b --- /dev/null +++ b/apps/web/src/services/renderer/media-recorder-exporter.ts @@ -0,0 +1,195 @@ +import type { FrameRate } from "opencut-wasm"; +import { TICKS_PER_SECOND } from "@/wasm"; +import { frameRateToFloat } from "@/fps/utils"; +import type { RootNode } from "./nodes/root-node"; +import type { CanvasRenderer } from "./canvas-renderer"; +import { detectMediaRecorderSupport } from "./media-recorder-support"; + +export { + detectMediaRecorderSupport, + isWebCodecsExportSupported, +} from "./media-recorder-support"; +export type { MediaRecorderSupport } from "./media-recorder-support"; + +/** + * MediaRecorder-based exporter used when WebCodecs encoders + * (VideoEncoder / AudioEncoder) are not available in the current browser. + * + * Captures the renderer's output canvas via `canvas.captureStream(fps)` and + * mixes the project audio buffer through a `MediaStreamAudioDestinationNode`. + * The combined MediaStream is recorded with `MediaRecorder`, producing a + * WebM blob (which `