diff --git a/.github/workflows/docker-build-media-server.yml b/.github/workflows/docker-build-media-server.yml index 1925be2822..6521617de0 100644 --- a/.github/workflows/docker-build-media-server.yml +++ b/.github/workflows/docker-build-media-server.yml @@ -13,17 +13,43 @@ on: - main paths: - "apps/media-server/**" + - "scripts/benchmark-instant-audio.py" + - "scripts/benchmark-audio-intelligibility.py" + - "scripts/test-audio-intelligibility.py" - ".github/workflows/docker-build-media-server.yml" pull_request: paths: - "apps/media-server/**" + - "scripts/benchmark-instant-audio.py" + - "scripts/benchmark-audio-intelligibility.py" + - "scripts/test-audio-intelligibility.py" - ".github/workflows/docker-build-media-server.yml" +permissions: {} + concurrency: group: media-server-${{ github.head_ref || github.ref_name }}-${{ inputs.tag || 'latest' }} cancel-in-progress: true jobs: + audio-metrics: + name: Audio intelligibility alignment + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + contents: read + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 + with: + persist-credentials: false + - name: Verify aligned audio metrics + env: + PYTHONDONTWRITEBYTECODE: "1" + run: | + python3 -m venv "$RUNNER_TEMP/audio-metrics" + "$RUNNER_TEMP/audio-metrics/bin/python" -m pip install --disable-pip-version-check --no-input --only-binary=:all: numpy==2.4.1 scipy==1.18.1 pystoi==0.4.1 + "$RUNNER_TEMP/audio-metrics/bin/python" scripts/test-audio-intelligibility.py + build: name: Build Docker Image (${{ matrix.platform }}) runs-on: ${{ matrix.runner }} @@ -77,6 +103,11 @@ jobs: env: MEDIA_IMAGE: ${{ github.event_name == 'pull_request' && 'cap-media-server:verification' || format('ghcr.io/{0}/cap-media-server@{1}', env.REPOSITORY_OWNER, steps.build.outputs.digest) }} run: | + docker run --rm --network none --entrypoint bun "$MEDIA_IMAGE" test \ + src/__tests__/lib/audio-quality-policy.test.ts \ + src/__tests__/lib/audio-quality.integration.test.ts \ + src/__tests__/lib/audio-quality-formats.integration.test.ts \ + src/__tests__/lib/audio-quality-benchmark.test.ts docker run --rm --network none --entrypoint bun "$MEDIA_IMAGE" test \ src/__tests__/lib/recording-verification.integration.test.ts \ src/__tests__/lib/job-manager.test.ts diff --git a/apps/media-server/Dockerfile b/apps/media-server/Dockerfile index 04f8d241bd..e7fc94c90b 100644 --- a/apps/media-server/Dockerfile +++ b/apps/media-server/Dockerfile @@ -11,7 +11,9 @@ RUN bun install --frozen-lockfile --production COPY apps/media-server/src ./src -RUN bun test src/__tests__/lib/drive-resumable-upload.test.ts src/__tests__/lib/storage-upload.test.ts src/__tests__/lib/container-memory.test.ts \ +RUN bun test src/__tests__/lib/audio-quality-policy.test.ts src/__tests__/lib/audio-quality.integration.test.ts \ + && bun test src/__tests__/lib/audio-quality-formats.integration.test.ts src/__tests__/lib/audio-quality-benchmark.test.ts \ + && bun test src/__tests__/lib/drive-resumable-upload.test.ts src/__tests__/lib/storage-upload.test.ts src/__tests__/lib/container-memory.test.ts \ && bun test src/__tests__/lib/media-size.test.ts src/__tests__/lib/media-probe.integration.test.ts \ && bun test src/__tests__/lib/recording-verification.integration.test.ts src/__tests__/lib/job-manager.test.ts \ && bun test src/__tests__/lib/media-transfer.test.ts \ diff --git a/apps/media-server/audio-quality-benchmark-summary.json b/apps/media-server/audio-quality-benchmark-summary.json new file mode 100644 index 0000000000..799105046c --- /dev/null +++ b/apps/media-server/audio-quality-benchmark-summary.json @@ -0,0 +1,85 @@ +{ + "baseline": { + "count": 60, + "hours": 4.463996666666667, + "medianLufs": -29.205, + "belowMinus24": 40, + "peakAboveZero": 9, + "alreadyLoud": 9 + }, + "voice": { + "count": 60, + "passing": 42, + "rejected": 1, + "unchanged": 17, + "errors": [], + "skipReasons": { + "unsafe-levels": 12, + "already-loud": 5 + }, + "medianInputLufs": -31.744999999999997, + "medianOutputLufs": -16.84, + "medianGain": 14.495, + "maxTruePeak": -1.34, + "maxDurationDeltaMs": 20.999999999958163, + "sampleCountsExact": true, + "medianRealtimeFactor": 0.08116900157594112 + }, + "levels": { + "count": 60, + "passing": 40, + "rejected": 0, + "unchanged": 20, + "errors": [], + "skipReasons": { + "unsafe-levels": 12, + "already-loud": 5, + "insufficient-headroom": 3 + }, + "medianInputLufs": -32.07, + "medianOutputLufs": -23.08, + "medianGain": 10.64, + "maxTruePeak": -1.95, + "maxDurationDeltaMs": 20.999999999958163, + "sampleCountsExact": true, + "medianRealtimeFactor": 0.09984551719406805 + }, + "holdoutVoice": { + "count": 21, + "passing": 15, + "rejected": 1, + "unchanged": 5, + "errors": [], + "skipReasons": { + "unsafe-levels": 2, + "already-loud": 3 + }, + "medianInputLufs": -30.92, + "medianOutputLufs": -16.78, + "medianGain": 13.339999999999998, + "maxTruePeak": -1.34, + "maxDurationDeltaMs": 20.999999999958163, + "sampleCountsExact": true, + "medianRealtimeFactor": 0.08261003124306479 + }, + "codeHash": "40ef7500ef8643decd898509879bfe0293ced1f5d124f8a93535f9080f1bf609", + "scope": "Offline public/transcribed cohort; passing technical gates does not establish perceptual quality or production eligibility.", + "productionEnabled": false, + "reviewValidation": { + "workerCodeHash": "d1dbc4b3be3ff1b833b51a5a87b0933b2b7c82eb0816e7eba690fee12c7a7cf1", + "existingCohortRetested": 60, + "existingOutputHashesIdentical": true, + "additionalProductionFiles": 16, + "additionalProductionFilesUnchanged": 16, + "additionalProductionOriginalHashesPreserved": true, + "localTests": 46, + "localAssertions": 148, + "alignmentUnitTests": 2, + "alignedIntelligibility": { + "count": 39, + "medianStoiDelta": -0.00025973077349839, + "worstStoiDelta": -0.005509516622390076, + "belowMinusPointZeroOne": 0 + } + } +} diff --git a/apps/media-server/audio-quality-benchmarks.md b/apps/media-server/audio-quality-benchmarks.md new file mode 100644 index 0000000000..36f7ae241b --- /dev/null +++ b/apps/media-server/audio-quality-benchmarks.md @@ -0,0 +1,155 @@ +# Instant audio quality experiment + +This is an offline, shadow-only experiment. No route, recording finalizer, player, +export, upload, desktop capture path, or production flag imports the worker. +`mode: "off"` returns before filesystem access. There is no publishing mode. + +The proposed first rollout is bounded, constant level correction, after production +validation. EQ and denoising remain experimental because consistent perceptual +improvement has not been established. This PR does not enable either profile. + +## Evidence and limits + +The September 7–8, 2026 study measured 60 additional public, unprotected Instant +recordings with completed transcripts: 20 from each of September 5, 6, and 7, +all from different owners and separate from the initial 12-recording study. +The expanded cohort contains 4.464 hours of audio. Median playback-compensated +loudness is -29.205 LUFS; 40/60 recordings are below -24 LUFS, 9 already exceed +-18 LUFS, and 9 have true peaks above 0 dBTP. + +A fixed 39-recording tuning / 21-recording holdout split preceded processing. +The final voice policy passed technical gates on 42/60, left 17 unchanged, and +rejected one short holdout clip for excessive gain. Among the 42 passing clips, +median loudness moved from -31.745 to -16.84 LUFS, median gain was 14.495 dB, +and the highest encoded true peak was -1.34 dBTP. All decoded sample counts were +preserved; container duration changes were at most 21 ms. The reserved holdout +alone had 15 passes, five unchanged, and that one rejected candidate. +Constant-gain processing passed 40/60 and left 20 unchanged, with no rejected +outputs. These counts measure technical eligibility, not listening preference. + +Five initial policies produced 195 comparisons. Naive dynamic normalization, +EQ/compression, and denoising each changed container duration by more than 25 ms +on 29/39 recordings. Aggressive processing also damaged synthetic intelligibility +scores. These failures remain in the local evidence; they are not shipping presets. + +The adjusted policy uses a 60 Hz high-pass, -0.75 dB at 250 Hz, +0.75 dB at 2.5 kHz, +6 dB adaptive FFT denoising, and loudness normalization. It has no extra compressor. +Input and output gates bound gain and peak level. Separate constant-gain processing +is available for content whose suitability for voice processing is unknown. + +The exact final worker was tested on 39 controlled cases at 48 kHz: three reference +voice excerpts, four additive noise types, and three SNRs, plus the unmodified +references. Median STOI change was -0.000260, worst -0.005510; none exceeded the +chosen -0.01 regression tolerance. Six inputs were conservatively left unchanged. +These are relative tests against existing recordings, not clean studio ground +truth, subjective quality ratings, or a matched Loom comparison. The calibration +run with 12 dB denoising exceeded that tolerance in four cases, motivating 6 dB. + +Volume-matched RMS in uncaptioned intervals changed by a median +0.096 dB across +41 passing clips, with a maximum increase of 6.841 dB. Twelve voice candidates +changed LRA by more than two LU. Those observations require listening review for +background noise swelling and altered dynamics before enabling voice processing. + +The reviewed intelligibility scorer aligns reference, noisy input, and processed +audio to the same overlapping sample interval before computing STOI and SI-SDR. +It retains unaligned scores and the measured lag separately; aligning a score does +not waive timing gates. The 39 exact-worker cases were rescored from their original +artifacts after this correction, with zero cases below the -0.01 tolerance. + +Full-recording LUFS and caption-aligned RMS answer different questions. Caption +intervals approximate speech activity; uncaptioned audio is not necessarily noise +or silence. A completed transcript does not establish that a recording contains +only microphone speech. Mono loudness uses FFmpeg's `dual_mono=true` playback +compensation consistently; it must not be mixed with uncompensated mono metrics. + +## Source and timing guarantees + +The worker only reads an absolute regular local source, hashes it before and after, +and writes into a unique temporary directory. It copies video packets and verifies +them with the existing packet-proof helper. Existing finalization checks are +unchanged. Results carry source/output hashes, metrics, version, and validation +failures; every nonempty validation failure list disqualifies that candidate. + +The worker restricts demuxers and protocols to local media files, rejecting playlists +instead of following their references. Only AAC inputs are eligible for processing; +other codecs are left unchanged. A MOV/PCM fixture exposed a video preservation +mismatch, so the first rollout deliberately bypasses that format. + +The worker skips silence, extreme levels, existing clipping, unsupported formats, +already loud content, nonzero audio start times, discontinuous source timestamps, +and mismatched source audio/video durations. Voice processing additionally requires +`speechOnlyConfirmed`; the benchmark explicitly overrides this only for local +research. There is no production content classifier in this change. + +FFmpeg's denoiser delays content by two sample-advance blocks without adjusting +PTS. Padding the tail and trimming that delay preserves boundary speech. Integer +sample timebases avoid timestamp rounding at 44.1 kHz. The encoded AAC result is +remeasured, with one bounded peak correction rendered from the original if needed. +Failed validation never authorizes publication. Cancellation, timeouts, and exceptions +clean up only the worker's own temporary files. + +46 tests cover policy gates, mono/stereo, 44.1/48 kHz, both profiles, speech-like +markers at clip boundaries, exact video packets, source preservation, silence, +nonzero/discontinuous timestamps, cancellation, and timeout. Scoped TypeScript and +Biome checks also pass. Measurements used macOS FFmpeg 8.0.1 and Bun 1.4.0; +the reviewed format suite also runs in both production-image architectures and +in the Railway Docker build. The full production cohort was rerun locally. + +## Reproducing + +Keep source media, transcripts, per-recording measurements, and customer identifiers +outside the repository. Aggregate results and the frozen worker source hash are in +[audio-quality-benchmark-summary.json](audio-quality-benchmark-summary.json). +The local study retains `final-summary.json`, `final-voice-results.json`, +`final-levels-results.json`, and `report.md`, plus per-recording run receipts. +Interrupted runs and retries are retained separately. + +The manifest is a JSON array with `id`, `split` (`tuning` or `holdout`), `stratum`, +`createdAt`, and `duration`. Sources are `sources/.m4a`; transcripts are +`sources/.vtt`. Initial download uses the authenticated Cap CLI for existing +transcripts and the public playlist for audio. No new transcription is requested. + +```sh +python3 scripts/benchmark-instant-audio.py /absolute/study --phase baseline +python3 scripts/benchmark-instant-audio.py /absolute/study --phase tuning --policies gain6 gain12 dynamic equalized clean +bun apps/media-server/scripts/benchmark-audio-quality.ts /absolute/study tuning unique-label voice +bun apps/media-server/scripts/benchmark-audio-quality.ts /absolute/study holdout another-label levels +``` + +Use a new label per run; the worker benchmark will not overwrite existing results. +Rejected candidates are recorded with their validation failures but are not copied +into the output set. Earlier historical runs retained rejected files for diagnosis. +The intelligibility calibration requires NumPy, SciPy, and pystoi. Supply three +reference IDs with `--reference-ids`; their M4A files must be two directories above +the output directory. Output-directory suffix `-v2` selects the corrected mild +policy; a name containing `strength` selects the 6/12 dB comparison. This calibration +script records historical filter alternatives; the TypeScript worker benchmark is +the authoritative final implementation. + +## Production-data revalidation + +The tighter local-input restrictions were applied to the original 60-recording +cohort again. All 40 accepted outputs had identical hashes to the benchmark outputs; +20 sources were left unchanged. Sixteen additional public production files, including +browser captures and recordings without completed transcripts, were left unchanged +by stream, level, headroom, or timestamp gates. All original hashes were preserved. +These are bounded compatibility checks, not proof of safety for every possible file. + +## Before serving any enhanced audio + +Human review of the 12 volume-matched A/B excerpts is still required. Speech-only +eligibility, noisy and mixed-system-audio cases, and recordings excluded by the +public/transcribed selection need broader coverage. The short holdout clip rejected +for excessive gain must remain on its original audio; do not relax its gate to make +the benchmark pass. + +Run the exact policy in the production Linux image, then verify actual share-page, +embed, seeking, downloads, edits, transcript alignment, and fallback behavior. +Measure worker memory, throughput, storage, and tail latency before rollout. + +Future integration should create a separately versioned derivative after the +original is available, using a durable idempotent job bound to the source hash. +Publish atomically only after validation and only if the source still matches. +Keep the original available throughout processing, on failure, and for rollback. +Existing desktop installs could then benefit server-side without a capture update; +this experiment does not yet implement that serving integration. diff --git a/apps/media-server/scripts/benchmark-audio-quality.ts b/apps/media-server/scripts/benchmark-audio-quality.ts new file mode 100644 index 0000000000..4fe5f62ab0 --- /dev/null +++ b/apps/media-server/scripts/benchmark-audio-quality.ts @@ -0,0 +1,104 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { isAbsolute, join } from "node:path"; +import { z } from "zod"; +import { createAudioQualityCandidate } from "../src/lib/audio-quality"; +import { retainAudioQualityBenchmarkResult } from "../src/lib/audio-quality-benchmark"; + +const [root, split, label, profile] = process.argv.slice(2); +if ( + !root || + !isAbsolute(root) || + !["tuning", "holdout"].includes(split) || + !label || + !/^[a-z0-9-]+$/.test(label) || + (profile !== "levels" && profile !== "voice") +) + throw new Error( + "Usage: benchmark-audio-quality.ts ABS_ROOT tuning|holdout LABEL levels|voice", + ); + +const rows = z + .array( + z.object({ + id: z.string().regex(/^[a-z0-9]{15}$/), + split: z.enum(["tuning", "holdout"]), + }), + ) + .parse(JSON.parse(await readFile(join(root, "cohort.json"), "utf8"))) + .filter((row) => row.split === split); +const destination = join(root, label); +await mkdir(destination); +const codeHash = createHash("sha256") + .update( + await readFile(new URL("../src/lib/audio-quality.ts", import.meta.url)), + ) + .update( + await readFile( + new URL("../src/lib/audio-quality-policy.ts", import.meta.url), + ), + ) + .digest("hex"); +await writeFile( + join(destination, "run.json"), + JSON.stringify( + { + split, + profile, + codeHash, + count: rows.length, + forcedContentGateForOfflineExperiment: profile === "voice", + startedAt: new Date().toISOString(), + }, + null, + 2, + ), + { flag: "wx" }, +); +let next = 0; +const results: Record[] = []; +async function worker() { + for (;;) { + const row = rows[next++]; + if (!row) return; + const started = performance.now(); + let receipt: Record; + try { + const result = await createAudioQualityCandidate( + join(root, "sources", `${row.id}.m4a`), + { + mode: "shadow", + profile: profile as "levels" | "voice", + speechOnlyConfirmed: profile === "voice", + }, + ); + const evidence = await retainAudioQualityBenchmarkResult( + result, + join(destination, `${row.id}.mp4`), + ); + receipt = { id: row.id, ...evidence, codeHash }; + } catch (error) { + receipt = { + id: row.id, + status: "failed", + error: error instanceof Error ? error.message : String(error), + codeHash, + }; + } + receipt.wallMs = performance.now() - started; + await writeFile( + join(destination, `${row.id}.json`), + JSON.stringify(receipt, null, 2), + { flag: "wx" }, + ); + results.push(receipt); + console.log(JSON.stringify(receipt)); + } +} +await Promise.all([worker(), worker()]); +await writeFile( + join(destination, "results.json"), + JSON.stringify(results, null, 2), + { flag: "wx" }, +); +if (results.some((result) => result.status === "failed")) process.exitCode = 1; diff --git a/apps/media-server/src/__tests__/lib/audio-quality-benchmark.test.ts b/apps/media-server/src/__tests__/lib/audio-quality-benchmark.test.ts new file mode 100644 index 0000000000..028cb58dcc --- /dev/null +++ b/apps/media-server/src/__tests__/lib/audio-quality-benchmark.test.ts @@ -0,0 +1,73 @@ +import { expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AudioQualityResult } from "../../lib/audio-quality"; +import { retainAudioQualityBenchmarkResult } from "../../lib/audio-quality-benchmark"; + +test.each(["accepted", "rejected", "existing"])( + "benchmark output gate: %s", + async (mode) => { + const directory = await mkdtemp( + join(tmpdir(), "cap-audio-benchmark-test-"), + ); + const path = join(directory, "candidate.mp4"); + const destination = join(directory, "result.mp4"); + let cleaned = false; + const measurements = { + lufs: -20, + truePeak: -2, + lra: 4, + duration: 10, + channels: 1, + sampleRate: 48000, + sampleCount: 480000, + }; + const result: AudioQualityResult = { + status: "shadow-candidate", + path, + sourceSha256: "source", + outputSha256: "output", + input: measurements, + output: measurements, + profile: "levels", + version: "audio-quality-v3", + peakCorrectionDb: 0, + validationFailures: mode === "rejected" ? ["excessive-gain"] : [], + elapsedMs: 1, + cleanup: async () => { + cleaned = true; + await rm(path); + }, + }; + try { + await writeFile(path, "processed"); + if (mode === "existing") { + await writeFile(destination, "original"); + await expect( + retainAudioQualityBenchmarkResult(result, destination), + ).rejects.toThrow(); + expect(await readFile(destination, "utf8")).toBe("original"); + } else { + const receipt = await retainAudioQualityBenchmarkResult( + result, + destination, + ); + if (mode === "rejected") { + expect(receipt).toMatchObject({ + status: "rejected", + validationFailures: ["excessive-gain"], + }); + await expect(stat(destination)).rejects.toThrow(); + } else { + expect(receipt.status).toBe("shadow-candidate"); + expect(await readFile(destination, "utf8")).toBe("processed"); + } + } + expect(cleaned).toBe(true); + await expect(stat(path)).rejects.toThrow(); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }, +); diff --git a/apps/media-server/src/__tests__/lib/audio-quality-formats.integration.test.ts b/apps/media-server/src/__tests__/lib/audio-quality-formats.integration.test.ts new file mode 100644 index 0000000000..c5bd7ea215 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/audio-quality-formats.integration.test.ts @@ -0,0 +1,242 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createAudioQualityCandidate } from "../../lib/audio-quality"; + +let directory: string; + +async function fixture( + name: string, + inputs: readonly string[], + output: readonly string[], +) { + const path = join(directory, name); + const process = Bun.spawn( + [ + "ffmpeg", + "-v", + "error", + "-nostdin", + "-n", + ...inputs, + ...output, + "-threads", + "1", + path, + ], + { stdout: "ignore", stderr: "pipe", stdin: "ignore" }, + ); + const [code, error] = await Promise.all([ + process.exited, + new Response(process.stderr).text(), + ]); + if (code !== 0) throw new Error(error); + return path; +} + +function audio( + expression = "0.02*sin(2*PI*440*t)", + rate = 48000, + duration = 3, +) { + return [ + "-f", + "lavfi", + "-i", + `aevalsrc=${expression}:s=${rate}:d=${duration}`, + ]; +} + +const video = ["-f", "lavfi", "-i", "testsrc2=size=160x90:rate=15:duration=3"]; +const h264 = ["-c:v", "libx264", "-preset", "ultrafast"]; +const aac = ["-c:a", "aac", "-b:a", "192k"]; + +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), "cap-audio-format-test-")); +}); + +afterAll(async () => { + await rm(directory, { recursive: true, force: true }); +}); + +describe("audio level correction recording formats", () => { + test.each([ + { + name: "screen-mono.mp4", + inputs: [...video, ...audio()], + output: [...h264, ...aac], + expected: "shadow-candidate", + }, + { + name: "system-music.m4a", + inputs: audio( + "0.01*sin(2*PI*220*t)+0.01*sin(2*PI*880*t)|0.01*sin(2*PI*330*t)+0.01*sin(2*PI*1100*t)", + 44100, + ), + output: aac, + expected: "shadow-candidate", + }, + { + name: "pcm.mov", + inputs: [...video, ...audio()], + output: [...h264, "-c:a", "pcm_s16le"], + expected: "unchanged", + }, + { + name: "silent.mp4", + inputs: [...video, ...audio("0")], + output: [...h264, ...aac], + expected: "unchanged", + }, + { + name: "video-only.mp4", + inputs: video, + output: h264, + expected: "unchanged", + }, + { + name: "surround.m4a", + inputs: audio( + "0.01*sin(2*PI*440*t)|0.01*sin(2*PI*550*t)|0.01*sin(2*PI*660*t)|0.01*sin(2*PI*770*t)|0.01*sin(2*PI*880*t)|0.01*sin(2*PI*990*t)", + ), + output: aac, + expected: "unchanged", + }, + { + name: "dual-audio.mp4", + inputs: [...video, ...audio(), ...audio()], + output: ["-map", "0:v", "-map", "1:a", "-map", "2:a", ...h264, ...aac], + expected: "unchanged", + }, + { + name: "low-rate.m4a", + inputs: audio(undefined, 32000), + output: aac, + expected: "unchanged", + }, + { + name: "short.m4a", + inputs: audio(undefined, 48000, 1), + output: aac, + expected: "unchanged", + }, + { + name: "loud.m4a", + inputs: audio("0.8*sin(2*PI*440*t)"), + output: aac, + expected: "unchanged", + }, + { + name: "mismatched-duration.mp4", + inputs: [...video, ...audio(undefined, 48000, 4)], + output: [...h264, ...aac], + expected: "unchanged", + }, + { + name: "browser.webm", + inputs: [...video, ...audio()], + output: ["-c:v", "libvpx", "-deadline", "realtime", "-c:a", "libopus"], + expected: "unchanged", + }, + { + name: "legacy.mkv", + inputs: [...video, ...audio()], + output: [...h264, "-c:a", "flac"], + expected: "unchanged", + }, + { + name: "legacy.mp3", + inputs: audio(), + output: ["-c:a", "libmp3lame"], + expected: "unchanged", + }, + ])( + "preserves the source for $name", + async ({ name, inputs, output, expected }) => { + const path = await fixture(name, inputs, output); + const original = await readFile(path); + const sourceHash = createHash("sha256").update(original).digest("hex"); + const result = await createAudioQualityCandidate(path, { + mode: "shadow", + profile: "levels", + }); + try { + expect(result.status).toBe(expected); + if (result.status === "shadow-candidate") { + expect(result.validationFailures).toEqual([]); + expect(result.sourceSha256).toBe(sourceHash); + expect(result.output.truePeak).toBeLessThanOrEqual(-1); + expect( + Math.abs(result.output.lra - result.input.lra), + ).toBeLessThanOrEqual(1); + } + expect(await readFile(path)).toEqual(original); + } finally { + if (result.status === "shadow-candidate") await result.cleanup(); + } + }, + 30_000, + ); + + test("preserves subtitle-bearing recordings without dropping a stream", async () => { + const subtitle = join(directory, "captions.srt"); + await writeFile(subtitle, "1\n00:00:00,000 --> 00:00:02,000\nSpeech\n"); + const source = await fixture( + "subtitles.mp4", + [...video, ...audio(), "-i", subtitle], + [ + "-map", + "0:v", + "-map", + "1:a", + "-map", + "2:s", + ...h264, + ...aac, + "-c:s", + "mov_text", + ], + ); + const before = await readFile(source); + expect( + await createAudioQualityCandidate(source, { + mode: "shadow", + profile: "levels", + }), + ).toEqual({ status: "unchanged", reason: "unsupported-streams" }); + expect(await readFile(source)).toEqual(before); + }, 30_000); + + test("rejects corrupt input without replacing it", async () => { + const source = join(directory, "corrupt.mp4"); + const original = Buffer.from("not a media container"); + await writeFile(source, original); + await expect( + createAudioQualityCandidate(source, { + mode: "shadow", + profile: "levels", + }), + ).rejects.toThrow(); + expect(await readFile(source)).toEqual(original); + }); + + test("rejects playlist input instead of following referenced media", async () => { + const media = await fixture( + "playlist-media.mp4", + [...video, ...audio()], + [...h264, ...aac], + ); + const source = join(directory, "playlist.m3u8"); + const original = `#EXTM3U\n#EXT-X-TARGETDURATION:3\n#EXTINF:3,\n${media}\n#EXT-X-ENDLIST\n`; + await writeFile(source, original); + await expect( + createAudioQualityCandidate(source, { + mode: "shadow", + profile: "levels", + }), + ).rejects.toThrow(); + expect(await readFile(source, "utf8")).toBe(original); + }, 30_000); +}); diff --git a/apps/media-server/src/__tests__/lib/audio-quality-policy.test.ts b/apps/media-server/src/__tests__/lib/audio-quality-policy.test.ts new file mode 100644 index 0000000000..bae8acf96a --- /dev/null +++ b/apps/media-server/src/__tests__/lib/audio-quality-policy.test.ts @@ -0,0 +1,84 @@ +import { describe, expect, test } from "bun:test"; +import { + type AudioQualityMeasurements, + planAudioQuality, + validateAudioQualityMeasurements, +} from "../../lib/audio-quality-policy"; + +const input: AudioQualityMeasurements = { + lufs: -35, + truePeak: -16, + lra: 6, + duration: 80, + channels: 1, + sampleRate: 48000, + sampleCount: 3840000, +}; + +describe("audio quality policy", () => { + test("is disabled without shadow mode", () => { + expect(planAudioQuality(input, { mode: "off", profile: "voice" })).toEqual({ + kind: "skip", + reason: "disabled", + }); + }); + + test("requires affirmative speech-only evidence for voice processing", () => { + expect( + planAudioQuality(input, { mode: "shadow", profile: "voice" }), + ).toEqual({ kind: "skip", reason: "unconfirmed-speech-only" }); + }); + + test("retains mixed audio dynamics and bounds gain", () => { + const plan = planAudioQuality(input, { mode: "shadow", profile: "levels" }); + expect(plan).toMatchObject({ + kind: "candidate", + filter: "volume=12.000000dB", + }); + expect( + planAudioQuality( + { ...input, truePeak: -4 }, + { mode: "shadow", profile: "levels" }, + ), + ).toMatchObject({ kind: "candidate", filter: "volume=2.000000dB" }); + }); + + test.each([ + { lufs: Number.NEGATIVE_INFINITY }, + { truePeak: Number.NaN }, + { truePeak: 10.84 }, + { lufs: -65 }, + { duration: 1 }, + { channels: 6 }, + { sampleRate: 8000 }, + { lufs: -14 }, + { truePeak: -2.5 }, + ])("leaves unsupported or risky input unchanged: %j", (patch) => { + expect( + planAudioQuality( + { ...input, ...patch }, + { mode: "shadow", profile: "levels" }, + ).kind, + ).toBe("skip"); + }); + + test("rejects measurable output regressions", () => { + const output = { ...input, truePeak: -0.2, lufs: -12, duration: 81 }; + expect(validateAudioQualityMeasurements(input, output, "levels")).toEqual([ + "audio-duration-changed", + "insufficient-peak-headroom", + "unexpected-loudness", + "excessive-gain", + ]); + }); + + test("accepts bounded constant gain without changing duration or dynamics", () => { + expect( + validateAudioQualityMeasurements( + input, + { ...input, lufs: -23, truePeak: -4 }, + "levels", + ), + ).toEqual([]); + }); +}); diff --git a/apps/media-server/src/__tests__/lib/audio-quality.integration.test.ts b/apps/media-server/src/__tests__/lib/audio-quality.integration.test.ts new file mode 100644 index 0000000000..5f45ee4b40 --- /dev/null +++ b/apps/media-server/src/__tests__/lib/audio-quality.integration.test.ts @@ -0,0 +1,224 @@ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createAudioQualityCandidate } from "../../lib/audio-quality"; + +let directory: string; + +async function command(args: string[]) { + const proc = Bun.spawn(args, { stdout: "pipe", stderr: "pipe" }); + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).arrayBuffer(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (code !== 0) throw new Error(stderr); + return stdout; +} + +async function fixture( + name: string, + rate = 48000, + channels = 1, + options: { silence?: boolean; offset?: number; gap?: boolean } = {}, +) { + const samples = rate * 3; + const bytes = samples * channels * 2; + const wav = Buffer.alloc(44 + bytes); + wav.write("RIFF", 0); + wav.writeUInt32LE(36 + bytes, 4); + wav.write("WAVEfmt ", 8); + wav.writeUInt32LE(16, 16); + wav.writeUInt16LE(1, 20); + wav.writeUInt16LE(channels, 22); + wav.writeUInt32LE(rate, 24); + wav.writeUInt32LE(rate * channels * 2, 28); + wav.writeUInt16LE(channels * 2, 32); + wav.writeUInt16LE(16, 34); + wav.write("data", 36); + wav.writeUInt32LE(bytes, 40); + for (let index = 0; index < samples; index++) { + const t = index / rate; + const envelope = [0.005, 0.15, 1.5, 2.99].reduce( + (sum, center) => sum + Math.exp(-(((t - center) / 0.003) ** 2)), + 0, + ); + for (let channel = 0; channel < channels; channel++) { + const value = options.silence + ? 0 + : (0.005 + envelope * 0.055) * + Math.cos(2 * Math.PI * (1000 + channel * 900) * t); + wav.writeInt16LE( + Math.round(value * 32767), + 44 + (index * channels + channel) * 2, + ); + } + } + const wavPath = join(directory, `${name}.wav`); + const source = join(directory, `${name}.mp4`); + await writeFile(wavPath, wav); + await command([ + "ffmpeg", + "-v", + "error", + "-f", + "lavfi", + "-i", + "testsrc2=size=160x90:rate=15:duration=3", + "-i", + wavPath, + "-c:v", + "libx264", + "-preset", + "ultrafast", + "-c:a", + "aac", + "-b:a", + "192k", + ...(options.gap ? ["-af", "asetpts=PTS+if(gte(T\\,1)\\,0.05/TB\\,0)"] : []), + ...(options.offset ? ["-output_ts_offset", String(options.offset)] : []), + source, + ]); + return source; +} + +async function decode(path: string, rate: number) { + const bytes = await command([ + "ffmpeg", + "-v", + "error", + "-i", + path, + "-map", + "0:a:0", + "-ac", + "1", + "-ar", + String(rate), + "-f", + "f32le", + "-", + ]); + return new Float32Array(bytes); +} + +function peakPosition(samples: Float32Array, center: number, rate: number) { + const start = Math.max(0, Math.floor((center - 0.04) * rate)); + const end = Math.min(samples.length, Math.ceil((center + 0.04) * rate)); + let peak = start; + for (let index = start; index < end; index++) { + if (Math.abs(samples[index]) > Math.abs(samples[peak])) peak = index; + } + return peak / rate; +} + +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), "cap-audio-quality-test-")); +}); + +afterAll(async () => { + await rm(directory, { recursive: true, force: true }); +}); + +describe("audio quality derivative", () => { + for (const rate of [44100, 48000]) { + for (const channels of [1, 2]) { + for (const profile of ["levels", "voice"] as const) { + test(`${profile} preserves video, channels, timing, and boundary pulses at ${rate}/${channels}`, async () => { + const source = await fixture( + `${profile}-${rate}-${channels}`, + rate, + channels, + ); + const before = await readFile(source); + const result = await createAudioQualityCandidate(source, { + mode: "shadow", + profile, + speechOnlyConfirmed: true, + }); + expect(result.status).toBe("shadow-candidate"); + if (result.status !== "shadow-candidate") return; + try { + expect(result.validationFailures).toEqual([]); + expect(result.output.channels).toBe(channels); + expect(result.output.truePeak).toBeLessThanOrEqual(-1); + const original = await decode(source, rate); + const processed = await decode(result.path, rate); + for (const position of [0.005, 0.15, 1.5, 2.99]) { + expect( + Math.abs( + peakPosition(original, position, rate) - + peakPosition(processed, position, rate), + ), + ).toBeLessThan(0.003); + } + expect(await readFile(source)).toEqual(before); + } finally { + await result.cleanup(); + } + }, 30_000); + } + } + } + + test("leaves nonzero source starts untouched", async () => { + const source = await fixture("offset", 48000, 1, { offset: 2 }); + const result = await createAudioQualityCandidate(source, { + mode: "shadow", + profile: "levels", + }); + expect(result).toEqual({ + status: "unchanged", + reason: "source-start-offset", + }); + }, 30_000); + + test("does not rewrite discontinuous source audio", async () => { + const source = await fixture("gap", 48000, 1, { gap: true }); + expect( + await createAudioQualityCandidate(source, { + mode: "shadow", + profile: "levels", + }), + ).toEqual({ status: "unchanged", reason: "source-timeline-discontinuous" }); + }, 30_000); + + test("preserves silence and does no work when disabled", async () => { + expect( + await createAudioQualityCandidate("/does-not-exist", { + mode: "off", + profile: "voice", + }), + ).toEqual({ status: "unchanged", reason: "disabled" }); + const source = await fixture("silence", 48000, 1, { silence: true }); + expect( + await createAudioQualityCandidate(source, { + mode: "shadow", + profile: "levels", + }), + ).toEqual({ status: "unchanged", reason: "unsafe-levels" }); + }, 30_000); + + test("handles cancellation without modifying the source", async () => { + const source = await fixture("cancel"); + const before = await readFile(source); + await expect( + createAudioQualityCandidate(source, { + mode: "shadow", + profile: "voice", + speechOnlyConfirmed: true, + abortSignal: AbortSignal.abort(), + }), + ).rejects.toThrow(); + await expect( + createAudioQualityCandidate(source, { + mode: "shadow", + profile: "voice", + speechOnlyConfirmed: true, + timeoutMs: 20, + }), + ).rejects.toThrow(); + expect(await readFile(source)).toEqual(before); + }, 30_000); +}); diff --git a/apps/media-server/src/lib/audio-quality-benchmark.ts b/apps/media-server/src/lib/audio-quality-benchmark.ts new file mode 100644 index 0000000000..9c4a5c0ea9 --- /dev/null +++ b/apps/media-server/src/lib/audio-quality-benchmark.ts @@ -0,0 +1,18 @@ +import { copyFile } from "node:fs/promises"; +import type { AudioQualityResult } from "./audio-quality"; + +export async function retainAudioQualityBenchmarkResult( + result: AudioQualityResult, + destination: string, +) { + if (result.status !== "shadow-candidate") return result; + try { + const { cleanup: _cleanup, path: _path, ...evidence } = result; + if (result.validationFailures.length) + return { ...evidence, status: "rejected" as const }; + await copyFile(result.path, destination, 1); + return evidence; + } finally { + await result.cleanup(); + } +} diff --git a/apps/media-server/src/lib/audio-quality-policy.ts b/apps/media-server/src/lib/audio-quality-policy.ts new file mode 100644 index 0000000000..432eddd760 --- /dev/null +++ b/apps/media-server/src/lib/audio-quality-policy.ts @@ -0,0 +1,113 @@ +export interface AudioQualityMeasurements { + lufs: number; + truePeak: number; + lra: number; + duration: number; + channels: number; + sampleRate: number; + sampleCount: number; +} + +export type AudioQualityProfile = "levels" | "voice"; + +export type AudioQualityPlan = + | { kind: "skip"; reason: string } + | { + kind: "candidate"; + profile: AudioQualityProfile; + version: "audio-quality-v3"; + filter: string; + gainDb: number; + }; + +export function planAudioQuality( + measurements: AudioQualityMeasurements, + options: { + mode: "off" | "shadow"; + profile: AudioQualityProfile; + speechOnlyConfirmed?: boolean; + }, +): AudioQualityPlan { + if (options.mode !== "shadow") return { kind: "skip", reason: "disabled" }; + if ( + !Object.values(measurements).every(Number.isFinite) || + measurements.lufs < -50 || + measurements.lufs > 0 || + measurements.lra < 0 || + measurements.lra > 50 || + measurements.truePeak < -100 || + measurements.truePeak > 0 + ) + return { kind: "skip", reason: "unsafe-levels" }; + if ( + measurements.duration < 3 || + measurements.duration > 3600 || + !Number.isSafeInteger(measurements.sampleCount) || + measurements.sampleCount < 1 || + ![1, 2].includes(measurements.channels) || + ![44100, 48000].includes(measurements.sampleRate) + ) + return { kind: "skip", reason: "unsupported-format" }; + if (measurements.lufs >= -18) return { kind: "skip", reason: "already-loud" }; + if (options.profile === "voice" && !options.speechOnlyConfirmed) + return { kind: "skip", reason: "unconfirmed-speech-only" }; + const gainDb = Math.min( + 12, + -16 - measurements.lufs, + -2 - measurements.truePeak, + ); + if (options.profile === "levels" && gainDb < 1) + return { kind: "skip", reason: "insufficient-headroom" }; + const preGain = Math.min(18, Math.max(0, -20 - measurements.lufs)); + const targetLufs = Math.min(-16, measurements.lufs + 18); + // afftdn delays content by two sample-advance blocks without compensating PTS. + const denoiseDelay = 2 * Math.floor(measurements.sampleRate / 80); + return { + kind: "candidate", + profile: options.profile, + version: "audio-quality-v3", + gainDb: options.profile === "levels" ? gainDb : preGain, + filter: + options.profile === "levels" + ? `volume=${gainDb.toFixed(6)}dB` + : [ + `volume=${preGain.toFixed(6)}dB`, + "highpass=f=60", + "equalizer=f=250:t=q:w=0.8:g=-0.75", + "equalizer=f=2500:t=q:w=0.7:g=0.75", + `apad=pad_len=${denoiseDelay}`, + "afftdn=nr=6:nf=-45:tn=1:gs=5", + `atrim=start_sample=${denoiseDelay}:end_sample=${measurements.sampleCount + denoiseDelay}`, + "asetpts=N/SR/TB", + `loudnorm=I=${targetLufs.toFixed(2)}:TP=-2:LRA=11:dual_mono=true`, + ].join(","), + }; +} + +export function validateAudioQualityMeasurements( + input: AudioQualityMeasurements, + output: AudioQualityMeasurements, + profile: AudioQualityProfile, +): string[] { + const failures: string[] = []; + if (!Object.values(output).every(Number.isFinite)) + return ["invalid-output-measurements"]; + if (input.channels !== output.channels) + failures.push("channel-count-changed"); + if (input.sampleRate !== output.sampleRate) + failures.push("sample-rate-changed"); + if (Math.abs(input.duration - output.duration) > 0.025) + failures.push("audio-duration-changed"); + if ( + output.sampleCount < input.sampleCount || + output.sampleCount - input.sampleCount >= 1024 + ) + failures.push("decoded-sample-count-changed"); + if (output.truePeak > -1) failures.push("insufficient-peak-headroom"); + if (output.lufs < input.lufs - 0.5 || output.lufs > -14) + failures.push("unexpected-loudness"); + if (output.lufs - input.lufs > 18.75) failures.push("excessive-gain"); + if (profile === "levels" && Math.abs(output.lra - input.lra) > 1) + failures.push("dynamics-changed"); + return failures; +} diff --git a/apps/media-server/src/lib/audio-quality.ts b/apps/media-server/src/lib/audio-quality.ts new file mode 100644 index 0000000000..d5e381f686 --- /dev/null +++ b/apps/media-server/src/lib/audio-quality.ts @@ -0,0 +1,412 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { lstat, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join } from "node:path"; +import { spawn } from "bun"; +import { z } from "zod"; +import { + type AudioQualityMeasurements, + type AudioQualityProfile, + planAudioQuality, + validateAudioQualityMeasurements, +} from "./audio-quality-policy"; +import { proveRecordingPackets } from "./recording-packet-proof"; +import { registerSubprocess, unregisterSubprocess } from "./subprocess"; + +const streamSchema = z.object({ + codec_type: z.string(), + codec_name: z.string(), + channels: z.number().int().optional(), + sample_rate: z.string().optional(), + duration: z.string().optional(), + start_time: z.string().optional(), +}); +const probeSchema = z.object({ streams: z.array(streamSchema) }); +const loudnessSchema = z.object({ + input_i: z.string(), + input_tp: z.string(), + input_lra: z.string(), +}); + +const localInputOptions = [ + "-protocol_whitelist", + "file", + "-format_whitelist", + "mov,matroska,webm,avi,wav,mp3,flac,ogg,aac", +]; + +async function readBounded( + stream: ReadableStream, + limit: number, +): Promise { + const reader = stream.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + let exceeded = false; + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > limit) exceeded = true; + else chunks.push(value); + } + } finally { + reader.releaseLock(); + } + if (exceeded) + throw new Error("Audio quality subprocess output exceeded limit"); + return Buffer.concat(chunks).toString("utf8"); +} + +async function run( + args: string[], + signal: AbortSignal, + stdoutLimit = 1024 * 1024, +) { + signal.throwIfAborted(); + const proc = registerSubprocess( + spawn({ cmd: args, stdout: "pipe", stderr: "pipe", stdin: "ignore" }), + ); + const abort = () => { + if (proc.exitCode === null) proc.kill("SIGKILL"); + }; + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + try { + const [stdout, stderr, code] = await Promise.all([ + readBounded(proc.stdout, stdoutLimit), + readBounded(proc.stderr, 128 * 1024), + proc.exited, + ]); + signal.throwIfAborted(); + if (code !== 0) throw new Error(`Audio quality subprocess exited ${code}`); + return { stdout, stderr }; + } finally { + if (proc.exitCode === null) proc.kill("SIGKILL"); + await proc.exited; + signal.removeEventListener("abort", abort); + unregisterSubprocess(proc); + } +} + +async function fingerprint(path: string, signal: AbortSignal) { + const before = await lstat(path, { bigint: true }); + if (!before.isFile()) + throw new Error("Audio quality requires a regular file"); + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path, { signal })) + hash.update(chunk); + const after = await lstat(path, { bigint: true }); + if ( + before.ino !== after.ino || + before.dev !== after.dev || + before.size !== after.size || + before.mtimeNs !== after.mtimeNs || + before.ctimeNs !== after.ctimeNs + ) + throw new Error("Audio quality source changed during hashing"); + return hash.digest("hex"); +} + +async function probe(path: string, signal: AbortSignal) { + const result = await run( + [ + "ffprobe", + "-v", + "error", + ...localInputOptions, + "-show_streams", + "-of", + "json", + path, + ], + signal, + ); + return probeSchema.parse(JSON.parse(result.stdout)).streams; +} + +async function hasContinuousAudioTimeline( + path: string, + measurements: AudioQualityMeasurements, + startTime: number, + signal: AbortSignal, +): Promise { + const result = await run( + [ + "ffprobe", + "-v", + "error", + "-select_streams", + "a:0", + "-show_entries", + "frame=pts_time,nb_samples", + "-of", + "csv=p=0", + ...localInputOptions, + path, + ], + signal, + 8 * 1024 * 1024, + ); + let samples = 0; + for (const line of result.stdout.trim().split("\n")) { + const fields = line.trim().split(","); + if (fields.length !== 2 || !fields[0] || !fields[1]) return false; + const pts = Number(fields[0]); + const count = Number(fields[1]); + if ( + !Number.isFinite(pts) || + !Number.isSafeInteger(count) || + count <= 0 || + Math.abs(pts - startTime - samples / measurements.sampleRate) > + 1 / measurements.sampleRate + ) + return false; + samples += count; + } + return samples === measurements.sampleCount; +} + +export async function measureAudioQuality( + path: string, + signal: AbortSignal, +): Promise { + const streams = await probe(path, signal); + const audio = streams.filter((stream) => stream.codec_type === "audio"); + const stream = audio[0]; + if (audio.length !== 1 || !stream) + throw new Error("Audio quality requires exactly one audio stream"); + const result = await run( + [ + "ffmpeg", + "-hide_banner", + "-nostdin", + "-nostats", + "-threads", + "1", + ...localInputOptions, + "-i", + path, + "-map", + "0:a:0", + "-af", + "astats=measure_perchannel=none:measure_overall=Number_of_samples,loudnorm=I=-16:TP=-2:LRA=11:dual_mono=true:print_format=json", + "-f", + "null", + "-", + ], + signal, + ); + const start = result.stderr.lastIndexOf("{"); + const end = result.stderr.lastIndexOf("}"); + const values = loudnessSchema.parse( + JSON.parse(result.stderr.slice(start, end + 1)), + ); + return { + lufs: Number(values.input_i), + truePeak: Number(values.input_tp), + lra: Number(values.input_lra), + duration: Number(stream.duration), + channels: stream.channels ?? 0, + sampleRate: Number(stream.sample_rate), + sampleCount: Number(result.stderr.match(/Number of samples: (\d+)/)?.[1]), + }; +} + +export type AudioQualityResult = + | { status: "unchanged"; reason: string } + | { + status: "shadow-candidate"; + path: string; + sourceSha256: string; + outputSha256: string; + input: AudioQualityMeasurements; + output: AudioQualityMeasurements; + profile: AudioQualityProfile; + version: "audio-quality-v3"; + peakCorrectionDb: number; + validationFailures: string[]; + elapsedMs: number; + cleanup: () => Promise; + }; + +export async function createAudioQualityCandidate( + sourcePath: string, + options: { + mode: "off" | "shadow"; + profile: AudioQualityProfile; + speechOnlyConfirmed?: boolean; + abortSignal?: AbortSignal; + timeoutMs?: number; + }, +): Promise { + if (options.mode !== "shadow") + return { status: "unchanged", reason: "disabled" }; + if (!isAbsolute(sourcePath)) + throw new Error("Audio quality requires an absolute local source path"); + const controller = new AbortController(); + const timer = setTimeout( + () => controller.abort(new Error("Audio quality timed out")), + Math.min(30 * 60_000, Math.max(1, options.timeoutMs ?? 10 * 60_000)), + ); + const signal = options.abortSignal + ? AbortSignal.any([controller.signal, options.abortSignal]) + : controller.signal; + const started = performance.now(); + let directory: string | undefined; + try { + const sourceSha256 = await fingerprint(sourcePath, signal); + const streams = await probe(sourcePath, signal); + const audioStreams = streams.filter((s) => s.codec_type === "audio"); + const videoStreams = streams.filter((s) => s.codec_type === "video"); + if ( + audioStreams.length !== 1 || + videoStreams.length > 1 || + streams.some((s) => !["audio", "video"].includes(s.codec_type)) + ) + return { status: "unchanged", reason: "unsupported-streams" }; + if (audioStreams[0]?.codec_name !== "aac") + return { status: "unchanged", reason: "unsupported-audio-codec" }; + const input = await measureAudioQuality(sourcePath, signal); + const plan = planAudioQuality(input, options); + if (plan.kind === "skip") + return { status: "unchanged", reason: plan.reason }; + const videoDuration = Number(videoStreams[0]?.duration); + const sourceStart = Number(audioStreams[0]?.start_time); + if (!Number.isFinite(sourceStart)) + return { status: "unchanged", reason: "unknown-source-start" }; + if (sourceStart !== 0) + return { status: "unchanged", reason: "source-start-offset" }; + if ( + !(await hasContinuousAudioTimeline( + sourcePath, + input, + sourceStart, + signal, + )) + ) + return { status: "unchanged", reason: "source-timeline-discontinuous" }; + if ( + videoStreams.length && + (!Number.isFinite(videoDuration) || + Math.abs(videoDuration - input.duration) > 0.1) + ) + return { status: "unchanged", reason: "source-duration-mismatch" }; + directory = await mkdtemp(join(tmpdir(), "cap-audio-quality-")); + let path = join(directory, "candidate.mp4"); + const renderArgs = [ + "ffmpeg", + "-hide_banner", + "-nostdin", + "-nostats", + "-v", + "error", + "-n", + "-copyts", + ...localInputOptions, + "-i", + sourcePath, + "-map", + "0", + "-map_metadata", + "0", + "-map_chapters", + "0", + "-c:v", + "copy", + "-af", + `${plan.filter},aresample=${input.sampleRate},atrim=end_sample=${input.sampleCount},asettb=1/${input.sampleRate},asetpts=N`, + "-filter_threads", + "1", + "-c:a", + "aac", + "-b:a", + input.channels === 1 ? "192k" : "256k", + "-ar", + String(input.sampleRate), + "-threads", + "1", + "-avoid_negative_ts", + "disabled", + "-movflags", + "+faststart", + path, + ]; + await run(renderArgs, signal); + let output = await measureAudioQuality(path, signal); + let peakCorrectionDb = 0; + if (Number.isFinite(output.truePeak) && output.truePeak > -1) { + const correction = -2 - output.truePeak; + if (correction >= -6) { + peakCorrectionDb = correction; + path = join(directory, "candidate-limited.mp4"); + const args = [...renderArgs]; + const filterIndex = args.indexOf("-af") + 1; + args[filterIndex] += `,volume=${correction.toFixed(6)}dB`; + args[args.length - 1] = path; + await run(args, signal); + output = await measureAudioQuality(path, signal); + } + } + await run( + [ + "ffmpeg", + "-v", + "error", + "-xerror", + "-nostdin", + ...localInputOptions, + "-i", + path, + "-f", + "null", + "-", + ], + signal, + ); + const validationFailures = validateAudioQualityMeasurements( + input, + output, + plan.profile, + ); + const outputStreams = await probe(path, signal); + const inputStart = Number(audioStreams[0]?.start_time); + const outputStart = Number( + outputStreams.find((s) => s.codec_type === "audio")?.start_time, + ); + if ( + !Number.isFinite(inputStart) || + !Number.isFinite(outputStart) || + Math.abs(inputStart - outputStart) > 1 / input.sampleRate + ) + validationFailures.push("audio-start-changed"); + if (!(await hasContinuousAudioTimeline(path, output, outputStart, signal))) + validationFailures.push("output-timeline-discontinuous"); + if (videoStreams.length) + await proveRecordingPackets(sourcePath, null, path, signal); + if ((await fingerprint(sourcePath, signal)) !== sourceSha256) + throw new Error("Audio quality source changed during processing"); + const outputSha256 = await fingerprint(path, signal); + const retainedDirectory = directory; + directory = undefined; + return { + status: "shadow-candidate", + path, + sourceSha256, + outputSha256, + input, + output, + profile: plan.profile, + version: plan.version, + peakCorrectionDb, + validationFailures, + elapsedMs: performance.now() - started, + cleanup: () => rm(retainedDirectory, { recursive: true, force: true }), + }; + } finally { + clearTimeout(timer); + if (directory) await rm(directory, { recursive: true, force: true }); + } +} diff --git a/scripts/benchmark-audio-intelligibility.py b/scripts/benchmark-audio-intelligibility.py new file mode 100644 index 0000000000..56b822d6c8 --- /dev/null +++ b/scripts/benchmark-audio-intelligibility.py @@ -0,0 +1,110 @@ +import argparse +import concurrent.futures +import importlib.util +import json +import pathlib + +import numpy as np +from pystoi import stoi +from scipy.io import wavfile +from scipy.signal import correlate, correlation_lags, lfilter + +spec = importlib.util.spec_from_file_location("benchmark", pathlib.Path(__file__).with_name("benchmark-instant-audio.py")) +benchmark = importlib.util.module_from_spec(spec) +spec.loader.exec_module(benchmark) + + +def decode(path): + result, _ = benchmark.execute(["ffmpeg", "-v", "error", "-nostdin", "-i", str(path), "-ac", "1", "-ar", "16000", "-f", "f32le", "-"]) + return np.frombuffer(result.stdout, dtype=np.float32).astype(np.float64) + + +def sdr(reference, output): + projection = np.dot(reference, output) / max(np.dot(reference, reference), 1e-20) * reference + return float(10 * np.log10(max(np.sum(projection ** 2), 1e-20) / max(np.sum((output - projection) ** 2), 1e-20))) + + +def score_signals(reference, noisy, output): + n = min(len(reference), len(noisy), len(output)) + a, baseline, b = reference[:n], noisy[:n], output[:n] + correlation = correlate(b, a, method="fft") + lags = correlation_lags(len(b), len(a)) + region = np.abs(lags) <= 1600 + lag = int(lags[region][np.argmax(correlation[region])]) + unaligned_input = float(stoi(a, baseline, 16000)) + unaligned_output = float(stoi(a, b, 16000)) + if lag > 0: + a, baseline, b = a[:-lag], baseline[:-lag], b[lag:] + elif lag < 0: + a, baseline, b = a[-lag:], baseline[-lag:], b[:lag] + input_stoi = float(stoi(a, baseline, 16000)) + output_stoi = float(stoi(a, b, 16000)) + return {"metricVersion": "aligned-v1", "inputStoi": input_stoi, "outputStoi": output_stoi, "stoiDelta": output_stoi - input_stoi, "inputSiSdr": sdr(a, baseline), "outputSiSdr": sdr(a, b), "lagMs": lag / 16, "evaluatedSamples": len(a), "unalignedInputStoi": unaligned_input, "unalignedOutputStoi": unaligned_output, "unalignedStoiDelta": unaligned_output - unaligned_input} + + +def evaluate(case, root): + name, reference, noisy = case + case_path = root / f"{name}.wav" + wavfile.write(case_path, 16000, noisy.astype(np.float32)) + base = benchmark.measure(case_path) + results = [] + policies = ["levels", "equalized-v2", "clean-v2"] if root.name.endswith("-v2") else ["levels", "equalized", "clean", "clean3"] + if "strength" in root.name: + policies = ["clean6-v2", "clean12-v2"] + for policy in policies: + gain = min(18, max(0, -20 - base["lufs"])) + filters = f"volume={gain:.6f}dB" + if policy.endswith("-v2"): + filters += ",highpass=f=60,equalizer=f=250:t=q:w=0.8:g=-0.75,equalizer=f=2500:t=q:w=0.7:g=0.75" + if policy.startswith("clean"): + strength = 6 if policy == "clean6-v2" else 12 if policy == "clean12-v2" else 3 + filters += f",apad=pad_len=400,afftdn=nr={strength}:nf=-45:tn=1:gs=5,atrim=start_sample=400:end_sample={len(reference)+400},asetpts=N/SR/TB" + elif policy != "levels": + filters += ",highpass=f=70,equalizer=f=250:t=q:w=0.8:g=-1.5,equalizer=f=2500:t=q:w=0.7:g=1.5" + if policy in ["clean", "clean3"]: + filters += f",afftdn=nr={6 if policy == 'clean' else 3}:nf=-45:tn=1:gs=5" + if policy != "levels" and not policy.endswith("-v2"): + filters += ",acompressor=threshold=0.125:ratio=2:attack=15:release=200:makeup=1" + filters += f",loudnorm=I=-16:TP=-2:LRA=11:dual_mono=true,aresample=16000,atrim=end_sample={len(reference)},asetpts=N/SR/TB" + output = root / f"{name}-{policy}.m4a" + _, elapsed = benchmark.execute(["ffmpeg", "-v", "error", "-nostdin", "-y", "-i", str(case_path), "-af", filters, "-ar", "16000", "-c:a", "aac", "-b:a", "96k", str(output)]) + y = decode(output) + results.append({"case": name, "policy": policy, **score_signals(reference, noisy, y), "sampleDelta": len(y) - len(reference), "processingSeconds": elapsed}) + print(json.dumps({"case": name, "stoiDeltas": {r["policy"]: round(r["stoiDelta"], 5) for r in results}}), flush=True) + return results + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("root", type=pathlib.Path) + parser.add_argument("--reference-ids", nargs=3, required=True) + args = parser.parse_args() + root = args.root + root.mkdir(parents=True, exist_ok=True) + initial = root.parent.parent + ids = args.reference_ids + cases = [] + rng = np.random.default_rng(20260907) + for index, vid in enumerate(ids): + x = decode(initial / f"{vid}.m4a")[8 * 16000:28 * 16000] + x *= 10 ** (-24 / 20) / np.sqrt(np.mean(x ** 2)) + t = np.arange(len(x)) / 16000 + white = rng.normal(size=len(x)) + pink = lfilter([0.049922035, -0.095993537, 0.050612699, -0.004408786], [1, -2.494956002, 2.017265875, -0.5221894], white) + hum = np.sin(2 * np.pi * 50 * t) + .4 * np.sin(2 * np.pi * 100 * t) + .2 * np.sin(2 * np.pi * 150 * t) + hiss = lfilter([1, -.95], [1], white) + cases.append((f"voice{index}-unchanged", x, x.copy())) + for name, noise in [("white", white), ("fan", pink), ("hum", hum), ("hiss", hiss)]: + noise /= np.sqrt(np.mean(noise ** 2)) + for snr in [0, 10, 20]: + noisy = x + noise * np.sqrt(np.mean(x ** 2)) / 10 ** (snr / 20) + cases.append((f"voice{index}-{name}-snr{snr}", x, noisy)) + results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + for result in pool.map(lambda case: evaluate(case, root), cases): + results.extend(result) + (root / "results.json").write_text(json.dumps(results, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark-instant-audio.py b/scripts/benchmark-instant-audio.py new file mode 100644 index 0000000000..f85d8e88a4 --- /dev/null +++ b/scripts/benchmark-instant-audio.py @@ -0,0 +1,128 @@ +import argparse +import concurrent.futures +import hashlib +import json +import pathlib +import re +import subprocess +import time + +import numpy as np + + +def execute(args, timeout=600): + started = time.monotonic() + result = subprocess.run(args, capture_output=True, timeout=timeout) + if result.returncode: + raise RuntimeError(f"{args[0]} exited {result.returncode}: {result.stderr.decode(errors='replace')[-500:]}") + return result, time.monotonic() - started + + +def measure(path): + probe, _ = execute(["ffprobe", "-v", "error", "-show_streams", "-show_format", "-of", "json", str(path)]) + metadata = json.loads(probe.stdout) + stream = next(s for s in metadata["streams"] if s["codec_type"] == "audio") + result, elapsed = execute(["ffmpeg", "-hide_banner", "-nostdin", "-i", str(path), "-vn", "-af", "loudnorm=I=-16:TP=-2:LRA=11:dual_mono=true:print_format=json", "-f", "null", "-"]) + log = result.stderr.decode() + values = json.loads(log[log.rfind("{"):log.rfind("}") + 1]) + return {"lufs": float(values["input_i"]), "truePeak": float(values["input_tp"]), "lra": float(values["input_lra"]), "threshold": float(values["input_thresh"]), "duration": float(stream.get("duration", metadata["format"]["duration"])), "channels": stream["channels"], "sampleRate": int(stream["sample_rate"]), "codec": stream["codec_name"], "bitrate": int(stream.get("bit_rate", 0)), "measurementSeconds": elapsed} + + +def speech_metrics(path, transcript): + result, _ = execute(["ffmpeg", "-v", "error", "-nostdin", "-i", str(path), "-vn", "-ac", "1", "-ar", "16000", "-f", "f32le", "-c:a", "pcm_f32le", "-"]) + samples = np.frombuffer(result.stdout, dtype=np.float32) + n = len(samples) // 320 + frames = samples[:n * 320].reshape(n, 320).astype(np.float64) + power = np.mean(frames * frames, axis=1) + centers = (np.arange(n) + .5) * .02 + speech = np.zeros(n, dtype=bool) + pauses = np.ones(n, dtype=bool) + cues = [] + for match in re.finditer(r"(\d+):(\d+):(\d+\.\d+) --> (\d+):(\d+):(\d+\.\d+)", transcript): + v = list(map(float, match.groups())) + start, end = v[0] * 3600 + v[1] * 60 + v[2], v[3] * 3600 + v[4] * 60 + v[5] + cues.append((start, end)) + speech |= (centers >= start) & (centers <= end) + pauses &= ~((centers >= start - .25) & (centers <= end + .25)) + def db(values): + return float(10 * np.log10(max(float(np.mean(values)), 1e-24))) if len(values) else None + return {"decodedDuration": len(samples) / 16000, "cueCount": len(cues), "lastCueEnd": max((e for _, e in cues), default=0), "captionCoverage": float(speech.mean()) if n else 0, "captionRms": db(power[speech]), "pauseRms": db(power[pauses]), "pauseSeconds": float(pauses.sum() * .02), "silentFrameFraction": float(np.mean(power < 1e-9)) if n else 1} + + +def baseline(row, root): + vid = row["id"] + out = root / "baseline" / f"{vid}.json" + if out.exists(): + return json.loads(out.read_text()) + media = root / "sources" / f"{vid}.m4a" + vtt = root / "sources" / f"{vid}.vtt" + if not vtt.exists(): + execute(["cap", "caps", "transcript", vid, "--format", "vtt", "--output", str(vtt), "--json"]) + if not media.exists(): + part = media.with_suffix(".partial.m4a") + execute(["ffmpeg", "-v", "error", "-nostdin", "-y", "-i", f"https://cap.so/api/playlist?videoId={vid}&videoType=mp4", "-map", "0:a:0", "-vn", "-c:a", "copy", str(part)]) + part.rename(media) + metrics = measure(media) + metrics.update(speech_metrics(media, vtt.read_text())) + metrics.update({"id": vid, "split": row["split"], "stratum": row["stratum"], "createdAt": row["createdAt"], "databaseDuration": row["duration"], "sourceSha256": hashlib.sha256(media.read_bytes()).hexdigest(), "transcriptSha256": hashlib.sha256(vtt.read_bytes()).hexdigest()}) + out.write_text(json.dumps(metrics, indent=2, allow_nan=True)) + print(json.dumps({"baseline": vid, "lufs": metrics["lufs"], "peak": metrics["truePeak"], "duration": metrics["duration"]}), flush=True) + return metrics + + +def candidate(row, root, policy): + out = root / policy / f"{row['id']}.json" + if out.exists(): + return json.loads(out.read_text()) + media = root / "sources" / f"{row['id']}.m4a" + gain = min(-16 - row["lufs"], 12, -2 - row["truePeak"]) + if policy == "gain6": + gain = min(gain, 6) + if policy.startswith("gain") and (not np.isfinite(gain) or gain < 1 or row["lufs"] < -50 or row["duration"] < 3 or row["channels"] > 2): + result = {"id": row["id"], "split": row["split"], "policy": policy, "status": "unchanged", "gain": 0, "input": row, "output": row, "processingSeconds": 0} + else: + filters = "loudnorm=I=-16:TP=-2:LRA=11:dual_mono=true" if policy == "dynamic" else f"volume={gain:.6f}dB" + if policy in ["equalized", "clean"]: + pre_gain = min(18, max(0, -20 - row["lufs"])) + filters = f"volume={pre_gain:.6f}dB,highpass=f=70,equalizer=f=250:t=q:w=0.8:g=-1.5,equalizer=f=2500:t=q:w=0.7:g=1.5" + if policy == "clean": + filters += ",afftdn=nr=6:nf=-45:tn=1:gs=5" + filters += ",acompressor=threshold=0.125:ratio=2:attack=15:release=200:makeup=1,loudnorm=I=-16:TP=-2:LRA=11:dual_mono=true" + path = root / policy / f"{row['id']}.m4a" + _, elapsed = execute(["ffmpeg", "-v", "error", "-nostdin", "-y", "-i", str(media), "-map", "0:a:0", "-af", filters, "-ar", str(row["sampleRate"]), "-c:a", "aac", "-b:a", "192k" if row["channels"] == 1 else "256k", str(path)]) + output = measure(path) + output.update(speech_metrics(path, (root / "sources" / f"{row['id']}.vtt").read_text())) + result = {"id": row["id"], "split": row["split"], "policy": policy, "status": "rendered", "gain": gain if policy != "dynamic" else None, "input": row, "output": output, "processingSeconds": elapsed, "realtimeFactor": elapsed / row["duration"], "durationDeltaMs": (output["duration"] - row["duration"]) * 1000, "lufsDelta": output["lufs"] - row["lufs"], "lraDelta": output["lra"] - row["lra"], "sourceUnchanged": hashlib.sha256(media.read_bytes()).hexdigest() == row["sourceSha256"]} + result["filters"] = filters + if row["pauseRms"] is not None and output["pauseRms"] is not None: + result["loudnessMatchedPauseChangeDb"] = output["pauseRms"] - row["pauseRms"] - result["lufsDelta"] + out.write_text(json.dumps(result, indent=2, allow_nan=True)) + print(json.dumps({"policy": policy, "id": row["id"], "status": result["status"], "outputLufs": result["output"]["lufs"]}), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("root", type=pathlib.Path) + parser.add_argument("--phase", choices=["baseline", "tuning", "holdout"], default="baseline") + parser.add_argument("--policies", nargs="+", default=["gain6", "gain12", "dynamic"]) + args = parser.parse_args() + for name in ["sources", "baseline", *args.policies]: + (args.root / name).mkdir(parents=True, exist_ok=True) + rows = json.loads((args.root / "cohort.json").read_text()) + jobs = [(row, None) for row in rows] if args.phase == "baseline" else [(json.loads((args.root / "baseline" / f"{row['id']}.json").read_text()), policy) for row in rows if row["split"] == args.phase for policy in args.policies] + results = [] + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + futures = {pool.submit(baseline, row, args.root) if policy is None else pool.submit(candidate, row, args.root, policy): row["id"] for row, policy in jobs} + for future in concurrent.futures.as_completed(futures): + try: + results.append(future.result()) + except Exception as error: + print(json.dumps({"id": futures[future], "error": str(error)}), flush=True) + (args.root / f"{args.phase}-results.json").write_text(json.dumps(results, indent=2, allow_nan=True)) + if len(results) != len(jobs): + raise SystemExit(f"Only {len(results)}/{len(jobs)} jobs succeeded") + + +if __name__ == "__main__": + main() diff --git a/scripts/test-audio-intelligibility.py b/scripts/test-audio-intelligibility.py new file mode 100644 index 0000000000..35556c0efb --- /dev/null +++ b/scripts/test-audio-intelligibility.py @@ -0,0 +1,42 @@ +import importlib.util +import pathlib +import unittest + +import numpy as np + +spec = importlib.util.spec_from_file_location("intelligibility", pathlib.Path(__file__).with_name("benchmark-audio-intelligibility.py")) +benchmark = importlib.util.module_from_spec(spec) +spec.loader.exec_module(benchmark) + + +class AlignmentTests(unittest.TestCase): + def test_delayed_or_advanced_audio_scores_the_same_overlapping_speech(self): + x = np.random.default_rng(2240).normal(0, .05, 64000) + for lag in [-400, 0, 400]: + with self.subTest(lag=lag): + y = np.roll(x, lag) * .8 + if lag > 0: + y[:lag] = 0 + elif lag < 0: + y[lag:] = 0 + result = benchmark.score_signals(x, x.copy(), y) + self.assertEqual(result["lagMs"], lag / 16) + self.assertEqual(result["evaluatedSamples"], len(x) - abs(lag)) + self.assertAlmostEqual(result["stoiDelta"], 0, places=6) + self.assertGreater(result["outputSiSdr"], 90) + if lag: + self.assertLess(result["unalignedStoiDelta"], -.1) + + def test_input_and_output_use_the_same_trimmed_reference(self): + rng = np.random.default_rng(2240) + x = rng.normal(0, .05, 64000) + noisy = x + rng.normal(0, .01, len(x)) + output = np.concatenate([np.zeros(400), noisy[:-400]]) + result = benchmark.score_signals(x, noisy, output) + self.assertEqual(result["lagMs"], 25) + self.assertAlmostEqual(result["stoiDelta"], 0, places=6) + self.assertAlmostEqual(result["inputSiSdr"], result["outputSiSdr"], places=6) + + +if __name__ == "__main__": + unittest.main()