From 3a7dadc99ca47d1436fc2678207de94e7ea793b2 Mon Sep 17 00:00:00 2001 From: antoine-debroye <144667691+antoine-debroye@users.noreply.github.com> Date: Wed, 28 Oct 2026 20:51:50 +0000 Subject: [PATCH] Cut adverts a host inserts as it serves, whatever their language Hosts such as Simplecast and Acast stitch adverts in per request, chosen by the listener's country, so a French show fetched from the UK carries English adverts. Language and proxies cannot remove them; a proxy only changes which adverts arrive. What every inserted advert has is that the host concatenates a file it already has: the same advert is the same MP3 frames in every episode, while the programme, encoded once per episode, never is. - src/lib/copied-audio.js finds stretches byte-identical across episodes, and the breaks between the slot markers Acast stitches around them, so adverts never heard elsewhere are cut too. New catalogue kind `inserted` (migration 014), one row per stretch or per break slot. - MP3 frame hashes are kept beside each fingerprint; older episodes are read once without decoding. - A feed whose host is seen inserting gets every new episode downloaded a second time a day later (same address and user agent), with a switch on the subscription form. - Guards: over 2.5 minutes, or a copy two episodes share mid-file, is offered not cut; breaks never exceed half an episode; an approved cut is never taken away for a row still waiting. Also fixes the jingle anchor cutting a show's spoken intro (present in 1.10.1): on Le Rendez-vous Tech it auto-confirmed the show's own theme and cut up to 1:33. A jingle starting more than 3 s after the host's inserted audio ends is no longer auto-confirmed, and an earlier auto-confirmation is taken back; owner-confirmed jingles are untouched. Measured on real feeds on 2026-10-28: RMC opener and closer in 30/30 episodes; Rendez-vous Tech pre-, mid- and post-rolls in 5/5 marked episodes, intro kept; RTL nothing. npm test 1476/1476, acceptance 65/65. Co-Authored-By: Claude Opus 5.5 --- README.md | 38 +- doc/selfpod-spec.md | 110 ++++- scripts/check-copied-audio.mjs | 60 +++ src/constants.js | 3 + src/db/migrations/014-inserted-audio.sql | 123 +++++ src/lib/audio-search.js | 10 +- src/lib/audio-tasks.js | 40 +- src/lib/auto-approve.js | 49 ++ src/lib/copied-audio.js | 417 +++++++++++++++++ src/lib/fingerprint-file.js | 42 ++ src/lib/present-segment.js | 4 + src/lib/present-subscription.js | 1 + src/lib/present-transcript.js | 46 ++ src/lib/segment-kind.js | 1 + src/services/ad-detect.js | 374 ++++++++++++++- src/services/ad-pipeline.js | 22 +- src/services/adverts-view.js | 7 +- src/services/remote-feeds.js | 31 +- src/services/subscriptions.js | 17 +- src/web/routes/fragments.js | 2 + src/web/views/partials/subscription-form.eta | 15 + src/workers/audio-search.js | 5 +- test/acceptance/run.sh | 82 ++++ test/helpers/mp3.js | 33 +- test/integration/ad-anchor.test.js | 11 +- test/integration/ad-detect.test.js | 8 +- test/integration/ad-pipeline.test.js | 20 +- test/integration/ad-segments-api.test.js | 3 +- test/integration/ad-segments-ui.test.js | 3 +- test/integration/ad-timeline-ui.test.js | 3 +- test/integration/dynamic-adverts.test.js | 63 ++- test/integration/inserted-audio.test.js | 453 +++++++++++++++++++ test/integration/subscriptions-ui.test.js | 22 + test/integration/transcript-detect.test.js | 3 +- test/integration/trim-publish.test.js | 5 +- test/integration/trimmer.test.js | 5 +- test/unit/auto-approve.test.js | 63 +++ test/unit/copied-audio.test.js | 183 ++++++++ test/unit/migrations.test.js | 97 ++++ 39 files changed, 2400 insertions(+), 74 deletions(-) create mode 100644 scripts/check-copied-audio.mjs create mode 100644 src/db/migrations/014-inserted-audio.sql create mode 100644 src/lib/copied-audio.js create mode 100644 test/integration/inserted-audio.test.js create mode 100644 test/unit/copied-audio.test.js diff --git a/README.md b/README.md index 8ecef2f..6a4b3c3 100644 --- a/README.md +++ b/README.md @@ -416,16 +416,34 @@ correctly, and proved on a GPU only on real hardware. ### Adverts a host inserts as it serves -Some podcasts insert adverts into the audio as it is downloaded, so the file is -different every time. SelfPod can spot these by downloading one episode a second -time, a day later, and comparing: whatever changed cannot be the theme tune, so it -is an advert by construction. - -This is done sparingly and on purpose. A second download counts again in the -publisher's listener figures, so SelfPod only does it for episodes whose audio shows -positive signs of having been stitched together, never sooner than a day, and only a -couple at a time. It will not disguise itself to defeat a host's caching — that -would be interfering with the publisher's measurement, which is not SelfPod's to do. +Some hosts — Simplecast, Acast and others — insert adverts into the audio as it is +downloaded, picked by the listener's country and by chance. A French show fetched from +the UK carries English adverts; from Germany, German ones. So SelfPod does not look for +a language or a country. It looks for what every inserted advert has in common: the +host splices in a file it already has, so **the same advert is the same bytes in every +episode it lands in** — while the programme, encoded afresh each time, never is. + +- **Copied audio.** A stretch whose frames are byte for byte the same in two or more + episodes is cut. RMC's sponsor opener and closer are found this way in every episode. +- **Breaks between markers.** Hosts like Acast stitch a short clip at both edges of + every break. SelfPod finds those markers and cuts what lies between them — even + adverts it has never heard before. Shown as *Pre-roll*, *Mid-roll* and *Post-roll* + breaks on the Adverts page, each with *Restore here*. +- **A second download, a day later.** For a feed whose host inserts, every new episode + is downloaded once more and compared, to learn adverts that rotate. SelfPod turns + this on by itself once it has seen the host insert; the subscription form has the + switch too. Each second download counts again in the publisher's figures. + +In automatic mode these are cut without asking, with two exceptions that are offered +instead: anything over two and a half minutes, and a stretch only two episodes share in +the middle (a "best of" could share real programme that way). SelfPod never disguises +itself to see different adverts — a proxy would only swap which adverts arrive. + +To see what it finds in episodes you downloaded yourself: + +```bash +node scripts/check-copied-audio.mjs path/to/folder-of-mp3s +``` Only for `.mp3` at present. Other formats are published untouched. diff --git a/doc/selfpod-spec.md b/doc/selfpod-spec.md index 6d5e406..5772959 100644 --- a/doc/selfpod-spec.md +++ b/doc/selfpod-spec.md @@ -1052,22 +1052,86 @@ per request. It is the stronger signal, and the only one that identifies an advert rather than something that merely repeats: a theme tune is in both copies, so it cannot be what differs between them. -### 19.3 The cost of looking twice, and why it is rationed - -A second download is a second IAB-countable listen, so it doubles the -publisher's figures for an episode taken once. Hosts also cache the -stitch per listener, keyed on requesting address and user agent, so two -requests from one container seconds apart are the same listener by -construction and would hit that cache. Defeating it would need a rotating -user agent or egress address; that is deliberate evasion of the -publisher's measurement and SelfPod will not do it. - -So a second download happens only when the first file carries a positive -signal — a sample-rate or channel-mode change part-way through, or a Xing -header disagreeing with the frames present — never sooner than a day, a -couple per tick across all subscriptions, and charged to the same daily -byte budget. The second copy is deleted; the file on the share is the one -the owner has. +### 19.3 Audio the host inserts as it serves + +Hosts such as Simplecast and Acast stitch adverts into the MP3 as each +request is served, chosen by the requester's country and by chance. A +French show fetched from the UK carries English adverts; fetched from +Germany, German ones; fetched from France, French ones. So nothing about +an inserted advert's language or content identifies it, and changing +where SelfPod asks from (a proxy, another address) would only change +which adverts arrive. The evidence has to be something every inserted +advert has, whatever it says. + +**The same bytes.** A host inserts by concatenating MP3 frames from a +file it already has; it does not re-encode the episode around them. So +an advert served to two episodes is *byte for byte* the same frames in +both, while the programme — mastered and encoded once per episode — +never shares a frame with another episode, its theme tune included. +Frames copied between episodes are therefore audio the host spliced in +(`src/lib/copied-audio.js`). Measured on 2026-10-28 from the UK: + +- RMC on Simplecast, 30 episodes (three of them best-ofs): a 13.9 s + opener and a 10.1 s closer, the same frames in 30 of 30; nothing else. + The opener is the sponsor tag *and* the "Vous écoutez RMC" ident, one + file; the owner chose to cut all of it. +- Le Rendez-vous Tech on Acast, 6 episodes: UK adverts in English in + pre-, mid- and post-rolls, and a 2.2 s marker stitched at both edges of + every break. +- Laurent Gerra (RTL) on Audiomeans: nothing copied, nothing inserted. + +**Breaks between markers.** A short copied clip found more than once in +one episode is a host's slot marker. What lies between two markers, or +between one and the edge of the file, and is no longer than two and a +half minutes, is a break — cut whether or not any advert in it was ever +heard elsewhere. Never when the breaks would add up to half the episode. +Breaks are catalogued one row per slot (pre-roll, mid-roll *n*, +post-roll), so a row is in each episode once and *Restore here* restores +one break. + +**What is cut without asking**, and why the guards differ from §19.5's: +a copied stretch carries none of the ambiguity of a merely repeated one, +so no position guard and no fifteen-second floor (the RMC opener sits at +0:00 and its closer is ten seconds). What remains: two and a half minutes +at most; and a copy in only two episodes, mid-file, is offered rather +than cut — a best-of assembled from an earlier episode's file without +re-encoding would share programme exactly so. An advert reaches a third +episode soon enough, or sits where hosts put them, at the very start or +end of the file. + +**One row per stretch.** The acoustic search (§19.2) hears inserted audio +too; it leaves alone what an inserted row explains, but never takes the +occurrences of a row that is already cutting. Audio another approved row +already cuts is not offered again. + +**Frame hashes are kept** beside each fingerprint (`{episode}.h1.frames` +in the fingerprint folder) so this costs no decoding. An episode +fingerprinted before they were kept is read once, without decoding, and +only if its bytes are still the ones its fingerprint describes. + +**Looking twice.** Comparing two downloads of one episode finds what a +host changes from day to day — the only way to learn an advert served to +one episode only, outside any marked break. A second download is a +second IAB-countable listen, so it is rationed: made when the first file +carries a stitch signal (a sample-rate or channel-mode change part-way +through, a Xing header that disagrees with the frames present), or when +the subscription is known to insert — set by SelfPod the first time it +finds copied audio in the show or a second download comes back +different, or by the owner on the subscription form. Never sooner than a +day, a couple per tick across all subscriptions, charged to the same +daily byte budget; the second copy is deleted. + +Hosts cache the stitch per listener, keyed on address and user agent, so +a second request seconds later is the same listener by construction +(three downloads of one RMC episode, minutes apart, two user agents: +byte-identical). Defeating that would need a rotating user agent or +egress address. That is deliberate evasion of the publisher's +measurement, and it would not help: it changes which adverts arrive, +not whether they do. SelfPod asks again, later, as itself. + +**What none of this finds:** an advert inserted into one episode only, +outside any marked break, served identically to both downloads. The words +(§19.6) and a taught boundary remain for that. ### 19.4 Cutting @@ -1297,6 +1361,20 @@ point at the jingle by hand. Confirming a jingle does not guess where a *differe kind of ident sits partway into an episode; it only ever cuts from 0:00 to where the sound was heard. +**Not confirmed by itself when it starts after the host's audio ends.** The proposal +is the earliest sound every recent episode shares, and that need not be a station +ident. On Le Rendez-vous Tech (Acast, 2026-10-28) one sponsored episode carried no +adverts and so no break marker, which left the show's own theme as that sound — after +the host's spoken intro. Confirmed automatically, 1.10 cut the intro from every +episode, up to 1:33. Now, in any recent episode where the host's inserted audio +(§19.3: copied frames, or a break between markers) starts before the jingle but ends +more than three seconds before it, the jingle is not confirmed automatically: what +lies between is the show's own. It stays a proposal the owner can still confirm, and +the breaks are cut regardless. A jingle SelfPod confirmed by itself before this rule is +taken back the same way, with its cut; one the owner confirmed, or a taught boundary +linked, is never touched. RMC is unaffected: its "Vous écoutez RMC" ident is inside the +inserted opener, so the jingle starts before the inserted audio ends. + ### 19.8 What a cut is, putting one back, and when SelfPod looks **Added in 1.9.0.** diff --git a/scripts/check-copied-audio.mjs b/scripts/check-copied-audio.mjs new file mode 100644 index 0000000..9652115 --- /dev/null +++ b/scripts/check-copied-audio.mjs @@ -0,0 +1,60 @@ +#!/usr/bin/env node +/** + * What the copied-audio search finds in a folder of real episodes. + * + * node scripts/check-copied-audio.mjs [--json] + * + * Runs `findCopiedAudio` exactly as the detector does, on files you downloaded from the + * host yourself (not SelfPod's served copies, which are already cut). Prints every + * stretch found in two or more episodes, which of them are slot markers, and the breaks + * between markers — so a change to the search can be checked against real audio before + * it is trusted with anyone's feed. + */ +import { readdir, readFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { findCopiedAudio } from '../src/lib/copied-audio.js'; +import { frameProfile } from '../src/lib/mp3-frames.js'; + +const args = process.argv.slice(2); +const folder = args.find((arg) => !arg.startsWith('--')); +if (!folder) { + console.error('usage: node scripts/check-copied-audio.mjs [--json]'); + process.exit(2); +} + +const names = (await readdir(folder)).filter((name) => name.toLowerCase().endsWith('.mp3')).sort(); +const episodes = []; +for (const name of names) { + const profile = frameProfile(await readFile(join(folder, name))); + if (!profile?.frameCount) { + console.warn(`${name}: no MP3 frames, skipped`); + continue; + } + episodes.push({ id: name, hashes: profile.hashes, frameMs: profile.durationMs / profile.frameCount, durationMs: profile.durationMs }); +} + +const started = Date.now(); +const result = findCopiedAudio(episodes); +const elapsed = Date.now() - started; + +if (args.includes('--json')) { + console.log(JSON.stringify(result, null, 2)); + process.exit(0); +} + +const seconds = (ms) => (ms / 1000).toFixed(1); +const durations = Object.fromEntries(episodes.map((episode) => [episode.id, episode.durationMs])); +const where = (o) => { + const tail = durations[o.episodeId] - o.endMs < 1000 ? ' (to the end)' : ''; + return `${o.episodeId}@${seconds(o.startMs)}–${seconds(o.endMs)}${tail}`; +}; + +console.log(`${episodes.length} episodes, searched in ${elapsed} ms`); +console.log(`\nCopied stretches: ${result.copied.length}`); +for (const row of result.copied) { + const marker = result.markers.includes(row.signature) ? ' [marker]' : ''; + console.log(` ${seconds(row.durationMs)} s in ${row.episodeCount} episodes, ${row.occurrenceCount} times${marker}`); + for (const o of row.occurrences) console.log(` ${where(o)}`); +} +console.log(`\nBreaks between markers: ${result.blocks.length}`); +for (const block of result.blocks) console.log(` ${where(block)} (${seconds(block.endMs - block.startMs)} s)`); diff --git a/src/constants.js b/src/constants.js index 22642ec..7037fb5 100644 --- a/src/constants.js +++ b/src/constants.js @@ -509,6 +509,9 @@ export const SEGMENT_KINDS = Object.freeze({ REPEATED_AUDIO: 'repeated_audio', DIFF: 'diff', TAUGHT_RANGE: 'taught_range', + // Audio the host spliced in as it served the file: the same MP3 frames in two or + // more episodes, or a break between the slot markers a host stitches (copied-audio.js). + INSERTED: 'inserted', }); /** The `source` 1.8 would have written for a kind — kept so a rollback can read rows. */ diff --git a/src/db/migrations/014-inserted-audio.sql b/src/db/migrations/014-inserted-audio.sql new file mode 100644 index 0000000..81957e9 --- /dev/null +++ b/src/db/migrations/014-inserted-audio.sql @@ -0,0 +1,123 @@ +-- Audio a host inserts as it serves, found by its MP3 frames (spec §19.3). +-- +-- A new kind of catalogue row, `inserted`: a stretch whose frames are byte for byte the +-- same in two or more episodes (a file the host spliced in), or a break between the slot +-- markers a host stitches around one. And a flag on a subscription saying its host is +-- known to insert, so every new episode of it is downloaded a second time. +-- +-- Additive in effect, not in form, exactly as 009 was. `ad_segments.kind` is a CHECK +-- constraint and SQLite cannot widen one in place, so the catalogue and the two tables +-- that point at it are rebuilt with the same rows. The migration runner keeps foreign +-- keys ON inside its transaction, so the order matters: the replacements are built and +-- filled first, the old children are dropped while nothing references them, the old +-- parent only once it has no children left for an implicit DELETE to cascade into, and +-- the renames come last — renaming the parent is what rewrites the new children's +-- foreign keys to point at `ad_segments`. +-- +-- An older image run against this schema reads an `inserted` row as one more row it +-- does not have a special case for, and cuts it like any other approved row. Rolling +-- back stays non-destructive, the property every migration since 007 preserves. + +CREATE TABLE ad_segments_v3 ( + id TEXT PRIMARY KEY, + show_id TEXT NOT NULL REFERENCES shows(id) ON DELETE CASCADE, + signature TEXT NOT NULL, + -- 'transcript' is found by what was said. For a repeated read it is the hash of the + -- words; for a boundary the owner taught it is `marker:{id}`. + source TEXT NOT NULL CHECK (source IN ('corpus', 'diff', 'transcript')), + status TEXT NOT NULL DEFAULT 'candidate' + CHECK (status IN ('candidate', 'approved', 'rejected')), + auto_approved INTEGER NOT NULL DEFAULT 0, + hold_reason TEXT, + duration_ms INTEGER NOT NULL, + episode_count INTEGER NOT NULL DEFAULT 0, + occurrence_count INTEGER NOT NULL DEFAULT 0, + exemplar_episode_id TEXT REFERENCES episodes(id) ON DELETE SET NULL, + exemplar_start_ms INTEGER, + exemplar_end_ms INTEGER, + first_seen_at TEXT NOT NULL, + decided_at TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + + -- What was said, for segments found by — or later matched to — the words. + text TEXT, + raw_text TEXT, + cue_score REAL, + cues TEXT, + language TEXT, + + -- What kind of thing the row is (011). `inserted` is new here: audio the host + -- spliced in, found by its frames rather than by how it sounds. + kind TEXT NOT NULL DEFAULT 'repeated_audio' + CHECK (kind IN ('jingle', 'boundary_words', 'remembered_words', 'repeated_words', + 'repeated_audio', 'diff', 'taught_range', 'inserted')), + marker_id TEXT REFERENCES ad_markers(id) ON DELETE CASCADE, + anchor_id TEXT REFERENCES ad_anchors(id) ON DELETE CASCADE, + first_cut_at TEXT, + + UNIQUE(show_id, signature) +); + +CREATE TABLE ad_segment_occurrences_v3 ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + segment_id TEXT NOT NULL REFERENCES ad_segments_v3(id) ON DELETE CASCADE, + episode_id TEXT NOT NULL REFERENCES episodes(id) ON DELETE CASCADE, + start_frame INTEGER NOT NULL, + end_frame INTEGER NOT NULL, + start_ms INTEGER NOT NULL, + end_ms INTEGER NOT NULL, + UNIQUE(segment_id, episode_id, start_frame) +); + +CREATE TABLE ad_cut_overrides_v3 ( + id TEXT PRIMARY KEY, + episode_id TEXT NOT NULL REFERENCES episodes(id) ON DELETE CASCADE, + segment_id TEXT REFERENCES ad_segments_v3(id) ON DELETE SET NULL, + start_ms INTEGER NOT NULL, + end_ms INTEGER NOT NULL CHECK (end_ms > start_ms), + created_at TEXT NOT NULL +); + +INSERT INTO ad_segments_v3 + (id, show_id, signature, source, status, auto_approved, hold_reason, duration_ms, + episode_count, occurrence_count, exemplar_episode_id, exemplar_start_ms, exemplar_end_ms, + first_seen_at, decided_at, created_at, updated_at, text, raw_text, cue_score, cues, language, + kind, marker_id, anchor_id, first_cut_at) +SELECT + id, show_id, signature, source, status, auto_approved, hold_reason, duration_ms, + episode_count, occurrence_count, exemplar_episode_id, exemplar_start_ms, exemplar_end_ms, + first_seen_at, decided_at, created_at, updated_at, text, raw_text, cue_score, cues, language, + kind, marker_id, anchor_id, first_cut_at +FROM ad_segments; + +INSERT INTO ad_segment_occurrences_v3 + (id, segment_id, episode_id, start_frame, end_frame, start_ms, end_ms) +SELECT id, segment_id, episode_id, start_frame, end_frame, start_ms, end_ms +FROM ad_segment_occurrences; + +INSERT INTO ad_cut_overrides_v3 (id, episode_id, segment_id, start_ms, end_ms, created_at) +SELECT id, episode_id, segment_id, start_ms, end_ms, created_at +FROM ad_cut_overrides; + +DROP TABLE ad_cut_overrides; +DROP TABLE ad_segment_occurrences; +DROP TABLE ad_segments; + +ALTER TABLE ad_segments_v3 RENAME TO ad_segments; +ALTER TABLE ad_segment_occurrences_v3 RENAME TO ad_segment_occurrences; +ALTER TABLE ad_cut_overrides_v3 RENAME TO ad_cut_overrides; + +CREATE INDEX idx_ad_segments_show ON ad_segments(show_id, status); +CREATE INDEX idx_ad_segments_kind ON ad_segments(show_id, kind); +CREATE INDEX idx_ad_segments_marker ON ad_segments(marker_id); +CREATE INDEX idx_ad_segments_anchor ON ad_segments(anchor_id); +CREATE INDEX idx_ad_occurrences_episode ON ad_segment_occurrences(episode_id); +CREATE INDEX idx_ad_cut_overrides_episode ON ad_cut_overrides(episode_id); +CREATE INDEX idx_ad_cut_overrides_segment ON ad_cut_overrides(segment_id); + +-- The host of this feed has been seen inserting audio — set by SelfPod the first time +-- it finds some, or by the owner — so every new episode is downloaded a second time a +-- day later to learn the adverts that rotate (spec §19.3). Never a disguise: the same +-- address and the same user agent as the first download. +ALTER TABLE feed_subscriptions ADD COLUMN inserts_audio INTEGER NOT NULL DEFAULT 0; diff --git a/src/lib/audio-search.js b/src/lib/audio-search.js index e9a6629..c528a5f 100644 --- a/src/lib/audio-search.js +++ b/src/lib/audio-search.js @@ -1,7 +1,8 @@ import { Worker } from 'node:worker_threads'; import { findHeadAnchors } from './audio-anchor.js'; -import { decodeWindowToWav, fingerprintFile, profileFile } from './audio-tasks.js'; +import { decodeWindowToWav, fingerprintFile, frameHashesFile, profileFile } from './audio-tasks.js'; +import { findCopiedAudio } from './copied-audio.js'; import { findRepeatedAudio } from './repeated-audio.js'; const WORKER_URL = new URL('../workers/audio-search.js', import.meta.url); @@ -114,6 +115,13 @@ export function createAudioWorker({ logger = null, inline = false, fileTimeoutMs headAnchors(episodes, options) { return run('headAnchors', { episodes, options }, () => findHeadAnchors(episodes, options)); }, + copiedAudio(episodes, options) { + return run('copiedAudio', { episodes, options }, () => findCopiedAudio(episodes, options)); + }, + /** @see frameHashesFile in audio-tasks.js */ + frameHashesFile(payload) { + return run('frameHashesFile', payload, () => unwrap(() => frameHashesFile(payload)), { timeoutMs: fileTimeoutMs }); + }, /** @see fingerprintFile in audio-tasks.js */ fingerprintFile(payload) { return run('fingerprintFile', payload, () => unwrap(() => fingerprintFile(payload)), { timeoutMs: fileTimeoutMs }); diff --git a/src/lib/audio-tasks.js b/src/lib/audio-tasks.js index a225a68..3058bf4 100644 --- a/src/lib/audio-tasks.js +++ b/src/lib/audio-tasks.js @@ -65,11 +65,15 @@ export async function fingerprintFile({ path, maxBytes, expectSha256 = null }) { const decoded = await decodeToMono(bytes, profile.frames, (samples) => fingerprinter.push(samples)); const hashes = fingerprinter.finish(); + // The frames' own hashes come free with the profile, and they are what finds audio a + // host spliced in byte for byte (copied-audio.js) — so they travel with the fingerprint. + const frameHashes = Uint32Array.from(profile.hashes); return { result: { sha256, bytes: bytes.length, hashes, + frameHashes, frameCount: profile.frameCount, durationMs: profile.durationMs, sampleRate: profile.sampleRate, @@ -77,7 +81,41 @@ export async function fingerprintFile({ path, maxBytes, expectSha256 = null }) { decodeErrors: decoded.errors, discontinuities: profile.discontinuities.length, }, - transfer: [hashes.buffer], + transfer: [hashes.buffer, frameHashes.buffer], + }; +} + +/** + * Reads an episode and hashes its MP3 frames, without decoding a sample. + * + * For an episode fingerprinted before frame hashes were kept: the fingerprint is still + * good, and decoding an hour of audio again to get at hashes of bytes the file already + * holds would be a minute of a NAS's time per episode for nothing. + */ +export async function frameHashesFile({ path, maxBytes }) { + const { handle, size } = await openGatedFile(path, { maxBytes }); + let bytes; + try { + bytes = await readRange(handle, 0, size); + } finally { + await handle.close(); + } + const sha256 = createHash('sha256').update(bytes).digest('hex'); + const profile = frameProfile(bytes); + if (!profile) return { result: { sha256, bytes: bytes.length, noFrames: true } }; + if (profile.truncated) return { result: { sha256, bytes: bytes.length, tooLong: true } }; + const frameHashes = Uint32Array.from(profile.hashes); + return { + result: { + sha256, + bytes: bytes.length, + frameHashes, + frameCount: profile.frameCount, + durationMs: profile.durationMs, + sampleRate: profile.sampleRate, + samplesPerFrame: profile.frames[0]?.samplesPerFrame ?? 1152, + }, + transfer: [frameHashes.buffer], }; } diff --git a/src/lib/auto-approve.js b/src/lib/auto-approve.js index b98927c..a995d18 100644 --- a/src/lib/auto-approve.js +++ b/src/lib/auto-approve.js @@ -54,6 +54,18 @@ function positionGuard(segment, episodeDurations) { return null; } +/** Whether every occurrence touches the start or the end of its episode — a pre- or post-roll. */ +function atAnEdgeEverywhere(segment, episodeDurations) { + const durations = episodeDurations ?? {}; + const occurrences = segment.occurrences ?? []; + if (!occurrences.length) return false; + return occurrences.every((occurrence) => { + if ((occurrence.startMs ?? Infinity) < 1000) return true; + const total = durations[occurrence.episodeId]; + return Boolean(total) && total - (occurrence.endMs ?? 0) < 1000; + }); +} + export function safeToApproveAutomatically( segment, { episodeDurations, minEpisodes = 3, source = 'corpus' } = {}, @@ -82,6 +94,43 @@ export function safeToApproveAutomatically( return { safe: true, reason: null }; } + /* + * Audio the host spliced in: the same MP3 frames in two or more episodes, or a break + * between the markers a host stitches around one (copied-audio.js). Safe by + * construction for the reason a diff is: a programme is mastered and encoded once per + * episode, so its theme tune never comes out as the same frames twice — only a file + * the host already had and concatenated in does. So the position guards are not + * applied (RMC's inserted opener sits at 0:00 in every episode, which is exactly what + * they would refuse), and neither is the fifteen-second minimum (its closer is ten). + * + * The length guard still applies, for the diff's reason: what a host inserts is its + * choice, and two and a half minutes of it is more likely a re-used programme + * segment than a break. A copied stretch must also really be in two episodes; a + * break carries its markers as its evidence instead. + */ + if (source === 'inserted') { + const seconds = segment.durationMs ? segment.durationMs / 1000 : null; + if (seconds !== null && seconds > DIFF_MAX_SECONDS) return { safe: false, reason: 'too_long_to_be_an_advert' }; + if (segment.betweenMarkers) return { safe: true, reason: null }; + if ((segment.episodeCount ?? 0) < 2) return { safe: false, reason: 'only_heard_once' }; + /* + * In exactly two episodes, and somewhere in the middle of them, a copy has one other + * innocent explanation: a "best of" assembled from an earlier episode's file without + * re-encoding it, which shares a stretch of *programme* with that episode byte for + * byte. RMC's best-ofs were re-encoded (30 episodes checked, 2026-10-28), but nothing + * makes every publisher do that. An advert a host inserts reaches a third episode + * soon enough, or sits where a host puts one — at the very start or end of the file + * — so only that shape is cut unasked; the rest is offered. + */ + // At the edges, a minute at most: a pre- or post-roll is that, and a longer stretch + // two episodes share at their edges is as likely a recap or a reused segment. + const shortEdge = seconds !== null && seconds <= 60 && atAnEdgeEverywhere(segment, episodeDurations); + if ((segment.episodeCount ?? 0) < 3 && !shortEdge) { + return { safe: false, reason: 'seen_too_few_times' }; + } + return { safe: true, reason: null }; + } + /* * Found by the words. Repetition is still the evidence — a stretch heard once is * offered and never cut, whatever it says — but the wording is the tie-breaker the diff --git a/src/lib/copied-audio.js b/src/lib/copied-audio.js new file mode 100644 index 0000000..84a32cf --- /dev/null +++ b/src/lib/copied-audio.js @@ -0,0 +1,417 @@ +import { createHash } from 'node:crypto'; + +/** + * Finding audio a host spliced into a show, by its MP3 frames (spec §19.3). + * + * A host that inserts adverts as it serves does not re-encode the episode around them: + * it concatenates frames from a file it already has. So the same advert lands in every + * episode it is served with as the *same bytes*, while the programme — mastered and + * encoded once per episode — never shares a frame with another episode. A theme tune + * played in every episode is re-encoded every time and does not match here; an advert, + * or an ident the host adds, always does. + * + * That makes byte-identical frames shared between two episodes the one signal that + * tells an inserted stretch from a repeated part of the programme — which the acoustic + * search (`repeated-audio.js`) cannot do, because a theme tune and an advert sound + * equally alike from one episode to the next. Measured on 2026-10-28 from the UK: + * Simplecast's RMC opener and closer were the same frames in 8 of 8 episodes; Acast's + * pre-, mid- and post-rolls on Le Rendez-vous Tech were copied between episodes, and + * a 2.2 s clip was stitched at the edge of every one of its blocks. + * + * Two answers come back: + * + * - **copied** — every stretch found in two or more episodes, with where it is. + * - **blocks** — what lies between a host's slot markers. A short copied clip that + * turns up several times inside one episode is a marker the host stitches at the + * edge of each break; what sits between two of them, or between one and the edge of + * the file, is a break when it is short — including an advert never heard before. + */ + +/** Shorter than this is a coincidence or a single repeated frame, not a file. */ +export const MIN_COPIED_MS = 1500; + +/** The longest a host's slot marker is taken to be. Acast's was 2.2 s. */ +export const MAX_MARKER_MS = 5000; + +/** + * The longest stretch between markers that is called a break. + * + * The same figure `auto-approve.js` gives a difference between two downloads, for the + * same reason: what lies between markers is chosen by whoever serves the file, and a + * longer stretch is far more likely to be programme than a break. + */ +export const MAX_BLOCK_MS = 150_000; + +/** + * Below this share of distinct frames a stretch is silence or a held tone. + * + * Silence encodes to the same frame over and over, and two episodes that both contain a + * second of it would otherwise "share" it. + */ +export const MIN_DISTINCT_FRACTION = 0.5; + +/** + * Only every STRIDE-th frame position goes into the index. + * + * A match of at least `MIN_COPIED_MS` is ~57 frames, so it always contains several + * indexed positions, and scanning every position of the other side finds it. This cuts + * the index sixteen-fold — a 100-minute episode is 230,000 frames, and a window of 28 + * of them is six million. + */ +const STRIDE = 16; + +/** + * A key indexed more often than this identifies nothing: silence, or a frame a whole + * file is made of. Generous, because a marker is meant to recur — six times an episode + * across 28 episodes is 168 occurrences, about ten of them indexed per key. + */ +const MAX_KEY_OCCURRENCES = 64; + +/** Overlap past which a later find is the same stretch as an earlier one. */ +const SAME_STRETCH = 0.9; + +const POSITION_BITS = 2 ** 24; + +/** + * @param {Array<{id: string, hashes: Uint32Array|number[], frameMs: number}>} episodes + * `hashes` are `frameProfile(...).hashes`; `frameMs` is one frame's duration. + * @param {{minCopiedMs?: number, maxMarkerMs?: number, maxBlockMs?: number}} [options] + */ +export function findCopiedAudio(episodes, options = {}) { + const minCopiedMs = options.minCopiedMs ?? MIN_COPIED_MS; + const maxMarkerMs = options.maxMarkerMs ?? MAX_MARKER_MS; + const maxBlockMs = options.maxBlockMs ?? MAX_BLOCK_MS; + + const usable = episodes.filter((episode) => episode.hashes?.length && episode.frameMs > 0); + if (usable.length < 2) return { copied: [], markers: [], blocks: [] }; + + const index = buildIndex(usable); + const claimed = usable.map(() => []); + const rows = []; + + for (let e = 0; e < usable.length; e += 1) { + const a = usable[e].hashes; + const minFrames = Math.ceil(minCopiedMs / usable[e].frameMs); + for (let i = 0; i < a.length; i += 1) { + const claim = claimAt(claimed[e], i); + if (claim) { + i = claim.end - 1; + continue; + } + const run = longestPartnerRun(usable, index, e, i); + if (!run || run.end - run.start < minFrames) continue; + + const content = a.subarray ? a.subarray(run.start, run.end) : a.slice(run.start, run.end); + if (distinctFraction(content) < MIN_DISTINCT_FRACTION) { + // Claimed all the same, so the scan does not test every frame of it again. + claimed[e].push({ start: run.start, end: run.end }); + i = run.end - 1; + continue; + } + const occurrences = occurrencesOf(usable, index, content); + for (const occurrence of occurrences) claimed[occurrence.e].push({ start: occurrence.start, end: occurrence.end }); + if (new Set(occurrences.map((o) => o.e)).size >= 2) { + rows.push({ signature: signatureOf(content), frameCount: content.length, content, occurrences }); + } + i = run.end - 1; + } + } + + const copied = foldOverlapping(rows).map((row) => describeRow(row, usable)); + const markers = copied.filter((row) => isMarker(row, maxMarkerMs)); + const blocks = blocksBetween(markers, usable, maxBlockMs); + return { copied, markers: markers.map((row) => row.signature), blocks }; +} + +function buildIndex(usable) { + const index = new Map(); + usable.forEach((episode, e) => { + const hashes = episode.hashes; + for (let i = 0; i < hashes.length; i += STRIDE) { + const key = hashes[i]; + const at = index.get(key); + if (at) { + if (at.length <= MAX_KEY_OCCURRENCES) at.push(e * POSITION_BITS + i); + } else index.set(key, [e * POSITION_BITS + i]); + } + }); + // Left in the map but marked: a key past the cap is never looked up again. + for (const [key, at] of index) if (at.length > MAX_KEY_OCCURRENCES) index.set(key, null); + return index; +} + +function unpack(packed) { + const e = Math.floor(packed / POSITION_BITS); + return { e, at: packed - e * POSITION_BITS }; +} + +function claimAt(claims, i) { + for (const claim of claims) if (i >= claim.start && i < claim.end) return claim; + return null; +} + +/** The longest exact run through frame `i` of episode `e` shared with another episode. */ +function longestPartnerRun(usable, index, e, i) { + const a = usable[e].hashes; + const hits = index.get(a[i]); + if (!hits) return null; + let best = null; + for (const packed of hits) { + const { e: o, at: j } = unpack(packed); + if (o === e) continue; + const b = usable[o].hashes; + let back = 0; + while (i - back - 1 >= 0 && j - back - 1 >= 0 && a[i - back - 1] === b[j - back - 1]) back += 1; + let forward = 1; + while (i + forward < a.length && j + forward < b.length && a[i + forward] === b[j + forward]) forward += 1; + const run = { start: i - back, end: i + forward }; + if (!best || run.end - run.start > best.end - best.start) best = run; + } + return best; +} + +/** Every place `content` occurs, byte for byte, in any episode — its own included. */ +function occurrencesOf(usable, index, content) { + const seen = new Set(); + const found = []; + const probes = Math.min(STRIDE, content.length); + for (let k = 0; k < probes; k += 1) { + const hits = index.get(content[k]); + if (!hits) continue; + for (const packed of hits) { + const { e, at } = unpack(packed); + const start = at - k; + const id = e * POSITION_BITS + start; + if (start < 0 || seen.has(id)) continue; + seen.add(id); + const hashes = usable[e].hashes; + if (start + content.length > hashes.length) continue; + let same = true; + for (let n = 0; n < content.length; n += 1) { + if (hashes[start + n] !== content[n]) { + same = false; + break; + } + } + if (same) found.push({ e, start, end: start + content.length }); + } + } + return found.sort((x, y) => x.e - y.e || x.start - y.start); +} + +/** + * One stretch, one row. + * + * Two ways one advert is found twice, both seen on the real RMC show: + * + * - **Where it is.** Two episodes share one frame more of an opener than the rest + * do, and the longer run is found first. Its occurrences sit on top of the shorter + * row's in the same episodes. + * - **What it is.** The host keeps two versions of the opener whose last frame differs + * — 533 frames each, 532 in common. Each version is exact in its own four episodes, + * so the two rows never share an episode and only their frames show they are one. + * + * Either way, "the same" means at least nine-tenths of the *longer* of the two: a + * two-second marker stitched beside a thirty-second advert overlaps it completely, and + * folding them would stretch the marker to the advert's length. + */ +function foldOverlapping(rows) { + const sorted = [...rows].sort( + (x, y) => episodeCount(y) - episodeCount(x) || y.frameCount - x.frameCount, + ); + const kept = []; + for (const row of sorted) { + const byPlace = kept.find((candidate) => + row.occurrences.every((occurrence) => + candidate.occurrences.some( + (other) => + other.e === occurrence.e && + Math.min(other.end, occurrence.end) - Math.max(other.start, occurrence.start) >= + SAME_STRETCH * Math.max(other.end - other.start, occurrence.end - occurrence.start), + ), + ), + ); + if (byPlace) { + for (const occurrence of row.occurrences) { + const other = byPlace.occurrences.find( + (o) => o.e === occurrence.e && Math.min(o.end, occurrence.end) > Math.max(o.start, occurrence.start), + ); + other.start = Math.min(other.start, occurrence.start); + other.end = Math.max(other.end, occurrence.end); + } + continue; + } + const byContent = kept.find( + (candidate) => + longestCommonRun(candidate.content, row.content) >= SAME_STRETCH * Math.max(candidate.frameCount, row.frameCount), + ); + if (byContent) { + for (const occurrence of row.occurrences) { + const clash = byContent.occurrences.some( + (o) => o.e === occurrence.e && Math.min(o.end, occurrence.end) > Math.max(o.start, occurrence.start), + ); + if (!clash) byContent.occurrences.push({ ...occurrence }); + } + continue; + } + kept.push({ ...row, occurrences: row.occurrences.map((o) => ({ ...o })) }); + } + return kept; +} + +/** The longest stretch of frames two contents have in common, at any offset. */ +function longestCommonRun(a, b) { + const positions = new Map(); + for (let i = 0; i < a.length; i += 1) { + const at = positions.get(a[i]); + if (at) at.push(i); + else positions.set(a[i], [i]); + } + let best = 0; + const tried = new Set(); + for (let j = 0; j < b.length; j += 1) { + for (const i of positions.get(b[j]) ?? []) { + const diagonal = i - j; + if (tried.has(diagonal)) continue; + tried.add(diagonal); + let run = 0; + for (let n = Math.max(0, -diagonal); n < b.length && n + diagonal < a.length; n += 1) { + if (a[n + diagonal] === b[n]) { + run += 1; + if (run > best) best = run; + } else run = 0; + } + } + } + return best; +} + +function episodeCount(row) { + return new Set(row.occurrences.map((o) => o.e)).size; +} + +function describeRow(row, usable) { + const occurrences = row.occurrences.map((o) => ({ + episodeId: usable[o.e].id, + startFrame: o.start, + endFrame: o.end, + startMs: Math.round(o.start * usable[o.e].frameMs), + endMs: Math.round(o.end * usable[o.e].frameMs), + })); + const frameMs = usable[row.occurrences[0].e].frameMs; + return { + signature: row.signature, + durationMs: Math.round(row.frameCount * frameMs), + episodeCount: new Set(occurrences.map((o) => o.episodeId)).size, + occurrenceCount: occurrences.length, + occurrences, + }; +} + +/** Short, and found more than once inside a single episode. */ +function isMarker(row, maxMarkerMs) { + if (row.durationMs > maxMarkerMs) return false; + const perEpisode = new Map(); + for (const o of row.occurrences) perEpisode.set(o.episodeId, (perEpisode.get(o.episodeId) ?? 0) + 1); + return [...perEpisode.values()].some((count) => count >= 2); +} + +/** + * The stretches between markers that are short enough to be a break. + * + * What makes the short stretches breaks is the long ones: programme runs for many + * minutes between breaks. An episode where the short stretches would add up to most of + * it — a two-minute episode with markers in it, say, whose programme is itself shorter + * than a break — cannot tell the two apart this way, and nothing is said about it. + */ +function blocksBetween(markers, usable, maxBlockMs) { + const byEpisode = new Map(); + for (const marker of markers) { + for (const o of marker.occurrences) { + if (!byEpisode.has(o.episodeId)) byEpisode.set(o.episodeId, []); + byEpisode.get(o.episodeId).push(o); + } + } + const blocks = []; + for (const episode of usable) { + const marks = mergeRanges(byEpisode.get(episode.id) ?? []); + if (!marks.length) continue; + const total = episode.hashes.length; + const maxFrames = Math.floor(maxBlockMs / episode.frameMs); + const gaps = []; + let from = 0; + for (const mark of marks) { + gaps.push({ start: from, end: mark.startFrame }); + from = mark.endFrame; + } + gaps.push({ start: from, end: total }); + + const breaks = gaps.filter((gap) => gap.end - gap.start > 0 && gap.end - gap.start <= maxFrames); + const cutFrames = breaks.reduce((sum, gap) => sum + gap.end - gap.start, 0); + // Never most of an episode, whatever the markers say: that is an episode whose + // programme is no longer than its breaks, or markers that are not a host's. + if (cutFrames > total / 2) continue; + + /* + * A break is reported with the markers around it, and breaks that touch are one. + * Acast stitched two markers back to back with a tenth of a second between them; + * reported apart, that is three pieces on the page for one break, and the tenth of + * a second is a click left in the episode if any piece is restored on its own. + */ + const pieces = [...breaks, ...marks.map((mark) => ({ start: mark.startFrame, end: mark.endFrame, marker: true }))] + .sort((x, y) => x.start - y.start); + const joined = []; + for (const piece of pieces) { + const last = joined[joined.length - 1]; + if (last && piece.start <= last.end) { + last.end = Math.max(last.end, piece.end); + last.breaks += piece.marker ? 0 : 1; + } else joined.push({ start: piece.start, end: piece.end, breaks: piece.marker ? 0 : 1 }); + } + const middle = joined.filter((range) => range.breaks && range.start > 0 && range.end < total); + for (const range of joined) { + // A marker on its own is already a copied stretch; only a break is new here. + if (!range.breaks) continue; + // Which break of the episode this is: the catalogue keeps one row per slot, so a + // row is in each episode at most once and "Restore here" means one break. + const slot = range.start === 0 ? 'pre' : range.end >= total ? 'post' : `mid-${middle.indexOf(range) + 1}`; + const content = episode.hashes.subarray + ? episode.hashes.subarray(range.start, range.end) + : episode.hashes.slice(range.start, range.end); + blocks.push({ + signature: signatureOf(content, 'break'), + slot, + episodeId: episode.id, + startFrame: range.start, + endFrame: range.end, + startMs: Math.round(range.start * episode.frameMs), + endMs: Math.round(range.end * episode.frameMs), + }); + } + } + return blocks; +} + +function mergeRanges(ranges) { + const sorted = [...ranges].sort((x, y) => x.startFrame - y.startFrame); + const merged = []; + for (const range of sorted) { + const last = merged[merged.length - 1]; + if (last && range.startFrame <= last.endFrame) last.endFrame = Math.max(last.endFrame, range.endFrame); + else merged.push({ startFrame: range.startFrame, endFrame: range.endFrame }); + } + return merged; +} + +function distinctFraction(content) { + if (!content.length) return 0; + return new Set(content).size / content.length; +} + +/** The same frames give the same signature, in whichever episode they are found. */ +export function signatureOf(content, prefix = 'copied') { + const hash = createHash('sha256'); + const bytes = Buffer.alloc(content.length * 4); + for (let n = 0; n < content.length; n += 1) bytes.writeUInt32BE(content[n] >>> 0, n * 4); + return `${prefix}:${hash.update(bytes).digest('hex').slice(0, 24)}`; +} diff --git a/src/lib/fingerprint-file.js b/src/lib/fingerprint-file.js index d5eb9b7..c088d6a 100644 --- a/src/lib/fingerprint-file.js +++ b/src/lib/fingerprint-file.js @@ -115,3 +115,45 @@ export function msToFrame(ms, { sampleRate, samplesPerFrame }) { if (!sampleRate || !samplesPerFrame) return 0; return Math.floor((ms * sampleRate) / (samplesPerFrame * 1000)); } + +/* + * MP3 frame hashes, kept beside the fingerprint (spec §19.3). + * + * A file of their own, not a field of the fingerprint, so that adding them did not + * invalidate every fingerprint already computed: those cost a full decode each, and + * these cost only a read. Same header layout, its own magic and version. + */ +const FRAMES_MAGIC = 0x53504648; // "SPFH" +export const FRAME_HASHES_VERSION = 1; + +/** @param {{hashes: Uint32Array|number[], sampleRate: number, samplesPerFrame: number, durationMs: number}} frames */ +export function encodeFrameHashes({ hashes, sampleRate, samplesPerFrame, durationMs }) { + const buffer = Buffer.alloc(HEADER_BYTES + hashes.length * 4); + buffer.writeUInt32BE(FRAMES_MAGIC, 0); + buffer.writeUInt32BE(FRAME_HASHES_VERSION, 4); + buffer.writeUInt32BE(hashes.length, 8); + buffer.writeUInt32BE(sampleRate || 0, 12); + buffer.writeUInt32BE(samplesPerFrame || 0, 16); + buffer.writeUInt32BE(Math.min(durationMs || 0, 0xffffffff), 20); + buffer.writeUInt32BE(0, 24); + for (let i = 0; i < hashes.length; i += 1) buffer.writeUInt32BE(hashes[i] >>> 0, HEADER_BYTES + i * 4); + return buffer; +} + +/** Reads a frame-hash file, or null for anything it does not recognise — it is a cache. */ +export function decodeFrameHashes(buffer) { + if (!buffer || buffer.length < HEADER_BYTES) return null; + if (buffer.readUInt32BE(0) !== FRAMES_MAGIC) return null; + if (buffer.readUInt32BE(4) !== FRAME_HASHES_VERSION) return null; + const count = buffer.readUInt32BE(8); + if (buffer.length < HEADER_BYTES + count * 4) return null; + const bytes = new Uint8Array(count * 4); + bytes.set(buffer.subarray(HEADER_BYTES, HEADER_BYTES + count * 4)); + if (LITTLE_ENDIAN) Buffer.from(bytes.buffer, 0, bytes.length).swap32(); + return { + hashes: new Uint32Array(bytes.buffer, 0, count), + sampleRate: buffer.readUInt32BE(12) || null, + samplesPerFrame: buffer.readUInt32BE(16) || null, + durationMs: buffer.readUInt32BE(20) || null, + }; +} diff --git a/src/lib/present-segment.js b/src/lib/present-segment.js index 0aa9534..528cbdf 100644 --- a/src/lib/present-segment.js +++ b/src/lib/present-segment.js @@ -66,6 +66,10 @@ export function presentSegment(segment, { episodes, transcripts = null, mode = ' ? 'The boundary you set' : isAnchor ? 'The station jingle' + : kind === SEGMENT_KINDS.INSERTED + ? (segment.signature ?? '').startsWith('break:') + ? 'Between the host’s markers' + : `Spliced in by the host — the same bytes in ${segment.episode_count} ${segment.episode_count === 1 ? 'episode' : 'episodes'}` : segment.source === SEGMENT_SOURCES.DIFF ? 'Changed between two downloads of the same episode' : segment.source === SEGMENT_SOURCES.TRANSCRIPT diff --git a/src/lib/present-subscription.js b/src/lib/present-subscription.js index 39316a6..4965c93 100644 --- a/src/lib/present-subscription.js +++ b/src/lib/present-subscription.js @@ -29,6 +29,7 @@ export function presentSubscription(subscription, { subscriptions }) { minDurationSeconds: subscription.min_duration_seconds, maxDurationSeconds: subscription.max_duration_seconds, backfillCount: subscription.backfill_count, + insertsAudio: Boolean(subscription.inserts_audio), bootstrapped: Boolean(subscription.bootstrapped_at), pollIntervalSeconds: subscription.poll_interval_seconds, lastPolledAt: subscription.last_polled_at, diff --git a/src/lib/present-transcript.js b/src/lib/present-transcript.js index 2db36fb..0536abd 100644 --- a/src/lib/present-transcript.js +++ b/src/lib/present-transcript.js @@ -149,6 +149,29 @@ function repetitionPhrase(segment, positionLabel) { } /** "40 seconds" / "2:10": how long a stretch of sound is, for a sentence. */ +/** The break a `break:` row stands for, from its slot: pre, post, or mid-N. */ +export function breakSlotName(signature = '') { + const slot = signature.startsWith('break:') ? signature.slice('break:'.length) : null; + if (!slot) return null; + if (slot === 'pre') return 'Pre-roll'; + if (slot === 'post') return 'Post-roll'; + const n = Number(slot.replace(/^mid-/, '')); + return n > 1 ? `Mid-roll ${n}` : 'Mid-roll'; +} + +/** + * What an `inserted` row is, in a phrase. Both kinds are the host's own doing, and the + * evidence is said plainly because it is the reason SelfPod is sure: a programme is + * never the same bytes twice; a break is what sits between the host's markers. + */ +function insertedPhrase(segment) { + const count = segment.episode_count ?? 0; + const episodes = `${count} episode${count === 1 ? '' : 's'}`; + const slot = breakSlotName(segment.signature); + if (slot) return `${slot} break between the host’s markers, in ${episodes}`; + return `The same ${lengthPhrase(segment.duration_ms)} of audio spliced into ${episodes} by the host`; +} + function lengthPhrase(ms) { const seconds = Math.round((ms ?? 0) / 1000); return seconds < 90 ? `${seconds} second${seconds === 1 ? '' : 's'}` : formatClock(ms); @@ -202,6 +225,29 @@ export function describeVerdict(segment, { mode = 'review', positionLabel = null offerMarker: false, }; } + if (kind === SEGMENT_KINDS.INSERTED) { + const what = insertedPhrase(segment); + if (segment.status === 'approved') { + return { + verdict: 'will_cut', + key: segment.auto_approved ? 'inserted_auto' : 'inserted_removed', + sentence: segment.auto_approved ? `${what}, cut automatically.` : `${what} — you removed it.`, + offerMarker: false, + }; + } + if (segment.status === 'rejected') { + return { verdict: 'will_leave', key: 'inserted_kept', sentence: `${what} — you kept it.`, offerMarker: false }; + } + return { + verdict: 'asking', + key: segment.hold_reason === 'too_long_to_be_an_advert' ? 'inserted_too_long' : 'inserted', + sentence: + segment.hold_reason === 'too_long_to_be_an_advert' + ? `${what} — longer than any advert, so SelfPod will not cut it unasked.` + : `${what}.`, + offerMarker: false, + }; + } if (kind === SEGMENT_KINDS.REPEATED_AUDIO && !segment.text) { const sound = `The same ${lengthPhrase(segment.duration_ms)} of sound in ${count} episode${count === 1 ? '' : 's'}`; if (segment.status === 'approved') { diff --git a/src/lib/segment-kind.js b/src/lib/segment-kind.js index 111b840..0b7d01d 100644 --- a/src/lib/segment-kind.js +++ b/src/lib/segment-kind.js @@ -10,6 +10,7 @@ import { SEGMENT_KINDS, SEGMENT_SOURCES } from '../constants.js'; export function inferKind({ signature = '', source = null, cues = null } = {}) { if (signature.startsWith('marker:')) return SEGMENT_KINDS.BOUNDARY_WORDS; if (signature.startsWith('anchor:')) return SEGMENT_KINDS.JINGLE; + if (signature.startsWith('copied:') || signature.startsWith('break:')) return SEGMENT_KINDS.INSERTED; if (source === SEGMENT_SOURCES.DIFF) return SEGMENT_KINDS.DIFF; if (source === SEGMENT_SOURCES.TRANSCRIPT) { return cues === null || cues === undefined ? SEGMENT_KINDS.REMEMBERED_WORDS : SEGMENT_KINDS.REPEATED_WORDS; diff --git a/src/services/ad-detect.js b/src/services/ad-detect.js index ab306bc..31b4937 100644 --- a/src/services/ad-detect.js +++ b/src/services/ad-detect.js @@ -20,9 +20,12 @@ import { badRequest, notFound } from '../lib/errors.js'; import { EVENTS } from '../lib/events.js'; import { FINGERPRINT_HEADER_BYTES, + FRAME_HASHES_VERSION, decodeFingerprint, decodeFingerprintHeader, + decodeFrameHashes, encodeFingerprint, + encodeFrameHashes, msToFrame, } from '../lib/fingerprint-file.js'; import { createAudioWorker } from '../lib/audio-search.js'; @@ -88,6 +91,9 @@ export function createAdDetect({ db, config, events, logger, shows, episodes, tr 'SELECT * FROM ad_segments WHERE show_id = ? ORDER BY episode_count DESC, duration_ms DESC', ); const selectSegment = db.prepare('SELECT * FROM ad_segments WHERE id = ?'); + const markInsertingHost = db.prepare( + 'UPDATE feed_subscriptions SET inserts_audio = 1, updated_at = ? WHERE show_id = ? AND inserts_audio = 0', + ); const selectBySignature = db.prepare( 'SELECT * FROM ad_segments WHERE show_id = ? AND signature = ?', ); @@ -96,6 +102,11 @@ export function createAdDetect({ db, config, events, logger, shows, episodes, tr return join(config.fingerprintDir, showId, `${episodeId}.${FINGERPRINT_VERSION}.fp`); } + /** Beside the fingerprint, so `derived-audio.js` removes it with the episode. */ + function frameHashesPath(showId, episodeId) { + return join(config.fingerprintDir, showId, `${episodeId}.h${FRAME_HASHES_VERSION}.frames`); + } + /* ---- fingerprints -------------------------------------------------------- */ /** * Reads an episode's frames and stores the fingerprint. @@ -215,6 +226,20 @@ export function createAdDetect({ db, config, events, logger, shows, episodes, tr const target = fingerprintPath(show.id, episode.id); await mkdir(join(config.fingerprintDir, show.id), { recursive: true }); + // The frame hashes first: a fingerprint row that exists promises they describe the + // same file, and a crash between the two writes must not leave last version's + // hashes beside this version's fingerprint. + if (read.frameHashes?.length) { + await writeFile( + frameHashesPath(show.id, episode.id), + encodeFrameHashes({ + hashes: read.frameHashes, + sampleRate: read.sampleRate, + samplesPerFrame: read.samplesPerFrame, + durationMs: read.durationMs, + }), + ); + } await writeFile(target, encoded); upsertFingerprint.run({ @@ -298,6 +323,58 @@ export function createAdDetect({ db, config, events, logger, shows, episodes, tr } } + /** + * An episode's MP3 frame hashes, with how long one frame lasts, or null. + * + * Written beside every fingerprint computed since frame hashes were kept. For an + * episode fingerprinted before that, they are read from the file once — a read and a + * walk of its frames, no decode — and kept. Only for the very file the fingerprint + * describes: if the digest differs, the episode changed since, the next fingerprint + * pass will write both afresh, and until then there is nothing to say about it. + */ + async function loadFrameHashes(episode) { + const row = selectFingerprint.get(episode.id); + if (!row || row.algorithm_version !== FINGERPRINT_VERSION) return null; + const path = frameHashesPath(episode.show_id, episode.id); + let decoded = null; + try { + decoded = decodeFrameHashes(await readFile(path)); + } catch { + decoded = null; + } + if (!decoded) { + const show = shows.get(episode.show_id); + if (!show) return null; + const gate = await gateEpisodeFile(shows.dirFor(show), episode.filename, { maxBytes: MAX_PIPELINE_FILE_BYTES }); + if (gate.refused) return null; + let read; + try { + read = await search.frameHashesFile({ path: gate.path, maxBytes: MAX_PIPELINE_FILE_BYTES }); + } catch (error) { + logger?.debug({ err: error, episodeId: episode.id }, 'could not read frame hashes'); + return null; + } + if (!read?.frameHashes?.length || read.sha256 !== row.sha256) return null; + decoded = { + hashes: read.frameHashes, + sampleRate: read.sampleRate, + samplesPerFrame: read.samplesPerFrame, + durationMs: read.durationMs, + }; + await mkdir(join(config.fingerprintDir, show.id), { recursive: true }); + await writeFile(path, encodeFrameHashes(decoded)).catch((error) => { + logger?.debug({ err: error, episodeId: episode.id }, 'could not keep frame hashes'); + }); + } + if (!decoded.sampleRate || !decoded.samplesPerFrame) return null; + return { + episodeId: episode.id, + hashes: decoded.hashes, + frameMs: (decoded.samplesPerFrame / decoded.sampleRate) * 1000, + durationMs: decoded.durationMs, + }; + } + /* * Everything detection reads that detection does not itself write, as one string. * @@ -897,6 +974,60 @@ export function createAdDetect({ db, config, events, logger, shows, episodes, tr /** How many of a show's newest episodes are searched when proposing a jingle. */ const MAX_PROPOSAL_EPISODES = 20; + /** + * How far past the end of the host's inserted audio a jingle may start and still be + * the start of the programme. A host's opener runs straight into the show; three + * seconds allows for a breath of silence or a sting, and no sentence. + */ + const JINGLE_AFTER_INSERT_MS = 3000; + + /** + * Per episode, the stretches the host inserted — copied frames and breaks between + * markers — as `[startMs, endMs]`, read from the frames afresh rather than from the + * catalogue, so the answer does not depend on which detector happened to write first + * or on rows a confirmed jingle already overlaps. + */ + async function insertedStretches(show) { + const corpus = []; + for (const episode of episodes.listByShow(show.id)) { + if (corpus.length >= corpusWindow()) break; + const frames = await loadFrameHashes(episode); + if (frames?.hashes?.length) corpus.push({ id: episode.id, hashes: frames.hashes, frameMs: frames.frameMs }); + } + const stretches = new Map(); + if (corpus.length < 2) return stretches; + work.corpusSearches += 1; + const found = await search.copiedAudio(corpus); + const add = (episodeId, startMs, endMs) => { + if (!stretches.has(episodeId)) stretches.set(episodeId, []); + stretches.get(episodeId).push([startMs, endMs]); + }; + for (const row of found.copied) for (const o of row.occurrences) add(o.episodeId, o.startMs, o.endMs); + for (const block of found.blocks) add(block.episodeId, block.startMs, block.endMs); + return stretches; + } + + /** + * An episode where the jingle starts well after the host's inserted audio has ended, + * or null. Then what lies between is the show's own — Le Rendez-vous Tech's host + * introducing the episode before the theme — and "cut everything before the jingle" + * would cut it. Only inserted audio that starts before the jingle counts; an episode + * with none of it at its head says nothing either way, which is the case the jingle + * search was built for (a pre-roll heard once, then the station ident). + */ + async function jingleFollowsProgramme(show, onsetByEpisode, candidates) { + const stretches = await insertedStretches(show); + for (const episode of candidates) { + const onset = onsetByEpisode.get(episode.id); + if (onset === undefined || onset < MIN_ANCHOR_CUT_MS) continue; + const before = (stretches.get(episode.id) ?? []).filter(([start]) => start < onset); + if (!before.length) continue; + const insertedEnd = Math.max(...before.map(([, end]) => end)); + if (onset - insertedEnd > JINGLE_AFTER_INSERT_MS) return { episodeId: episode.id, gapMs: onset - insertedEnd }; + } + return null; + } + const selectAnchors = db.prepare('SELECT * FROM ad_anchors WHERE show_id = ? ORDER BY created_at'); const selectAnchor = db.prepare('SELECT * FROM ad_anchors WHERE id = ?'); const insertAnchorRow = db.prepare( @@ -1210,6 +1341,31 @@ export function createAdDetect({ db, config, events, logger, shows, episodes, tr ([from, to]) => occurrence.start >= from && occurrence.end <= to, ); + /* + * And ground the copied-frames search already explains: audio the host inserted + * sounds the same in every episode too, so this search finds it again — as "audio + * this show repeats", held for review because it sits at 0:00 — beside the row + * that already cuts it. By overlap rather than containment, because this search + * pushes its edges outwards and the frames' edges are exact. + */ + const insertedByEpisode = new Map(); + for (const row of selectSegments.all(show.id)) { + if (row.kind !== SEGMENT_KINDS.INSERTED || row.status === SEGMENT_STATUS.REJECTED) continue; + for (const o of selectOccurrencesOf.all(row.id)) { + if (!insertedByEpisode.has(o.episode_id)) insertedByEpisode.set(o.episode_id, []); + insertedByEpisode.get(o.episode_id).push([o.start_frame, o.end_frame]); + } + } + const coveredByInserted = (occurrence) => { + const length = occurrence.end - occurrence.start; + if (length <= 0) return false; + const inside = (insertedByEpisode.get(occurrence.episodeId) ?? []).reduce( + (sum, [from, to]) => sum + Math.max(0, Math.min(to, occurrence.end) - Math.max(from, occurrence.start)), + 0, + ); + return inside >= SAME_THING_OVERLAP * length; + }; + const updatedThisPass = new Set(); let recorded = 0; // Counted apart from `recorded`, because "found three things" and "found three @@ -1254,7 +1410,19 @@ export function createAdDetect({ db, config, events, logger, shows, episodes, tr // show repeats." A brand new segment left with fewer than two occurrences by // that filtering is not worth creating; an existing one is still upserted with // whatever survives, so it shrinks (or empties) rather than going stale. - const inSearch = occurrences.filter((occurrence) => !coveredByAnchor(occurrence)); + // A row already cutting keeps what it cuts, whatever the copied search found + // since: it is the owner's decision (or automatic mode's), and taking its + // occurrences away for a row still waiting to be asked about would put the + // advert back until someone answers. + // + // And only when the copied rows explain *every* occurrence. A theme copied byte + // for byte into two episodes and re-encoded in a third is heard in three: the + // third is evidence the copies are not, and dropping the two copied occurrences + // would leave one and lose the finding (acceptance step 22). + const cutting = known?.status === SEGMENT_STATUS.APPROVED; + const unanchored = occurrences.filter((occurrence) => !coveredByAnchor(occurrence)); + const explained = !cutting && unanchored.length > 0 && unanchored.every(coveredByInserted); + const inSearch = explained ? [] : unanchored; if (!known && inSearch.length < 2) continue; // Occurrences in episodes this search did not look at are carried forward // untouched: not being compared this pass is not the same as not being there. @@ -1293,6 +1461,167 @@ export function createAdDetect({ db, config, events, logger, shows, episodes, tr return { segments: recorded, newSegments: fresh, episodes: corpus.length }; }, + /** + * Finds audio the host spliced in, by its MP3 frames, and updates the catalogue. + * + * Two kinds of row, both `inserted` (copied-audio.js says why each is evidence): + * every stretch whose frames are byte for byte the same in two or more of the newest + * episodes, one row each; and the breaks between a host's slot markers, one row for + * the show with an occurrence per break. Cheap next to the acoustic search — no + * decoding, the frame hashes are kept beside the fingerprint — and run first, so + * the acoustic search can leave alone what this has already explained. + * + * Rows are recognised again by where they are as well as by their frames: the host + * may serve a variant that differs in a frame, and the row the owner already decided + * about must stay the one that row is. + */ + async detectCopiedAudio(showId) { + const show = shows.getOrThrow(showId); + if (show.ad_trim_mode === 'off') return { segments: 0, skipped: 'mode_off' }; + + const corpus = []; + for (const episode of episodes.listByShow(show.id)) { + if (corpus.length >= corpusWindow()) break; + const frames = await loadFrameHashes(episode); + if (frames?.hashes?.length) corpus.push({ id: episode.id, hashes: frames.hashes, frameMs: frames.frameMs }); + } + const durations = Object.fromEntries(corpus.map((entry) => [entry.id, Math.round(entry.hashes.length * entry.frameMs)])); + if (corpus.length < 2) return { segments: 0, skipped: 'not_enough_episodes' }; + const inWindow = new Set(corpus.map((entry) => entry.id)); + + work.corpusSearches += 1; + const found = await search.copiedAudio(corpus); + + const known = selectSegments + .all(show.id) + .filter((row) => row.kind === SEGMENT_KINDS.INSERTED) + .map((row) => ({ row, occurrences: selectOccurrencesOf.all(row.id) })); + const knownFor = (occurrences) => { + for (const { row, occurrences: stored } of known) { + const same = occurrences.some((found_) => + stored.some( + (o) => + o.episode_id === found_.episodeId && + Math.min(o.end_frame, found_.endFrame) - Math.max(o.start_frame, found_.startFrame) >= + SAME_THING_OVERLAP * Math.max(found_.endFrame - found_.startFrame, o.end_frame - o.start_frame), + ), + ); + if (same) return row; + } + return null; + }; + // Occurrences in episodes this search did not look at stay as they were: not being + // compared this pass is not the same as not being there. + const carriedFor = (row) => + row + ? selectOccurrencesOf + .all(row.id) + .filter((o) => !inWindow.has(o.episode_id)) + .map((o) => ({ episodeId: o.episode_id, start: o.start_frame, end: o.end_frame, startMs: o.start_ms, endMs: o.end_ms })) + : []; + const asOccurrence = (o) => ({ episodeId: o.episodeId, start: o.startFrame, end: o.endFrame, startMs: o.startMs, endMs: o.endMs }); + + const write = (existing, signature, occurrences, durationMs, { betweenMarkers = false } = {}) => { + const all = [...occurrences.map(asOccurrence), ...carriedFor(existing)]; + const episodeCount = new Set(all.map((o) => o.episodeId)).size; + const verdict = safeToApproveAutomatically( + { durationMs, episodeCount, betweenMarkers, occurrences: all }, + { source: 'inserted', episodeDurations: durations }, + ); + const auto = show.ad_trim_mode === 'auto' && verdict.safe; + return upsertSegment(show.id, { + signature: existing?.signature ?? signature, + kind: SEGMENT_KINDS.INSERTED, + durationMs, + episodeCount, + occurrenceCount: all.length, + occurrences: all, + status: auto ? SEGMENT_STATUS.APPROVED : SEGMENT_STATUS.CANDIDATE, + autoApproved: auto, + holdReason: verdict.safe ? null : verdict.reason, + }); + }; + + /* + * A copied stretch that sits inside a break is already cut by the break's row. + * Written as well, it would be a second row for the same audio — and, being in + * two episodes mid-file more often than not, one that waits to be asked about and + * holds those episodes back while the break beside it is cut unasked. + */ + const breaksIn = new Map(); + for (const block of found.blocks) { + if (!breaksIn.has(block.episodeId)) breaksIn.set(block.episodeId, []); + breaksIn.get(block.episodeId).push(block); + } + const insideABreak = (o) => + (breaksIn.get(o.episodeId) ?? []).some((block) => o.startFrame >= block.startFrame && o.endFrame <= block.endFrame); + + /* + * Audio another approved row already cuts — a read the owner removed by ear before + * this search existed, a taught range — is not offered again: a second row would + * be a question about something already answered, and in review mode it would hold + * the episode back until it was answered twice. + */ + const cutByOthers = new Map(); + for (const row of selectSegments.all(show.id)) { + if (row.kind === SEGMENT_KINDS.INSERTED || row.status !== SEGMENT_STATUS.APPROVED) continue; + for (const o of selectOccurrencesOf.all(row.id)) { + if (!cutByOthers.has(o.episode_id)) cutByOthers.set(o.episode_id, []); + cutByOthers.get(o.episode_id).push([o.start_frame, o.end_frame]); + } + } + const alreadyCut = (o) => { + const length = o.endFrame - o.startFrame; + const covered = (cutByOthers.get(o.episodeId) ?? []).reduce( + (sum, [from, to]) => sum + Math.max(0, Math.min(to, o.endFrame) - Math.max(from, o.startFrame)), + 0, + ); + return length > 0 && covered >= SAME_THING_OVERLAP * length; + }; + + let recorded = 0; + let fresh = 0; + const written = new Set(); + for (const row of found.copied) { + if (row.occurrences.every((o) => insideABreak(o) || alreadyCut(o))) continue; + const existing = selectBySignature.get(show.id, row.signature) ?? knownFor(row.occurrences); + // The first find to reach a stored row is it; a second rewriting the same row + // would replace its occurrences with a smaller set and re-cut every episode. + if (existing && written.has(existing.id)) continue; + const stored = write(existing, row.signature, row.occurrences, row.durationMs); + written.add(stored.id); + recorded += 1; + if (stored.isNew) fresh += 1; + } + /* + * The breaks, one row per slot — the pre-roll, each mid-roll by its place, the + * post-roll — not one per break. A row per break would be a hundred rows for one + * finding; one row for all of them would be in each episode three times, and + * "Restore here" restores one occurrence of a row in an episode. A slot is in each + * episode at most once. + */ + const slots = new Map(); + for (const block of found.blocks) { + if (alreadyCut(block)) continue; + if (!slots.has(block.slot)) slots.set(block.slot, []); + slots.get(block.slot).push(block); + } + for (const [slot, blocks] of slots) { + const longest = Math.max(...blocks.map((block) => block.endMs - block.startMs)); + const signature = `break:${slot}`; + const stored = write(selectBySignature.get(show.id, signature), signature, blocks, longest, { betweenMarkers: true }); + recorded += 1; + if (stored.isNew) fresh += 1; + } + + // The host inserts: every new episode of this feed is worth a second download. + if (recorded) markInsertingHost.run(nowIso(), show.id); + + if (recorded) events?.emit(EVENTS.SHOW_CHANGED, { showId: show.id }); + logger?.info({ showId: show.id, segments: recorded, fresh, episodes: corpus.length }, 'looked for inserted audio'); + return { segments: recorded, newSegments: fresh, episodes: corpus.length }; + }, + /** * Records what differed between two downloads of one episode. * @@ -1914,26 +2243,59 @@ export function createAdDetect({ db, config, events, logger, shows, episodes, tr * automatically confirmed jingle dismisses it, so it is never proposed again. * In review mode it stays a question. */ + /* + * …unless the host's own inserted audio shows the "jingle" is not where the + * programme starts (`jingleFollowsProgramme`). Found on Le Rendez-vous Tech: its + * one sponsored episode has no adverts and so no break marker, which left the + * show's theme as the only sound every episode opens with — after the host's + * spoken intro. Confirmed automatically, it cut that intro from every episode. + * It stays a proposal the owner can still confirm; the breaks are cut regardless. + */ + const recent = fingerprinted.slice(0, MAX_PROPOSAL_EPISODES); + let takenBack = false; if (!anchor.confirmed_at && show.ad_trim_mode === 'auto' && anchor.origin === 'proposed') { - const recent = fingerprinted.slice(0, MAX_PROPOSAL_EPISODES); - if (recent.length >= 2 && recent.every((episode) => onsetByEpisode.has(episode.id))) { + if ( + recent.length >= 2 && + recent.every((episode) => onsetByEpisode.has(episode.id)) && + !(await jingleFollowsProgramme(show, onsetByEpisode, recent)) + ) { db.prepare( 'UPDATE ad_anchors SET confirmed_at = @now, auto_confirmed = 1, updated_at = @now WHERE id = @id', ).run({ now: nowIso(), id: anchor.id }); anchor = selectAnchor.get(anchor.id); autoConfirmed = true; } + } else if (anchor.confirmed_at && anchor.auto_confirmed && anchor.origin === 'proposed' && !autoConfirmed) { + /* + * And one SelfPod confirmed by itself before this check existed is taken back + * the same way — never one the owner confirmed or a taught boundary linked. + * Its cut goes with it, and the episodes it was in are cut again without it. + */ + const wrong = await jingleFollowsProgramme(show, onsetByEpisode, recent); + if (wrong) { + const segment = selectBySignature.get(show.id, `anchor:${anchor.id}`); + if (segment) { + markForRecut(segment.id); + db.prepare('DELETE FROM ad_segments WHERE id = ?').run(segment.id); + } + db.prepare( + 'UPDATE ad_anchors SET confirmed_at = NULL, auto_confirmed = 0, updated_at = @now WHERE id = @id', + ).run({ now: nowIso(), id: anchor.id }); + anchor = selectAnchor.get(anchor.id); + takenBack = true; + logger?.info({ showId: show.id, anchorId: anchor.id, ...wrong }, 'took back a jingle that starts after the programme does'); + } } // Whether this call wrote anything the other detectors read: a proposal, a // confirmation, or a fresh hit row. The pipeline runs the detectors again // within the same pass when it did, so a marker linked here is cut to today. - const changed = proposed || linked || autoConfirmed || located > 0; + const changed = proposed || linked || autoConfirmed || takenBack || located > 0; // A proposal is not a decision. Stopping here leaves the hits recorded — the // review card can already say how many episodes the jingle was heard in — but // cuts nothing until the owner, or a linked marker, actually decides. - if (!anchor.confirmed_at) return { skipped: 'awaiting_decision', anchorId: anchor.id, proposed, changed, located }; + if (!anchor.confirmed_at) return { skipped: 'awaiting_decision', anchorId: anchor.id, proposed, changed, located, takenBack }; const occurrences = []; for (const [episodeId, onsetMs] of onsetByEpisode) { @@ -2489,7 +2851,7 @@ export function createAdDetect({ db, config, events, logger, shows, episodes, tr */ const READ_ONLY = new Set([ 'fingerprintEpisode', 'loadFingerprint', 'fingerprintDigest', 'close', 'countFingerprinted', - 'fingerprintShow', 'detectForShow', 'detectFromTranscripts', 'detectAnchors', 'reconcileKinds', + 'fingerprintShow', 'detectForShow', 'detectCopiedAudio', 'detectFromTranscripts', 'detectAnchors', 'reconcileKinds', 'foldDuplicateReads', 'sponsorSuggestions', 'exemplarTranscripts', 'spokenIn', 'anchorCutFor', 'listSegments', 'getSegment', 'listAnchors', 'getAnchor', 'anchorSummary', 'anchorStatusFor', 'listMarkers', 'getMarker', 'restoresIn', 'isRestored', 'cutListFor', 'detectionInputs', 'catalogueDigest', 'workDone', diff --git a/src/services/ad-pipeline.js b/src/services/ad-pipeline.js index d1dde6d..17c9551 100644 --- a/src/services/ad-pipeline.js +++ b/src/services/ad-pipeline.js @@ -208,7 +208,7 @@ export function createAdPipeline({ if (!activity) return; const refused = counts.refused ?? []; if ( - !counts.found && !counts.trimmed && !counts.failed && !counts.released && + !counts.found && !counts.inserted && !counts.trimmed && !counts.failed && !counts.released && !counts.transcribed && !counts.heard && !counts.rememberedCuts && !counts.markerCuts && !counts.transcriptionFailed && !counts.foldedIn && !counts.anchorProposed && !counts.anchorMissed && !refused.length @@ -218,6 +218,9 @@ export function createAdPipeline({ if (counts.transcribed) { parts.push(`${counts.transcribed} ${counts.transcribed === 1 ? 'episode' : 'episodes'} listened to`); } + if (counts.inserted) { + parts.push(`${counts.inserted} ${counts.inserted === 1 ? 'stretch' : 'stretches'} the host inserted found`); + } if (counts.found) parts.push(`${counts.found} repeated ${counts.found === 1 ? 'segment' : 'segments'} found`); if (counts.heard) parts.push(`${counts.heard} sponsor ${counts.heard === 1 ? 'read' : 'reads'} heard`); if (counts.rememberedCuts) parts.push(`${counts.rememberedCuts} cut from memory`); @@ -412,6 +415,8 @@ export function createAdPipeline({ // to decide" and "not yet asked" are not the same answer. const inputs = adDetect.detectionInputs(showId); let detected = { skipped: 'unchanged' }; + let inserted = { skipped: 'unchanged' }; + let insertedNew = 0; let heard = transcriber ? { skipped: 'unchanged' } : null; // 'ran' when any detector did: the jingle search alone can be what a pass // needed, and what it wrote can settle the catalogue before the others look. @@ -427,11 +432,24 @@ export function createAdPipeline({ * pass to see the other's result; now the pass asks again when the jingle * search changed something, and stops when it did not. */ + // Once, before the rounds: it reads only frames, which the rounds do not + // change, and what it finds is what the acoustic search must leave alone. + // After the jingle search above, so a jingle that search took back no + // longer covers the pre-roll this must now cut. + inserted = await adDetect.detectCopiedAudio(showId); + insertedNew = inserted?.newSegments ?? 0; for (let round = 0; round < 2; round += 1) { detected = await adDetect.detectForShow(showId); heard = transcriber ? await adDetect.detectFromTranscripts(showId) : null; const again = await adDetect.detectAnchors(showId); anchored = mergeAnchored(anchored, again); + // A jingle taken back here (when the first jingle search was skipped this + // pass) no longer covers the pre-roll, which the copied search left alone + // while it did: ask it again, or that pre-roll goes out uncut. + if (again?.takenBack) { + inserted = await adDetect.detectCopiedAudio(showId); + insertedNew += inserted?.newSegments ?? 0; + } if (!again.changed) { settled = true; break; @@ -465,6 +483,7 @@ export function createAdPipeline({ refused: [...refused.values()], corpus: holds.corpusSize, found: detected?.newSegments ?? 0, + inserted: insertedNew, transcribed: transcribed?.transcribed ?? 0, transcriptionFailed: transcribed?.failed ?? 0, heard: heard?.newSegments ?? 0, @@ -491,6 +510,7 @@ export function createAdPipeline({ fingerprinted: fingerprinted?.fingerprinted ?? 0, transcribed: transcribed?.transcribed ?? 0, segments: detected?.segments ?? 0, + inserted: inserted?.segments ?? 0, spoken: heard?.segments ?? 0, trimmed: trimmed.trimmed, failed: trimmed.failed, diff --git a/src/services/adverts-view.js b/src/services/adverts-view.js index 3347f30..2078ae9 100644 --- a/src/services/adverts-view.js +++ b/src/services/adverts-view.js @@ -5,6 +5,7 @@ import { escapeHtml } from '../lib/html.js'; import { presentSegment, describeComparability } from '../lib/present-segment.js'; import { describeAdvertStage, + breakSlotName, describeListenScope, describeVerdict, flattenTranscript, @@ -749,7 +750,11 @@ export function createAdvertsView({ db, config = null, adDetect, transcriber, ep ? `${formatClock(segment.duration_ms)} you marked as an advert` : segment.kind === SEGMENT_KINDS.DIFF ? `${formatClock(segment.duration_ms)} inserted by the host` - : words ?? `${formatClock(segment.duration_ms)} of repeated sound`; + : segment.kind === SEGMENT_KINDS.INSERTED + ? breakSlotName(segment.signature) + ? `${breakSlotName(segment.signature)} breaks between the host’s markers` + : `${formatClock(segment.duration_ms)} spliced in by the host` + : words ?? `${formatClock(segment.duration_ms)} of repeated sound`; rows.push({ key: segment.id, kind: segment.kind, diff --git a/src/services/remote-feeds.js b/src/services/remote-feeds.js index fd5cbb3..5f6d3d7 100644 --- a/src/services/remote-feeds.js +++ b/src/services/remote-feeds.js @@ -422,20 +422,32 @@ export function createRemoteFeeds({ * fine, SelfPod simply will not be re-checking it, and an exception thrown for that * would throw away a file that is already on disk and already accepted. */ - async function planRecheck(path, filename, declaredDurationSeconds) { + async function planRecheck(path, filename, declaredDurationSeconds, { insertsAudio = false } = {}) { if (!filename.toLowerCase().endsWith('.mp3')) return {}; + const after = () => new Date(Date.now() + RECHECK_DELAY_MS).toISOString(); try { const signals = describeStitchSignals(frameProfile(await readFile(path)), { declaredDurationSeconds, }); - if (!signals.likely) return {}; + if (signals.likely) return { recheck_reason: signals.detail, recheck_after: after() }; + } catch { + // Unreadable frames say nothing either way; what is known about the host still does. + } + /* + * A host already seen inserting audio is asked again for every episode, stitch marks + * or not. Simplecast and Acast splice cleanly — no sample-rate change, no Xing + * mismatch, measured on 2026-10-28 — so the marks above never fire for them, and the + * advert that rotates by tomorrow is only learnt by asking tomorrow. Still the same + * address, the same user agent, a day later and charged to the same byte budget: + * this asks more often, it never disguises the asking (spec §19.3). + */ + if (insertsAudio) { return { - recheck_reason: signals.detail, - recheck_after: new Date(Date.now() + RECHECK_DELAY_MS).toISOString(), + recheck_reason: 'This feed’s host inserts audio as it serves, so every episode is compared with a second download.', + recheck_after: after(), }; - } catch { - return {}; } + return {}; } /** @@ -593,7 +605,9 @@ export function createRemoteFeeds({ // one looks like it had adverts joined on rather than mixed in. It decides one // thing: whether to spend a second download on it in a day's time. See // lib/stitch-signals.js for why that bar is deliberately high. - const recheck = await planRecheck(staged, extension.filename, row.declared_duration_seconds); + const recheck = await planRecheck(staged, extension.filename, row.declared_duration_seconds, { + insertsAudio: Boolean(subscription.inserts_audio), + }); subscriptions.markItem(row.id, { identity_key: identityKey, @@ -840,6 +854,9 @@ export function createRemoteFeeds({ // programme at an offset nobody would think to check. const ranges = runsToRanges(diff.onlyInA, original.frames); segments = adDetect ? (adDetect.recordDiffSegments(episode, ranges, { timing: null })?.segments ?? 0) : 0; + // A host that served two different files has shown it inserts audio, so + // every later episode of the feed is compared too. + if (ranges.length || !diff.identical) subscriptions.markInsertsAudio(item.subscription_id); outcome = ranges.length ? 'differs' : diff.identical diff --git a/src/services/subscriptions.js b/src/services/subscriptions.js index e200ce6..71080e3 100644 --- a/src/services/subscriptions.js +++ b/src/services/subscriptions.js @@ -57,11 +57,11 @@ export function createSubscriptions({ db, config, events, logger }) { `INSERT INTO feed_subscriptions (id, show_id, feed_url, remote_title, enabled, include_keywords, exclude_keywords, min_duration_seconds, max_duration_seconds, backfill_count, catch_up_mode, - poll_interval_seconds, next_poll_at, created_at, updated_at) + poll_interval_seconds, inserts_audio, next_poll_at, created_at, updated_at) VALUES (@id, @show_id, @feed_url, @remote_title, @enabled, @include_keywords, @exclude_keywords, @min_duration_seconds, @max_duration_seconds, @backfill_count, @catch_up_mode, - @poll_interval_seconds, @next_poll_at, @created_at, @updated_at)`, + @poll_interval_seconds, @inserts_audio, @next_poll_at, @created_at, @updated_at)`, ); const selectDue = db.prepare( @@ -227,6 +227,13 @@ export function createSubscriptions({ db, config, events, logger }) { fields.enabled = isTrue(patch.enabled) ? 1 : 0; } + // The host inserts audio as it serves: every new episode is downloaded a second time + // a day later and compared (spec §19.3). SelfPod sets it on its own the first time it + // sees the host do so; this is the owner saying so first, or changing their mind. + if (patch.insertsAudio !== undefined) { + fields.inserts_audio = isTrue(patch.insertsAudio) ? 1 : 0; + } + if (patch.catchUpMode !== undefined) { const mode = String(patch.catchUpMode); if (!['resume', 'catch_up'].includes(mode)) { @@ -297,6 +304,7 @@ export function createSubscriptions({ db, config, events, logger }) { backfill_count: fields.backfill_count ?? DEFAULT_BACKFILL_COUNT, catch_up_mode: fields.catch_up_mode ?? 'resume', poll_interval_seconds: fields.poll_interval_seconds ?? null, + inserts_audio: fields.inserts_audio ?? 0, // Due immediately: someone who has just pasted a URL wants to see it work. next_poll_at: null, created_at: nowIso(), @@ -323,6 +331,11 @@ export function createSubscriptions({ db, config, events, logger }) { * complete download to reach, and re-opening it on a one-character keyword edit * would silently re-fetch gigabytes. The UI offers it as an explicit action instead. */ + /** What a second download that came back different proves: the host inserts. */ + markInsertsAudio(id) { + db.prepare('UPDATE feed_subscriptions SET inserts_audio = 1, updated_at = ? WHERE id = ? AND inserts_audio = 0').run(nowIso(), id); + }, + update(id, patch = {}) { const existing = api.getOrThrow(id); const fields = validate(patch, { partial: true, existing }); diff --git a/src/web/routes/fragments.js b/src/web/routes/fragments.js index 8445436..1ae5e11 100644 --- a/src/web/routes/fragments.js +++ b/src/web/routes/fragments.js @@ -633,6 +633,8 @@ export default async function fragmentRoutes(fastify, services) { minDurationSeconds: minutesToSeconds(body.minDurationMinutes), maxDurationSeconds: minutesToSeconds(body.maxDurationMinutes), backfillCount: body.backfillCount, + // A switch posts nothing when it is off, and this form always posts every field. + insertsAudio: body.insertsAudio === '1' || body.insertsAudio === 'on', }; const existing = services.subscriptions.getForShow(show.id); diff --git a/src/web/views/partials/subscription-form.eta b/src/web/views/partials/subscription-form.eta index d816ccb..060dba9 100644 --- a/src/web/views/partials/subscription-form.eta +++ b/src/web/views/partials/subscription-form.eta @@ -86,6 +86,21 @@ <% } %> +
+ +
+ aria-describedby="f-inserts-hint"> + + <%= sub && sub.insertsAudio ? 'Yes' : 'No' %> +
+
+ When on, each new episode is downloaded once more a day later, from the same address, and + whatever changed between the two is cut. SelfPod turns this on by itself the first time it + finds audio the host inserted. Each second download counts again in the publisher’s figures. +
+
+ <% if (it.reopened) { %>
diff --git a/src/workers/audio-search.js b/src/workers/audio-search.js index ff6aa4c..2b9caf5 100644 --- a/src/workers/audio-search.js +++ b/src/workers/audio-search.js @@ -1,7 +1,8 @@ import { parentPort } from 'node:worker_threads'; import { findHeadAnchors } from '../lib/audio-anchor.js'; -import { decodeWindowToWav, fingerprintFile, profileFile } from '../lib/audio-tasks.js'; +import { decodeWindowToWav, fingerprintFile, frameHashesFile, profileFile } from '../lib/audio-tasks.js'; +import { findCopiedAudio } from '../lib/copied-audio.js'; import { findRepeatedAudio } from '../lib/repeated-audio.js'; /** @@ -23,7 +24,9 @@ import { findRepeatedAudio } from '../lib/repeated-audio.js'; const TASKS = { repeatedAudio: ({ episodes, options }) => ({ result: findRepeatedAudio(episodes, options) }), headAnchors: ({ episodes, options }) => ({ result: findHeadAnchors(episodes, options) }), + copiedAudio: ({ episodes, options }) => ({ result: findCopiedAudio(episodes, options) }), fingerprintFile, + frameHashesFile, profileFile, decodeWindow: decodeWindowToWav, }; diff --git a/test/acceptance/run.sh b/test/acceptance/run.sh index 96a3f10..e014afc 100755 --- a/test/acceptance/run.sh +++ b/test/acceptance/run.sh @@ -960,6 +960,88 @@ if issues: || fail "the recogniser is reported unhealthy" # ---------------------------------------------------------------------- report +step "24. Audio a host spliced in is cut, whatever language it is in" + +# The two shapes measured on real hosts on 2026-10-28 (spec §19.3): the same opener +# spliced in at 0:00 of every episode (Simplecast, RMC), and a short marker stitched at +# both edges of every break, around adverts no other episode carried (Acast). Byte- +# identical frames are the evidence, so this is generated rather than encoded: the +# adverts here have no language at all, which is the point. +mkdir -p "${WORK}/data/shows/host-club" +python3 - "${WORK}/data/shows/host-club" "${WORK}/host-expected" <<'PY' +import sys, pathlib + +FRAME_BYTES = 417 +FRAME_MS = 1152 / 44100 * 1000 + +def frame(seed): + header = bytes([0xFF, 0xFB, (9 << 4) | (0 << 2), (1 << 6)]) + payload = bytearray(FRAME_BYTES - 4) + x = (seed * 2654435761) & 0xFFFFFFFF + for i in range(len(payload)): + x = (x ^ (x << 13)) & 0xFFFFFFFF + x ^= x >> 17 + x = (x ^ (x << 5)) & 0xFFFFFFFF + payload[i] = x & 0xFE + return header + bytes(payload) + +def seconds(n): + return round(n * 1000 / FRAME_MS) + +def segment(start, count): + return b"".join(frame(start + i) for i in range(count)) + +out, expected = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) +expected.mkdir(parents=True, exist_ok=True) +opener = segment(30_000, seconds(14)) # the same file at 0:00 of every episode +marker = segment(70_000, seconds(2.2)) # the host's slot marker +for n in range(3): + first = segment(1_000_000 + n * 100_000, seconds(200)) + second = segment(1_050_000 + n * 100_000, seconds(200)) + midroll = segment(200_000 + n * 10_000, seconds(30)) # heard in this episode only + (out / f"episode-{n + 1}.mp3").write_bytes(opener + first + marker + midroll + marker + second) + (expected / f"episode-{n + 1}.mp3").write_bytes(first + second) +PY + +wait_for_scan 12 +HOST_SHOW_ID="$(api "${BASE}/api/shows" | json ' +print(next(s["id"] for s in json.load(sys.stdin)["shows"] if s["slug"] == "host-club"))')" +api -X PATCH -H 'Content-Type: application/json' -d '{"mode":"auto"}' \ + "${BASE}/api/shows/${HOST_SHOW_ID}/ad-trim" >/dev/null +api -X POST "${BASE}/api/shows/${HOST_SHOW_ID}/ad-detect" >/dev/null + +HOST_STATE="$(api "${BASE}/api/shows/${HOST_SHOW_ID}/ad-segments")" +printf '%s' "$HOST_STATE" | json ' +d = json.load(sys.stdin) +kinds = sorted((s.get("kind"), s["status"]) for s in d["segments"]) +inserted = [k for k in kinds if k[0] == "inserted"] +ok = len(inserted) >= 2 and all(status == "approved" for _, status in inserted) and d["held"] == 0 +print(kinds, "held", d["held"]) +sys.exit(0 if ok else 1)' >/dev/null \ + && pass "the opener and the break are found and cut without asking, and nothing is held" \ + || fail "the inserted audio was not cut on its own ($(printf '%s' "$HOST_STATE" | head -c 160))" + +HOST_FEED_TOKEN="$(api "${BASE}/api/shows/${HOST_SHOW_ID}" | json ' +print(json.load(sys.stdin)["show"]["feedToken"])')" +HOST_FEED="$(curl -s "${BASE}/feeds/host-club/${HOST_FEED_TOKEN}.xml")" +HOST_OK=0 +for n in 1 2 3; do + URL="$(printf '%s' "$HOST_FEED" | json " +import re +body = sys.stdin.read() +print(next(u for u in re.findall(r'url=\"([^\"]+)\"', body) if 'episode-${n}.mp3' in u).replace('&', '&'))")" + curl -s -o "${WORK}/host-served-${n}.mp3" "$URL" + python3 - "${WORK}/host-served-${n}.mp3" "${WORK}/host-expected/episode-${n}.mp3" <<'PY' && HOST_OK=$((HOST_OK + 1)) +import sys, pathlib +served, expected = (pathlib.Path(p).read_bytes() for p in sys.argv[1:3]) +# The programme, every byte of it, and nothing else but a header a cut may add. +sys.exit(0 if expected in served and len(served) - len(expected) < 2000 else 1) +PY +done +[ "$HOST_OK" = "3" ] \ + && pass "each episode is served as exactly its programme: opener, markers and mid-roll gone" \ + || fail "only ${HOST_OK} of 3 episodes were served as exactly their programme" + step "Result" printf ' %d passed, %d failed\n\n' "$PASS" "$FAIL" [ "$FAIL" -eq 0 ] || exit 1 diff --git a/test/helpers/mp3.js b/test/helpers/mp3.js index 9f2a258..a0b0db1 100644 --- a/test/helpers/mp3.js +++ b/test/helpers/mp3.js @@ -1,3 +1,5 @@ +import { readFrames } from '../../src/lib/mp3-frames.js'; + /** * Builds MP3 files frame by frame, for tests. * @@ -14,13 +16,19 @@ const FRAME_BYTES = 417; export const FRAME_MS = (1152 / 44100) * 1000; -function frameHeader(channelMode = 'joint', bitrateIndex = 9, sampleRateIndex = 0) { +/** + * `variant` sets the private, copyright and original bits (0–7): flags a decoder + * ignores, so the audio is the same while the bytes are not. That is what a real + * programme's repeated theme is from one episode to the next — the same sound, encoded + * afresh — and what a host's spliced-in advert is not (copied-audio.js). + */ +function frameHeader(channelMode = 'joint', bitrateIndex = 9, sampleRateIndex = 0, variant = 0) { const modes = { stereo: 0, joint: 1, dual: 2, mono: 3 }; return Buffer.from([ 0xff, 0xfb, // MPEG-1, Layer III, no CRC - (bitrateIndex << 4) | (sampleRateIndex << 2), - (modes[channelMode] << 6) | 0x00, + (bitrateIndex << 4) | (sampleRateIndex << 2) | (variant & 4 ? 0x01 : 0), + (modes[channelMode] << 6) | (variant & 1 ? 0x08 : 0) | (variant & 2 ? 0x04 : 0), ]); } @@ -30,8 +38,8 @@ function frameHeader(channelMode = 'joint', bitrateIndex = 9, sampleRateIndex = * Deterministic so a test can rebuild the same "advert" twice, and distinct across * seeds so two different frames never hash alike by construction. */ -export function frame(seed, { channelMode = 'joint', bitrateIndex = 9 } = {}) { - const header = frameHeader(channelMode, bitrateIndex); +export function frame(seed, { channelMode = 'joint', bitrateIndex = 9, variant = 0 } = {}) { + const header = frameHeader(channelMode, bitrateIndex, 0, variant); const payload = Buffer.alloc(FRAME_BYTES - 4); let x = (seed * 2654435761) >>> 0; for (let i = 0; i < payload.length; i += 1) { @@ -69,3 +77,18 @@ export function id3v2(payloadSize = 100) { // Filled with a byte that is not 0xFF, so nothing inside the tag looks like audio. return Buffer.concat([header, Buffer.alloc(payloadSize, 0x41)]); } + +/** + * The same audio, different bytes: a real MP3 with `variant` written into every frame + * header the way `frame()` writes it. For a fixture file repeated across episodes that + * stands for audio a programme carries — re-encoded every episode — rather than a file a + * host splices in unchanged. + */ +export function restamp(buffer, variant) { + const out = Buffer.from(buffer); + for (const frame of readFrames(out).frames) { + out[frame.offset + 2] = (out[frame.offset + 2] & 0xfe) | (variant & 4 ? 0x01 : 0); + out[frame.offset + 3] = (out[frame.offset + 3] & 0xf3) | (variant & 1 ? 0x08 : 0) | (variant & 2 ? 0x04 : 0); + } + return out; +} diff --git a/test/integration/ad-anchor.test.js b/test/integration/ad-anchor.test.js index 4f1cbf7..c6d21fd 100644 --- a/test/integration/ad-anchor.test.js +++ b/test/integration/ad-anchor.test.js @@ -6,6 +6,7 @@ import { afterEach, describe, it } from 'node:test'; import { SEGMENT_SOURCES, SEGMENT_STATUS } from '../../src/constants.js'; import { frameProfile } from '../../src/lib/mp3-frames.js'; +import { restamp } from '../helpers/mp3.js'; import { FIXTURE_DIR } from '../helpers/harness.js'; import { createTestServer } from '../helpers/http.js'; import { cannedWhisper, whisperJson } from '../helpers/whisper.js'; @@ -47,8 +48,16 @@ async function makeShow({ mode = 'review', minEpisodes = 3, whisper = null } = { return { show: server.shows.get(created.id), dir }; } +/* + * Each episode's copy of a fixture gets header bits of its own (`restamp`): the same + * sound, different bytes — a jingle mastered into each episode, as the real one is. + * Copied byte for byte, the four files would be audio a host spliced in, which the + * copied-frames search cuts before this file's jingle search is ever asked. + */ +let episodesAdded = 0; async function addEpisode(dir, name, ...parts) { - await writeFile(join(dir, name), Buffer.concat(parts)); + const variant = episodesAdded++ % 8; + await writeFile(join(dir, name), Buffer.concat(parts.map((part) => restamp(part, variant)))); await server.scanner.scanAllNow('manual'); const show = server.shows.getBySlug('jingle-show'); return server.episodes.listByShow(show.id).find((row) => row.filename === name); diff --git a/test/integration/ad-detect.test.js b/test/integration/ad-detect.test.js index 0860741..70bebf9 100644 --- a/test/integration/ad-detect.test.js +++ b/test/integration/ad-detect.test.js @@ -56,9 +56,11 @@ describe('fingerprinting an episode', () => { assert.ok(result.frameCount > 0); // Derived data lives with the other derived data, never in the user's own folder. - const stored = await readdir(join(app.config.fingerprintDir, show.id)); - assert.equal(stored.length, 1); + // The fingerprint, and beside it the episode's MP3 frame hashes (copied-audio.js). + const stored = (await readdir(join(app.config.fingerprintDir, show.id))).sort(); + assert.equal(stored.length, 2, stored.join(', ')); assert.match(stored[0], new RegExp(`^${episode.id}\\.\\d+\\.fp$`)); + assert.match(stored[1], new RegExp(`^${episode.id}\\.h\\d+\\.frames$`)); const files = await readdir(showDir); assert.ok( @@ -131,7 +133,7 @@ describe('fingerprinting an episode', () => { const show = await makeEpisodes(1); const [episode] = app.episodes.listByShow(show.id); await app.adDetect.fingerprintEpisode(episode); - assert.equal((await readdir(join(app.config.fingerprintDir, show.id))).length, 1); + assert.equal((await readdir(join(app.config.fingerprintDir, show.id))).length, 2, 'the fingerprint and the frame hashes'); await app.episodes.deleteWithFile(episode.id); diff --git a/test/integration/ad-pipeline.test.js b/test/integration/ad-pipeline.test.js index 25d9cca..206a4b0 100644 --- a/test/integration/ad-pipeline.test.js +++ b/test/integration/ad-pipeline.test.js @@ -501,15 +501,17 @@ describe('what the trimmed audio actually is', () => { const original = episodeBytes(0); assert.equal(response.statusCode, 200); - // Within half a second, not to the frame. Segments are now found by what the audio - // *sounds* like, and each sub-fingerprint describes a 372ms window — so a boundary - // is placed to a fraction of a second rather than exactly. The error is deliberately - // biased towards removing a little programme rather than leaving a little advert. - const removed = (frameProfile(original).frameCount - frameProfile(served).frameCount) * (FRAME_MS / 1000); - assert.ok( - removed >= 40 && removed < 40.8, - `expected the 40s read gone with a little margin, ${removed.toFixed(2)}s went`, - ); + // To the frame. The read is the same MP3 frames in every episode — the way a host + // splices one in — so the copied-frames search finds exactly its frames and the cut + // takes exactly those: not the half-second margin the acoustic search adds when it + // is the one placing an edge (that case: "a theme tune that was re-encoded"). + const programme = [ + ...frameProfile(segment(100_000, framesFor(45))).hashes, + ...frameProfile(segment(600_000, framesFor(45))).hashes, + ]; + const removed = frameProfile(original).frameCount - frameProfile(served).frameCount; + assert.equal(removed, framesFor(40), `expected the read's ${framesFor(40)} frames gone, ${removed} went`); + assert.deepEqual([...frameProfile(served).hashes], programme, 'what is served is not exactly the programme'); // The programme before the advert is untouched, byte for byte. assert.equal( Buffer.compare(served.subarray(0, 30_000), original.subarray(0, 30_000)), diff --git a/test/integration/ad-segments-api.test.js b/test/integration/ad-segments-api.test.js index 105b39b..0b02a78 100644 --- a/test/integration/ad-segments-api.test.js +++ b/test/integration/ad-segments-api.test.js @@ -34,7 +34,8 @@ async function makeShow({ mode = 'review', count = 3 } = {}) { join(showDir, `episode-${n}.mp3`), stitch( segment(100_000 + n * 50_000, framesFor(45)), - segment(2_000, framesFor(40)), + // The same sound in each episode, re-encoded each time: see `variant` in helpers/mp3.js. + segment(2_000, framesFor(40), { variant: n % 8 }), segment(600_000 + n * 50_000, framesFor(45)), ), ); diff --git a/test/integration/ad-segments-ui.test.js b/test/integration/ad-segments-ui.test.js index 7e4bf81..620f6ad 100644 --- a/test/integration/ad-segments-ui.test.js +++ b/test/integration/ad-segments-ui.test.js @@ -32,7 +32,8 @@ async function makeShow({ mode = 'review', count = 3 } = {}) { join(showDir, `episode-${n}.mp3`), stitch( segment(100_000 + n * 50_000, framesFor(45)), - segment(2_000, framesFor(40)), + // The same sound in each episode, re-encoded each time: see `variant` in helpers/mp3.js. + segment(2_000, framesFor(40), { variant: n % 8 }), segment(600_000 + n * 50_000, framesFor(45)), ), ); diff --git a/test/integration/ad-timeline-ui.test.js b/test/integration/ad-timeline-ui.test.js index 48a3414..5221a53 100644 --- a/test/integration/ad-timeline-ui.test.js +++ b/test/integration/ad-timeline-ui.test.js @@ -43,7 +43,8 @@ async function makeShow({ mode = 'review', count = 3, process = true } = {}) { for (let n = 0; n < count; n += 1) { await writeFile( join(showDir, `episode-${n}.mp3`), - stitch(segment(100_000 + n * 50_000, framesFor(45)), segment(2_000, framesFor(40)), segment(600_000 + n * 50_000, framesFor(45))), + // The shared read is the same sound re-encoded per episode (`variant`, helpers/mp3.js). + stitch(segment(100_000 + n * 50_000, framesFor(45)), segment(2_000, framesFor(40), { variant: n % 8 }), segment(600_000 + n * 50_000, framesFor(45))), ); } await server.scanner.scanAllNow('manual'); diff --git a/test/integration/dynamic-adverts.test.js b/test/integration/dynamic-adverts.test.js index 2483095..9780022 100644 --- a/test/integration/dynamic-adverts.test.js +++ b/test/integration/dynamic-adverts.test.js @@ -19,6 +19,8 @@ let audioRequests; let advert; /** How long the advert the sentinel is currently inserting runs for. */ let advertSeconds; +/** Whether the host encodes its adverts like the programme, leaving no stitch marks. */ +let cleanSplice; /** * One episode, with whichever advert the host is currently inserting. @@ -31,7 +33,7 @@ let advertSeconds; function episode(advertSeed, advertSeconds = 20) { return stitch( segment(10_000, framesFor(30), { channelMode: 'joint' }), - segment(advertSeed, framesFor(advertSeconds), { channelMode: 'stereo' }), + segment(advertSeed, framesFor(advertSeconds), { channelMode: cleanSplice ? 'joint' : 'stereo' }), segment(90_000, framesFor(30), { channelMode: 'joint' }), ); } @@ -51,6 +53,7 @@ beforeEach(async () => { audioRequests = []; advert = 50_000; advertSeconds = 20; + cleanSplice = false; server = createServer((req, res) => { if (req.url.startsWith('/audio/')) { @@ -76,12 +79,13 @@ afterEach(async () => { }); /** Subscribes, downloads the one episode, and returns the show and the ledger row. */ -async function take({ mode = 'review' } = {}) { +async function take({ mode = 'review', insertsAudio = false } = {}) { const show = app.shows.getBySlug('tape-club'); app.db.prepare('UPDATE shows SET ad_trim_mode = ? WHERE id = ?').run(mode, show.id); const subscription = app.subscriptions.create(show.id, { feedUrl: `${origin}/feed.xml`, backfillCount: 10, + insertsAudio, }); await app.remoteFeeds.pollNow(subscription.id); const [item] = app.subscriptions.items({ subscriptionId: subscription.id }); @@ -422,3 +426,58 @@ describe('what the difference is treated as', () => { assert.equal(found.status, SEGMENT_STATUS.CANDIDATE); }); }); + +describe('a host that splices its adverts in without a mark', () => { + // Simplecast and Acast, measured on 2026-10-28: the advert is encoded like the + // programme, so no stitch signal fires and the second download was never made. + beforeEach(() => { + cleanSplice = true; + }); + + it('is not downloaded twice while nothing is known about the host', async () => { + const { item } = await take(); + assert.equal(item.decision, 'downloaded'); + assert.equal(item.recheck_after, null, 'a clean file was re-downloaded on no evidence'); + }); + + it('is compared a day later once the feed is known to insert, and the advert found', async () => { + const { show, item } = await take({ insertsAudio: true }); + assert.ok(item.recheck_after, 'no second look was planned'); + assert.match(item.recheck_reason, /inserts audio/); + const wait = new Date(item.recheck_after) - Date.now(); + assert.ok(wait > 23 * 3600_000, `only waiting ${Math.round(wait / 3600_000)}h`); + + advert = 70_000; // the next day's advert + makeDue(item.id); + await app.remoteFeeds.recheckDue(); + + const [after] = app.subscriptions.items({ subscriptionId: item.subscription_id }); + assert.equal(after.recheck_outcome, 'differs'); + const diff = app.adDetect.listSegments(show.id).filter((row) => row.source === SEGMENT_SOURCES.DIFF); + assert.equal(diff.length, 1); + const [occurrence] = diff[0].occurrences; + assert.deepEqual([occurrence.start_frame, occurrence.end_frame], [framesFor(30), framesFor(30) + framesFor(20)]); + // The same address both times: asked again, never disguised. + assert.deepEqual(new Set(audioRequests), new Set(['/audio/a.mp3'])); + }); + + it('learns that the host inserts from a second download that came back different', async () => { + cleanSplice = false; // stitch marks this time, so the second look happens anyway + const { item, subscription } = await take(); + assert.equal(app.subscriptions.get(subscription.id).inserts_audio, 0); + advert = 70_000; + makeDue(item.id); + await app.remoteFeeds.recheckDue(); + assert.equal(app.subscriptions.get(subscription.id).inserts_audio, 1, 'the feed was not marked as one that inserts'); + }); + + it('leaves the feed unmarked when the second download is the same', async () => { + cleanSplice = false; + const { item, subscription } = await take(); + makeDue(item.id); + await app.remoteFeeds.recheckDue(); + const [after] = app.subscriptions.items({ subscriptionId: subscription.id }); + assert.equal(after.recheck_outcome, 'identical'); + assert.equal(app.subscriptions.get(subscription.id).inserts_audio, 0); + }); +}); diff --git a/test/integration/inserted-audio.test.js b/test/integration/inserted-audio.test.js new file mode 100644 index 0000000..ab07824 --- /dev/null +++ b/test/integration/inserted-audio.test.js @@ -0,0 +1,453 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { readdir, rm, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, it } from 'node:test'; + +import { SEGMENT_KINDS, SEGMENT_STATUS } from '../../src/constants.js'; +import { frameProfile } from '../../src/lib/mp3-frames.js'; +import { createTestServer } from '../helpers/http.js'; +import { FRAME_MS, segment, stitch } from '../helpers/mp3.js'; + +/** + * Audio a host inserts as it serves, through the whole app: files in a show folder, the + * scan, a pass of the advert pipeline, and the audio a podcast app is then given. + * + * The shapes are the two real hosts measured on 2026-10-28 (spec §19.3): RMC on + * Simplecast, with the same opener and closer spliced into every episode; Acast, with a + * 2.2 s marker around every break, and adverts in the breaks that no other episode had. + * Frame numbers are spaced so that no two parts share a frame by accident. + */ +const frames = (seconds) => Math.round((seconds * 1000) / FRAME_MS); +const programme = (n, seconds, part = 0) => segment(1_000_000 + n * 100_000 + part * 50_000, frames(seconds)); +const advert = (k, seconds) => segment(k * 10_000, frames(seconds)); +const MARKER = () => advert(7, 2.2); + +let server; +let showDir; + +beforeEach(async () => { + server = await createTestServer(); + showDir = await server.makeShowFolder('chroniques'); +}); + +afterEach(async () => { + await server.cleanup(); +}); + +async function makeShow(mode, files) { + await writeFile(join(showDir, '.keep'), ''); + await server.scanner.scanAllNow('manual'); + const created = server.shows.getBySlug('chroniques'); + server.db.prepare('UPDATE shows SET ad_trim_mode = ? WHERE id = ?').run(mode, created.id); + for (const [name, bytes] of Object.entries(files)) await writeFile(join(showDir, name), bytes); + await server.scanner.scanAllNow('manual'); + return server.shows.get(created.id); +} + +/** What a podcast app is given for an episode, at the address the feed advertises. */ +async function served(show, filename) { + const episode = server.episodes.listByShow(show.id).find((row) => row.filename === filename); + const version = episode.trimmed_etag ? `?v=${episode.trimmed_etag}` : ''; + const response = await server.app.inject({ + method: 'GET', + url: `/media/${show.slug}/${show.feed_token}/${episode.id}/${encodeURIComponent(filename)}${version}`, + }); + assert.equal(response.statusCode, 200); + return [...frameProfile(response.rawPayload).hashes]; +} + +const hashesOf = (...parts) => [...frameProfile(stitch(...parts)).hashes]; + +const rows = (show) => server.adDetect.listSegments(show.id).filter((row) => row.status !== SEGMENT_STATUS.REJECTED); + +describe('an opener and a closer the host splices into every episode', () => { + // RMC: the same 14 s opener at 0:00 and 10 s closer at the end, in every episode. + const rmc = (n) => stitch(advert(1, 14), programme(n, 200), advert(2, 10)); + const files = { 'a.mp3': rmc(1), 'b.mp3': rmc(2), 'c.mp3': rmc(3) }; + + it('is cut from every episode, exactly, leaving exactly the programme', async () => { + const show = await makeShow('auto', files); + await server.adPipeline.processShow(show.id); + + for (const [n, name] of [[1, 'a.mp3'], [2, 'b.mp3'], [3, 'c.mp3']]) { + assert.deepEqual(await served(show, name), hashesOf(programme(n, 200)), `${name} is not exactly its programme`); + } + const inserted = rows(show).filter((row) => row.kind === SEGMENT_KINDS.INSERTED); + assert.equal(inserted.length, 2, 'the opener and the closer, one row each'); + for (const row of inserted) { + assert.equal(row.status, SEGMENT_STATUS.APPROVED); + assert.equal(row.auto_approved, 1); + assert.equal(row.episode_count, 3); + } + }); + + it('is not offered again as "audio this show repeats" beside the row that cuts it', async () => { + // The acoustic search hears the same opener in all three episodes too. Before, it + // was the only thing that did, and held it as a theme tune at 0:00. + const show = await makeShow('auto', files); + await server.adPipeline.processShow(show.id); + + const others = rows(show).filter((row) => row.kind !== SEGMENT_KINDS.INSERTED && row.occurrences.length); + assert.deepEqual( + others.map((row) => `${row.kind} ${row.duration_ms} ms`), + [], + 'the inserted audio came back as a second row', + ); + }); + + it('is only offered, and nothing cut, while the show is set to review', async () => { + const show = await makeShow('review', files); + await server.adPipeline.processShow(show.id); + + const inserted = rows(show).filter((row) => row.kind === SEGMENT_KINDS.INSERTED); + assert.equal(inserted.length, 2); + for (const row of inserted) assert.equal(row.status, SEGMENT_STATUS.CANDIDATE); + assert.ok(server.episodes.listByShow(show.id).every((row) => !row.trimmed_etag), 'something was cut unasked'); + }); + + it('stays out of one episode the owner restored it in, and only that one', async () => { + const show = await makeShow('auto', files); + await server.adPipeline.processShow(show.id); + const opener = rows(show).find((row) => row.kind === SEGMENT_KINDS.INSERTED && row.occurrences[0].start_ms === 0); + const b = server.episodes.listByShow(show.id).find((row) => row.filename === 'b.mp3'); + + server.adDetect.restoreHere({ segmentId: opener.id, episodeId: b.id }); + await server.adPipeline.processShow(show.id); + await server.adPipeline.processShow(show.id); + + assert.deepEqual(await served(show, 'b.mp3'), hashesOf(advert(1, 14), programme(2, 200)), 'the restore did not hold'); + assert.deepEqual(await served(show, 'a.mp3'), hashesOf(programme(1, 200)), 'the restore leaked to another episode'); + }); + + it('is not looked for again on a pass where nothing changed', async () => { + // Detection counts as an owner's action unless it is declared read-only, and an + // owner's action means "look again next pass" — for ever, on every pass. + const show = await makeShow('auto', files); + await server.adPipeline.processShow(show.id); + await server.adPipeline.processShow(show.id); + const searches = server.adDetect.workDone().corpusSearches; + + await server.adPipeline.processShow(show.id); + + assert.equal(server.adDetect.workDone().corpusSearches, searches, 'an unchanged show was searched again'); + }); + + it('marks a followed feed as one whose host inserts audio', async () => { + const show = await makeShow('auto', files); + server.db + .prepare( + `INSERT INTO feed_subscriptions (id, show_id, feed_url, created_at, updated_at) + VALUES ('sub', ?, 'https://feeds.example.com/chroniques.xml', 'x', 'x')`, + ) + .run(show.id); + await server.adPipeline.processShow(show.id); + assert.equal(server.db.prepare(`SELECT inserts_audio FROM feed_subscriptions WHERE id = 'sub'`).get().inserts_audio, 1); + }); +}); + +describe('breaks between the markers a host stitches around them', () => { + // Acast: a different advert in every break, a marker at every edge, two markers back + // to back where the post-roll starts. + const acast = (n) => + stitch( + advert(100 + n * 10, 30), MARKER(), + programme(n, 240), + MARKER(), advert(101 + n * 10, 30), MARKER(), + programme(n, 240, 1), + MARKER(), MARKER(), advert(102 + n * 10, 25), + ); + + it('cuts every break, markers included, though no advert in them was heard twice', async () => { + const show = await makeShow('auto', { 'a.mp3': acast(1), 'b.mp3': acast(2), 'c.mp3': acast(3) }); + await server.adPipeline.processShow(show.id); + + for (const [n, name] of [[1, 'a.mp3'], [2, 'b.mp3'], [3, 'c.mp3']]) { + assert.deepEqual( + await served(show, name), + hashesOf(programme(n, 240), programme(n, 240, 1)), + `${name} is not exactly its programme`, + ); + } + const slots = rows(show) + .filter((row) => row.signature.startsWith('break:')) + .map((row) => [row.signature, row.episode_count, row.occurrences.length]); + assert.deepEqual(slots.sort(), [['break:mid-1', 3, 3], ['break:post', 3, 3], ['break:pre', 3, 3]]); + }); +}); + +describe('a library fingerprinted before frame hashes were kept', () => { + // Every episode already on an owner's NAS: fingerprinted by 1.10, with no frame-hash + // file beside its fingerprint. Decoding them all again would be a minute a NAS episode. + const rmc = (n) => stitch(advert(1, 14), programme(n, 200), advert(2, 10)); + const frameFiles = async (show) => + (await readdir(join(server.config.fingerprintDir, show.id))).filter((name) => name.endsWith('.frames')); + + it('reads the hashes from the files once, without fingerprinting them again, and cuts', async () => { + const show = await makeShow('review', { 'a.mp3': rmc(1), 'b.mp3': rmc(2), 'c.mp3': rmc(3) }); + await server.adPipeline.processShow(show.id); + for (const name of await frameFiles(show)) await rm(join(server.config.fingerprintDir, show.id, name)); + server.db.prepare('DELETE FROM ad_segments WHERE show_id = ?').run(show.id); + const computed = server.adDetect.workDone().fingerprintsComputed; + + const found = await server.adDetect.detectCopiedAudio(show.id); + + assert.equal(found.segments, 2, 'the opener and the closer were not found from the files'); + assert.equal((await frameFiles(show)).length, 3, 'the hashes were not kept'); + assert.equal(server.adDetect.workDone().fingerprintsComputed, computed, 'it decoded the episodes again'); + }); + + it('does not trust a file that changed since its fingerprint', async () => { + const show = await makeShow('review', { 'a.mp3': rmc(1), 'b.mp3': rmc(2) }); + await server.adPipeline.processShow(show.id); + for (const name of await frameFiles(show)) await rm(join(server.config.fingerprintDir, show.id, name)); + server.db.prepare('DELETE FROM ad_segments WHERE show_id = ?').run(show.id); + // Replaced on the share, and no pass has fingerprinted it since. + await writeFile(join(showDir, 'b.mp3'), stitch(programme(9, 200))); + + const found = await server.adDetect.detectCopiedAudio(show.id); + + assert.equal(found.skipped, 'not_enough_episodes', 'hashes of the new file were paired with the old fingerprint'); + }); +}); + +describe('what the copied frames must not cut unasked', () => { + it('offers a stretch of programme two episodes share mid-file, and cuts nothing', async () => { + // A "best of" assembled from an earlier episode's file without re-encoding shares a + // stretch of programme with that episode, byte for byte, in the middle of both. + const clip = programme(9, 45); + const show = await makeShow('auto', { + 'original.mp3': stitch(programme(1, 120), clip, programme(1, 120, 1)), + 'best-of.mp3': stitch(programme(2, 60), clip, programme(2, 60, 1)), + 'other.mp3': stitch(programme(3, 200)), + }); + await server.adPipeline.processShow(show.id); + + const [row] = rows(show).filter((r) => r.kind === SEGMENT_KINDS.INSERTED); + assert.ok(row, 'the shared stretch was not found at all'); + assert.equal(row.status, SEGMENT_STATUS.CANDIDATE); + assert.equal(row.hold_reason, 'seen_too_few_times'); + assert.ok(server.episodes.listByShow(show.id).every((e) => !e.trimmed_etag), 'programme was cut unasked'); + }); + + it('leaves alone what the owner already removed by ear, and asks nothing new about it', async () => { + // An owner in review mode who removed a read before this search existed. The cut + // they approved goes on cutting, and the same audio is not offered a second time — + // which would hold every episode back until it was answered again. + const read = advert(3, 40); + const files = { + 'a.mp3': stitch(programme(1, 45), read, programme(1, 45, 1)), + 'b.mp3': stitch(programme(2, 45), read, programme(2, 45, 1)), + 'c.mp3': stitch(programme(3, 45), read, programme(3, 45, 1)), + }; + const show = await makeShow('review', files); + await server.adDetect.fingerprintShow(show.id); + await server.adDetect.detectForShow(show.id); + const [byEar] = rows(show).filter((r) => r.kind === SEGMENT_KINDS.REPEATED_AUDIO); + assert.ok(byEar, 'the acoustic search did not find the read'); + server.adDetect.decide(byEar.id, SEGMENT_STATUS.APPROVED); + + await server.adPipeline.processShow(show.id); + await server.adPipeline.processShow(show.id); + + assert.deepEqual(rows(show).filter((r) => r.kind === SEGMENT_KINDS.INSERTED), [], 'the same audio was offered twice'); + assert.ok(server.episodes.listByShow(show.id).every((e) => e.publish_hold === null), 'an episode is held for a question already answered'); + const stillCutting = server.adDetect.listSegments(show.id).find((r) => r.id === byEar.id); + assert.equal(stillCutting.occurrences.length, 3, 'the approved cut lost its occurrences to a row nobody approved'); + const cut = server.adDetect.cutListFor(server.episodes.listByShow(show.id).find((e) => e.filename === 'a.mp3').id); + assert.ok(cut.length >= 1, 'the read is no longer cut'); + }); +}); + +describe('a cut the owner approved, beside a copied row still waiting', () => { + it('keeps cutting, even where the waiting row covers it', async () => { + // Reached when the two overlap only in part — the copied row is wider than what the + // ear found — so the copied search does write its row, and it waits to be asked. + const read = advert(3, 40); + const show = await makeShow('review', { + 'a.mp3': stitch(programme(1, 45), read, programme(1, 45, 1)), + 'b.mp3': stitch(programme(2, 45), read, programme(2, 45, 1)), + 'c.mp3': stitch(programme(3, 45), read, programme(3, 45, 1)), + }); + await server.adDetect.fingerprintShow(show.id); + await server.adDetect.detectForShow(show.id); + const [byEar] = rows(show).filter((r) => r.kind === SEGMENT_KINDS.REPEATED_AUDIO); + server.adDetect.decide(byEar.id, SEGMENT_STATUS.APPROVED); + + // A waiting copied row a quarter-minute wider each side than the read. + const id = 'waiting-copy'; + server.db + .prepare( + `INSERT INTO ad_segments (id, show_id, signature, source, status, duration_ms, episode_count, occurrence_count, + first_seen_at, created_at, updated_at, kind) + VALUES (?, ?, 'copied:wider', 'corpus', 'candidate', 70000, 3, 3, 'x', 'x', 'x', 'inserted')`, + ) + .run(id, show.id); + const pad = frames(15); + for (const e of server.episodes.listByShow(show.id)) { + server.db + .prepare( + `INSERT INTO ad_segment_occurrences (segment_id, episode_id, start_frame, end_frame, start_ms, end_ms) + VALUES (?, ?, ?, ?, ?, ?)`, + ) + .run(id, e.id, frames(45) - pad, frames(85) + pad, 30_000, 100_000); + } + + await server.adDetect.detectForShow(show.id); + + const after = server.adDetect.listSegments(show.id).find((r) => r.id === byEar.id); + assert.equal(after.occurrences.length, 3, 'the approved cut lost its occurrences to a row nobody approved'); + }); +}); + +describe('copies inside a break', () => { + it('are cut with the break, not offered again as rows of their own', async () => { + // Acast serves the same advert to two episodes now and then; inside a break it is + // already cut, and a row of its own would wait to be asked about and hold them back. + // In the mid-roll: mid-file and in two episodes, the shape that is only offered. + const shared = advert(5, 30); + const acastWith = (n, midroll) => + stitch(advert(100 + n * 10, 30), MARKER(), programme(n, 240), MARKER(), midroll, MARKER(), programme(n, 240, 1), MARKER(), MARKER(), advert(102 + n * 10, 25)); + const show = await makeShow('auto', { + 'a.mp3': acastWith(1, shared), + 'b.mp3': acastWith(2, shared), + 'c.mp3': acastWith(3, advert(130, 30)), + }); + await server.adPipeline.processShow(show.id); + + const inserted = rows(show).filter((r) => r.kind === SEGMENT_KINDS.INSERTED); + assert.ok(inserted.every((r) => r.signature.startsWith('break:') || r.status === SEGMENT_STATUS.APPROVED), inserted.map((r) => `${r.signature} ${r.status}`).join('; ')); + assert.equal(inserted.filter((r) => r.status === SEGMENT_STATUS.CANDIDATE).length, 0); + assert.ok(server.episodes.listByShow(show.id).every((e) => e.publish_hold === null), 'an episode is held'); + }); +}); + +describe('a theme copied into some episodes and re-encoded in another', () => { + // Acceptance step 22's files: the same 48k theme bytes in two episodes, an 80k encode + // of it in the third. Heard in all three, and the copied row explains only two. + const fixture = (name) => readFileSync(join(import.meta.dirname, '../fixtures/audio', name)); + + it('is still found by ear in all three', async () => { + const [a, b, c, t48, t80] = ['prog-a.mp3', 'prog-b.mp3', 'prog-c.mp3', 'theme-48k.mp3', 'theme-80k.mp3'].map(fixture); + const show = await makeShow('review', { + 'part-1.mp3': stitch(a, t48, b), + 'part-2.mp3': stitch(b, t48, c), + 'part-3.mp3': stitch(c, t80, a), + }); + server.db.prepare('UPDATE shows SET ad_auto_min_episodes = 2 WHERE id = ?').run(show.id); + await server.adPipeline.processShow(show.id); + + const byEar = rows(show).filter((r) => r.kind === SEGMENT_KINDS.REPEATED_AUDIO && r.duration_ms >= 4000); + const best = byEar.sort((x, y) => y.episode_count - x.episode_count)[0]; + assert.ok(best, `nothing found by ear: ${rows(show).map((r) => `${r.kind} ${r.episode_count}`).join(', ')}`); + assert.equal(best.episode_count, 3); + assert.ok(best.duration_ms >= 4500 && best.duration_ms <= 8000, `${best.duration_ms} ms`); + }); +}); + +describe('a show whose own theme follows a spoken intro', () => { + /* + * Le Rendez-vous Tech on Acast, 2026-10-28: pre-roll, marker, then the host's spoken + * intro ("Aujourd'hui on va parler de… c'est parti"), then the show's theme. One + * sponsored episode carries no adverts and so no marker, which makes the theme — not + * the marker — the only sound every episode opens with, and the jingle search + * proposed it. In automatic mode it confirmed itself and cut the intro (1.10.1 too). + */ + const theme = (n) => segment(5_000_000, frames(6), { variant: n % 8 }); + // Above every programme range (episodes 1–9 end below 2,000,000), so no intro shares + // a frame with anything else. + const intro = (n) => segment(3_000_000 + n * 10_000, frames(14 + n * 3)); + const marked = (n) => + stitch( + advert(100 + n * 10, 30), MARKER(), + intro(n), theme(n), programme(n, 240), + MARKER(), advert(101 + n * 10, 30), MARKER(), + programme(n, 240, 1), + MARKER(), MARKER(), advert(102 + n * 10, 25), + ); + const sponsored = (n) => stitch(intro(n), theme(n), programme(n, 500)); + const files = { 'e1.mp3': marked(1), 'e2.mp3': marked(2), 'e3.mp3': marked(3), 'e4.mp3': sponsored(4) }; + + const jingle = (show) => server.adDetect.listAnchors(show.id)[0] ?? null; + + it('keeps the intro: the theme is offered as a jingle, not confirmed, and only the adverts are cut', async () => { + const show = await makeShow('auto', files); + await server.adPipeline.processShow(show.id); + + const anchor = jingle(show); + assert.ok(anchor, 'no jingle was proposed: the scenario did not reproduce'); + assert.equal(anchor.confirmed_at, null, 'the theme was confirmed as the station jingle'); + assert.ok( + !rows(show).some((r) => r.kind === SEGMENT_KINDS.JINGLE), + 'a jingle row is cutting', + ); + for (const n of [1, 2, 3]) { + assert.deepEqual( + await served(show, `e${n}.mp3`), + hashesOf(intro(n), theme(n), programme(n, 240), programme(n, 240, 1)), + `e${n} is not exactly its intro, theme and programme`, + ); + } + }); + + it('takes back a jingle it confirmed by itself before, and cuts the pre-roll instead', async () => { + // An owner already on 1.10: the theme was confirmed automatically on an earlier pass. + const show = await makeShow('auto', files); + await server.adPipeline.processShow(show.id); + // What 1.10 left behind: the anchor confirmed by itself, and its row cutting from + // 0:00 to where the theme starts — written here as 1.10 wrote it, because this + // version's detector would take it back the moment it ran. + const anchor = jingle(show); + server.db + .prepare('UPDATE ad_anchors SET confirmed_at = ?, auto_confirmed = 1 WHERE id = ?') + .run('2026-10-01T00:00:00.000Z', anchor.id); + server.db + .prepare( + `INSERT INTO ad_segments (id, show_id, signature, source, kind, anchor_id, status, duration_ms, episode_count, + occurrence_count, first_seen_at, created_at, updated_at) + VALUES ('old-jingle', ?, ?, 'corpus', 'jingle', ?, 'approved', 50000, 3, 3, 'x', 'x', 'x')`, + ) + .run(show.id, `anchor:${anchor.id}`, anchor.id); + for (const hit of server.db.prepare('SELECT * FROM ad_anchor_hits WHERE anchor_id = ? AND heard = 1').all(anchor.id)) { + const onset = hit.at_ms - anchor.lead_ms; + if (onset < 2000) continue; + server.db + .prepare( + `INSERT INTO ad_segment_occurrences (segment_id, episode_id, start_frame, end_frame, start_ms, end_ms) + VALUES ('old-jingle', ?, 0, ?, 0, ?)`, + ) + .run(hit.episode_id, Math.floor(onset / FRAME_MS), onset); + server.db.prepare(`UPDATE episodes SET trim_status = 'pending' WHERE id = ?`).run(hit.episode_id); + } + await server.trimmer.trimShow(show.id); + assert.notDeepEqual( + await served(show, 'e1.mp3'), + hashesOf(intro(1), theme(1), programme(1, 240), programme(1, 240, 1)), + 'setup: the intro was not cut the way 1.10 cut it', + ); + + // A new episode arriving is what starts the next pass for real. + await writeFile(join(showDir, 'e5.mp3'), marked(5)); + await server.scanner.scanAllNow('manual'); + await server.adPipeline.processShow(show.id); + + assert.equal(jingle(show).confirmed_at, null, 'the automatic confirmation stood'); + assert.ok(!rows(show).some((r) => r.kind === SEGMENT_KINDS.JINGLE), 'the jingle still cuts'); + assert.deepEqual( + await served(show, 'e1.mp3'), + hashesOf(intro(1), theme(1), programme(1, 240), programme(1, 240, 1)), + 'e1 is not exactly its intro, theme and programme', + ); + }); + + it('leaves a jingle the owner confirmed alone', async () => { + const show = await makeShow('auto', files); + await server.adPipeline.processShow(show.id); + server.db + .prepare(`UPDATE ad_anchors SET confirmed_at = ?, auto_confirmed = 0, origin = 'pointed_at' WHERE show_id = ?`) + .run('2026-10-01T00:00:00.000Z', show.id); + await server.adDetect.detectAnchors(show.id); + assert.ok(jingle(show).confirmed_at, 'the owner’s decision was taken back'); + assert.ok(rows(show).some((r) => r.kind === SEGMENT_KINDS.JINGLE)); + }); +}); diff --git a/test/integration/subscriptions-ui.test.js b/test/integration/subscriptions-ui.test.js index 4e29935..36ca0de 100644 --- a/test/integration/subscriptions-ui.test.js +++ b/test/integration/subscriptions-ui.test.js @@ -135,6 +135,28 @@ describe('following a feed without JavaScript', () => { }); }); +describe('saying the host inserts audio', () => { + it('is saved from the switch, and turned off again by leaving it off', async () => { + // A switch that is off posts nothing at all, and this form posts every field — so + // "absent" has to mean off, or the switch could be turned on and never off. + await post('/ui/shows/tape-club/subscription', { feedUrl: `${origin}/feed.xml`, backfillCount: '10', insertsAudio: '1' }); + assert.equal(server.subscriptions.getForShow(show.id).inserts_audio, 1); + + const page = await server.get('/shows/tape-club/subscription'); + assert.match(page.body, /id="f-inserts"[^>]*checked/, 'the form does not show it on'); + + await post('/ui/shows/tape-club/subscription', { feedUrl: `${origin}/feed.xml`, backfillCount: '10' }); + assert.equal(server.subscriptions.getForShow(show.id).inserts_audio, 0); + }); + + it('uses no name an ad blocker hides', async () => { + // EasyList blanked the whole Adverts page once over a class called ad-panel. + const page = await server.get('/shows/tape-club/subscription'); + const names = [...page.body.matchAll(/\b(?:id|class)="([^"]*)"/g)].flatMap((m) => m[1].split(/\s+/)); + assert.deepEqual(names.filter((name) => /(^|[-_])(ad|ads|advert|sponsor)([-_]|$)/i.test(name)), []); + }); +}); + describe('following a feed with htmx', () => { it('returns the re-rendered form rather than a redirect', async () => { const response = await post( diff --git a/test/integration/transcript-detect.test.js b/test/integration/transcript-detect.test.js index 8d2569c..4b0816a 100644 --- a/test/integration/transcript-detect.test.js +++ b/test/integration/transcript-detect.test.js @@ -304,7 +304,8 @@ describe('hearing a sponsor read in the words', () => { describe('a pre-roll first found by ear', () => { /** Programme of its own, with the same forty-second pre-roll audio in front. */ function withPreRoll(n) { - return stitch(segment(2_000, framesFor(40)), segment(500_000 + n * 60_000, framesFor(30))); + // Re-encoded per episode (`variant`, helpers/mp3.js): found by ear, as the test says. + return stitch(segment(2_000, framesFor(40), { variant: n % 8 }), segment(500_000 + n * 60_000, framesFor(30))); } const preRollWords = (n) => whisperJson([ diff --git a/test/integration/trim-publish.test.js b/test/integration/trim-publish.test.js index 3ea70ae..e937ae1 100644 --- a/test/integration/trim-publish.test.js +++ b/test/integration/trim-publish.test.js @@ -30,9 +30,10 @@ async function makeShow() { join(showDir, `episode-${n}.mp3`), stitch( segment(100_000 + n * 50_000, framesFor(40)), - segment(2_000, framesFor(30)), + // The same sound in each episode, re-encoded each time: see `variant` in helpers/mp3.js. + segment(2_000, framesFor(30), { variant: n % 8 }), segment(600_000 + n * 50_000, framesFor(40)), - segment(3_000, framesFor(25)), + segment(3_000, framesFor(25), { variant: n % 8 }), ), ); } diff --git a/test/integration/trimmer.test.js b/test/integration/trimmer.test.js index 33f9b97..8e7e7d1 100644 --- a/test/integration/trimmer.test.js +++ b/test/integration/trimmer.test.js @@ -365,7 +365,8 @@ describe('when an episode or a show goes', () => { await app.adDetect.fingerprintShow(show.id); const { readdir } = await import('node:fs/promises'); assert.equal((await readdir(join(app.config.trimmedDir, show.id))).length, 3); - assert.equal((await readdir(join(app.config.fingerprintDir, show.id))).length, 3); + // A fingerprint and the frame hashes beside it, per episode. + assert.equal((await readdir(join(app.config.fingerprintDir, show.id))).length, 6); const [episode] = app.episodes.listByShow(show.id); await app.episodes.deleteWithFile(episode.id); @@ -375,7 +376,7 @@ describe('when an episode or a show goes', () => { 2, 'a deleted episode left its trimmed copy behind', ); - assert.equal((await readdir(join(app.config.fingerprintDir, show.id))).length, 2); + assert.equal((await readdir(join(app.config.fingerprintDir, show.id))).length, 4); await app.shows.remove(show.id, { deleteFiles: false }); diff --git a/test/unit/auto-approve.test.js b/test/unit/auto-approve.test.js index fb79aea..ac90e15 100644 --- a/test/unit/auto-approve.test.js +++ b/test/unit/auto-approve.test.js @@ -62,3 +62,66 @@ describe('a segment found by ear, once the words are known', () => { assert.deepEqual(safeToApproveAutomatically(theme, options), { safe: false, reason: 'always_at_the_start' }); }); }); + +describe('audio the host inserted, found by its frames', () => { + // RMC's real opener: 13.9 s at 0:00 of every episode. The acoustic branch refuses it + // twice over — at the start, and under fifteen seconds — and is right to: a theme tune + // has exactly that shape. Copied frames are what tell the two apart. + const opener = () => segment({ startMs: 0, durationMs: 13_900 }); + + it('cuts an opener at 0:00 that the acoustic branch would hold as a theme tune', () => { + assert.equal(safeToApproveAutomatically(opener(), { episodeDurations: durations }).safe, false); + assert.deepEqual(safeToApproveAutomatically(opener(), { source: 'inserted' }), { safe: true, reason: null }); + }); + + it('cuts a closer of ten seconds, under the acoustic floor', () => { + const closer = segment({ startMs: 590_000, durationMs: 10_100 }); + assert.equal(safeToApproveAutomatically(closer, { source: 'inserted' }).safe, true); + }); + + it('offers a copy in exactly two episodes, mid-file — a best-of may reuse programme so', () => { + const twice = segment({ episodes: ['a', 'b'], startMs: 200_000, durationMs: 45_000 }); + assert.deepEqual( + safeToApproveAutomatically(twice, { source: 'inserted', episodeDurations: durations }), + { safe: false, reason: 'seen_too_few_times' }, + ); + }); + + it('cuts a copy in two episodes where a host puts one: the very start, or the very end', () => { + const preRoll = segment({ episodes: ['a', 'b'], startMs: 0, durationMs: 30_000 }); + const postRoll = segment({ episodes: ['a', 'b'], startMs: 570_000, durationMs: 30_000 }); + for (const copy of [preRoll, postRoll]) { + assert.equal(safeToApproveAutomatically(copy, { source: 'inserted', episodeDurations: durations }).safe, true); + } + }); + + it('offers a copy in two episodes at their edges when it runs over a minute', () => { + // A recap or a reused segment opening one episode and closing another. + const long = segment({ episodes: ['a', 'b'], startMs: 0, durationMs: 90_000 }); + assert.deepEqual( + safeToApproveAutomatically(long, { source: 'inserted', episodeDurations: durations }), + { safe: false, reason: 'seen_too_few_times' }, + ); + }); + + it('cuts a copy mid-file once it reaches a third episode', () => { + const thrice = segment({ episodes: ['a', 'b', 'c'], startMs: 200_000, durationMs: 45_000 }); + assert.equal(safeToApproveAutomatically(thrice, { source: 'inserted', episodeDurations: durations }).safe, true); + }); + + it('holds a copy that is not really in two episodes', () => { + const once = segment({ episodes: ['a'], startMs: 0, durationMs: 20_000 }); + assert.deepEqual(safeToApproveAutomatically(once, { source: 'inserted' }), { safe: false, reason: 'only_heard_once' }); + }); + + it('cuts a break between markers, whose markers are its evidence', () => { + const block = { durationMs: 72_300, episodeCount: 1, betweenMarkers: true }; + assert.equal(safeToApproveAutomatically(block, { source: 'inserted' }).safe, true); + }); + + it('holds anything longer than two and a half minutes, however it was found', () => { + for (const evidence of [segment({ durationMs: 151_000 }), { durationMs: 151_000, episodeCount: 1, betweenMarkers: true }]) { + assert.deepEqual(safeToApproveAutomatically(evidence, { source: 'inserted' }), { safe: false, reason: 'too_long_to_be_an_advert' }); + } + }); +}); diff --git a/test/unit/copied-audio.test.js b/test/unit/copied-audio.test.js new file mode 100644 index 0000000..bbe4a05 --- /dev/null +++ b/test/unit/copied-audio.test.js @@ -0,0 +1,183 @@ +import assert from 'node:assert/strict'; +import { describe, it } from 'node:test'; + +import { findCopiedAudio, signatureOf } from '../../src/lib/copied-audio.js'; +import { frameProfile } from '../../src/lib/mp3-frames.js'; +import { FRAME_MS, frame, segment, stitch } from '../helpers/mp3.js'; + +/** + * Synthesised episodes, frame by frame. A seed range is "the same audio, encoded the same + * way": two episodes built with the same seeds share those frames byte for byte, exactly + * as an advert a host concatenates in does — and episodes built with different seeds + * share nothing, as two separately mastered programmes do. + * + * Nothing here can stand in for a real host. The shapes are taken from what two real + * ones served on 2026-10-28 (scripts/check-copied-audio.mjs): RMC's opener at 0:00 in + * two versions differing in their last frame; Acast's 2.2 s marker at the edge of every + * break, twice back to back at the start of a post-roll. + */ +const frames = (seconds) => Math.round((seconds * 1000) / FRAME_MS); + +function episode(id, ...parts) { + const profile = frameProfile(stitch(...parts)); + return { id, hashes: profile.hashes, frameMs: profile.durationMs / profile.frameCount }; +} + +/* + * Seeds are frame numbers, and a part of N frames uses N consecutive ones — so parts + * are spaced further apart than any part is long, or two "different" adverts would share + * frames and the tests would be about that instead. Adverts (at most 100 s, 3,830 + * frames) sit 10,000 apart below a million; programme halves (at most 900 s, 34,456 + * frames) 50,000 apart above it, and episodes 100,000 apart. + */ +const programme = (n, seconds, part = 0) => segment(1_000_000 + n * 100_000 + part * 50_000, frames(seconds)); +const advert = (seed, seconds) => segment(seed * 10, frames(seconds)); +const MARKER = () => advert(7_000, 2.2); + +describe('finding audio a host copied into several episodes', () => { + it('finds a stretch that is the same frames in two episodes, exactly where it is', () => { + const result = findCopiedAudio([ + episode('a', programme(1, 60), advert(500, 30), programme(1, 60, 1)), + episode('b', programme(2, 45), advert(500, 30), programme(2, 90, 1)), + ]); + assert.equal(result.copied.length, 1); + const [row] = result.copied; + assert.equal(row.episodeCount, 2); + const byEpisode = Object.fromEntries(row.occurrences.map((o) => [o.episodeId, [o.startFrame, o.endFrame]])); + assert.deepEqual(byEpisode, { a: [frames(60), frames(60) + frames(30)], b: [frames(45), frames(45) + frames(30)] }); + }); + + it('finds nothing between episodes that share no frames — separately mastered programmes', () => { + const result = findCopiedAudio([ + episode('a', programme(1, 120)), + episode('b', programme(2, 120)), + episode('c', programme(3, 120)), + ]); + assert.deepEqual(result, { copied: [], markers: [], blocks: [] }); + }); + + it('does not call shared silence a copy', () => { + // Silence encodes to the same frame over and over; two episodes with a second of it + // each would otherwise "share" it. + const silence = Buffer.concat(Array.from({ length: frames(3) }, () => frame(42))); + const result = findCopiedAudio([ + episode('a', programme(1, 30), silence, programme(1, 30, 1)), + episode('b', programme(2, 30), silence, programme(2, 30, 1)), + ]); + assert.deepEqual(result.copied, []); + }); + + it('ignores a shared stretch shorter than a second and a half', () => { + const result = findCopiedAudio([ + episode('a', programme(1, 30), advert(500, 1.2), programme(1, 30, 1)), + episode('b', programme(2, 30), advert(500, 1.2), programme(2, 30, 1)), + ]); + assert.deepEqual(result.copied, []); + }); + + it('is one row for an opener the host keeps in two versions differing in the last frame', () => { + // RMC: 533 frames in each version, 532 in common, each version in four episodes. + const head = advert(500, 13); + const versionA = stitch(head, frame(900_001)); + const versionB = stitch(head, frame(900_002)); + const result = findCopiedAudio([ + episode('a1', versionA, programme(1, 200)), + episode('a2', versionA, programme(2, 200)), + episode('b1', versionB, programme(3, 200)), + episode('b2', versionB, programme(4, 200)), + ]); + assert.equal(result.copied.length, 1, `one opener came back as ${result.copied.length} rows`); + assert.equal(result.copied[0].episodeCount, 4); + for (const o of result.copied[0].occurrences) assert.equal(o.startFrame, 0); + }); + + it('gives the same frames the same signature in whichever episode they are found', () => { + const a = findCopiedAudio([episode('a', advert(500, 20), programme(1, 60)), episode('b', advert(500, 20), programme(2, 60))]); + const b = findCopiedAudio([episode('x', programme(8, 30), advert(500, 20)), episode('y', advert(500, 20), programme(9, 60))]); + assert.equal(a.copied[0].signature, b.copied[0].signature); + assert.match(a.copied[0].signature, /^copied:[0-9a-f]{24}$/); + assert.equal(signatureOf([1, 2, 3], 'break').slice(0, 6), 'break:'); + }); +}); + +describe('breaks between the markers a host stitches around them', () => { + /** Acast's shape: pre-roll, a mid-roll, a post-roll that runs to the end. */ + const acast = (n, preroll, midroll, postroll) => + episode( + `e${n}`, + preroll, MARKER(), + programme(n, 600), + MARKER(), midroll, MARKER(), + programme(n, 600, 1), + MARKER(), MARKER(), postroll, + ); + + it('cuts the breaks, markers included — even adverts heard nowhere else', () => { + const result = findCopiedAudio([ + acast(1, advert(10_000, 30), advert(11_000, 60), advert(12_000, 40)), + acast(2, advert(20_000, 45), advert(21_000, 30), advert(22_000, 35)), + ]); + assert.equal(result.markers.length, 1, 'the 2.2 s clip was not taken for a marker'); + const e1 = result.blocks.filter((b) => b.episodeId === 'e1').map((b) => [b.startFrame, b.endFrame]); + const m = frames(2.2); + const pre = frames(30) + m; + const mid = [pre + frames(600), pre + frames(600) + m + frames(60) + m]; + const postFrom = mid[1] + frames(600); + assert.deepEqual(e1, [[0, pre], mid, [postFrom, postFrom + 2 * m + frames(40)]]); + for (const block of result.blocks) assert.match(block.signature, /^break:/); + // One slot each, so the catalogue can keep a row per slot and a row is in each + // episode once — which is what "Restore here" restores. + assert.deepEqual(result.blocks.filter((b) => b.episodeId === 'e1').map((b) => b.slot), ['pre', 'mid-1', 'post']); + }); + + it('keeps the marker a marker when an advert next to it is copied too', () => { + // Found on the real show: a 30 s advert copied with the marker beside it was folded + // into the marker's row and stretched every marker to 33 seconds. It takes the + // marker being in more episodes than the advert, as it was there — the row in more + // episodes is the one others are folded into. + const shared = advert(30_000, 30); + const result = findCopiedAudio([ + acast(1, shared, advert(11_000, 60), advert(12_000, 40)), + acast(2, shared, advert(21_000, 30), advert(22_000, 35)), + acast(3, advert(31_000, 30), advert(32_000, 30), advert(33_000, 35)), + ]); + const marker = result.copied.find((row) => result.markers.includes(row.signature)); + assert.ok(marker, 'no marker row'); + for (const o of marker.occurrences) { + assert.ok(o.endFrame - o.startFrame <= frames(2.2) + 1, `a marker occurrence is ${o.endMs - o.startMs} ms long`); + } + }); + + it('says nothing about an episode with no stretch long enough to be the programme', () => { + // A two-minute episode with markers in it cannot tell programme from break this way: + // every stretch in it is short enough to be a break, so they add up to most of it. + const short = episode('short', advert(10_000, 20), MARKER(), programme(1, 100), MARKER(), advert(11_000, 20)); + const other = episode('long', advert(20_000, 20), MARKER(), programme(2, 900), MARKER(), MARKER(), advert(21_000, 20)); + const result = findCopiedAudio([short, other]); + assert.ok(result.markers.length, 'no marker'); + assert.deepEqual(result.blocks.filter((b) => b.episodeId === 'short'), []); + assert.ok(result.blocks.some((b) => b.episodeId === 'long')); + }); + + it('never makes breaks of most of an episode', () => { + // Markers close together everywhere but one long stretch: more break than programme. + const parts = []; + for (let i = 0; i < 12; i += 1) parts.push(advert(40_000 + i * 5_000, 100), MARKER()); + const busy = episode('busy', ...parts, programme(1, 200)); + const other = episode('other', MARKER(), programme(2, 900), MARKER()); + const result = findCopiedAudio([busy, other]); + assert.deepEqual(result.blocks.filter((b) => b.episodeId === 'busy'), []); + }); + + it('does not take a short copy heard once per episode for a marker', () => { + // An inserted ident at 0:00 of every episode is copied audio, not a slot edge. + const result = findCopiedAudio([ + episode('a', advert(500, 3), programme(1, 600)), + episode('b', advert(500, 3), programme(2, 600)), + episode('c', advert(500, 3), programme(3, 600)), + ]); + assert.equal(result.copied.length, 1); + assert.deepEqual(result.markers, []); + assert.deepEqual(result.blocks, []); + }); +}); diff --git a/test/unit/migrations.test.js b/test/unit/migrations.test.js index 03e23ab..d8f2d0f 100644 --- a/test/unit/migrations.test.js +++ b/test/unit/migrations.test.js @@ -443,3 +443,100 @@ describe('migration 012 indexes the advert tables\' foreign keys', () => { assert.match(plan, /idx_ad_segments_anchor/, `a lookup by anchor does not use the index: ${plan}`); }); }); + +/** + * Migration 014 rebuilds the catalogue again, to let `kind` say `inserted`. The same + * stake as 009: a rebuild that dropped a decision, an occurrence or a restore would + * republish adverts the owner removed, or remove audio they put back — silently. + */ +describe('migration 014 lets the catalogue hold inserted audio without losing anything', () => { + function seededAt013() { + const db = seededAt010(); + runMigrations(db, { upTo: 13 }); + assert.equal(db.pragma('user_version', { simple: true }), 13); + db.prepare( + `INSERT INTO ad_cut_overrides (id, episode_id, segment_id, start_ms, end_ms, created_at) + VALUES ('restore', 'e1', 'audio', 0, 9000, '2026-10-01T00:00:00.000Z')`, + ).run(); + db.prepare( + `INSERT INTO feed_subscriptions (id, show_id, feed_url, created_at, updated_at) + VALUES ('sub', 's1', 'https://example.com/feed.xml', 'x', 'x')`, + ).run(); + return db; + } + + const snapshot = (db) => ({ + segments: db.prepare('SELECT * FROM ad_segments ORDER BY id').all(), + occurrences: db.prepare('SELECT * FROM ad_segment_occurrences ORDER BY id').all(), + overrides: db.prepare('SELECT * FROM ad_cut_overrides ORDER BY id').all(), + }); + + it('keeps every segment, occurrence and restore exactly as it was', () => { + const db = seededAt013(); + const before = snapshot(db); + // Eight rows of every kind 1.10 writes (011 tidied two of the ten seeded away), one restore. + assert.deepEqual([before.segments.length, before.occurrences.length, before.overrides.length], [8, 8, 1]); + runMigrations(db, { upTo: 14 }); + assert.equal(db.pragma('user_version', { simple: true }), 14); + assert.deepEqual(snapshot(db), before); + assert.deepEqual(db.pragma('foreign_key_check'), []); + }); + + it('accepts the new kind and still refuses one that does not exist', () => { + const db = seededAt013(); + runMigrations(db, { upTo: 14 }); + const insert = db.prepare( + `INSERT INTO ad_segments (id, show_id, signature, source, status, duration_ms, first_seen_at, created_at, updated_at, kind) + VALUES (?, 's1', ?, 'corpus', 'candidate', 1000, 'x', 'x', 'x', ?)`, + ); + insert.run('ins', 'copied:abc', 'inserted'); + assert.throws(() => insert.run('bad', 'copied:def', 'not_a_kind'), /CHECK constraint failed/); + }); + + it('points the children at the rebuilt table: deleting a row cascades and releases its restore', () => { + const db = seededAt013(); + runMigrations(db, { upTo: 14 }); + for (const table of ['ad_segment_occurrences', 'ad_cut_overrides']) { + const targets = db.pragma(`foreign_key_list(${table})`).filter((fk) => fk.from === 'segment_id').map((fk) => fk.table); + assert.deepEqual(targets, ['ad_segments'], `${table} points at ${targets}`); + } + db.prepare(`DELETE FROM ad_segments WHERE id = 'audio'`).run(); + assert.equal(db.prepare(`SELECT COUNT(*) AS n FROM ad_segment_occurrences WHERE segment_id = 'audio'`).get().n, 0); + assert.equal(db.prepare(`SELECT segment_id FROM ad_cut_overrides WHERE id = 'restore'`).get().segment_id, null); + }); + + it('keeps the indexes, continues the occurrence ids, and flags no subscription yet', () => { + const db = seededAt013(); + const lastId = db.prepare('SELECT MAX(id) AS id FROM ad_segment_occurrences').get().id; + runMigrations(db, { upTo: 14 }); + for (const name of [ + 'idx_ad_segments_show', 'idx_ad_segments_kind', 'idx_ad_segments_marker', 'idx_ad_segments_anchor', + 'idx_ad_occurrences_episode', 'idx_ad_cut_overrides_episode', 'idx_ad_cut_overrides_segment', + ]) { + assert.ok(db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?").get(name), `${name} is missing`); + } + const next = db + .prepare(`INSERT INTO ad_segment_occurrences (segment_id, episode_id, start_frame, end_frame, start_ms, end_ms) + VALUES ('diff', 'e2', 0, 10, 0, 261)`) + .run().lastInsertRowid; + assert.ok(Number(next) > lastId, 'a new occurrence reused an id'); + assert.equal(db.prepare(`SELECT inserts_audio FROM feed_subscriptions WHERE id = 'sub'`).get().inserts_audio, 0); + }); + + it('still takes the statements 1.10 runs, so rolling back is harmless', () => { + const db = seededAt013(); + runMigrations(db, { upTo: 14 }); + // What 1.10's upsertSegment and reconcileKinds write, unchanged. + db.prepare( + `INSERT INTO ad_segments (id, show_id, signature, source, status, auto_approved, hold_reason, duration_ms, + episode_count, occurrence_count, exemplar_episode_id, exemplar_start_ms, exemplar_end_ms, + first_seen_at, created_at, updated_at, kind) + VALUES ('old', 's1', 'feedfacefeedfacefeedface', 'corpus', 'candidate', 0, NULL, 5000, 2, 2, 'e1', 0, 5000, + 'x', 'x', 'x', 'repeated_audio')`, + ).run(); + db.prepare( + `UPDATE ad_segments SET kind = 'diff' WHERE show_id = 's1' AND kind = 'repeated_audio' AND source = 'diff'`, + ).run(); + assert.equal(db.prepare(`SELECT kind FROM ad_segments WHERE id = 'old'`).get().kind, 'repeated_audio'); + }); +});