From 8046bd68ccf23a945d180fff8511ccb7cea0af22 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 12:53:44 +0100 Subject: [PATCH 01/20] Add shared video pipeline helpers, test runner, and justfile recipes - scripts/videolib.py: frontmatter and VTT parsing, logging setup (Rich in a terminal, plain UTC-timestamped lines in CI), RunSummary with exit codes 0/1/2 and --summary-json, atomic writes, and retries with backoff - scripts/videotesting.py: fakes for YouTube, yt-dlp, Whisper, and Anthropic - scripts/run-tests.sh and 'just test' run every scripts/test-*.py - Remove the stale download-youtube-audio recipe, pass arguments through to the video scripts, and add 'just update-videos' for local runs --- justfile | 46 ++- scripts/run-tests.sh | 25 ++ scripts/test-validate-blog-posts.py | 4 + scripts/test-videolib.py | 380 ++++++++++++++++++ scripts/videolib.py | 577 ++++++++++++++++++++++++++++ scripts/videotesting.py | 373 ++++++++++++++++++ 6 files changed, 1394 insertions(+), 11 deletions(-) create mode 100755 scripts/run-tests.sh mode change 100644 => 100755 scripts/test-validate-blog-posts.py create mode 100755 scripts/test-videolib.py create mode 100644 scripts/videolib.py create mode 100644 scripts/videotesting.py diff --git a/justfile b/justfile index abc229f8c..f1029c13d 100644 --- a/justfile +++ b/justfile @@ -70,27 +70,51 @@ extract-software-colors: create-hex-logos: ./scripts/create-hex-logos.py -# Update videos -update-youtube-videos: - ./scripts/update-youtube-videos.py +# Fetch video metadata from YouTube into data/videos.toml (pass --refresh-stats to update view counts) +update-youtube-videos *args: + ./scripts/update-youtube-videos.py "$@" # Sync video entries from data/videos.toml into content -sync-videos: - ./scripts/sync-videos.py +sync-videos *args: + ./scripts/sync-videos.py "$@" # Sync courses and learning paths from Posit Academy (needs SKILLJAR_API_KEY) sync-courses *args: ./scripts/sync-courses.py {{args}} -download-youtube-audio *args: - ./scripts/download-youtube-audio.py {{args}} - -transcribe: - ./scripts/transcribe.py +# Download audio and transcribe new videos with Whisper (needs OPENAI_API_KEY) +transcribe *args: + ./scripts/transcribe.py "$@" # Format VTT transcriptions into HTML using the Anthropic API format-transcriptions *args: - ./scripts/format-transcriptions.py {{args}} + ./scripts/format-transcriptions.py "$@" + +# Run the whole video pipeline locally (fallback for the nightly workflow), e.g. `just update-videos 5 chrome` +update-videos limit="15" browser="": + #!/usr/bin/env bash + set -euo pipefail + # Exit code 2 means some videos failed; keep going and fix them later. + step() { + "$@" && return 0 + status=$? + if [ "$status" -eq 2 ]; then + echo "⚠ $1 finished with errors (exit code 2); continuing" >&2 + return 0 + fi + echo "✗ $1 failed (exit code $status); stopping" >&2 + exit "$status" + } + cookies=() + if [ -n "{{ browser }}" ]; then cookies=(--cookies-from-browser "{{ browser }}"); fi + step ./scripts/update-youtube-videos.py + step ./scripts/sync-videos.py + step ./scripts/transcribe.py --limit "{{ limit }}" ${cookies[@]+"${cookies[@]}"} + step ./scripts/format-transcriptions.py --limit "{{ limit }}" + +# Run the tests of the Python scripts (extra arguments go to pytest) +test *args: + ./scripts/run-tests.sh "$@" quarto-preview: uv run quarto preview diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh new file mode 100755 index 000000000..0130c4a7e --- /dev/null +++ b/scripts/run-tests.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Run every scripts/test-*.py. Each test file is a PEP 723 uv script that +# declares its own dependencies, so each one runs in its own environment. +# Extra arguments are passed on to pytest, e.g. `./scripts/run-tests.sh -q -x`. + +set -uo pipefail +cd "$(dirname "$0")/.." + +failed=() +for test_file in scripts/test-*.py; do + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then echo "::group::$test_file"; else echo "==> $test_file"; fi + if ! uv run --quiet --script "$test_file" "$@"; then + failed+=("$test_file") + fi + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then echo "::endgroup::"; fi +done + +if ((${#failed[@]})); then + echo + echo "Failed test files:" + printf ' %s\n' "${failed[@]}" + exit 1 +fi +echo +echo "All test files passed." diff --git a/scripts/test-validate-blog-posts.py b/scripts/test-validate-blog-posts.py old mode 100644 new mode 100755 index 8c8b51bfc..820498e22 --- a/scripts/test-validate-blog-posts.py +++ b/scripts/test-validate-blog-posts.py @@ -566,3 +566,7 @@ def test_subdirectory_image(self, tmp_path): (p.parent / "assets" / "hero.png").touch() fm = {"image": "assets/hero.png"} assert v.check_image_exists(p, fm, ctx) == [] + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/test-videolib.py b/scripts/test-videolib.py new file mode 100755 index 000000000..f3fff98fc --- /dev/null +++ b/scripts/test-videolib.py @@ -0,0 +1,380 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for videolib.py, the shared helpers of the video pipeline.""" + +import asyncio +import json +import logging +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 +from videotesting import RateLimitError, StatusError, no_sleep # noqa: E402 + +# --- VTT --- + + +class TestVttTimestamps: + @pytest.mark.parametrize( + "ts, seconds", + [ + ("00:00:00.000", 0.0), + ("00:01:30.500", 90.5), + ("01:02:03.100", 3723.1), + ("02:05.250", 125.25), # no hours + ], + ) + def test_parse(self, ts, seconds): + assert videolib.parse_vtt_timestamp(ts) == pytest.approx(seconds) + + @pytest.mark.parametrize( + "seconds, ts", + [ + (0, "00:00:00.000"), + (90.5, "00:01:30.500"), + (3723.1, "01:02:03.100"), + (59.9996, "00:01:00.000"), # rounds up instead of "00:00:60.000" + ], + ) + def test_format(self, seconds, ts): + assert videolib.format_vtt_timestamp(seconds) == ts + + +class TestParseVtt: + def test_basic(self): + vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:02.500\nHello\n\n00:00:03.000 --> 00:00:04.000\nWorld\n" + assert videolib.parse_vtt(vtt) == [(1.0, 2.5, "Hello"), (3.0, 4.0, "World")] + + def test_multiline_cue_and_identifier(self): + vtt = "WEBVTT\n\ncue-1\n00:00:01.000 --> 00:00:02.000\nline one\nline two\n" + assert videolib.parse_vtt(vtt) == [(1.0, 2.0, "line one\nline two")] + + def test_empty_cues_dropped(self): + vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\n\n\n00:00:03.000 --> 00:00:04.000\nkept\n" + assert videolib.parse_vtt(vtt) == [(3.0, 4.0, "kept")] + + def test_crlf(self): + vtt = "WEBVTT\r\n\r\n00:00:01.000 --> 00:00:02.000\r\nHi\r\n" + assert videolib.parse_vtt(vtt) == [(1.0, 2.0, "Hi")] + + def test_not_vtt(self): + assert videolib.parse_vtt("") == [] + assert videolib.parse_vtt("just some text") == [] + + def test_round_trip(self): + cues = [(0.0, 4.0, "a"), (4.0, 9.5, "b c"), (3600.25, 3605.0, "d")] + assert videolib.parse_vtt(videolib.render_vtt(cues)) == cues + + +# --- Frontmatter --- + + +class TestFrontmatter: + def test_split(self): + fm, section, rest = videolib.split_frontmatter("---\ntitle: X\n---\nbody\n") + assert fm == {"title": "X"} + assert section == "title: X" + assert rest == "body\n" + + @pytest.mark.parametrize( + "content", + [ + "no frontmatter", + "---\ntitle: X\nnever closed\n", + "---\ntitle: [unclosed\n---\n", + "---\n- a list\n---\n", + ], + ) + def test_invalid(self, content): + fm, section, rest = videolib.split_frontmatter(content) + assert fm == {} and section == "" and rest == content + + def test_read_missing_file(self, tmp_path): + assert videolib.read_frontmatter(tmp_path / "nope.md") == {} + + +# --- Files --- + + +class TestAtomicWrite: + def test_writes(self, tmp_path): + p = tmp_path / "sub" / "f.txt" + videolib.atomic_write_text(p, "héllo") + assert p.read_text(encoding="utf-8") == "héllo" + + def test_failure_leaves_original_and_no_temp_files(self, tmp_path, monkeypatch): + p = tmp_path / "f.txt" + p.write_text("original") + + def boom(*_): + raise OSError("disk full") + + monkeypatch.setattr(videolib.os, "replace", boom) + with pytest.raises(OSError): + videolib.atomic_write_text(p, "new") + assert p.read_text() == "original" + assert list(tmp_path.iterdir()) == [p] + + +# --- Video directories --- + + +class TestVideoDirs: + def test_newest_first(self, tmp_path): + for name in ["2024-01-01_a", "2026-05-01_b", "2025-03-03_c"]: + (tmp_path / name).mkdir() + (tmp_path / "file.txt").write_text("x") + assert [d.name for d in videolib.video_dirs(tmp_path)] == [ + "2026-05-01_b", + "2025-03-03_c", + "2024-01-01_a", + ] + + def test_resolve(self, tmp_path): + (tmp_path / "2026-01-01_talk-one").mkdir() + (tmp_path / "2026-01-01_talk-two").mkdir() + (tmp_path / "2025-01-01_other").mkdir() + assert videolib.resolve_video_dir("2025", tmp_path).name == "2025-01-01_other" + assert videolib.resolve_video_dir("2026-01-01_talk-one", tmp_path).name == "2026-01-01_talk-one" + with pytest.raises(ValueError, match="Ambiguous"): + videolib.resolve_video_dir("2026-01-01_talk", tmp_path) + with pytest.raises(ValueError, match="not found"): + videolib.resolve_video_dir("1999", tmp_path) + + @pytest.mark.parametrize( + "url, vid", + [ + ("https://www.youtube.com/watch?v=ohUZNk5WCGQ", "ohUZNk5WCGQ"), + ("https://youtu.be/ohUZNk5WCGQ", "ohUZNk5WCGQ"), + ("https://www.youtube.com/shorts/ohUZNk5WCGQ", "ohUZNk5WCGQ"), + ("https://example.com", None), + (None, None), + ], + ) + def test_youtube_id(self, url, vid): + assert videolib.youtube_id(url) == vid + + +# --- Run summary --- + + +class TestRunSummary: + def test_exit_codes(self): + s = videolib.RunSummary("x") + s.add("created", slug="a") + assert s.exit_code == videolib.EXIT_OK + s.add("error", slug="b", reason="boom") + assert s.exit_code == videolib.EXIT_PARTIAL + s.fatal = "no API key" + assert s.exit_code == videolib.EXIT_FATAL + + def test_counts_merge_items_and_tallies(self): + s = videolib.RunSummary("x") + s.add("created", slug="a") + s.add("created", slug="b") + s.tally("unchanged", 1700) + s.tally("created") + assert s.counts() == {"created": 3, "unchanged": 1700} + + def test_none_fields_dropped(self): + s = videolib.RunSummary("x") + item = s.add("error", slug="a", reason=None, detail="d") + assert item == {"status": "error", "slug": "a", "detail": "d"} + + def test_json_schema(self, tmp_path): + s = videolib.RunSummary("sync-videos") + s.add("error", slug="a", reason="boom") + s.extra["dry_run"] = False + path = tmp_path / "out" / "summary.json" + s.write(path) + data = json.loads(path.read_text()) + assert set(data) == { + "script", + "started_at", + "finished_at", + "exit_code", + "fatal", + "counts", + "extra", + "environment", + "items", + } + assert data["script"] == "sync-videos" + assert data["exit_code"] == 2 + assert data["counts"] == {"error": 1} + assert data["finished_at"] is not None + + def test_run_main_turns_exceptions_into_fatal(self, tmp_path, caplog): + s = videolib.RunSummary("x") + path = tmp_path / "s.json" + + def body(): + raise RuntimeError("kaput") + + with caplog.at_level(logging.ERROR): + code = videolib.run_main(s, logging.getLogger("t"), path, body) + assert code == 1 + assert json.loads(path.read_text())["fatal"] == "RuntimeError: kaput" + assert "Unhandled error" in caplog.text + + def test_run_main_ok(self, tmp_path): + s = videolib.RunSummary("x") + assert videolib.run_main(s, logging.getLogger("t"), None, lambda: None) == 0 + + +# --- Retries --- + + +class TestRetryableErrors: + @pytest.mark.parametrize( + "error, retryable", + [ + (RateLimitError(), True), + (StatusError(500), True), + (StatusError(529), True), # Anthropic "overloaded" + (StatusError(408), True), + (StatusError(400), False), + (StatusError(401), False), + (TimeoutError(), True), + (ValueError("bad"), False), + ], + ) + def test_is_retryable(self, error, retryable): + assert videolib.is_retryable_api_error(error) is retryable + + def test_connection_error_by_class_name(self): + class APIConnectionError(Exception): + pass + + class APITimeoutError(APIConnectionError): + pass + + assert videolib.is_retryable_api_error(APITimeoutError()) + + def test_describe(self): + assert videolib.describe_api_error(StatusError(429, "slow down")) == ( + "StatusError status=429 request_id=req_429: slow down" + ) + assert videolib.describe_api_error(ValueError()) == "ValueError" + + +class TestRetryAsync: + def run(self, fn, **kwargs): + sleeps = [] + + async def fake_sleep(s): + sleeps.append(s) + + result = asyncio.run( + videolib.retry_async( + fn, label="video-x", logger=logging.getLogger("t"), sleep=fake_sleep, **kwargs + ) + ) + return result, sleeps + + def test_retries_then_succeeds(self, caplog): + outcomes = [RateLimitError(), StatusError(503), "ok"] + + async def fn(): + r = outcomes.pop(0) + if isinstance(r, Exception): + raise r + return r + + with caplog.at_level(logging.WARNING): + result, sleeps = self.run(fn, base_delay=1.0) + assert result == "ok" + assert len(sleeps) == 2 + assert 1.0 <= sleeps[0] < 2.0 and 2.0 <= sleeps[1] < 3.0 # exponential + jitter + assert "video-x: attempt 1/5 failed: RateLimitError status=429" in caplog.text + assert "attempt 2/5 failed: StatusError status=503" in caplog.text + + def test_non_retryable_raises_immediately(self, caplog): + calls = [] + + async def fn(): + calls.append(1) + raise StatusError(400, "bad request") + + with caplog.at_level(logging.ERROR), pytest.raises(StatusError): + self.run(fn) + assert calls == [1] + assert "not retryable" in caplog.text + + def test_gives_up(self, caplog): + calls = [] + + async def fn(): + calls.append(1) + raise StatusError(500) + + with caplog.at_level(logging.ERROR), pytest.raises(StatusError): + self.run(fn, attempts=3) + assert len(calls) == 3 + assert "giving up after 3 attempts" in caplog.text + + +class TestRateLimiter: + def test_spacing(self): + async def go(): + limiter = videolib.RateLimiter(min_interval=0.05) + loop = asyncio.get_running_loop() + times = [] + for _ in range(3): + await limiter.wait() + times.append(loop.time()) + return times + + times = asyncio.run(go()) + assert times[1] - times[0] >= 0.045 + assert times[2] - times[1] >= 0.045 + + +# --- Logging --- + + +class TestLogging: + def test_plain_format_in_ci(self, monkeypatch, capsys): + monkeypatch.setenv("CI", "true") + logger = videolib.setup_logging("test-logger") + logger.info("hello %s", "world") + err = capsys.readouterr().err + assert "INFO test-logger: hello world" in err + assert err[:4].isdigit() and "Z " in err # UTC timestamp first + + def test_verbose_and_env_level(self, monkeypatch): + monkeypatch.setenv("CI", "true") + assert videolib.setup_logging("a", verbose=True).getEffectiveLevel() == logging.DEBUG + monkeypatch.setenv("LOG_LEVEL", "warning") + assert videolib.setup_logging("a").getEffectiveLevel() == logging.WARNING + + def test_noisy_loggers_quieted(self, monkeypatch): + monkeypatch.setenv("CI", "true") + videolib.setup_logging("a") + assert logging.getLogger("httpx").level == logging.WARNING + + def test_log_environment(self, caplog): + with caplog.at_level(logging.INFO): + env = videolib.log_environment( + logging.getLogger("t"), packages=["pyyaml", "no-such-package-xyz"], tools=["no-such-tool-xyz"] + ) + assert env["packages"]["pyyaml"] + assert env["packages"]["no-such-package-xyz"] is None + assert env["tools"]["no-such-tool-xyz"] is None + assert "tool no-such-tool-xyz: NOT FOUND" in caplog.text + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/videolib.py b/scripts/videolib.py new file mode 100644 index 000000000..9f6483426 --- /dev/null +++ b/scripts/videolib.py @@ -0,0 +1,577 @@ +""" +Shared helpers for the video pipeline scripts: + + update-youtube-videos.py -> sync-videos.py -> transcribe.py -> format-transcriptions.py + +The scripts are PEP 723 uv scripts that import this module from their own +directory. Only pyyaml and rich are required here (imported lazily where +possible), so every script must declare both in its inline dependencies. + +Conventions shared by the scripts: + +- Exit codes: 0 = everything succeeded, 1 = fatal error (nothing useful was + written), 2 = partial success (some videos failed, the rest was written). +- ``--summary-json PATH`` writes a machine-readable run summary (see + ``RunSummary``), which ``video-sync-report.py`` turns into the PR body. +- Logging goes to stderr. In a terminal it is colored by Rich; elsewhere (CI) + it is plain, timestamped text in UTC so that logs can be correlated. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import platform +import random +import re +import shutil +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Awaitable, Callable, TypeVar + +T = TypeVar("T") + +EXIT_OK = 0 +EXIT_FATAL = 1 +EXIT_PARTIAL = 2 + +DEFAULT_PROJECT_ROOT = Path(__file__).resolve().parent.parent + +log = logging.getLogger("videolib") + + +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- + +_console = None + + +def is_interactive() -> bool: + """True when stderr is a terminal and we're not running in CI.""" + return sys.stderr.isatty() and not os.getenv("CI") + + +def get_console(): + """The Rich console shared by the log handler and progress bars.""" + global _console + if _console is None: + from rich.console import Console + + _console = Console(stderr=True) + return _console + + +class _UTCFormatter(logging.Formatter): + converter = time.gmtime + + def formatTime(self, record, datefmt=None): # noqa: N802 (logging API) + t = time.gmtime(record.created) + return time.strftime("%Y-%m-%dT%H:%M:%S", t) + f".{int(record.msecs):03d}Z" + + +NOISY_LOGGERS = [ + "httpx", + "httpcore", + "openai", + "anthropic", + "googleapiclient.discovery", + "googleapiclient.discovery_cache", + "urllib3", +] + + +def setup_logging(name: str, verbose: bool = False) -> logging.Logger: + """Configure the root logger and return the logger for ``name``. + + The level is DEBUG with ``verbose``, otherwise ``$LOG_LEVEL`` or INFO. + """ + level_name = "DEBUG" if verbose else os.getenv("LOG_LEVEL", "INFO").upper() + level = getattr(logging, level_name, logging.INFO) + + root = logging.getLogger() + for handler in list(root.handlers): + root.removeHandler(handler) + + if is_interactive(): + from rich.logging import RichHandler + + handler: logging.Handler = RichHandler( + console=get_console(), + show_path=False, + markup=False, + rich_tracebacks=True, + log_time_format="[%X]", + ) + else: + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter( + _UTCFormatter("%(asctime)s %(levelname)-7s %(name)s: %(message)s") + ) + root.addHandler(handler) + root.setLevel(level) + + # Third-party HTTP clients are chatty at INFO/DEBUG; only show their + # warnings unless we're debugging. + for noisy in NOISY_LOGGERS: + logging.getLogger(noisy).setLevel(level if verbose else logging.WARNING) + + return logging.getLogger(name) + + +def make_progress(*columns_extra): + """A Rich progress bar that is disabled outside interactive terminals.""" + from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, + ) + + return Progress( + SpinnerColumn(), + TextColumn("{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TimeElapsedColumn(), + *columns_extra, + console=get_console(), + disable=not is_interactive(), + transient=True, + ) + + +def add_common_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--summary-json", + type=Path, + metavar="PATH", + help="write a machine-readable run summary to PATH", + ) + parser.add_argument( + "--verbose", "-v", action="store_true", help="enable debug logging" + ) + # Hidden: lets the tests run a script against a temporary fixture repo. + parser.add_argument( + "--project-root", + type=Path, + default=DEFAULT_PROJECT_ROOT, + help=argparse.SUPPRESS, + ) + + +def log_environment( + logger: logging.Logger, + packages: list[str] | None = None, + tools: list[str] | None = None, +) -> dict[str, Any]: + """Log interpreter, package, and external tool versions; return them.""" + from importlib import metadata + + env: dict[str, Any] = { + "python": platform.python_version(), + "platform": platform.platform(), + "ci": bool(os.getenv("CI")), + "packages": {}, + "tools": {}, + } + for package in packages or []: + try: + env["packages"][package] = metadata.version(package) + except metadata.PackageNotFoundError: + env["packages"][package] = None + for tool in tools or []: + env["tools"][tool] = tool_version(tool) + + logger.info( + "Environment: python %s on %s%s", + env["python"], + env["platform"], + " (CI)" if env["ci"] else "", + ) + for package, version in env["packages"].items(): + logger.info(" package %s: %s", package, version or "not installed") + for tool, version in env["tools"].items(): + logger.info(" tool %s: %s", tool, version or "NOT FOUND") + return env + + +def tool_version(tool: str) -> str | None: + """First line of ``tool --version`` (``-version`` for ffmpeg), or None.""" + path = shutil.which(tool) + if not path: + return None + flag = "-version" if tool in ("ffmpeg", "ffprobe") else "--version" + try: + result = subprocess.run( + [path, flag], capture_output=True, text=True, timeout=15 + ) + except (OSError, subprocess.SubprocessError) as e: + return f"error: {e}" + out = (result.stdout or result.stderr).strip().splitlines() + return out[0] if out else "unknown" + + +# --------------------------------------------------------------------------- +# Run summary +# --------------------------------------------------------------------------- + + +def utcnow_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +@dataclass +class RunSummary: + """Collects per-video results of one script run. + + ``items`` hold the interesting results (new, changed, failed, ...), each + a dict with at least ``status``. ``tallies`` count bulk outcomes that + aren't worth listing one by one (e.g. 1,700 "already transcribed"). + An item with status ``error`` makes the run partial (exit code 2); + ``fatal`` makes it fatal (exit code 1). + """ + + script: str + started_at: str = field(default_factory=utcnow_iso) + finished_at: str | None = None + items: list[dict[str, Any]] = field(default_factory=list) + tallies: dict[str, int] = field(default_factory=dict) + fatal: str | None = None + extra: dict[str, Any] = field(default_factory=dict) + environment: dict[str, Any] = field(default_factory=dict) + + def add(self, status: str, **fields: Any) -> dict[str, Any]: + item = {"status": status, **{k: v for k, v in fields.items() if v is not None}} + self.items.append(item) + return item + + def tally(self, status: str, n: int = 1) -> None: + self.tallies[status] = self.tallies.get(status, 0) + n + + def counts(self) -> dict[str, int]: + counts = dict(self.tallies) + for item in self.items: + counts[item["status"]] = counts.get(item["status"], 0) + 1 + return dict(sorted(counts.items())) + + def by_status(self, *statuses: str) -> list[dict[str, Any]]: + return [i for i in self.items if i["status"] in statuses] + + @property + def errors(self) -> list[dict[str, Any]]: + return self.by_status("error") + + @property + def exit_code(self) -> int: + if self.fatal: + return EXIT_FATAL + if self.errors: + return EXIT_PARTIAL + return EXIT_OK + + def to_dict(self) -> dict[str, Any]: + return { + "script": self.script, + "started_at": self.started_at, + "finished_at": self.finished_at, + "exit_code": self.exit_code, + "fatal": self.fatal, + "counts": self.counts(), + "extra": self.extra, + "environment": self.environment, + "items": self.items, + } + + def write(self, path: Path | None) -> None: + self.finished_at = self.finished_at or utcnow_iso() + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + atomic_write_text(path, json.dumps(self.to_dict(), indent=2, default=str) + "\n") + + def log_counts(self, logger: logging.Logger) -> None: + counts = self.counts() + if counts: + logger.info( + "Summary: %s", + ", ".join(f"{status}={n}" for status, n in counts.items()), + ) + else: + logger.info("Summary: nothing to do") + if self.fatal: + logger.error("Fatal: %s", self.fatal) + elif self.errors: + logger.warning("%d error(s); exit code %d", len(self.errors), self.exit_code) + + +def run_main( + summary: RunSummary, + logger: logging.Logger, + summary_path: Path | None, + body: Callable[[], None], +) -> int: + """Run ``body``, turning uncaught exceptions into a fatal summary. + + The summary is always written, so the report can explain a crash. + """ + try: + body() + except KeyboardInterrupt: + summary.fatal = "interrupted" + raise + except Exception as e: # noqa: BLE001 - we want every failure in the summary + logger.exception("Unhandled error") + summary.fatal = f"{type(e).__name__}: {e}" + finally: + summary.write(summary_path) + summary.log_counts(logger) + return summary.exit_code + + +# --------------------------------------------------------------------------- +# Files +# --------------------------------------------------------------------------- + + +def atomic_write_bytes(path: Path, data: bytes) -> None: + """Write ``data`` to a temp file next to ``path``, then rename it.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) + tmp = Path(tmp_name) + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + os.replace(tmp, path) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + +def atomic_write_text(path: Path, text: str) -> None: + atomic_write_bytes(path, text.encode("utf-8")) + + +# --------------------------------------------------------------------------- +# Frontmatter +# --------------------------------------------------------------------------- + + +def split_frontmatter(content: str) -> tuple[dict[str, Any], str, str]: + """Return ``(frontmatter, yaml_section, remaining_content)``. + + Without (valid) frontmatter, returns ``({}, "", content)``. + """ + import yaml + + lines = content.split("\n") + if not lines or lines[0].strip() != "---": + return {}, "", content + end_idx = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) + if end_idx is None: + return {}, "", content + yaml_section = "\n".join(lines[1:end_idx]) + try: + frontmatter = yaml.safe_load(yaml_section) or {} + except yaml.YAMLError as e: + log.warning("Failed to parse YAML frontmatter: %s", e) + return {}, "", content + if not isinstance(frontmatter, dict): + return {}, "", content + return frontmatter, yaml_section, "\n".join(lines[end_idx + 1 :]) + + +def parse_frontmatter(content: str) -> dict[str, Any]: + return split_frontmatter(content)[0] + + +def read_frontmatter(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + return parse_frontmatter(path.read_text(encoding="utf-8")) + + +# --------------------------------------------------------------------------- +# Video directories +# --------------------------------------------------------------------------- + + +def video_dirs(videos_dir: Path) -> list[Path]: + """Video directories, newest first (names start with the YYYY-MM-DD date).""" + return sorted((d for d in videos_dir.iterdir() if d.is_dir()), reverse=True) + + +def resolve_video_dir(name_or_path: str, videos_dir: Path) -> Path: + """Find a video directory by path, exact name, or unique name prefix.""" + p = Path(name_or_path) + if p.is_absolute() and p.is_dir(): + return p + candidate = videos_dir / name_or_path + if candidate.is_dir(): + return candidate + matches = [ + d for d in videos_dir.iterdir() if d.is_dir() and d.name.startswith(name_or_path) + ] + if len(matches) == 1: + return matches[0] + if len(matches) > 1: + listed = "\n ".join(m.name for m in sorted(matches)) + raise ValueError(f"Ambiguous video name, matches:\n {listed}") + raise ValueError(f"Video directory not found: {name_or_path}") + + +def youtube_id(url: str | None) -> str | None: + if not url: + return None + m = re.search(r"(?:v=|youtu\.be/|/shorts/)([\w-]{11})", url) + return m.group(1) if m else None + + +# --------------------------------------------------------------------------- +# WebVTT +# --------------------------------------------------------------------------- + +Cue = tuple[float, float, str] + + +def parse_vtt_timestamp(ts: str) -> float: + parts = ts.strip().split(":") + if len(parts) == 3: + return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2]) + return int(parts[0]) * 60 + float(parts[1]) + + +def format_vtt_timestamp(seconds: float) -> str: + # Work in whole milliseconds so 59.9999 doesn't render as "00:00:60.000". + ms = round(seconds * 1000) + h, ms = divmod(ms, 3_600_000) + m, ms = divmod(ms, 60_000) + s, ms = divmod(ms, 1000) + return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}" + + +def parse_vtt(content: str) -> list[Cue]: + cues: list[Cue] = [] + for block in re.split(r"\n{2,}", content.replace("\r\n", "\n").strip()): + lines = block.strip().splitlines() + arrow = next((i for i, line in enumerate(lines) if "-->" in line), None) + if arrow is None: + continue + m = re.match(r"([\d:.]+)\s+-->\s+([\d:.]+)", lines[arrow].strip()) + if not m: + continue + start = parse_vtt_timestamp(m.group(1)) + end = parse_vtt_timestamp(m.group(2)) + text = "\n".join(lines[arrow + 1 :]).strip() + if text: + cues.append((start, end, text)) + return cues + + +def render_vtt(cues: list[Cue]) -> str: + lines = ["WEBVTT", ""] + for start, end, text in cues: + lines.append(f"{format_vtt_timestamp(start)} --> {format_vtt_timestamp(end)}") + lines.append(text) + lines.append("") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# API calls: rate limiting and retries +# --------------------------------------------------------------------------- + + +class RateLimiter: + """Ensures at least ``min_interval`` seconds between API requests.""" + + def __init__(self, min_interval: float = 1.0) -> None: + self._lock = asyncio.Lock() + self._last: float = 0.0 + self._interval = min_interval + + async def wait(self) -> None: + async with self._lock: + loop = asyncio.get_running_loop() + elapsed = loop.time() - self._last + if elapsed < self._interval: + await asyncio.sleep(self._interval - elapsed) + self._last = asyncio.get_running_loop().time() + + +RETRYABLE_ERROR_NAMES = {"APIConnectionError", "APITimeoutError", "RateLimitError"} + + +def is_retryable_api_error(e: BaseException) -> bool: + """Rate limits, timeouts, connection errors, and 5xx responses. + + Works for both the OpenAI and Anthropic SDKs without importing either. + """ + if any(cls.__name__ in RETRYABLE_ERROR_NAMES for cls in type(e).__mro__): + return True + status = getattr(e, "status_code", None) + if isinstance(status, int): + return status in (408, 409, 429) or status >= 500 + return isinstance(e, (TimeoutError, ConnectionError)) + + +def describe_api_error(e: BaseException) -> str: + """``RateLimitError status=429 request_id=req_123: message``""" + parts = [type(e).__name__] + status = getattr(e, "status_code", None) + if status is not None: + parts.append(f"status={status}") + request_id = getattr(e, "request_id", None) + if request_id: + parts.append(f"request_id={request_id}") + message = str(e).strip().splitlines()[0][:300] if str(e).strip() else "" + return " ".join(parts) + (f": {message}" if message else "") + + +async def retry_async( + fn: Callable[[], Awaitable[T]], + *, + label: str, + logger: logging.Logger, + attempts: int = 5, + base_delay: float = 1.0, + is_retryable: Callable[[BaseException], bool] = is_retryable_api_error, + describe: Callable[[BaseException], str] = describe_api_error, + sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, +) -> T: + """Call ``fn`` with exponential backoff (plus jitter) on retryable errors. + + Every failed attempt is logged with the error description, so a flaky + API is visible in the logs even when the call eventually succeeds. + """ + for attempt in range(1, attempts + 1): + try: + return await fn() + except Exception as e: + if not is_retryable(e): + logger.error("%s: %s (not retryable)", label, describe(e)) + raise + if attempt == attempts: + logger.error("%s: %s (giving up after %d attempts)", label, describe(e), attempts) + raise + delay = base_delay * (2 ** (attempt - 1)) + random.uniform(0, base_delay) + logger.warning( + "%s: attempt %d/%d failed: %s; retrying in %.1fs", + label, + attempt, + attempts, + describe(e), + delay, + ) + await sleep(delay) + raise AssertionError("unreachable") diff --git a/scripts/videotesting.py b/scripts/videotesting.py new file mode 100644 index 000000000..12d839f1c --- /dev/null +++ b/scripts/videotesting.py @@ -0,0 +1,373 @@ +""" +Test helpers for the video pipeline tests (scripts/test-*.py). + +Provides a loader for the hyphenated scripts, a tiny fixture repository, and +fakes for every external service the pipeline talks to: the YouTube Data +API, thumbnail downloads, yt-dlp, OpenAI Whisper, and the Anthropic API. +Nothing in here touches the network. +""" + +from __future__ import annotations + +import importlib.util +import re +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +SCRIPTS_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPTS_DIR)) + +import videolib # noqa: E402 + + +def load_script(filename: str): + """Import a hyphenated script (e.g. ``sync-videos.py``) as a module.""" + name = filename.removesuffix(".py").replace("-", "_") + if name in sys.modules: + return sys.modules[name] + spec = importlib.util.spec_from_file_location(name, SCRIPTS_DIR / filename) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +# --------------------------------------------------------------------------- +# Fixture repository +# --------------------------------------------------------------------------- + + +def write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def make_repo(root: Path) -> Path: + """A minimal project tree with people, software, and YouTube config.""" + write(root / "content/people/hadley-wickham/_index.md", "---\ntitle: Hadley Wickham\n---\n") + write(root / "content/people/mine-cetinkaya-rundel/_index.md", "---\ntitle: Mine Çetinkaya-Rundel\n---\n") + write(root / "content/software/quarto/_index.md", "---\ntitle: Quarto\n---\n") + write(root / "content/software/shiny/_index.md", "---\ntitle: Shiny\n---\n") + write(root / "content/software/commons/_index.md", "---\ntitle: commons\n---\n") + write(root / "data/software-match-ignore.toml", 'names = ["commons"]\n') + write( + root / "data/youtube.toml", + 'channels = ["PositPBC"]\nplaylists = ["PLtest"]\nvideos = ["indiv000001"]\n', + ) + (root / "content/resources/videos").mkdir(parents=True, exist_ok=True) + return root + + +def write_video_dir( + root: Path, + slug: str, + *, + url: str = "https://www.youtube.com/watch?v=abcdefghijk", + title: str = "A talk", + duration: int = 600, + extra: str = "", + vtt: str | None = None, + html: str | None = None, +) -> Path: + """Write a video directory as sync-videos.py would have produced it.""" + d = root / "content/resources/videos" / slug + write( + d / "_index.md", + f"---\ntitle: {title}\nresource_type: video\nsoftware:\n- quarto\n{extra}" + f"external:\n url: {url}\n duration: {duration}\n---\n", + ) + if vtt is not None: + write(d / "transcription.vtt", vtt) + if html is not None: + write(d / "transcription.txt", html) + return d + + +SAMPLE_VTT = """WEBVTT + +00:00:00.000 --> 00:00:04.000 +Hello everyone, today we talk about Quarto. + +00:00:04.000 --> 00:00:09.500 +It was built by the team that also made Shiny. + +00:00:20.000 --> 00:00:25.000 +Hadley Wickham joins us later. +""" + + +def vtt_to_html(vtt: str) -> str: + """What a well-behaved model returns: one

with one per cue.""" + cues = videolib.parse_vtt(vtt) + anchors = " ".join(f'{text}' for start, _, text in cues) + return f"

{anchors}

" + + +# --------------------------------------------------------------------------- +# YouTube Data API fake +# --------------------------------------------------------------------------- + + +def yt_item( + video_id: str, + title: str = "A talk", + *, + published: str = "2026-10-01T12:00:00Z", + description: str = "", + duration: str = "PT10M", + views: int = 100, + likes: int = 10, + comments: int = 1, + tags: list[str] | None = None, + channel_title: str = "Posit PBC", +) -> dict[str, Any]: + return { + "id": video_id, + "snippet": { + "title": title, + "publishedAt": published, + "description": description, + "channelTitle": channel_title, + "tags": tags or [], + "thumbnails": {"high": {"url": f"https://i.ytimg.com/vi/{video_id}/hqdefault.jpg"}}, + "defaultAudioLanguage": "en", + }, + "contentDetails": {"duration": duration, "caption": "false", "definition": "hd"}, + "statistics": {"viewCount": str(views), "likeCount": str(likes), "commentCount": str(comments)}, + } + + +class HttpError(Exception): + """Stand-in for googleapiclient.errors.HttpError.""" + + def __init__(self, status: int, message: str = "error"): + super().__init__(f"") + self.status_code = status + + +class _Request: + def __init__(self, fn, kwargs): + self._fn = fn + self.kwargs = kwargs + + def execute(self, num_retries: int = 0): + return self._fn(**self.kwargs) + + +class FakeYouTube: + """Mimics the parts of ``googleapiclient`` the metadata script uses. + + ``channels``: handle -> (uploads playlist id, channel title) + ``playlists``: playlist id -> list of video ids + ``videos``: video id -> API item (missing ids are "private/deleted") + ``page_size``: items per playlistItems page, to exercise pagination + ``fail_playlists`` / ``fail_videos``: raise HttpError for these + """ + + def __init__(self, channels=None, playlists=None, videos=None, page_size=50): + self.channels_data = channels or {} + self.playlists_data = playlists or {} + self.videos_data = videos or {} + self.page_size = page_size + self.fail_playlists: set[str] = set() + self.fail_videos = False + self.calls: list[str] = [] + + # channels().list(...) + def channels(self): + def list_(part, forUsername=None, forHandle=None): # noqa: N803 - API names + def run(): + self.calls.append("channels.list") + key = forUsername or (forHandle or "").lstrip("@") + if forUsername and key not in self.channels_data: + return {} + if key in self.channels_data: + uploads, title = self.channels_data[key] + return { + "items": [ + { + "contentDetails": {"relatedPlaylists": {"uploads": uploads}}, + "snippet": {"title": title}, + } + ] + } + return {} + + return _Request(lambda: run(), {}) + + return SimpleNamespace(list=list_) + + # playlistItems().list(...) / list_next(...) + def playlistItems(self): # noqa: N802 - API name + def page(playlistId, start): # noqa: N803 + self.calls.append("playlistItems.list") + if playlistId in self.fail_playlists or playlistId not in self.playlists_data: + raise HttpError(404, f"playlist {playlistId} not found") + ids = self.playlists_data[playlistId] + chunk = ids[start : start + self.page_size] + response = {"items": [{"contentDetails": {"videoId": v}} for v in chunk]} + if start + self.page_size < len(ids): + response["nextPageToken"] = str(start + self.page_size) + return response + + def list_(part, playlistId, maxResults=50, pageToken=None): # noqa: N803 + start = int(pageToken or 0) + req = _Request(lambda: page(playlistId, start), {}) + req.playlist_id = playlistId + return req + + def list_next(request, response): + token = response.get("nextPageToken") + if not token: + return None + return list_("contentDetails", request.playlist_id, pageToken=token) + + return SimpleNamespace(list=list_, list_next=list_next) + + # videos().list(...) + def videos(self): + def list_(part, id): # noqa: A002 - API name + def run(): + self.calls.append("videos.list") + if self.fail_videos: + raise HttpError(403, "quotaExceeded") + ids = id.split(",") + return {"items": [self.videos_data[v] for v in ids if v in self.videos_data]} + + return _Request(lambda: run(), {}) + + return SimpleNamespace(list=list_) + + +# --------------------------------------------------------------------------- +# HTTP (thumbnails) +# --------------------------------------------------------------------------- + + +class FakeHttp: + """``get(url, timeout)`` returning a response with ``content``.""" + + def __init__(self, fail_urls: set[str] | None = None): + self.fail_urls = fail_urls or set() + self.requested: list[str] = [] + + def __call__(self, url: str, timeout: float = 15): + self.requested.append(url) + if url in self.fail_urls: + raise OSError(f"connection refused: {url}") + return SimpleNamespace(content=b"JPEG:" + url.encode(), raise_for_status=lambda: None) + + +# --------------------------------------------------------------------------- +# yt-dlp fake +# --------------------------------------------------------------------------- + + +class FakeYtDlp: + """Stands in for ``subprocess.run`` when transcribe.py calls yt-dlp. + + ``outcomes`` maps a video URL to a list of results, consumed in order; + each result is either ``"ok"`` (an audio file is written) or an stderr + string (exit code 1). URLs without outcomes always succeed. + """ + + def __init__(self, outcomes: dict[str, list[str]] | None = None, size: int = 1024): + self.outcomes = {k: list(v) for k, v in (outcomes or {}).items()} + self.size = size + self.calls: list[list[str]] = [] + + def __call__(self, cmd, capture_output=True, text=True, **kwargs): + self.calls.append(cmd) + url = cmd[-1] + queue = self.outcomes.get(url) + result = queue.pop(0) if queue else "ok" + if result != "ok": + return subprocess.CompletedProcess(cmd, 1, stdout="", stderr=result) + template = cmd[cmd.index("--output") + 1] + out = Path(template.replace("%(ext)s", "webm")) + out.write_bytes(b"\0" * self.size) + return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") + + +# --------------------------------------------------------------------------- +# OpenAI / Anthropic fakes +# --------------------------------------------------------------------------- + + +class StatusError(Exception): + """Mimics an SDK ``APIStatusError`` (has ``status_code`` and ``request_id``).""" + + def __init__(self, status: int, message: str = "boom"): + super().__init__(message) + self.status_code = status + self.request_id = f"req_{status}" + + +class RateLimitError(StatusError): + def __init__(self): + super().__init__(429, "rate limited") + + +class FakeOpenAI: + """``client.audio.transcriptions.create(file=..., **kwargs)`` returning VTT. + + ``responses`` is a list consumed in order; an Exception instance is + raised instead of returned. When exhausted, ``default`` is returned. + """ + + def __init__(self, responses: list | None = None, default: str = SAMPLE_VTT): + self.responses = list(responses or []) + self.default = default + self.calls: list[dict[str, Any]] = [] + self.audio = SimpleNamespace(transcriptions=SimpleNamespace(create=self._create)) + + async def _create(self, file, **kwargs): + self.calls.append({"file": getattr(file, "name", None), **kwargs}) + result = self.responses.pop(0) if self.responses else self.default + if isinstance(result, BaseException): + raise result + return result + + +class FakeAnthropic: + """``client.messages.create(...)`` returning a message-like object. + + ``responses`` entries may be a string (the text), a ``(text, + stop_reason)`` tuple, or an Exception to raise. When exhausted, the + response is generated from the VTT in the user message. + """ + + def __init__(self, responses: list | None = None): + self.responses = list(responses or []) + self.calls: list[dict[str, Any]] = [] + self.messages = SimpleNamespace(create=self._create) + + async def _create(self, **kwargs): + self.calls.append(kwargs) + if self.responses: + result = self.responses.pop(0) + else: + user = kwargs["messages"][0]["content"] + vtt = user.split("\n\n---\n\n", 1)[-1] + result = vtt_to_html(vtt) + if isinstance(result, BaseException): + raise result + text, stop_reason = result if isinstance(result, tuple) else (result, "end_turn") + return SimpleNamespace( + content=[SimpleNamespace(type="text", text=text)], + stop_reason=stop_reason, + usage=SimpleNamespace(input_tokens=1000, output_tokens=500), + _request_id="req_fake", + ) + + +async def no_sleep(_seconds: float) -> None: + return None + + +def strip_ansi(text: str) -> str: + return re.sub(r"\x1b\[[0-9;]*m", "", text) From 6e80698d2eacb62eb5326ba0297412311db0e376 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 12:56:43 +0100 Subject: [PATCH 02/20] Only record real metadata changes in update-youtube-videos.py - Fetch all tracked videos every run, but only change an entry (and its last_updated) when a field actually changed - Keep stored view/like/comment counts unless --refresh-stats is passed - Report videos that went missing or private instead of silently keeping them - Resolve duplicate slugs deterministically, preferring the stored entry - Treat an unreadable videos.toml or a failure of every source as fatal (exit 1, nothing written); a single failed source is partial (exit 2) - Retry API calls, log quota use, and add --dry-run and --summary-json - Write videos.toml once at the end, atomically, and only when it changed Against the live API a second run in a row now changes nothing; before, every run rewrote all ~1,700 entries. --- scripts/test-update-youtube-videos.py | 373 ++++++++++++++ scripts/update-youtube-videos.py | 697 +++++++++++++++----------- 2 files changed, 778 insertions(+), 292 deletions(-) create mode 100755 scripts/test-update-youtube-videos.py diff --git a/scripts/test-update-youtube-videos.py b/scripts/test-update-youtube-videos.py new file mode 100755 index 000000000..eae8029b7 --- /dev/null +++ b/scripts/test-update-youtube-videos.py @@ -0,0 +1,373 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "google-api-python-client>=2.100.0", +# "tomli-w>=1.0.0", +# "python-dotenv>=1.0.0", +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for update-youtube-videos.py.""" + +import json +import sys +import tomllib +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from videotesting import FakeYouTube, load_script, make_repo, yt_item # noqa: E402 + +u = load_script("update-youtube-videos.py") + +NOW = "2026-11-12T05:00:00+00:00" +EARLIER = "2026-10-01T00:00:00+00:00" + + +# --- Pure helpers --- + + +@pytest.mark.parametrize( + "title, slug", + [ + ("Hello, World!", "hello-world"), + ("Mine Çetinkaya-Rundel | Teaching R", "mine-cetinkaya-rundel-teaching-r"), + ("posit::conf(2025) — keynote", "positconf2025-keynote"), + (" spaces and--dashes ", "spaces-and-dashes"), + ], +) +def test_slugify(title, slug): + assert u.slugify(title) == slug + + +@pytest.mark.parametrize( + "duration, seconds", + [ + ("PT1H2M3S", 3723), + ("PT45S", 45), + ("PT10M", 600), + ("P1DT2H", 93600), + ("P0D", 0), # live streams + ("", 0), + ("garbage", 0), + ], +) +def test_parse_iso8601_duration(duration, seconds): + assert u.parse_iso8601_duration(duration) == seconds + + +def test_best_thumbnail(): + assert u.best_thumbnail({"high": {"url": "h"}, "maxres": {"url": "m"}}) == "m" + assert u.best_thumbnail({"default": {"url": "d"}}) == "d" + assert u.best_thumbnail({}) == "" + + +def test_transform_video(): + item = yt_item("abcdefghijk", "My Talk", tags=["R", "#quarto"], views=5, duration="PT1M5S") + v = u.transform_video(item, "Posit PBC", "PLx", NOW) + assert v == { + "url": "https://www.youtube.com/watch?v=abcdefghijk", + "title": "My Talk", + "slug": "2026-10-01_my-talk", + "date": "2026-10-01T12:00:00Z", + "description": "", + "duration": 65, + "channel": "Posit PBC", + "playlist": "PLx", + "tags": ["R", "#quarto"], + "view_count": 5, + "like_count": 10, + "comment_count": 1, + "thumbnail": "https://i.ytimg.com/vi/abcdefghijk/hqdefault.jpg", + "language": "en", + "has_captions": False, + "definition": "hd", + "publish": True, + "last_updated": NOW, + } + + +def test_transform_uses_channel_title_when_unknown(): + v = u.transform_video(yt_item("abcdefghijk", channel_title="Other"), "", "", NOW) + assert v["channel"] == "Other" + + +# --- merge_video --- + + +def stored(**overrides): + v = u.transform_video(yt_item("abcdefghijk", "My Talk"), "Posit PBC", "", EARLIER) + v.update(overrides) + return v + + +def fetched(**item_kwargs): + return u.transform_video(yt_item("abcdefghijk", **{"title": "My Talk", **item_kwargs}), "Posit PBC", "", NOW) + + +class TestMergeVideo: + def test_new(self): + video, changed, stats = u.merge_video(None, fetched(), refresh_stats=False, now=NOW) + assert video["last_updated"] == NOW and changed == [] and stats == [] + + def test_unchanged_keeps_timestamp(self): + video, changed, stats = u.merge_video(stored(), fetched(), refresh_stats=False, now=NOW) + assert changed == [] and stats == [] + assert video["last_updated"] == EARLIER + assert video == stored() + + def test_title_change_bumps_timestamp_and_slug(self): + video, changed, _ = u.merge_video(stored(), fetched(title="New Title"), refresh_stats=False, now=NOW) + assert changed == ["title", "slug"] + assert video["slug"] == "2026-10-01_new-title" + assert video["last_updated"] == NOW + + def test_stats_kept_without_refresh(self): + video, changed, stats = u.merge_video(stored(), fetched(views=999), refresh_stats=False, now=NOW) + assert changed == [] and stats == [] + assert video["view_count"] == 100 + assert video["last_updated"] == EARLIER + + def test_stats_refreshed(self): + video, changed, stats = u.merge_video(stored(), fetched(views=999, likes=11), refresh_stats=True, now=NOW) + assert changed == [] + assert stats == ["view_count", "like_count"] + assert video["view_count"] == 999 and video["last_updated"] == NOW + + def test_refresh_without_stat_changes_keeps_timestamp(self): + video, _, stats = u.merge_video(stored(), fetched(), refresh_stats=True, now=NOW) + assert stats == [] and video["last_updated"] == EARLIER + + def test_publish_false_preserved(self): + video, changed, _ = u.merge_video(stored(publish=False), fetched(), refresh_stats=False, now=NOW) + assert video["publish"] is False and changed == [] + + def test_manual_keys_preserved(self): + video, changed, _ = u.merge_video(stored(note="keep me"), fetched(), refresh_stats=False, now=NOW) + assert video["note"] == "keep me" and changed == [] + + +# --- deduplicate_videos --- + + +class TestDeduplicate: + def videos(self): + a = {"slug": "s", "date": "2026-01-01T10:00:00Z"} + b = {"slug": "s", "date": "2026-01-01T11:00:00Z"} + c = {"slug": "other", "date": "2026-01-01T10:00:00Z"} + return {"url-a": a, "url-b": b, "url-c": c} + + def test_prefers_existing_entry(self): + result, discarded = u.deduplicate_videos(self.videos(), existing_urls={"url-a"}) + assert set(result) == {"url-a", "url-c"} + assert discarded == [{"url": "url-b", "slug": "s", "kept": "url-a"}] + + def test_then_later_date(self): + result, _ = u.deduplicate_videos(self.videos(), existing_urls=set()) + assert set(result) == {"url-b", "url-c"} + + def test_stable_across_runs(self): + first, _ = u.deduplicate_videos(self.videos(), existing_urls={"url-a", "url-b"}) + second, _ = u.deduplicate_videos(self.videos(), existing_urls=set(first)) + assert set(first) == set(second) + + def test_preserves_order(self): + result, _ = u.deduplicate_videos(self.videos(), existing_urls={"url-a"}) + assert list(result) == ["url-a", "url-c"] + + +# --- Fake API --- + + +def fake_api(**overrides) -> FakeYouTube: + yt = FakeYouTube( + channels={"PositPBC": ("UUposit", "Posit PBC")}, + playlists={"UUposit": ["video000001", "video000002", "video000003"], "PLtest": ["video000003"]}, + videos={ + "video000001": yt_item("video000001", "First talk", published="2026-10-01T10:00:00Z"), + "video000002": yt_item("video000002", "Second talk", published="2026-10-02T10:00:00Z"), + "video000003": yt_item("video000003", "Third talk", published="2026-10-03T10:00:00Z"), + "indiv000001": yt_item("indiv000001", "Elsewhere", published="2025-01-01T10:00:00Z"), + }, + page_size=2, # exercise pagination + ) + for k, v in overrides.items(): + setattr(yt, k, v) + return yt + + +def run(repo, yt, *args): + summary = repo / "summary.json" + code = u.main(["--project-root", str(repo), "--summary-json", str(summary), *args], youtube=yt) + data = json.loads(summary.read_text()) + return code, data + + +def read_videos(repo): + with open(repo / "data/videos.toml", "rb") as f: + return {v["url"].rsplit("=", 1)[1]: v for v in tomllib.load(f)["videos"]} + + +@pytest.fixture +def repo(tmp_path): + return make_repo(tmp_path) + + +class TestMain: + def test_first_run(self, repo): + code, data = run(repo, fake_api()) + assert code == 0 + videos = read_videos(repo) + assert set(videos) == {"video000001", "video000002", "video000003", "indiv000001"} + assert videos["video000003"]["playlist"] == "PLtest" + assert videos["video000003"]["channel"] == "Posit PBC" + assert videos["indiv000001"]["channel"] == "Posit PBC" # from snippet.channelTitle + assert data["counts"] == {"new": 4} + new = {i["video_id"]: i for i in data["items"]} + assert new["video000001"]["slug"] == "2026-10-01_first-talk" + assert new["video000001"]["title"] == "First talk" + assert data["extra"]["quota_used"] > 0 + + def test_second_run_is_byte_identical_and_not_rewritten(self, repo): + run(repo, fake_api()) + path = repo / "data/videos.toml" + before = path.read_bytes() + mtime = path.stat().st_mtime_ns + code, data = run(repo, fake_api()) + assert code == 0 + assert path.read_bytes() == before + assert path.stat().st_mtime_ns == mtime + assert data["counts"] == {"unchanged": 4} + assert data["extra"]["written"] is False + + def test_stats_only_change_without_refresh(self, repo): + run(repo, fake_api()) + before = (repo / "data/videos.toml").read_bytes() + yt = fake_api() + yt.videos_data["video000001"] = yt_item("video000001", "First talk", published="2026-10-01T10:00:00Z", views=5000) + code, data = run(repo, yt) + assert (repo / "data/videos.toml").read_bytes() == before + assert data["counts"] == {"unchanged": 4} + + def test_stats_change_with_refresh(self, repo): + run(repo, fake_api()) + yt = fake_api() + yt.videos_data["video000001"] = yt_item("video000001", "First talk", published="2026-10-01T10:00:00Z", views=5000) + code, data = run(repo, yt, "--refresh-stats") + assert read_videos(repo)["video000001"]["view_count"] == 5000 + assert data["counts"] == {"stats_refreshed": 1, "unchanged": 3} + assert data["extra"]["refresh_stats"] is True + + def test_title_change(self, repo): + run(repo, fake_api()) + yt = fake_api() + yt.videos_data["video000002"] = yt_item("video000002", "Renamed talk", published="2026-10-02T10:00:00Z") + code, data = run(repo, yt) + (item,) = data["items"] + assert item["status"] == "changed" + assert item["changed_fields"] == ["title", "slug"] + assert item["old_slug"] == "2026-10-02_second-talk" + assert item["slug"] == "2026-10-02_renamed-talk" + + def test_missing_and_unavailable(self, repo): + run(repo, fake_api()) + yt = fake_api() + yt.playlists_data["UUposit"] = ["video000002", "video000003"] # 1 removed from channel + del yt.videos_data["video000003"] # 3 went private + code, data = run(repo, yt) + assert code == 0 + statuses = {i["video_id"]: i["status"] for i in data["items"]} + assert statuses == {"video000001": "missing", "video000003": "unavailable"} + # Nothing is removed automatically. + assert set(read_videos(repo)) == {"video000001", "video000002", "video000003", "indiv000001"} + + def test_publish_false_survives(self, repo): + run(repo, fake_api()) + path = repo / "data/videos.toml" + path.write_text(path.read_text().replace("publish = true", "publish = false", 1)) + before = path.read_bytes() + run(repo, fake_api()) + assert path.read_bytes() == before + + def test_one_source_fails_is_partial(self, repo): + yt = fake_api() + yt.fail_playlists = {"PLtest"} + code, data = run(repo, yt) + assert code == 2 + (error,) = [i for i in data["items"] if i["status"] == "error"] + assert error["source"] == "playlist:PLtest" + assert "PLtest" in error["detail"] + # Everything else is still written. + assert "video000001" in read_videos(repo) + + def test_all_sources_fail_is_fatal_and_writes_nothing(self, repo): + yt = fake_api(channels_data={}) + yt.fail_playlists = {"PLtest"} + code, data = run(repo, yt) + assert code == 1 + assert "All 2 channels and playlists failed" in data["fatal"] + assert not (repo / "data/videos.toml").exists() + + def test_videos_api_failure_is_fatal(self, repo): + run(repo, fake_api()) + before = (repo / "data/videos.toml").read_bytes() + yt = fake_api() + yt.fail_videos = True + code, data = run(repo, yt) + assert code == 1 + assert "quotaExceeded" in data["fatal"] + assert (repo / "data/videos.toml").read_bytes() == before + + def test_corrupt_videos_toml_is_fatal(self, repo): + (repo / "data/videos.toml").write_text("this is [not toml") + code, data = run(repo, fake_api()) + assert code == 1 + assert "Could not parse" in data["fatal"] + assert (repo / "data/videos.toml").read_text() == "this is [not toml" + + def test_missing_config_is_fatal(self, repo): + (repo / "data/youtube.toml").unlink() + code, data = run(repo, fake_api()) + assert code == 1 and "Config file not found" in data["fatal"] + + def test_missing_api_key_is_fatal(self, repo, monkeypatch): + monkeypatch.delenv("YOUTUBE_API_KEY", raising=False) + summary = repo / "s.json" + code = u.main(["--project-root", str(repo), "--summary-json", str(summary)]) + assert code == 1 + assert "YOUTUBE_API_KEY" in json.loads(summary.read_text())["fatal"] + + def test_dry_run_writes_nothing(self, repo): + code, data = run(repo, fake_api(), "--dry-run") + assert code == 0 + assert not (repo / "data/videos.toml").exists() + assert data["counts"] == {"new": 4} + + def test_duplicate_slug_is_stable(self, repo): + yt = fake_api() + yt.playlists_data["UUposit"].append("video000004") + yt.videos_data["video000004"] = yt_item("video000004", "First talk", published="2026-10-01T23:00:00Z") + run(repo, yt) + first = read_videos(repo) + code, data = run(repo, yt) + assert read_videos(repo) == first + dupes = [i for i in data["items"] if i["status"] == "duplicate_slug"] + assert len(dupes) == 1 and dupes[0]["slug"] == "2026-10-01_first-talk" + + def test_logs_changes(self, repo, capsys, monkeypatch): + monkeypatch.setenv("CI", "true") + run(repo, fake_api()) + err = capsys.readouterr().err + assert "New: 2026-10-01_first-talk (video000001) 'First talk'" in err + assert "API quota used" in err + assert "Summary: new=4" in err + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/update-youtube-videos.py b/scripts/update-youtube-videos.py index 29ebe4459..087f411cd 100755 --- a/scripts/update-youtube-videos.py +++ b/scripts/update-youtube-videos.py @@ -1,9 +1,11 @@ -#!/usr/bin/env -S uv run +#!/usr/bin/env -S uv run --script # /// script +# requires-python = ">=3.11" # dependencies = [ # "google-api-python-client>=2.100.0", # "tomli-w>=1.0.0", # "python-dotenv>=1.0.0", +# "pyyaml>=6.0", # "rich>=13.0.0", # ] # /// @@ -11,38 +13,63 @@ """ Fetch video metadata from YouTube channels and playlists. -Reads channel names and playlist IDs from data/youtube.toml, -fetches metadata via the YouTube Data API v3, and writes -video information to data/videos.toml. +Reads channel names, playlist IDs, and individual video IDs from +data/youtube.toml, fetches metadata via the YouTube Data API v3, and writes +it to data/videos.toml. + +Every run fetches metadata for all tracked videos (~40 quota units), but a +video's entry, including its `last_updated` timestamp, only changes when one +of its fields actually changed. View, like, and comment counts change all +the time, so they're only refreshed with --refresh-stats (the nightly +workflow does this once a week); new videos always get them. + +Videos that disappear from YouTube are reported but never removed. + +Requires YOUTUBE_API_KEY in the environment or .env. + +Exit codes: 0 = success, 1 = fatal (nothing written), 2 = partial (a channel +or playlist could not be fetched; everything else was written). """ +from __future__ import annotations + import argparse +import logging import os import re import sys import tomllib import unicodedata -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone from pathlib import Path from typing import Any import tomli_w -from dotenv import load_dotenv -from googleapiclient.discovery import build -from rich.console import Console -from rich.progress import ( - BarColumn, - Progress, - SpinnerColumn, - TaskProgressColumn, - TextColumn, - TimeRemainingColumn, -) - -console = Console(stderr=True) + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 + +log = logging.getLogger("update-youtube-videos") THUMBNAIL_PREFERENCE = ["maxres", "high", "standard", "medium", "default"] +# Fields that change constantly; only refreshed with --refresh-stats. +STATS_FIELDS = ("view_count", "like_count", "comment_count") +# Bookkeeping fields that never count as a change by themselves. +VOLATILE_FIELDS = ("last_updated",) + +API_RETRIES = 3 + + +class FatalError(Exception): + """Something is wrong enough that videos.toml must not be written.""" + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + def slugify(text: str) -> str: text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("ascii") @@ -53,30 +80,132 @@ def slugify(text: str) -> str: def parse_iso8601_duration(duration: str) -> int: - match = re.match(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", duration) + match = re.fullmatch(r"P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?", duration or "") if not match: return 0 - hours = int(match.group(1) or 0) - minutes = int(match.group(2) or 0) - seconds = int(match.group(3) or 0) - return hours * 3600 + minutes * 60 + seconds + days, hours, minutes, seconds = (int(g or 0) for g in match.groups()) + return days * 86400 + hours * 3600 + minutes * 60 + seconds -def get_api_key() -> str: - load_dotenv() - key = os.getenv("YOUTUBE_API_KEY") - if not key: - console.print( - "[bold red]Error:[/] YOUTUBE_API_KEY not found in .env or environment" - ) - sys.exit(1) - return key +def best_thumbnail(thumbnails: dict) -> str: + for key in THUMBNAIL_PREFERENCE: + if key in thumbnails: + return thumbnails[key]["url"] + return "" + + +def transform_video(item: dict, channel: str, playlist: str, now: str) -> dict[str, Any]: + snippet = item.get("snippet", {}) + content = item.get("contentDetails", {}) + stats = item.get("statistics", {}) + title = snippet.get("title", "") + date = snippet.get("publishedAt", "") + date_prefix = date[:10] if len(date) >= 10 else "" + slug = f"{date_prefix}_{slugify(title)}" if date_prefix else slugify(title) + return { + "url": f"https://www.youtube.com/watch?v={item['id']}", + "title": title, + "slug": slug, + "date": date, + "description": snippet.get("description", ""), + "duration": parse_iso8601_duration(content.get("duration", "")), + "channel": channel or snippet.get("channelTitle", ""), + "playlist": playlist, + "tags": snippet.get("tags", []), + "view_count": int(stats.get("viewCount", 0)), + "like_count": int(stats.get("likeCount", 0)), + "comment_count": int(stats.get("commentCount", 0)), + "thumbnail": best_thumbnail(snippet.get("thumbnails", {})), + "language": snippet.get("defaultAudioLanguage", ""), + "has_captions": content.get("caption") == "true", + "definition": content.get("definition", ""), + "publish": True, + "last_updated": now, + } + + +def merge_video( + existing: dict[str, Any] | None, + fetched: dict[str, Any], + *, + refresh_stats: bool, + now: str, +) -> tuple[dict[str, Any], list[str], list[str]]: + """Merge freshly fetched metadata into the stored entry. + + Returns ``(video, changed_fields, changed_stats)``. ``last_updated`` only + moves when a field changed. Without ``refresh_stats`` the stored counts + are kept. ``publish = false`` and any keys this script doesn't produce + (added by hand) are preserved. + """ + if existing is None: + return {**fetched, "last_updated": now}, [], [] + + merged = dict(fetched) + for key, value in existing.items(): + merged.setdefault(key, value) + if existing.get("publish") is False: + merged["publish"] = False + if not refresh_stats: + for f in STATS_FIELDS: + if f in existing: + merged[f] = existing[f] + + keys = [k for k in merged if k not in VOLATILE_FIELDS] + changed = [k for k in keys if k not in STATS_FIELDS and merged.get(k) != existing.get(k)] + changed_stats = [k for k in keys if k in STATS_FIELDS and merged.get(k) != existing.get(k)] + + if changed or changed_stats: + merged["last_updated"] = now + else: + merged["last_updated"] = existing.get("last_updated", now) + return merged, changed, changed_stats + + +def deduplicate_videos( + videos: dict[str, dict[str, Any]], + existing_urls: set[str], +) -> tuple[dict[str, dict[str, Any]], list[dict[str, str]]]: + """Resolve slug collisions deterministically. + + Two videos with the same title on the same day get the same slug. Prefer + the entry already in videos.toml (so the winner doesn't flip from night + to night), then the later publish time, then the URL. + """ + + def rank(url: str) -> tuple: + v = videos[url] + return (url in existing_urls, v.get("date", ""), url) + + winners: dict[str, str] = {} + for url, video in videos.items(): + slug = video.get("slug", "") + if slug not in winners or rank(url) > rank(winners[slug]): + winners[slug] = url + + keep = set(winners.values()) + result = {url: v for url, v in videos.items() if url in keep} + discarded = [ + {"url": url, "slug": v.get("slug", ""), "kept": winners[v.get("slug", "")]} + for url, v in videos.items() + if url not in keep + ] + return result, discarded + + +def render_videos(videos: dict[str, dict[str, Any]]) -> str: + videos_list = [{k: v for k, v in video.items() if v is not None} for video in videos.values()] + return tomli_w.dumps({"videos": videos_list}) + + +# --------------------------------------------------------------------------- +# Files +# --------------------------------------------------------------------------- def load_youtube_config(config_path: Path) -> dict[str, list[str]]: if not config_path.exists(): - console.print(f"[bold red]Error:[/] Config file not found: {config_path}") - sys.exit(1) + raise FatalError(f"Config file not found: {config_path}") with open(config_path, "rb") as f: data = tomllib.load(f) return { @@ -92,315 +221,299 @@ def load_existing_videos(output_path: Path) -> dict[str, dict[str, Any]]: try: with open(output_path, "rb") as f: data = tomllib.load(f) - return {v["url"]: v for v in data.get("videos", [])} - except Exception as e: - console.print(f"[yellow]Warning:[/] Could not load existing videos: {e}") - return {} + except tomllib.TOMLDecodeError as e: + # Never continue with an empty list: that would wipe the file. + raise FatalError(f"Could not parse {output_path}: {e}") from e + return {v["url"]: v for v in data.get("videos", [])} -def deduplicate_videos( - videos: dict[str, dict[str, Any]], -) -> dict[str, dict[str, Any]]: - """Remove slug collisions, keeping the entry with the more recent last_updated.""" - seen: dict[str, str] = {} # slug -> url - result: dict[str, dict[str, Any]] = {} - for url, video in videos.items(): - slug = video.get("slug", "") - if slug not in seen: - seen[slug] = url - result[url] = video - else: - prev_url = seen[slug] - prev = result[prev_url] - if video.get("last_updated", "") > prev.get("last_updated", ""): - del result[prev_url] - seen[slug] = url - result[url] = video - console.print( - f" [yellow]Warning:[/] Duplicate slug '{slug}': " - f"replaced {prev_url} with {url}" - ) - else: - console.print( - f" [yellow]Warning:[/] Duplicate slug '{slug}': " - f"kept {prev_url}, discarded {url}" - ) - return result +# --------------------------------------------------------------------------- +# YouTube API +# --------------------------------------------------------------------------- -def write_videos(videos: dict[str, dict[str, Any]], output_path: Path) -> None: - videos_list = [ - {k: v for k, v in video.items() if v is not None} for video in videos.values() - ] - with open(output_path, "wb") as f: - tomli_w.dump({"videos": videos_list}, f) +class Quota: + """Rough quota bookkeeping: list calls cost 1 unit each.""" + def __init__(self) -> None: + self.calls: dict[str, int] = {} -def should_update_video(existing: dict[str, Any] | None, force: bool) -> bool: - if force or not existing: - return True - last_updated_str = existing.get("last_updated") - if not last_updated_str: - return True - try: - last_updated = datetime.fromisoformat(last_updated_str) - return (datetime.now(timezone.utc) - last_updated) >= timedelta(hours=12) - except Exception: - return True + def add(self, call: str, units: int = 1) -> None: + self.calls[call] = self.calls.get(call, 0) + units + @property + def total(self) -> int: + return sum(self.calls.values()) -def resolve_channel_uploads(youtube: Any, channel_name: str) -> tuple[str, str] | None: - for params in [ - {"forUsername": channel_name}, - {"forHandle": f"@{channel_name}"}, - ]: - try: - resp = ( - youtube.channels() - .list(part="contentDetails,snippet", **params) - .execute() - ) - if resp.get("items"): - item = resp["items"][0] - uploads_id = item["contentDetails"]["relatedPlaylists"]["uploads"] - title = item["snippet"]["title"] - console.print( - f" [green]\u2713[/] Resolved uploads playlist: {uploads_id}" - ) - return uploads_id, title - except Exception as e: - console.print(f" [yellow]Warning:[/] API error for {params}: {e}") - console.print( - f" [bold red]Error:[/] Could not resolve channel '{channel_name}', skipping" - ) + +def resolve_channel_uploads(youtube: Any, channel_name: str, quota: Quota) -> tuple[str, str] | None: + for params in [{"forUsername": channel_name}, {"forHandle": f"@{channel_name}"}]: + quota.add("channels.list") + resp = ( + youtube.channels() + .list(part="contentDetails,snippet", **params) + .execute(num_retries=API_RETRIES) + ) + if resp.get("items"): + item = resp["items"][0] + uploads_id = item["contentDetails"]["relatedPlaylists"]["uploads"] + title = item["snippet"]["title"] + log.info(" Resolved channel %r (%s) to uploads playlist %s", channel_name, title, uploads_id) + return uploads_id, title + log.debug(" No channel found for %s", params) return None -def fetch_playlist_video_ids(youtube: Any, playlist_id: str) -> list[str]: +def fetch_playlist_video_ids(youtube: Any, playlist_id: str, quota: Quota) -> list[str]: video_ids: list[str] = [] request = youtube.playlistItems().list( part="contentDetails", playlistId=playlist_id, maxResults=50 ) while request: - response = request.execute() + quota.add("playlistItems.list") + response = request.execute(num_retries=API_RETRIES) for item in response.get("items", []): - vid = item["contentDetails"]["videoId"] - video_ids.append(vid) + video_ids.append(item["contentDetails"]["videoId"]) request = youtube.playlistItems().list_next(request, response) return video_ids -def fetch_video_metadata(youtube: Any, video_ids: list[str]) -> list[dict]: +def fetch_video_metadata(youtube: Any, video_ids: list[str], quota: Quota) -> list[dict]: items: list[dict] = [] for i in range(0, len(video_ids), 50): batch = video_ids[i : i + 50] + quota.add("videos.list") resp = ( youtube.videos() .list(part="snippet,contentDetails,statistics", id=",".join(batch)) - .execute() + .execute(num_retries=API_RETRIES) ) items.extend(resp.get("items", [])) + log.debug(" Fetched metadata batch %d (%d videos)", i // 50 + 1, len(batch)) return items -def best_thumbnail(thumbnails: dict) -> str: - for key in THUMBNAIL_PREFERENCE: - if key in thumbnails: - return thumbnails[key]["url"] - return "" +def collect_video_ids( + youtube: Any, + config: dict[str, list[str]], + quota: Quota, + summary: videolib.RunSummary, +) -> dict[str, tuple[str, str]]: + """Map video id -> (channel title, playlist id) over all sources. + A source that fails is recorded as an error; if every channel and + playlist fails, something is wrong globally (API key, quota, network). + """ + video_context: dict[str, tuple[str, str]] = {} + sources = len(config["channels"]) + len(config["playlists"]) + failed = 0 -def transform_video(item: dict, channel: str, playlist: str) -> dict[str, Any]: - snippet = item.get("snippet", {}) - content = item.get("contentDetails", {}) - stats = item.get("statistics", {}) - title = snippet.get("title", "") - date = snippet.get("publishedAt", "") - date_prefix = date[:10] if len(date) >= 10 else "" - slug = f"{date_prefix}_{slugify(title)}" if date_prefix else slugify(title) - return { - "url": f"https://www.youtube.com/watch?v={item['id']}", - "title": title, - "slug": slug, - "date": date, - "description": snippet.get("description", ""), - "duration": parse_iso8601_duration(content.get("duration", "")), - "channel": channel or snippet.get("channelTitle", ""), - "playlist": playlist, - "tags": snippet.get("tags", []), - "view_count": int(stats.get("viewCount", 0)), - "like_count": int(stats.get("likeCount", 0)), - "comment_count": int(stats.get("commentCount", 0)), - "thumbnail": best_thumbnail(snippet.get("thumbnails", {})), - "language": snippet.get("defaultAudioLanguage", ""), - "has_captions": content.get("caption") == "true", - "definition": content.get("definition", ""), - "publish": True, - "last_updated": datetime.now(timezone.utc).isoformat(), - } + for channel_name in config["channels"]: + log.info("Channel %s", channel_name) + try: + resolved = resolve_channel_uploads(youtube, channel_name, quota) + if not resolved: + raise LookupError(f"channel {channel_name!r} not found by username or handle") + uploads_id, channel_title = resolved + ids = fetch_playlist_video_ids(youtube, uploads_id, quota) + except Exception as e: # noqa: BLE001 - recorded and reported + failed += 1 + log.error(" Could not fetch channel %s: %s", channel_name, e) + summary.add("error", reason="source_failed", source=f"channel:{channel_name}", detail=str(e)) + continue + log.info(" Found %d videos", len(ids)) + for vid in ids: + video_context.setdefault(vid, (channel_title, "")) + for playlist_id in config["playlists"]: + log.info("Playlist %s", playlist_id) + try: + ids = fetch_playlist_video_ids(youtube, playlist_id, quota) + except Exception as e: # noqa: BLE001 - recorded and reported + failed += 1 + log.error(" Could not fetch playlist %s: %s", playlist_id, e) + summary.add("error", reason="source_failed", source=f"playlist:{playlist_id}", detail=str(e)) + continue + log.info(" Found %d videos", len(ids)) + for vid in ids: + video_context[vid] = (video_context.get(vid, ("", ""))[0], playlist_id) + + if sources and failed == sources: + raise FatalError(f"All {sources} channels and playlists failed; check the API key and quota") + + if config["videos"]: + log.info("Individual videos: %d", len(config["videos"])) + for vid in config["videos"]: + if vid in video_context: + log.debug(" Already tracked: %s", vid) + else: + video_context[vid] = ("", "") + log.info(" Added %s", vid) -def main() -> None: - parser = argparse.ArgumentParser( - description="Fetch video metadata from YouTube channels and playlists." - ) - parser.add_argument( - "--force", - "-f", - action="store_true", - help="Force update all videos, ignoring last_updated timestamps.", - ) - args = parser.parse_args() + return video_context - console.print("\n[bold blue]YouTube Video Metadata Fetcher[/]\n") - script_dir = Path(__file__).parent - project_root = script_dir.parent - config_path = project_root / "data" / "youtube.toml" - output_path = project_root / "data" / "videos.toml" +# --------------------------------------------------------------------------- +# Core +# --------------------------------------------------------------------------- - console.print(f"[dim]Config: {config_path}[/]") - console.print(f"[dim]Output: {output_path}[/]\n") - config = load_youtube_config(config_path) - channels = config["channels"] - playlists = config["playlists"] - individual_videos = config["videos"] +def update_videos( + youtube: Any, + config: dict[str, list[str]], + existing: dict[str, dict[str, Any]], + *, + refresh_stats: bool, + summary: videolib.RunSummary, + now: str | None = None, +) -> dict[str, dict[str, Any]]: + """Return the new contents of videos.toml, recording results in ``summary``.""" + now = now or datetime.now(timezone.utc).isoformat() + quota = Quota() + + video_context = collect_video_ids(youtube, config, quota, summary) + log.info("Total unique videos in sources: %d", len(video_context)) + + log.info("Fetching metadata for %d videos", len(video_context)) + items = fetch_video_metadata(youtube, list(video_context), quota) + fetched_ids = {item["id"] for item in items} + + # Start from the stored order so the file diff stays minimal; new videos + # are appended. + videos: dict[str, dict[str, Any]] = dict(existing) + results: dict[str, dict[str, Any]] = {} # url -> summary fields + + for item in items: + vid = item["id"] + channel, playlist = video_context[vid] + fetched = transform_video(item, channel, playlist, now) + url = fetched["url"] + old = existing.get(url) + video, changed, changed_stats = merge_video(old, fetched, refresh_stats=refresh_stats, now=now) + videos[url] = video + + if old is None: + results[url] = {"status": "new", "slug": video["slug"], "title": video["title"]} + log.info("New: %s (%s) %r", video["slug"], vid, video["title"]) + elif changed: + result = {"status": "changed", "slug": video["slug"], "changed_fields": changed} + if "slug" in changed: + result["old_slug"] = old.get("slug") + results[url] = result + log.info("Changed: %s (%s): %s", video["slug"], vid, ", ".join(changed)) + for f in changed: + log.debug(" %s: %r -> %r", f, old.get(f), video.get(f)) + elif changed_stats: + results[url] = {"status": "stats_refreshed"} + else: + results[url] = {"status": "unchanged"} + + for vid in video_context: + if vid not in fetched_ids: + url = f"https://www.youtube.com/watch?v={vid}" + stored = existing.get(url, {}) + results[url] = {"status": "unavailable", "slug": stored.get("slug")} + log.warning("Unavailable: %s is listed but not returned by the API (private or deleted?)", vid) + + source_urls = {f"https://www.youtube.com/watch?v={vid}" for vid in video_context} + for url, stored in existing.items(): + if url not in source_urls: + results[url] = {"status": "missing", "slug": stored.get("slug"), "title": stored.get("title")} + log.warning( + "Missing: %s (%s) is in videos.toml but no longer in any channel or playlist", + stored.get("slug"), + videolib.youtube_id(url), + ) - console.print(f"[cyan]Channels:[/] {', '.join(channels) or '(none)'}") - console.print(f"[cyan]Playlists:[/] {', '.join(playlists) or '(none)'}") - console.print(f"[cyan]Videos:[/] {', '.join(individual_videos) or '(none)'}") + videos, discarded = deduplicate_videos(videos, set(existing)) + for d in discarded: + results[d["url"]] = {"status": "duplicate_slug", "slug": d["slug"], "kept": d["kept"]} + log.warning("Duplicate slug %s: kept %s, skipped %s", d["slug"], d["kept"], d["url"]) - if args.force: - console.print("[yellow]Force mode:[/] Ignoring last_updated timestamps") + for url, result in results.items(): + status = result.pop("status") + if status in ("unchanged", "stats_refreshed"): + summary.tally(status) + else: + summary.add(status, url=url, video_id=videolib.youtube_id(url), **result) - existing_videos = load_existing_videos(output_path) - if existing_videos: - console.print(f"[dim]Loaded {len(existing_videos)} existing videos[/]") + summary.extra["quota_used"] = quota.total + summary.extra["quota_calls"] = quota.calls + summary.extra["videos_in_sources"] = len(video_context) + log.info("API quota used: ~%d units (%s)", quota.total, quota.calls) + return videos - api_key = get_api_key() - youtube = build("youtube", "v3", developerKey=api_key) - # video_id -> (channel, playlist) - video_context: dict[str, tuple[str, str]] = {} - quota_used = 0 - - # Phase 2 + 3: Resolve channels and collect video IDs - for channel_name in channels: - console.print(f"\n[bold]Channel: {channel_name}[/]") - result = resolve_channel_uploads(youtube, channel_name) - quota_used += 1 - if not result: - continue - uploads_id, channel_title = result - video_ids = fetch_playlist_video_ids(youtube, uploads_id) - quota_used += (len(video_ids) // 50) + 1 - console.print(f" [green]\u2713[/] Found {len(video_ids)} videos") - for vid in video_ids: - if vid not in video_context: - video_context[vid] = (channel_title, "") - - for playlist_id in playlists: - console.print(f"\n[bold]Playlist: {playlist_id}[/]") +def build_youtube_client(project_root: Path) -> Any: + from dotenv import load_dotenv + from googleapiclient.discovery import build + + load_dotenv(project_root / ".env") + key = os.getenv("YOUTUBE_API_KEY") + if not key: + raise FatalError("YOUTUBE_API_KEY not found in .env or environment") + return build("youtube", "v3", developerKey=key, cache_discovery=False) + + +def main(argv: list[str] | None = None, youtube: Any = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.strip().split("\n\n")[0]) + parser.add_argument( + "--refresh-stats", + action="store_true", + help="also update view, like, and comment counts of existing videos", + ) + parser.add_argument("--force", "-f", action="store_true", help="alias for --refresh-stats") + parser.add_argument("--dry-run", action="store_true", help="show what would change; write nothing") + videolib.add_common_args(parser) + args = parser.parse_args(argv) + refresh_stats = args.refresh_stats or args.force + + videolib.setup_logging("update-youtube-videos", args.verbose) + summary = videolib.RunSummary("update-youtube-videos") + summary.extra.update(refresh_stats=refresh_stats, dry_run=args.dry_run) + + config_path = args.project_root / "data" / "youtube.toml" + output_path = args.project_root / "data" / "videos.toml" + + def body() -> None: + nonlocal youtube + summary.environment = videolib.log_environment( + log, packages=["google-api-python-client", "tomli-w"] + ) try: - video_ids = fetch_playlist_video_ids(youtube, playlist_id) - quota_used += (len(video_ids) // 50) + 1 - console.print(f" [green]\u2713[/] Found {len(video_ids)} videos") - for vid in video_ids: - video_context[vid] = (video_context.get(vid, ("", ""))[0], playlist_id) - except Exception as e: - console.print(f" [bold red]Error:[/] Could not fetch playlist: {e}") - - if individual_videos: - console.print(f"\n[bold]Individual videos: {len(individual_videos)}[/]") - for vid in individual_videos: - if vid not in video_context: - video_context[vid] = ("", "") - console.print(f" [green]\u2713[/] Added {vid}") - else: - console.print(f" [dim]○[/] Already tracked: {vid}") + config = load_youtube_config(config_path) + log.info("Config: %s", config_path) + for kind in ("channels", "playlists", "videos"): + log.info(" %s: %s", kind, ", ".join(config[kind]) or "(none)") + log.info("Stats: %s", "refreshing" if refresh_stats else "keeping stored counts") - console.print(f"\n[cyan]Total unique videos:[/] {len(video_context)}") + existing = load_existing_videos(output_path) + log.info("Loaded %d existing videos from %s", len(existing), output_path) - # Phase 4 + 5: Fetch metadata, deduplicate, merge - ids_to_fetch = [] - counters = {"new": 0, "updated": 0, "skipped": 0, "errors": 0} + if youtube is None: + youtube = build_youtube_client(args.project_root) - for vid, (channel, playlist) in video_context.items(): - url = f"https://www.youtube.com/watch?v={vid}" - existing = existing_videos.get(url) - if should_update_video(existing, args.force): - ids_to_fetch.append(vid) + videos = update_videos( + youtube, config, existing, refresh_stats=refresh_stats, summary=summary + ) + except FatalError as e: + summary.fatal = str(e) + log.error("%s", e) + return + + new_text = render_videos(videos) + old_text = output_path.read_text(encoding="utf-8") if output_path.exists() else None + if new_text == old_text: + log.info("%s is unchanged (%d videos)", output_path, len(videos)) + elif args.dry_run: + log.info("Dry run: would write %d videos to %s", len(videos), output_path) else: - counters["skipped"] += 1 + videolib.atomic_write_text(output_path, new_text) + log.info("Wrote %d videos to %s", len(videos), output_path) + summary.extra["written"] = new_text != old_text and not args.dry_run - if not ids_to_fetch: - console.print("\n[dim]All videos are up to date, nothing to fetch.[/]") - else: - console.print( - f"\n[bold]Fetching metadata for {len(ids_to_fetch)} videos[/]" - f" [dim]({counters['skipped']} skipped as fresh)[/]" - ) - - all_videos = dict(existing_videos) - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TaskProgressColumn(), - TimeRemainingColumn(), - console=console, - ) as progress: - task = progress.add_task("[cyan]Fetching...", total=len(ids_to_fetch)) - - for i in range(0, len(ids_to_fetch), 50): - batch = ids_to_fetch[i : i + 50] - items = fetch_video_metadata(youtube, batch) - quota_used += 1 - - fetched_ids = {item["id"] for item in items} - for vid in batch: - if vid not in fetched_ids: - console.print( - f" [yellow]Warning:[/] Video {vid} not returned by API (private/deleted?)" - ) - counters["errors"] += 1 - - for item in items: - vid = item["id"] - channel, playlist = video_context[vid] - url = f"https://www.youtube.com/watch?v={vid}" - existing = all_videos.get(url) - video = transform_video(item, channel, playlist) - if existing and existing.get("publish") is False: - video["publish"] = False - all_videos[url] = video - if existing: - counters["updated"] += 1 - else: - counters["new"] += 1 - - write_videos(all_videos, output_path) - progress.advance(task, len(batch)) - - existing_videos = all_videos - - # Write final state (includes skipped videos preserved from existing data) - existing_videos = deduplicate_videos(existing_videos) - write_videos(existing_videos, output_path) - - # Phase 6: Summary report - total = len(existing_videos) - console.print( - f"\n[bold green]\u2713 Done![/] {total} videos written to {output_path}" - ) - console.print( - f" New: {counters['new']} | Updated: {counters['updated']}" - f" | Skipped: {counters['skipped']} | Errors: {counters['errors']}" - ) - console.print(f" API quota used: ~{quota_used} units\n") + return videolib.run_main(summary, log, args.summary_json, body) if __name__ == "__main__": - main() + sys.exit(main()) From e1ef6a641f86e0dd7d9a4c525394aece63838606 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 12:59:36 +0100 Subject: [PATCH 03/20] Only write changed video pages in sync-videos.py and report changes - Compare the rendered _index.md with the file on disk and only write it when it differs; log and record which (external.) keys changed - Download the thumbnail again when its URL changes, retry once, and record a failed download as an item error instead of only printing it - Record renames, merges, updated references, and orphaned directories in the run summary - Write files atomically; add --dry-run and --summary-json; exit codes 0/1/2 On the current repo, a run now leaves 1,692 of 1,710 pages untouched; the 18 updates pick up people and software pages added since the last sync. --- scripts/sync-videos.py | 568 ++++++++++++++++++++---------------- scripts/test-sync-videos.py | 325 +++++++++++++++++++++ 2 files changed, 638 insertions(+), 255 deletions(-) create mode 100755 scripts/test-sync-videos.py diff --git a/scripts/sync-videos.py b/scripts/sync-videos.py index 9e87070ac..aa99addca 100755 --- a/scripts/sync-videos.py +++ b/scripts/sync-videos.py @@ -1,5 +1,6 @@ -#!/usr/bin/env -S uv run +#!/usr/bin/env -S uv run --script # /// script +# requires-python = ">=3.11" # dependencies = [ # "pyyaml>=6.0", # "rich>=13.0.0", @@ -16,31 +17,43 @@ and can be adjusted with include/exclude/override blocks in the _index.md. When a video's slug changes (e.g., because its title changed), the existing directory is renamed, or merged into the new one, and references are updated. + +Files are only written when their content changes, so a run without changes +on YouTube leaves the working tree untouched. The thumbnail is downloaded +again when its URL changes. + +Exit codes: 0 = success, 1 = fatal (nothing written), 2 = partial (some +videos failed; the rest was written). """ +from __future__ import annotations + +import argparse +import logging import re import shutil import sys +import time import tomllib from pathlib import Path -from typing import Any +from typing import Any, Callable -import requests import yaml -from rich.console import Console -from rich.progress import ( - BarColumn, - Progress, - SpinnerColumn, - TaskProgressColumn, - TextColumn, -) -console = Console(stderr=True) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 + +log = logging.getLogger("sync-videos") + +THUMBNAIL_ATTEMPTS = 2 + +parse_frontmatter = videolib.split_frontmatter # --------------------------------------------------------------------------- -# Shared utilities (mirrored from update-software-frontmatter.py) +# Frontmatter formatting (mirrored from update-software-frontmatter.py; the +# output must stay byte-for-byte stable, or every page shows up as changed) # --------------------------------------------------------------------------- @@ -49,34 +62,6 @@ def ignore_aliases(self, data): return True -def parse_frontmatter(content: str) -> tuple[dict[str, Any], str, str]: - lines = content.split("\n") - - if not lines or lines[0].strip() != "---": - return {}, "", content - - end_idx = None - for i in range(1, len(lines)): - if lines[i].strip() == "---": - end_idx = i - break - - if end_idx is None: - return {}, "", content - - yaml_lines = lines[1:end_idx] - yaml_section = "\n".join(yaml_lines) - - try: - frontmatter = yaml.safe_load(yaml_section) or {} - except yaml.YAMLError as e: - console.print(f"[yellow]Warning:[/] Failed to parse YAML: {e}") - return {}, "", content - - remaining_content = "\n".join(lines[end_idx + 1 :]) - return frontmatter, yaml_section, remaining_content - - def sort_dict_with_keys_at_end(data: Any, keys_at_end: list[str]) -> Any: if isinstance(data, dict): regular_keys = [] @@ -147,19 +132,17 @@ def format_frontmatter(frontmatter: dict[str, Any]) -> str: ) yaml_str = add_blank_lines_before_keys(yaml_str, keys_at_end) - yaml_str = add_comment_after_key( - yaml_str, "external", "updated automatically, do not edit" - ) + yaml_str = add_comment_after_key(yaml_str, "external", "updated automatically, do not edit") return yaml_str.strip() -def write_frontmatter( - file_path: Path, frontmatter: dict[str, Any], remaining_content: str -) -> None: - yaml_content = format_frontmatter(frontmatter) - new_content = f"---\n{yaml_content}\n---\n{remaining_content}" - file_path.write_text(new_content, encoding="utf-8") +def render_index(frontmatter: dict[str, Any], remaining_content: str) -> str: + return f"---\n{format_frontmatter(frontmatter)}\n---\n{remaining_content}" + + +def write_frontmatter(file_path: Path, frontmatter: dict[str, Any], remaining_content: str) -> None: + videolib.atomic_write_text(file_path, render_index(frontmatter, remaining_content)) def compute_top_level_keys( @@ -198,6 +181,23 @@ def compute_top_level_keys( return result +def changed_keys(old: dict[str, Any], new: dict[str, Any]) -> list[str]: + """Top-level keys that differ, with ``external.`` for the synced block.""" + changes = [] + for key in sorted(set(old) | set(new)): + if key == "external": + old_ext = old.get("external") or {} + new_ext = new.get("external") or {} + changes += [ + f"external.{k}" + for k in sorted(set(old_ext) | set(new_ext)) + if old_ext.get(k) != new_ext.get(k) + ] + elif old.get(key) != new.get(key): + changes.append(key) + return changes + + # --------------------------------------------------------------------------- # Data loading # --------------------------------------------------------------------------- @@ -209,16 +209,13 @@ def load_videos(videos_toml: Path) -> list[dict[str, Any]]: return data.get("videos", []) -def load_titles_by_slug( - directory: Path, ignore_slugs: set[str] | None = None -) -> dict[str, str]: +def load_titles_by_slug(directory: Path, ignore_slugs: set[str] | None = None) -> dict[str, str]: titles = {} for index_file in sorted(directory.glob("*/_index.md")): slug = index_file.parent.name if ignore_slugs and slug in ignore_slugs: continue - content = index_file.read_text(encoding="utf-8") - frontmatter, _, _ = parse_frontmatter(content) + frontmatter, _, _ = parse_frontmatter(index_file.read_text(encoding="utf-8")) title = str(frontmatter.get("title", "")).strip() if title: titles[slug] = title @@ -233,11 +230,6 @@ def load_software_match_ignore(ignore_toml: Path) -> set[str]: return set(data.get("names", [])) -# --------------------------------------------------------------------------- -# Filtering -# --------------------------------------------------------------------------- - - def should_process(video: dict[str, Any]) -> tuple[bool, str]: if video.get("publish") is False: return False, "publish=false" @@ -263,52 +255,49 @@ def detect_people(video: dict[str, Any], people_names: list[str]) -> list[str]: return find_matches(combined, people_names) -def detect_software( - video: dict[str, Any], software_titles: dict[str, str] -) -> list[str]: +def detect_software(video: dict[str, Any], software_titles: dict[str, str]) -> list[str]: # Match on display titles but return slugs, since Hugo builds # /software// from the stored value. combined = video.get("title", "") + " " + video.get("description", "") - return [ - slug - for slug, title in software_titles.items() - if find_matches(combined, [title]) - ] + return [slug for slug, title in software_titles.items() if find_matches(combined, [title])] # --------------------------------------------------------------------------- # Building external # --------------------------------------------------------------------------- +EXTERNAL_FIELDS = [ + "url", + "title", + "date", + "description", + "duration", + "channel", + "playlist", + "tags", + "view_count", + "like_count", + "comment_count", + "thumbnail", + "language", + "has_captions", + "definition", + "last_updated", +] + def build_external( video: dict[str, Any], detected_people: list[str], detected_software: list[str], ) -> dict[str, Any]: - fields = [ - "url", - "title", - "date", - "description", - "duration", - "channel", - "playlist", - "tags", - "view_count", - "like_count", - "comment_count", - "thumbnail", - "language", - "has_captions", - "definition", - "last_updated", - ] external: dict[str, Any] = { - f: video[f] for f in fields if f in video and video[f] is not None + f: video[f] for f in EXTERNAL_FIELDS if f in video and video[f] is not None } if "tags" in external and isinstance(external["tags"], list): - external["tags"] = [re.sub(r"[#?/]", "", t) for t in external["tags"] if re.sub(r"[#?/]", "", t)] + external["tags"] = [ + re.sub(r"[#?/]", "", t) for t in external["tags"] if re.sub(r"[#?/]", "", t) + ] if detected_people: external["people"] = detected_people if detected_software: @@ -316,24 +305,61 @@ def build_external( return external +def build_frontmatter( + video: dict[str, Any], + frontmatter: dict[str, Any], + people_names: list[str], + software_titles: dict[str, str], +) -> dict[str, Any]: + """Return the new frontmatter for ``video``, given the current one.""" + frontmatter = dict(frontmatter) + external = build_external( + video, detect_people(video, people_names), detect_software(video, software_titles) + ) + frontmatter["external"] = external + + include = frontmatter.get("include", {}) + exclude = frontmatter.get("exclude", {}) + override = frontmatter.get("override", {}) + top_level = compute_top_level_keys(external, include, exclude, override) + + if "date" in override: + date_str = str(override["date"])[:10] + else: + date_str = str(video["date"])[:10] + + frontmatter["title"] = top_level.get("title", video["title"]) + frontmatter["resource_type"] = "video" + frontmatter["date"] = date_str + frontmatter["description"] = top_level.get("description", video["description"]) + frontmatter["people"] = top_level.get("people", []) + frontmatter["software"] = top_level.get("software", []) + frontmatter["tags"] = top_level.get("tags", []) + frontmatter["image"] = "thumbnail.jpg" + if "resources" not in frontmatter: + frontmatter["resources"] = [] + return frontmatter + + # --------------------------------------------------------------------------- # Renamed videos # --------------------------------------------------------------------------- -def merge_video_dirs(old_dir: Path, new_dir: Path) -> None: - # Keep files and manual blocks from the old directory that the new one lacks. - for item in old_dir.iterdir(): +def merge_video_dirs(old_dir: Path, new_dir: Path) -> list[str]: + """Move files and manual blocks the new directory lacks; remove the old one. + + Returns the names of the files that were moved. + """ + moved = [] + for item in sorted(old_dir.iterdir()): if item.name != "_index.md" and not (new_dir / item.name).exists(): shutil.move(str(item), new_dir / item.name) + moved.append(item.name) - old_frontmatter, _, _ = parse_frontmatter( - (old_dir / "_index.md").read_text(encoding="utf-8") - ) + old_frontmatter, _, _ = parse_frontmatter((old_dir / "_index.md").read_text(encoding="utf-8")) new_index = new_dir / "_index.md" - new_frontmatter, _, remaining_content = parse_frontmatter( - new_index.read_text(encoding="utf-8") - ) + new_frontmatter, _, remaining_content = parse_frontmatter(new_index.read_text(encoding="utf-8")) merged = False for key in ["include", "exclude", "override", "resources"]: if old_frontmatter.get(key) and not new_frontmatter.get(key): @@ -343,22 +369,28 @@ def merge_video_dirs(old_dir: Path, new_dir: Path) -> None: write_frontmatter(new_index, new_frontmatter, remaining_content) shutil.rmtree(old_dir) + return moved def update_video_references(content_dir: Path, old_slug: str, new_slug: str) -> list[Path]: pattern = re.compile(r"videos/" + re.escape(old_slug) + r"(?![\w-])") updated = [] - for md_file in content_dir.rglob("*.md"): + for md_file in sorted(content_dir.rglob("*.md")): text = md_file.read_text(encoding="utf-8") new_text = pattern.sub(f"videos/{new_slug}", text) if new_text != text: - md_file.write_text(new_text, encoding="utf-8") + videolib.atomic_write_text(md_file, new_text) updated.append(md_file) return updated def handle_renamed_videos( - videos: list[dict[str, Any]], videos_dir: Path, content_dir: Path + videos: list[dict[str, Any]], + videos_dir: Path, + content_dir: Path, + summary: videolib.RunSummary, + *, + dry_run: bool = False, ) -> None: # Slugs are derived from YouTube titles, so a title change produces a new # slug. Match directories to videos by URL and move or merge stale ones. @@ -375,21 +407,37 @@ def handle_renamed_videos( url = (frontmatter.get("external") or {}).get("url") new_slug = slug_by_url.get(url) if not new_slug: - console.print( - f" [yellow]Warning:[/] {old_slug} is not in videos.toml, leaving as is" - ) + log.warning("Orphan: %s is not in videos.toml (or not published); leaving it as is", old_slug) + summary.add("orphan", slug=old_slug, url=url, video_id=videolib.youtube_id(url)) continue new_dir = videos_dir / new_slug - if new_dir.exists(): - merge_video_dirs(old_dir, new_dir) - console.print(f" [magenta]⇢[/] Merged {old_slug} into {new_slug}") + action = "merged" if new_dir.exists() else "renamed" + log.info("%s %s -> %s (%s)", "Merging" if action == "merged" else "Renaming", old_slug, new_slug, videolib.youtube_id(url)) + if dry_run: + summary.add("renamed", action=action, old_slug=old_slug, slug=new_slug, url=url, video_id=videolib.youtube_id(url), dry_run=True) + continue + + moved = None + if action == "merged": + moved = merge_video_dirs(old_dir, new_dir) + log.info(" Moved files: %s", ", ".join(moved) or "(none)") else: old_dir.rename(new_dir) - console.print(f" [magenta]⇢[/] Renamed {old_slug} to {new_slug}") - for md_file in update_video_references(content_dir, old_slug, new_slug): - console.print(f" [dim]Updated reference in {md_file}[/]") + updated = update_video_references(content_dir, old_slug, new_slug) + for md_file in updated: + log.info(" Updated reference in %s", md_file.relative_to(content_dir.parent)) + summary.add( + "renamed", + action=action, + old_slug=old_slug, + slug=new_slug, + url=url, + video_id=videolib.youtube_id(url), + moved_files=moved, + updated_references=[str(p.relative_to(content_dir.parent)) for p in updated], + ) # --------------------------------------------------------------------------- @@ -397,19 +445,26 @@ def handle_renamed_videos( # --------------------------------------------------------------------------- -def download_thumbnail(url: str, dest_dir: Path) -> bool: - dest = dest_dir / "thumbnail.jpg" - if dest.exists(): - return False - - try: - response = requests.get(url, timeout=15) - response.raise_for_status() - dest.write_bytes(response.content) - return True - except requests.RequestException as e: - console.print(f" [yellow]Warning:[/] Could not download thumbnail: {e}") - return False +def download_thumbnail( + url: str, + dest: Path, + http_get: Callable[..., Any], + *, + attempts: int = THUMBNAIL_ATTEMPTS, + retry_delay: float = 2.0, +) -> None: + """Download ``url`` to ``dest`` atomically; raise after ``attempts`` tries.""" + for attempt in range(1, attempts + 1): + try: + response = http_get(url, timeout=15) + response.raise_for_status() + videolib.atomic_write_bytes(dest, response.content) + return + except Exception as e: + if attempt == attempts: + raise + log.warning(" Thumbnail download failed (attempt %d/%d): %s; retrying", attempt, attempts, e) + time.sleep(retry_delay) # --------------------------------------------------------------------------- @@ -422,60 +477,60 @@ def process_video( videos_dir: Path, people_names: list[str], software_titles: dict[str, str], -) -> str: - try: - slug = video["slug"] - video_dir = videos_dir / slug - index_file = video_dir / "_index.md" - is_new = not video_dir.exists() - - if is_new: - video_dir.mkdir(parents=True, exist_ok=True) - frontmatter: dict[str, Any] = {} - remaining_content = "\n" - else: - content = index_file.read_text(encoding="utf-8") - frontmatter, _, remaining_content = parse_frontmatter(content) - - detected_people = detect_people(video, people_names) - detected_software = detect_software(video, software_titles) - - external = build_external(video, detected_people, detected_software) - frontmatter["external"] = external - - include = frontmatter.get("include", {}) - exclude = frontmatter.get("exclude", {}) - override = frontmatter.get("override", {}) - - top_level = compute_top_level_keys(external, include, exclude, override) - - if "date" in override: - date_str = str(override["date"])[:10] + *, + http_get: Callable[..., Any], + dry_run: bool = False, + retry_delay: float = 2.0, +) -> dict[str, Any]: + """Create or update one video directory; return a summary item.""" + slug = video["slug"] + video_dir = videos_dir / slug + index_file = video_dir / "_index.md" + is_new = not index_file.exists() + + if is_new: + old_frontmatter: dict[str, Any] = {} + remaining_content = "\n" + old_text = None + else: + old_text = index_file.read_text(encoding="utf-8") + old_frontmatter, _, remaining_content = parse_frontmatter(old_text) + + frontmatter = build_frontmatter(video, old_frontmatter, people_names, software_titles) + new_text = render_index(frontmatter, remaining_content) + + result: dict[str, Any] = {"slug": slug, "url": video.get("url"), "video_id": videolib.youtube_id(video.get("url"))} + if is_new: + result["status"] = "created" + result["title"] = frontmatter["title"] + elif new_text != old_text: + result["status"] = "updated" + result["changes"] = changed_keys(old_frontmatter, frontmatter) + else: + result["status"] = "unchanged" + + if new_text != old_text and not dry_run: + video_dir.mkdir(parents=True, exist_ok=True) + videolib.atomic_write_text(index_file, new_text) + + thumbnail = video_dir / "thumbnail.jpg" + old_thumbnail_url = (old_frontmatter.get("external") or {}).get("thumbnail") + thumbnail_url = video.get("thumbnail") + if thumbnail_url and (not thumbnail.exists() or (old_thumbnail_url and old_thumbnail_url != thumbnail_url)): + reason = "missing" if not thumbnail.exists() else "url changed" + if dry_run: + result["thumbnail"] = f"would download ({reason})" else: - date_val = video["date"] - date_str = str(date_val)[:10] - - frontmatter["title"] = top_level.get("title", video["title"]) - frontmatter["resource_type"] = "video" - frontmatter["date"] = date_str - frontmatter["description"] = top_level.get("description", video["description"]) - frontmatter["people"] = top_level.get("people", []) - frontmatter["software"] = top_level.get("software", []) - frontmatter["tags"] = top_level.get("tags", []) - frontmatter["image"] = "thumbnail.jpg" - if "resources" not in frontmatter: - frontmatter["resources"] = [] - - write_frontmatter(index_file, frontmatter, remaining_content) - download_thumbnail(video["thumbnail"], video_dir) - - return "created" if is_new else "updated" - - except Exception as e: - console.print( - f" [bold red]Error:[/] Failed to process {video.get('slug', '?')}: {e}" - ) - return "error" + try: + video_dir.mkdir(parents=True, exist_ok=True) + download_thumbnail(thumbnail_url, thumbnail, http_get, retry_delay=retry_delay) + result["thumbnail"] = f"downloaded ({reason})" + log.debug(" Downloaded thumbnail for %s (%s)", slug, reason) + except Exception as e: # noqa: BLE001 - recorded as an item error + result["status"] = "error" + result["reason"] = "thumbnail_failed" + result["detail"] = f"{type(e).__name__}: {e} ({thumbnail_url})" + return result # --------------------------------------------------------------------------- @@ -483,89 +538,92 @@ def process_video( # --------------------------------------------------------------------------- -def main() -> None: - console.print("\n[bold blue]Video Syncer[/]\n") - - script_dir = Path(__file__).parent - project_root = script_dir.parent - videos_toml = project_root / "data" / "videos.toml" - videos_dir = project_root / "content" / "resources" / "videos" - people_dir = project_root / "content" / "people" - software_dir = project_root / "content" / "software" - ignore_toml = project_root / "data" / "software-match-ignore.toml" - - for path, label in [ - (videos_toml, "videos TOML"), - (people_dir, "people directory"), - (software_dir, "software directory"), - ]: - if not path.exists(): - console.print(f"[bold red]Error:[/] {label} not found: {path}") - sys.exit(1) - - all_videos = load_videos(videos_toml) - people_names = list(load_titles_by_slug(people_dir).values()) - ignore_slugs = load_software_match_ignore(ignore_toml) - software_titles = load_titles_by_slug(software_dir, ignore_slugs=ignore_slugs) - - console.print( - f"[dim]Loaded {len(all_videos)} videos, " - f"{len(people_names)} people, " - f"{len(software_titles)} software entries " - f"({len(ignore_slugs)} ignored)[/]\n" - ) - - videos_dir.mkdir(parents=True, exist_ok=True) - - handle_renamed_videos( - [v for v in all_videos if should_process(v)[0]], - videos_dir, - project_root / "content", - ) - - created = updated = skipped = errors = 0 - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TaskProgressColumn(), - console=console, - ) as progress: - task = progress.add_task("[cyan]Syncing...", total=len(all_videos)) - - for video in all_videos: - slug = video.get("slug", "?") - progress.update(task, description=f"[cyan]{slug}") +def main(argv: list[str] | None = None, http_get: Callable[..., Any] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.strip().split("\n\n")[0]) + parser.add_argument("--dry-run", action="store_true", help="show what would change; write nothing") + videolib.add_common_args(parser) + args = parser.parse_args(argv) + + videolib.setup_logging("sync-videos", args.verbose) + summary = videolib.RunSummary("sync-videos") + summary.extra["dry_run"] = args.dry_run + + root = args.project_root + videos_toml = root / "data" / "videos.toml" + videos_dir = root / "content" / "resources" / "videos" + people_dir = root / "content" / "people" + software_dir = root / "content" / "software" + ignore_toml = root / "data" / "software-match-ignore.toml" + + def body() -> None: + nonlocal http_get + summary.environment = videolib.log_environment(log, packages=["pyyaml", "requests"]) + for path, label in [ + (videos_toml, "videos TOML"), + (people_dir, "people directory"), + (software_dir, "software directory"), + ]: + if not path.exists(): + summary.fatal = f"{label} not found: {path}" + return + if http_get is None: + import requests + + http_get = requests.get + + all_videos = load_videos(videos_toml) + people_names = list(load_titles_by_slug(people_dir).values()) + ignore_slugs = load_software_match_ignore(ignore_toml) + software_titles = load_titles_by_slug(software_dir, ignore_slugs=ignore_slugs) + log.info( + "Loaded %d videos, %d people, %d software entries (%d ignored)", + len(all_videos), + len(people_names), + len(software_titles), + len(ignore_slugs), + ) - ok, _ = should_process(video) - if not ok: - skipped += 1 + published = [v for v in all_videos if should_process(v)[0]] + summary.tally("unpublished", len(all_videos) - len(published)) + + videos_dir.mkdir(parents=True, exist_ok=True) + handle_renamed_videos(published, videos_dir, root / "content", summary, dry_run=args.dry_run) + + with videolib.make_progress() as progress: + task = progress.add_task("Syncing", total=len(published)) + for video in published: + progress.update(task, description=video.get("slug", "?")) + try: + result = process_video( + video, videos_dir, people_names, software_titles, http_get=http_get, dry_run=args.dry_run + ) + except Exception as e: # noqa: BLE001 - recorded as an item error + log.exception("Failed to process %s", video.get("slug", "?")) + result = { + "status": "error", + "slug": video.get("slug"), + "video_id": videolib.youtube_id(video.get("url")), + "reason": "exception", + "detail": f"{type(e).__name__}: {e}", + } + status = result.pop("status") + if status == "created": + log.info("Created %s (%s)", result["slug"], result["video_id"]) + elif status == "updated": + log.info("Updated %s (%s): %s", result["slug"], result["video_id"], ", ".join(result["changes"])) + elif status == "error": + log.error("Error %s (%s): %s", result["slug"], result.get("video_id"), result.get("detail")) + if "thumbnail" in result: + log.info(" Thumbnail %s: %s", result["slug"], result["thumbnail"]) + + if status == "unchanged" and "thumbnail" not in result: + summary.tally("unchanged") + else: + summary.add(status, **result) progress.advance(task) - continue - - result = process_video(video, videos_dir, people_names, software_titles) - if result == "created": - console.print(f" [green]✓[/] Created {slug}") - created += 1 - elif result == "updated": - console.print(f" [cyan]↑[/] Updated {slug}") - updated += 1 - elif result == "error": - errors += 1 - - progress.advance(task) - - console.print("\n[bold]Summary:[/]") - console.print(f" [green]✓[/] Created: {created}") - console.print(f" [cyan]↑[/] Updated: {updated}") - console.print(f" [dim]○[/] Skipped: {skipped}") - if errors: - console.print(f" [red]✗[/] Errors: {errors}") - sys.exit(1) - console.print("\n[bold green]✓ Done![/]\n") + return videolib.run_main(summary, log, args.summary_json, body) if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/scripts/test-sync-videos.py b/scripts/test-sync-videos.py new file mode 100755 index 000000000..3e226dd1a --- /dev/null +++ b/scripts/test-sync-videos.py @@ -0,0 +1,325 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "requests>=2.28.0", +# "tomli-w>=1.0.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for sync-videos.py.""" + +import json +import sys +from pathlib import Path + +import pytest +import tomli_w + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from videotesting import FakeHttp, load_script, make_repo, write # noqa: E402 + +s = load_script("sync-videos.py") + + +def video(vid="abcdefghijk", title="Intro to Quarto with Hadley Wickham", **overrides): + v = { + "url": f"https://www.youtube.com/watch?v={vid}", + "title": title, + "slug": f"2026-10-01_{title.lower().replace(' ', '-')}", + "date": "2026-10-01T12:00:00Z", + "description": "Learn Quarto and Shiny. #quarto", + "duration": 600, + "channel": "Posit PBC", + "playlist": "", + "tags": ["#quarto", "R/Python", "?", "data"], + "view_count": 1, + "like_count": 1, + "comment_count": 0, + "thumbnail": f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg", + "language": "en", + "has_captions": False, + "definition": "hd", + "publish": True, + "last_updated": "2026-11-01T00:00:00+00:00", + } + v.update(overrides) + return v + + +def write_videos(repo, videos): + (repo / "data/videos.toml").write_text(tomli_w.dumps({"videos": videos})) + + +def run(repo, http=None, *args): + summary = repo / "summary.json" + code = s.main( + ["--project-root", str(repo), "--summary-json", str(summary), *args], + http_get=http or FakeHttp(), + ) + return code, json.loads(summary.read_text()) + + +def items(data, status): + return [i for i in data["items"] if i["status"] == status] + + +@pytest.fixture +def repo(tmp_path): + return make_repo(tmp_path) + + +# --- Pure helpers --- + + +class TestComputeTopLevelKeys: + def test_external_only(self): + assert s.compute_top_level_keys({"title": "T", "people": ["A"]}, {}, {}, {}) == { + "title": "T", + "people": ["A"], + } + + def test_include_dedupes(self): + result = s.compute_top_level_keys({"software": ["quarto"]}, {"software": ["shiny", "quarto"]}, {}, {}) + assert result["software"] == ["quarto", "shiny"] + + def test_include_without_external_value(self): + assert s.compute_top_level_keys({}, {"people": ["A"]}, {}, {})["people"] == ["A"] + + def test_exclude(self): + result = s.compute_top_level_keys({"people": ["A", "B"]}, {}, {"people": ["B"]}, {}) + assert result["people"] == ["A"] + + def test_override_wins(self): + result = s.compute_top_level_keys({"title": "T"}, {"title": ["x"]}, {}, {"title": "Better"}) + assert result["title"] == "Better" + + +class TestDetection: + def test_people_word_boundaries(self): + v = {"title": "Talk by Hadley Wickham", "description": "Not Hadley Wickhamson"} + assert s.detect_people(v, ["Hadley Wickham", "Wickham Hadley"]) == ["Hadley Wickham"] + + def test_people_case_insensitive_and_unicode(self): + v = {"title": "mine çetinkaya-rundel on teaching", "description": ""} + assert s.detect_people(v, ["Mine Çetinkaya-Rundel"]) == ["Mine Çetinkaya-Rundel"] + + def test_no_partial_word_match(self): + assert s.find_matches("Shinylive apps", ["Shiny"]) == [] + assert s.find_matches("Shiny, live", ["Shiny"]) == ["Shiny"] + + def test_software_returns_slugs(self): + v = {"title": "Quarto tips", "description": "with shiny"} + assert s.detect_software(v, {"quarto": "Quarto", "shiny": "Shiny", "gt": "gt"}) == ["quarto", "shiny"] + + +def test_build_external_cleans_tags(): + ext = s.build_external(video(), ["Hadley Wickham"], ["quarto"]) + assert ext["tags"] == ["quarto", "RPython", "data"] + assert ext["people"] == ["Hadley Wickham"] + assert ext["software"] == ["quarto"] + assert "publish" not in ext and "slug" not in ext + + +def test_changed_keys(): + old = {"title": "A", "external": {"url": "u", "view_count": 1}, "tags": []} + new = {"title": "B", "external": {"url": "u", "view_count": 2, "people": ["X"]}, "tags": []} + assert s.changed_keys(old, new) == ["external.people", "external.view_count", "title"] + + +# --- main --- + + +class TestMain: + def test_create(self, repo): + write_videos(repo, [video()]) + http = FakeHttp() + code, data = run(repo, http) + assert code == 0 + d = repo / "content/resources/videos/2026-10-01_intro-to-quarto-with-hadley-wickham" + fm, _, _ = s.parse_frontmatter((d / "_index.md").read_text()) + assert fm["title"] == "Intro to Quarto with Hadley Wickham" + assert fm["date"] == "2026-10-01" + assert fm["people"] == ["Hadley Wickham"] + assert fm["software"] == ["quarto", "shiny"] # "commons" is in the ignore list + assert fm["image"] == "thumbnail.jpg" + assert fm["resources"] == [] + assert fm["external"]["url"] == video()["url"] + assert (d / "thumbnail.jpg").read_bytes() == b"JPEG:" + video()["thumbnail"].encode() + (created,) = items(data, "created") + assert created["video_id"] == "abcdefghijk" + assert created["thumbnail"] == "downloaded (missing)" + + def test_second_run_unchanged_and_untouched(self, repo): + write_videos(repo, [video()]) + run(repo) + index = next((repo / "content/resources/videos").glob("*/_index.md")) + mtime = index.stat().st_mtime_ns + http = FakeHttp() + code, data = run(repo, http) + assert code == 0 + assert data["counts"] == {"unchanged": 1, "unpublished": 0} + assert data["items"] == [] + assert index.stat().st_mtime_ns == mtime + assert http.requested == [] # no thumbnail download + + def test_update_reports_changed_keys(self, repo): + write_videos(repo, [video()]) + run(repo) + write_videos(repo, [video(view_count=99, description="Nothing else here")]) + code, data = run(repo) + (updated,) = items(data, "updated") + assert updated["changes"] == [ + "description", + "external.description", + "external.software", + "external.view_count", + "software", + ] + + def test_manual_blocks_and_body_preserved(self, repo): + write_videos(repo, [video()]) + run(repo) + index = next((repo / "content/resources/videos").glob("*/_index.md")) + text = index.read_text().replace("---\n", "---\noverride:\n title: Custom\nexclude:\n software:\n - shiny\n", 1) + index.write_text(text + "Some body text.\n") + run(repo) + fm, _, body = s.parse_frontmatter(index.read_text()) + assert fm["title"] == "Custom" + assert fm["software"] == ["quarto"] + assert "Some body text." in body + + def test_override_date(self, repo): + write_videos(repo, [video()]) + run(repo) + index = next((repo / "content/resources/videos").glob("*/_index.md")) + index.write_text(index.read_text().replace("---\n", "---\noverride:\n date: 2025-09-17\n", 1)) + run(repo) + assert s.parse_frontmatter(index.read_text())[0]["date"] == "2025-09-17" + + def test_unpublished_skipped(self, repo): + write_videos(repo, [video(publish=False)]) + code, data = run(repo) + assert data["counts"] == {"unpublished": 1} + assert list((repo / "content/resources/videos").iterdir()) == [] + + def test_rename_moves_dir_and_updates_references(self, repo): + write_videos(repo, [video()]) + run(repo) + videos_dir = repo / "content/resources/videos" + old = videos_dir / "2026-10-01_intro-to-quarto-with-hadley-wickham" + write(old / "transcription.vtt", "WEBVTT\n") + post = write( + repo / "content/blog/post/index.md", + "See [talk](/resources/videos/2026-10-01_intro-to-quarto-with-hadley-wickham/) " + "but not videos/2026-10-01_intro-to-quarto-with-hadley-wickham-part-2.\n", + ) + write_videos(repo, [video(title="Quarto deep dive", slug="2026-10-01_quarto-deep-dive")]) + code, data = run(repo) + assert code == 0 + assert not old.exists() + assert (videos_dir / "2026-10-01_quarto-deep-dive/transcription.vtt").exists() + assert post.read_text() == ( + "See [talk](/resources/videos/2026-10-01_quarto-deep-dive/) " + "but not videos/2026-10-01_intro-to-quarto-with-hadley-wickham-part-2.\n" + ) + (renamed,) = items(data, "renamed") + assert renamed["action"] == "renamed" + assert renamed["old_slug"] == "2026-10-01_intro-to-quarto-with-hadley-wickham" + assert renamed["slug"] == "2026-10-01_quarto-deep-dive" + assert renamed["updated_references"] == ["content/blog/post/index.md"] + + def test_merge_into_existing_dir(self, repo): + videos_dir = repo / "content/resources/videos" + write_videos(repo, [video()]) + run(repo) + old = videos_dir / "2026-10-01_intro-to-quarto-with-hadley-wickham" + write(old / "transcription.vtt", "WEBVTT\n") + index = old / "_index.md" + index.write_text(index.read_text().replace("---\n", "---\ninclude:\n people:\n - Someone\n", 1)) + # A directory for the new slug already exists (e.g. created by hand). + write(videos_dir / "2026-10-01_new/_index.md", "---\ntitle: placeholder\n---\n") + write_videos(repo, [video(title="New", slug="2026-10-01_new")]) + code, data = run(repo) + assert not old.exists() + new = videos_dir / "2026-10-01_new" + assert (new / "transcription.vtt").exists() + fm, _, _ = s.parse_frontmatter((new / "_index.md").read_text()) + assert fm["include"] == {"people": ["Someone"]} + assert "Someone" in fm["people"] + (renamed,) = items(data, "renamed") + assert renamed["action"] == "merged" + assert "transcription.vtt" in renamed["moved_files"] + + def test_orphan_reported(self, repo): + write( + repo / "content/resources/videos/2020-01-01_gone/_index.md", + "---\ntitle: Gone\nexternal:\n url: https://www.youtube.com/watch?v=gonegonegon\n---\n", + ) + write_videos(repo, [video()]) + code, data = run(repo) + assert code == 0 + (orphan,) = items(data, "orphan") + assert orphan["slug"] == "2020-01-01_gone" + assert (repo / "content/resources/videos/2020-01-01_gone").exists() + + def test_thumbnail_redownloaded_when_url_changes(self, repo): + write_videos(repo, [video()]) + run(repo) + new_url = "https://i.ytimg.com/vi/abcdefghijk/maxresdefault.jpg" + write_videos(repo, [video(thumbnail=new_url)]) + http = FakeHttp() + code, data = run(repo, http) + assert http.requested == [new_url] + thumb = next((repo / "content/resources/videos").glob("*/thumbnail.jpg")) + assert thumb.read_bytes() == b"JPEG:" + new_url.encode() + (updated,) = items(data, "updated") + assert updated["thumbnail"] == "downloaded (url changed)" + + def test_thumbnail_failure_is_partial(self, repo, monkeypatch): + monkeypatch.setattr(s.time, "sleep", lambda _: None) + write_videos(repo, [video(), video("zyxwvutsrqp", "Second")]) + http = FakeHttp(fail_urls={video()["thumbnail"]}) + code, data = run(repo, http) + assert code == 2 + (error,) = items(data, "error") + assert error["reason"] == "thumbnail_failed" + assert "connection refused" in error["detail"] + assert http.requested.count(video()["thumbnail"]) == 2 # retried once + # The page itself and the other video were still written. + assert len(list((repo / "content/resources/videos").glob("*/_index.md"))) == 2 + # The next run tries again because the file is still missing. + http2 = FakeHttp() + code, _ = run(repo, http2) + assert code == 0 and http2.requested == [video()["thumbnail"]] + + def test_dry_run_writes_nothing(self, repo): + write_videos(repo, [video()]) + http = FakeHttp() + code, data = run(repo, http, "--dry-run") + assert code == 0 + assert list((repo / "content/resources/videos").iterdir()) == [] + assert http.requested == [] + (created,) = items(data, "created") + assert created["thumbnail"] == "would download (missing)" + + def test_missing_videos_toml_is_fatal(self, repo): + code, data = run(repo) + assert code == 1 + assert "videos TOML not found" in data["fatal"] + + def test_logging(self, repo, capsys, monkeypatch): + monkeypatch.setenv("CI", "true") + write_videos(repo, [video()]) + run(repo) + err = capsys.readouterr().err + assert "Created 2026-10-01_intro-to-quarto-with-hadley-wickham (abcdefghijk)" in err + assert "Summary: created=1, unpublished=0" in err + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) From 116d834e39222c61d7386b5224f07fa129bfbdf8 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 13:03:52 +0100 Subject: [PATCH 04/20] Make transcribe.py robust for unattended runs - Respect transcribe: false and skip videos without a duration (live streams, premieres); process newest first - Add --limit, --max-audio-minutes, --video, --cookies, and --dry-run - Download into a temporary directory and split chunks there (replacing the insecure tempfile.mktemp), so no audio is left behind - Classify yt-dlp failures (bot_check, po_token, http_403, private, ...), log yt-dlp's full output, and retry network errors and 403s once - Stop downloading after three blocked downloads in a row - Retry Whisper calls with logged backoff; reject empty transcripts and warn when a transcript ends well before the video does - Log tool and package versions, timings, sizes, and estimated cost; write a run summary and use exit codes 0/1/2 --- scripts/test-transcribe.py | 452 ++++++++++++++++++ scripts/transcribe.py | 923 +++++++++++++++++++++++-------------- 2 files changed, 1031 insertions(+), 344 deletions(-) create mode 100755 scripts/test-transcribe.py diff --git a/scripts/test-transcribe.py b/scripts/test-transcribe.py new file mode 100755 index 000000000..2f6b4c642 --- /dev/null +++ b/scripts/test-transcribe.py @@ -0,0 +1,452 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "openai>=2.0.0", +# "python-dotenv>=1.0.0", +# "yt-dlp>=2025.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for transcribe.py. yt-dlp and Whisper are faked; ffmpeg is real when installed.""" + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 +from videotesting import ( # noqa: E402 + SAMPLE_VTT, + FakeOpenAI, + FakeYtDlp, + RateLimitError, + StatusError, + load_script, + make_repo, + no_sleep, + write_video_dir, +) + +t = load_script("transcribe.py") + +needs_ffmpeg = pytest.mark.skipif(not shutil.which("ffmpeg"), reason="ffmpeg not installed") + +# Real yt-dlp error output (shortened where it's long). +BOT_CHECK = ( + "ERROR: [youtube] dQw4w9WgXcQ: Sign in to confirm you’re not a bot. Use --cookies-from-browser " + "or --cookies for the authentication. See https://github.com/yt-dlp/yt-dlp/wiki/FAQ" +) +PO_TOKEN = ( + "WARNING: [youtube] dQw4w9WgXcQ: mweb client https formats require a GVS PO Token which was not " + "provided. They will be skipped as they may yield HTTP Error 403.\n" + "ERROR: [youtube] dQw4w9WgXcQ: Requested format is not available. Use --list-formats for a list of available formats" +) +NETWORK = ( + "ERROR: [youtube] dQw4w9WgXcQ: Unable to download API page: (caused by TransportError)" +) +HTTP_403 = "ERROR: unable to download video data: HTTP Error 403: Forbidden" + +URL_A = "https://www.youtube.com/watch?v=aaaaaaaaaaa" +URL_B = "https://www.youtube.com/watch?v=bbbbbbbbbbb" +URL_C = "https://www.youtube.com/watch?v=ccccccccccc" +URL_D = "https://www.youtube.com/watch?v=ddddddddddd" + + +@pytest.mark.parametrize( + "output, reason", + [ + (BOT_CHECK, "bot_check"), + ("ERROR: [youtube] x: Sign in to confirm you're not a bot. This helps protect our community.", "bot_check"), + (PO_TOKEN, "po_token"), + ( + "ERROR: [youtube] x: Sign in to confirm your age. This video may be inappropriate for some users.", + "age_restricted", + ), + ( + "ERROR: [youtube] x: Private video. Sign in if you've been granted access to this video", + "private", + ), + ( + "ERROR: [youtube] x: Join this channel to get access to members-only content like this video", + "members_only", + ), + ("ERROR: [youtube] x: This live event will begin in 3 hours.", "live_or_upcoming"), + ("ERROR: [youtube] x: Premieres in 2 hours", "live_or_upcoming"), + ("ERROR: [youtube] x: Video unavailable. This video has been removed by the uploader", "unavailable"), + (HTTP_403, "http_403"), + ("ERROR: unable to download video data: HTTP Error 429: Too Many Requests", "http_429"), + (NETWORK, "network"), + ("ERROR: [youtube] x: Read timed out.", "network"), + ( + "WARNING: [youtube] No supported JavaScript runtime could be found.\n" + "ERROR: [youtube] x: Requested format is not available.", + "js_runtime", + ), + ("ERROR: [youtube] x: Requested format is not available.", "format_unavailable"), + # An unrelated warning must not override a clear error. + ("WARNING: GVS PO Token which was not provided\nERROR: [youtube] x: Video unavailable", "unavailable"), + ("something else entirely", "unknown"), + ("", "unknown"), + ], +) +def test_classify_ytdlp_error(output, reason): + assert t.classify_ytdlp_error(output) == reason + + +def test_last_error_line(): + assert t.last_error_line(PO_TOKEN).startswith("ERROR: [youtube] dQw4w9WgXcQ: Requested format") + assert t.last_error_line("just a line\n\n") == "just a line" + assert t.last_error_line("") == "(no output)" + + +class TestCircuitBreaker: + def test_trips_after_consecutive_blocks(self): + b = t.CircuitBreaker(threshold=3) + for reason in ["bot_check", "po_token"]: + b.record(reason) + assert not b.tripped + b.record("http_429") + assert b.tripped + + def test_other_outcomes_reset(self): + b = t.CircuitBreaker(threshold=3) + for reason in ["bot_check", "bot_check", None, "bot_check", "private", "bot_check"]: + b.record(reason) + assert not b.tripped and b.consecutive == 1 + + +def test_build_ytdlp_cmd(tmp_path): + cmd = t.build_ytdlp_cmd(URL_A, tmp_path, "chrome", Path("c.txt")) + assert cmd[1:3] == ["-m", "yt_dlp"] + assert "--remote-components" in cmd and "ejs:github" in cmd + assert cmd[cmd.index("--output") + 1] == str(tmp_path / "audio.%(ext)s") + assert cmd[cmd.index("--cookies-from-browser") + 1] == "chrome" + assert cmd[cmd.index("--cookies") + 1] == "c.txt" + assert "--no-warnings" not in cmd # warnings explain failures + assert cmd[-1] == URL_A + + +class TestChunking: + def test_single_chunk(self): + assert t.compute_chunk_boundaries(100, 10, [50], max_bytes=100) == [(0.0, 100)] + + def test_cuts_at_last_silence_before_limit(self): + # 100 s at 1 byte/s, 40 bytes per chunk -> at most 40 s per chunk. + assert t.compute_chunk_boundaries(100, 100, [10, 35, 38, 70, 79], max_bytes=40) == [ + (0.0, 38), + (38, 70), + (70, 100), + ] + + def test_hard_cut_without_silence(self): + assert t.compute_chunk_boundaries(100, 100, [], max_bytes=40) == [(0.0, 40.0), (40.0, 80.0), (80.0, 100)] + + def test_merge_vtt_offsets(self): + part = "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nhi\n" + merged = t.merge_vtt([(part, 0.0), (part, 600.5)]) + assert videolib.parse_vtt(merged) == [(1.0, 2.0, "hi"), (601.5, 602.5, "hi")] + + +class TestCheckTranscript: + def test_empty(self): + with pytest.raises(t.TranscriptError) as e: + t.check_transcript("WEBVTT\n\n", 100) + assert e.value.reason == "empty_transcript" + + def test_ok(self): + cues, warnings = t.check_transcript(SAMPLE_VTT, 26) + assert len(cues) == 3 and warnings == [] + + def test_truncated(self): + _, warnings = t.check_transcript(SAMPLE_VTT, 600) + assert warnings == ["suspect_truncated: last cue ends at 00:00:25.000 of 00:10:00.000"] + + +def test_build_prompt(): + assert t.build_prompt({"title": "Quarto talk", "software": ["quarto", "shiny"]}) == "Quarto talk, quarto, shiny" + assert t.build_prompt({}) == "" + + +# --- Selection --- + + +@pytest.fixture +def repo(tmp_path): + return make_repo(tmp_path) + + +def videos_dir(repo): + return repo / "content/resources/videos" + + +class TestSelect: + def test_rules_and_order(self, repo): + write_video_dir(repo, "2026-01-01_old", url=URL_A) + write_video_dir(repo, "2026-03-01_new", url=URL_B) + write_video_dir(repo, "2026-02-01_done", url=URL_C, vtt=SAMPLE_VTT) + write_video_dir(repo, "2026-02-02_excluded", url=URL_D, extra="transcribe: false\n") + write_video_dir(repo, "2026-02-03_live", url=URL_D, duration=0) + (videos_dir(repo) / "2026-02-04_empty").mkdir() + summary = videolib.RunSummary("transcribe") + selected = t.select_videos(videolib.video_dirs(videos_dir(repo)), summary) + assert [c.slug for c in selected] == ["2026-03-01_new", "2026-01-01_old"] + assert summary.counts() == {"already_transcribed": 1, "excluded": 1, "skipped": 2} + reasons = {i["slug"]: i["reason"] for i in summary.items} + assert reasons == {"2026-02-04_empty": "no_index", "2026-02-03_live": "no_duration"} + + def test_limit(self, repo): + for i in range(5): + write_video_dir(repo, f"2026-01-0{i + 1}_v", url=URL_A) + summary = videolib.RunSummary("transcribe") + selected = t.select_videos(videolib.video_dirs(videos_dir(repo)), summary, limit=2) + assert [c.slug for c in selected] == ["2026-01-05_v", "2026-01-04_v"] + assert summary.tallies["deferred"] == 3 + assert summary.extra["deferred"] == ["2026-01-03_v", "2026-01-02_v", "2026-01-01_v"] + + def test_audio_budget(self, repo): + write_video_dir(repo, "2026-01-03_a", url=URL_A, duration=3600) + write_video_dir(repo, "2026-01-02_b", url=URL_B, duration=1800) + write_video_dir(repo, "2026-01-01_c", url=URL_C, duration=600) + summary = videolib.RunSummary("transcribe") + selected = t.select_videos(videolib.video_dirs(videos_dir(repo)), summary, max_audio_minutes=75) + # 60 min fits, 30 more doesn't, but 10 more does. + assert [c.slug for c in selected] == ["2026-01-03_a", "2026-01-01_c"] + assert summary.extra["selected_audio_minutes"] == 70.0 + + def test_first_video_over_budget_still_taken(self, repo): + write_video_dir(repo, "2026-01-03_long", url=URL_A, duration=4 * 3600) + summary = videolib.RunSummary("transcribe") + selected = t.select_videos(videolib.video_dirs(videos_dir(repo)), summary, max_audio_minutes=60) + assert [c.slug for c in selected] == ["2026-01-03_long"] + + +# --- main --- + + +def run(repo, *args, client=None, ytdlp=None, max_bytes=t.MAX_BYTES): + summary = repo / "summary.json" + code = t.main( + ["--project-root", str(repo), "--summary-json", str(summary), *args], + client=client if client is not None else FakeOpenAI(), + run=ytdlp or FakeYtDlp(), + sleep=no_sleep, + max_bytes=max_bytes, + min_request_interval=0, + ) + return code, json.loads(summary.read_text()) + + +def items(data, status): + return [i for i in data["items"] if i["status"] == status] + + +class TestMain: + def test_transcribes_and_cleans_up(self, repo, monkeypatch, tmp_path): + temp_root = tmp_path / "tmp" + temp_root.mkdir() + monkeypatch.setattr(t.tempfile, "tempdir", str(temp_root)) + a = write_video_dir(repo, "2026-01-02_a", url=URL_A, duration=26, title="Quarto talk") + b = write_video_dir(repo, "2026-01-01_b", url=URL_B, duration=26) + client = FakeOpenAI() + ytdlp = FakeYtDlp() + code, data = run(repo, client=client, ytdlp=ytdlp) + assert code == 0 + assert (a / "transcription.vtt").read_text() == SAMPLE_VTT + assert (b / "transcription.vtt").exists() + assert not list(a.glob("audio.*")) + assert list(temp_root.iterdir()) == [] # temporary download dirs removed + assert [c[-1] for c in ytdlp.calls] == [URL_A, URL_B] + assert client.calls[0]["model"] == "whisper-1" + assert client.calls[0]["response_format"] == "vtt" + assert client.calls[0]["prompt"] == "Quarto talk, quarto" + done = items(data, "transcribed") + assert [i["slug"] for i in done] == ["2026-01-02_a", "2026-01-01_b"] + assert done[0]["cues"] == 3 + assert done[0]["chunks"] == 1 + assert done[0]["estimated_cost_usd"] == pytest.approx(26 / 60 * 0.006, abs=1e-4) + assert "download_s" in done[0]["timings"] + assert data["extra"]["breaker_tripped"] is False + + def test_second_run_does_nothing(self, repo): + write_video_dir(repo, "2026-01-02_a", url=URL_A, duration=26) + run(repo) + ytdlp = FakeYtDlp() + code, data = run(repo, ytdlp=ytdlp) + assert code == 0 and ytdlp.calls == [] + assert data["counts"] == {"already_transcribed": 1} + + def test_circuit_breaker(self, repo): + for i, url in enumerate([URL_A, URL_B, URL_C, URL_D]): + write_video_dir(repo, f"2026-01-0{4 - i}_v{i}", url=url, duration=26) + ytdlp = FakeYtDlp({URL_A: [BOT_CHECK], URL_B: [PO_TOKEN], URL_C: [BOT_CHECK]}) + code, data = run(repo, "--jobs", "1", ytdlp=ytdlp) + assert code == 2 + assert [i["reason"] for i in items(data, "error")] == ["bot_check", "po_token", "bot_check"] + (skipped,) = items(data, "skipped") + assert skipped["reason"] == "download_blocked" and skipped["video_id"] == "ddddddddddd" + assert len(ytdlp.calls) == 3 + assert data["extra"]["breaker_tripped"] is True + + def test_download_error_details_in_summary_and_log(self, repo, capsys, monkeypatch): + monkeypatch.setenv("CI", "true") + write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + code, data = run(repo, ytdlp=FakeYtDlp({URL_A: [PO_TOKEN]})) + assert code == 2 + (error,) = items(data, "error") + assert error["stage"] == "download" + assert error["reason"] == "po_token" + assert error["detail"].startswith("ERROR: [youtube] dQw4w9WgXcQ: Requested format") + assert "GVS PO Token" in error["stderr"] + err = capsys.readouterr().err + assert "2026-01-01_a (aaaaaaaaaaa): download failed: po_token" in err + assert "yt-dlp: WARNING: [youtube] dQw4w9WgXcQ: mweb client" in err + + def test_network_error_retried(self, repo): + write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + ytdlp = FakeYtDlp({URL_A: [NETWORK, "ok"]}) + code, data = run(repo, ytdlp=ytdlp) + assert code == 0 and len(ytdlp.calls) == 2 + + def test_403_retried_once_then_error(self, repo): + write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + ytdlp = FakeYtDlp({URL_A: [HTTP_403, HTTP_403]}) + code, data = run(repo, ytdlp=ytdlp) + (error,) = items(data, "error") + assert error["reason"] == "http_403" and error["attempts"] == 2 + assert len(ytdlp.calls) == 2 + + def test_whisper_rate_limit_retried(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + client = FakeOpenAI([RateLimitError(), StatusError(503)]) + code, _ = run(repo, client=client) + assert code == 0 and len(client.calls) == 3 + assert (a / "transcription.vtt").exists() + + def test_whisper_client_error_not_retried(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + client = FakeOpenAI([StatusError(400, "Invalid file format")]) + code, data = run(repo, client=client) + assert code == 2 and len(client.calls) == 1 + (error,) = items(data, "error") + assert error["reason"] == "transcription_failed" + assert error["detail"] == "StatusError status=400 request_id=req_400: Invalid file format" + assert not (a / "transcription.vtt").exists() + + def test_empty_transcript_not_written(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + code, data = run(repo, client=FakeOpenAI(["WEBVTT\n\n"])) + assert code == 2 + assert items(data, "error")[0]["reason"] == "empty_transcript" + assert not (a / "transcription.vtt").exists() + + def test_truncated_is_a_warning(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=600) + code, data = run(repo) + assert code == 0 and (a / "transcription.vtt").exists() + assert items(data, "transcribed")[0]["warnings"][0].startswith("suspect_truncated") + + def test_existing_audio_used_and_deleted(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + (a / "audio.m4a").write_bytes(b"\0" * 100) + ytdlp = FakeYtDlp() + code, _ = run(repo, ytdlp=ytdlp) + assert code == 0 and ytdlp.calls == [] + assert not (a / "audio.m4a").exists() + + def test_existing_audio_kept_on_failure(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + (a / "audio.m4a").write_bytes(b"\0" * 100) + run(repo, client=FakeOpenAI([StatusError(400)])) + assert (a / "audio.m4a").exists() + + def test_video_option(self, repo): + write_video_dir(repo, "2026-01-02_a", url=URL_A, duration=26) + b = write_video_dir(repo, "2026-01-01_b", url=URL_B, duration=26) + code, data = run(repo, "--video", "2026-01-01") + assert [i["slug"] for i in items(data, "transcribed")] == ["2026-01-01_b"] + assert (b / "transcription.vtt").exists() + + def test_dry_run(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + ytdlp, client = FakeYtDlp(), FakeOpenAI() + code, data = run(repo, "--dry-run", ytdlp=ytdlp, client=client) + assert code == 0 and ytdlp.calls == [] and client.calls == [] + assert items(data, "would_transcribe")[0]["slug"] == "2026-01-01_a" + assert not (a / "transcription.vtt").exists() + + def test_missing_api_key_is_fatal(self, repo, monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + summary = repo / "s.json" + code = t.main(["--project-root", str(repo), "--summary-json", str(summary)], run=FakeYtDlp()) + assert code == 1 + assert "OPENAI_API_KEY" in json.loads(summary.read_text())["fatal"] + + def test_nothing_to_do_needs_no_api_key(self, repo, monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + code = t.main(["--project-root", str(repo)], run=FakeYtDlp()) + assert code == 0 + + def test_missing_ffmpeg_is_fatal(self, repo, monkeypatch): + real_which = shutil.which + monkeypatch.setattr(t.shutil, "which", lambda name: None if name == "ffprobe" else real_which(name)) + code, data = run(repo) + assert code == 1 and data["fatal"] == "Required tools not found: ffprobe" + + +# --- ffmpeg (real) --- + + +def make_tone_silence_tone(path: Path) -> None: + subprocess.run( + [ + "ffmpeg", "-y", "-v", "error", + "-f", "lavfi", "-i", "sine=frequency=440:duration=1", + "-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono:d=1", + "-f", "lavfi", "-i", "sine=frequency=440:duration=1", + "-filter_complex", "[0][1][2]concat=n=3:v=0:a=1", + "-ac", "1", "-ar", "44100", str(path), + ], + check=True, + ) # fmt: skip + + +@needs_ffmpeg +def test_detect_silence(tmp_path): + audio = tmp_path / "audio.wav" + make_tone_silence_tone(audio) + (point,) = t.detect_silence(audio) + assert 1.2 < point < 1.8 + assert t.get_audio_duration(audio) == pytest.approx(3.0, abs=0.05) + + +@needs_ffmpeg +def test_large_audio_split_and_merged(repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=3) + audio = a / "audio.wav" + make_tone_silence_tone(audio) + one_cue = "WEBVTT\n\n00:00:00.200 --> 00:00:00.900\npart\n" + client = FakeOpenAI(default=one_cue) + # Force a split: chunks may be at most ~60% of the file. + code, data = run(repo, client=client, max_bytes=int(audio.stat().st_size * 0.6)) + assert code == 0 + (done,) = items(data, "transcribed") + assert done["chunks"] == 2 + cues = videolib.parse_vtt((a / "transcription.vtt").read_text()) + assert len(cues) == 2 + assert cues[0][0] == pytest.approx(0.2) + assert 1.2 < cues[1][0] - 0.2 < 1.8 # second chunk starts at the silence + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/transcribe.py b/scripts/transcribe.py index c5d7d83d2..4ac2770a7 100755 --- a/scripts/transcribe.py +++ b/scripts/transcribe.py @@ -1,5 +1,6 @@ -#!/usr/bin/env -S uv run +#!/usr/bin/env -S uv run --script # /// script +# requires-python = ">=3.11" # dependencies = [ # "pyyaml>=6.0", # "rich>=13.0.0", @@ -9,86 +10,246 @@ # ] # /// - """ -Download audio from YouTube and transcribe it for each video under -content/resources/videos/. - -For each subdirectory: -- If transcription.vtt already exists, skip. -- If no audio file exists, download it from external.url in _index.md using - yt-dlp (pass --cookies-from-browser BROWSER if authentication is needed). -- If the audio exceeds the 25 MB API limit, split into chunks at silence - boundaries, transcribe each chunk, and merge the VTT output with corrected - timestamps. -- Delete the audio file after a successful transcription. - -Requires OPENAI_API_KEY in .env. -Runs up to --jobs transcriptions in parallel (default 5) with at least 1 second -between API requests and exponential backoff on rate-limit / server errors. +Download audio from YouTube and transcribe it with OpenAI Whisper. + +Processes video directories under content/resources/videos/, newest first, +that don't have a transcription.vtt yet. Skips videos with +`transcribe: false` in their frontmatter and videos without a duration +(live streams and premieres that haven't happened yet). + +For each video: +- Download the audio with yt-dlp into a temporary directory (an audio.* + file already in the video directory is used instead, and deleted after a + successful transcription). +- If the audio exceeds the 25 MB API limit, split it at silences, transcribe + each chunk, and merge the VTT output with corrected timestamps. +- Check the result (at least one cue; a warning if it ends well before the + video does) and write transcription.vtt. + +YouTube may block downloads from datacenter IPs (such as GitHub Actions +runners). Download errors are classified (bot_check, po_token, http_403, ...) +and logged with yt-dlp's full output. After three blocked downloads in a row +the remaining downloads are skipped, so we don't make matters worse. + +Requires OPENAI_API_KEY in the environment or .env, ffmpeg/ffprobe, and a +JavaScript runtime for yt-dlp (Deno). + +Exit codes: 0 = success, 1 = fatal, 2 = partial (some videos failed). """ +from __future__ import annotations + import argparse import asyncio +import logging import os -import random import re +import shutil import subprocess import sys import tempfile +import time +from dataclasses import dataclass, field from pathlib import Path -from typing import Any - -import openai -import yaml -from dotenv import load_dotenv -from rich.console import Console -from rich.progress import ( - BarColumn, - MofNCompleteColumn, - Progress, - SpinnerColumn, - TaskID, - TextColumn, - TimeElapsedColumn, -) - -console = Console(stderr=True) +from typing import Any, Awaitable, Callable -MAX_BYTES = 25 * 1024 * 1024 -MAX_RETRIES = 5 -BASE_BACKOFF = 1.0 - - -def parse_frontmatter(content: str) -> dict[str, Any]: - lines = content.split("\n") - if not lines or lines[0].strip() != "---": - return {} - end_idx = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) - if end_idx is None: - return {} - try: - return yaml.safe_load("\n".join(lines[1:end_idx])) or {} - except yaml.YAMLError: - return {} - - -def find_audio_file(video_dir: Path) -> Path | None: - for f in video_dir.glob("audio.*"): - if f.suffix != ".part": - return f - return None +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import videolib # noqa: E402 + +log = logging.getLogger("transcribe") + +MAX_BYTES = 25 * 1024 * 1024 +WHISPER_MODEL = "whisper-1" +WHISPER_USD_PER_MINUTE = 0.006 + +# Download failures that mean YouTube is blocking us, not this one video. +BLOCKING_REASONS = {"bot_check", "po_token", "http_429"} +BREAKER_THRESHOLD = 3 +# Download failures worth one more try. +RETRY_REASONS = {"network", "http_403"} +DOWNLOAD_RETRY_DELAY = 5.0 +# Warn when the transcript ends before this fraction of the video. +TRUNCATION_RATIO = 0.9 +MAX_STDERR_CHARS = 4000 + + +# --------------------------------------------------------------------------- +# Selecting videos +# --------------------------------------------------------------------------- + + +@dataclass +class Candidate: + dir: Path + url: str + duration: int + frontmatter: dict[str, Any] = field(default_factory=dict) + + @property + def slug(self) -> str: + return self.dir.name + + @property + def video_id(self) -> str | None: + return videolib.youtube_id(self.url) + + +def select_videos( + dirs: list[Path], + summary: videolib.RunSummary, + *, + limit: int | None = None, + max_audio_minutes: float | None = None, +) -> list[Candidate]: + """Pick the videos to transcribe this run, in the given order. + + ``limit`` caps the number of videos and ``max_audio_minutes`` the total + duration (a single video longer than the budget is still taken if it is + the first one, so it doesn't block the queue forever). + """ + selected: list[Candidate] = [] + minutes = 0.0 + deferred: list[str] = [] + + for d in dirs: + if (d / "transcription.vtt").exists(): + summary.tally("already_transcribed") + continue + index = d / "_index.md" + if not index.exists(): + summary.add("skipped", slug=d.name, reason="no_index") + continue + fm = videolib.read_frontmatter(index) + if fm.get("transcribe", True) is False: + summary.tally("excluded") + continue + external = fm.get("external") or {} + url = external.get("url") + if not url: + summary.add("skipped", slug=d.name, reason="no_url") + log.warning("Skipping %s: no external.url", d.name) + continue + duration = int(external.get("duration") or 0) + if duration <= 0: + summary.add("skipped", slug=d.name, video_id=videolib.youtube_id(url), reason="no_duration") + log.info("Skipping %s: no duration (live stream or premiere?)", d.name) + continue -def get_video_url(video_dir: Path) -> str | None: - index_file = video_dir / "_index.md" - if not index_file.exists(): - return None - frontmatter = parse_frontmatter(index_file.read_text(encoding="utf-8")) - return frontmatter.get("external", {}).get("url") + over_limit = limit is not None and len(selected) >= limit + over_budget = ( + max_audio_minutes is not None + and selected + and minutes + duration / 60 > max_audio_minutes + ) + if over_limit or over_budget: + deferred.append(d.name) + continue + selected.append(Candidate(d, url, duration, fm)) + minutes += duration / 60 + + if deferred: + summary.tally("deferred", len(deferred)) + summary.extra["deferred"] = deferred + log.info("Deferred %d videos to a later run (--limit / --max-audio-minutes)", len(deferred)) + summary.extra["selected_audio_minutes"] = round(minutes, 1) + return selected + + +# --------------------------------------------------------------------------- +# Downloading +# --------------------------------------------------------------------------- + +# Ordered: the first matching pattern wins. +YTDLP_ERROR_PATTERNS: list[tuple[str, str]] = [ + ("age_restricted", r"confirm your age|age[- ]restricted|inappropriate for some users"), + ("bot_check", r"confirm you(?:'|’)?re not a bot|Sign in to confirm you|not a bot"), + ("po_token", r"PO Token|po_token|GVS PO"), + ("members_only", r"members[- ]only|Join this channel"), + ("private", r"Private video|video is private"), + ("live_or_upcoming", r"live event will begin|Premieres in|This live event|is not available until|is_upcoming"), + ("js_runtime", r"JavaScript runtime|JS runtime|challenge solv|n challenge|Signature extraction failed"), + ("unavailable", r"Video unavailable|video is not available|has been removed|This video is no longer available|account associated with this video has been terminated"), + ("http_403", r"HTTP Error 403|403: Forbidden|status code 403"), + ("http_429", r"HTTP Error 429|Too Many Requests"), + ("format_unavailable", r"Requested format is not available|No video formats found"), + ( + "network", + r"timed out|Connection reset|Temporary failure in name resolution|Network is unreachable" + r"|Remote end closed|IncompleteRead|Connection refused|getaddrinfo failed|SSL", + ), +] + + +# Warnings that explain a vague ERROR line ("Requested format is not +# available", "HTTP Error 403") better than the error itself. +HINT_REASONS = ("bot_check", "po_token", "js_runtime") +VAGUE_REASONS = ("unknown", "format_unavailable", "http_403") + + +def _match(text: str, reasons: tuple[str, ...] | None = None) -> str: + for reason, pattern in YTDLP_ERROR_PATTERNS: + if (reasons is None or reason in reasons) and re.search(pattern, text, flags=re.IGNORECASE): + return reason + return "unknown" + + +def classify_ytdlp_error(output: str) -> str: + """Classify a failed yt-dlp run from its output. + + The ERROR lines decide; when they're vague, warnings such as "GVS PO + Token which was not provided" or "No supported JavaScript runtime" are + used instead. + """ + error_lines = "\n".join(line for line in output.splitlines() if "ERROR" in line) + reason = _match(error_lines or output) + if reason in VAGUE_REASONS: + hint = _match(output, HINT_REASONS) + if hint != "unknown": + return hint + return reason + + +def last_error_line(output: str) -> str: + """The most useful single line of yt-dlp output: the last ERROR line.""" + lines = [line.strip() for line in output.splitlines() if line.strip()] + errors = [line for line in lines if "ERROR" in line] + return (errors or lines or ["(no output)"])[-1] + + +class DownloadError(Exception): + def __init__(self, reason: str, stderr: str, attempts: int = 1): + super().__init__(last_error_line(stderr)) + self.reason = reason + self.stderr = stderr + self.attempts = attempts + + +class CircuitBreaker: + """Trips after ``threshold`` blocked downloads in a row.""" + + def __init__(self, threshold: int = BREAKER_THRESHOLD): + self.threshold = threshold + self.consecutive = 0 + self.tripped = False + + def record(self, reason: str | None) -> None: + if reason in BLOCKING_REASONS: + self.consecutive += 1 + if self.consecutive >= self.threshold and not self.tripped: + self.tripped = True + log.error( + "%d downloads in a row were blocked by YouTube (%s); " + "skipping the remaining downloads this run", + self.consecutive, + reason, + ) + else: + self.consecutive = 0 -def download_audio(url: str, dest_dir: Path, browser: str | None) -> None: +def build_ytdlp_cmd(url: str, dest_dir: Path, browser: str | None, cookies: Path | None) -> list[str]: cmd = [ sys.executable, "-m", @@ -100,39 +261,84 @@ def download_audio(url: str, dest_dir: Path, browser: str | None) -> None: "--output", str(dest_dir / "audio.%(ext)s"), "--no-playlist", + "--no-progress", "--quiet", - "--no-warnings", ] if browser: cmd += ["--cookies-from-browser", browser] + if cookies: + cmd += ["--cookies", str(cookies)] cmd.append(url) - result = subprocess.run(cmd, capture_output=True, text=True) + return cmd + + +def find_audio_file(directory: Path) -> Path | None: + for f in sorted(directory.glob("audio.*")): + if f.suffix not in (".part", ".ytdl"): + return f + return None + + +def download_audio( + url: str, + dest_dir: Path, + *, + browser: str | None, + cookies: Path | None, + run: Callable[..., subprocess.CompletedProcess], + label: str, +) -> Path: + cmd = build_ytdlp_cmd(url, dest_dir, browser, cookies) + log.debug("%s: running %s", label, " ".join(cmd)) + result = run(cmd, capture_output=True, text=True) + output = "\n".join(s for s in (result.stderr, result.stdout) if s and s.strip()) if result.returncode != 0: - raise RuntimeError(result.stderr.strip() or result.stdout.strip()) + raise DownloadError(classify_ytdlp_error(output), output) + for line in output.splitlines(): + log.debug("%s: yt-dlp: %s", label, line) + audio = find_audio_file(dest_dir) + if audio is None: + raise DownloadError("no_file", output or "yt-dlp succeeded but wrote no audio file") + return audio + + +async def download_with_retry( + candidate: Candidate, + dest_dir: Path, + args: argparse.Namespace, + run: Callable[..., subprocess.CompletedProcess], + sleep: Callable[[float], Awaitable[Any]], +) -> Path: + label = f"{candidate.slug} ({candidate.video_id})" + for attempt in (1, 2): + try: + return await asyncio.to_thread( + download_audio, + candidate.url, + dest_dir, + browser=args.cookies_from_browser, + cookies=args.cookies, + run=run, + label=label, + ) + except DownloadError as e: + e.attempts = attempt + if attempt == 1 and e.reason in RETRY_REASONS: + log.warning("%s: download failed (%s: %s); retrying in %.0fs", label, e.reason, e, DOWNLOAD_RETRY_DELAY) + await sleep(DOWNLOAD_RETRY_DELAY) + continue + raise + raise AssertionError("unreachable") -def build_prompt(frontmatter: dict[str, Any]) -> str: - parts = [] - title = frontmatter.get("title", "") - if title: - parts.append(title) - software = frontmatter.get("software", []) or [] - parts.extend(software) - return ", ".join(parts) +# --------------------------------------------------------------------------- +# Splitting large audio files +# --------------------------------------------------------------------------- def get_audio_duration(src: Path) -> float: result = subprocess.run( - [ - "ffprobe", - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - str(src), - ], + ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", str(src)], capture_output=True, text=True, check=True, @@ -140,20 +346,9 @@ def get_audio_duration(src: Path) -> float: return float(result.stdout.strip()) -def detect_silence( - src: Path, noise_db: float = -30, min_duration: float = 0.5 -) -> list[float]: +def detect_silence(src: Path, noise_db: float = -30, min_duration: float = 0.5) -> list[float]: result = subprocess.run( - [ - "ffmpeg", - "-i", - str(src), - "-af", - f"silencedetect=noise={noise_db}dB:d={min_duration}", - "-f", - "null", - "-", - ], + ["ffmpeg", "-i", str(src), "-af", f"silencedetect=noise={noise_db}dB:d={min_duration}", "-f", "null", "-"], capture_output=True, text=True, ) @@ -171,8 +366,9 @@ def compute_chunk_boundaries( duration: float, file_size: int, silence_points: list[float], + max_bytes: int = MAX_BYTES, ) -> list[tuple[float, float]]: - max_chunk_secs = MAX_BYTES / (file_size / duration) + max_chunk_secs = max_bytes / (file_size / duration) boundaries: list[tuple[float, float]] = [] start = 0.0 while start < duration: @@ -187,299 +383,338 @@ def compute_chunk_boundaries( return boundaries -def split_at_silence(src: Path, _offset: float = 0.0) -> list[tuple[Path, float]]: +def split_at_silence( + src: Path, work_dir: Path, _offset: float = 0.0, max_bytes: int = MAX_BYTES +) -> list[tuple[Path, float]]: + """Split ``src`` into chunks under ``max_bytes``; return (path, offset) pairs.""" duration = get_audio_duration(src) silence = detect_silence(src) - boundaries = compute_chunk_boundaries(duration, src.stat().st_size, silence) + boundaries = compute_chunk_boundaries(duration, src.stat().st_size, silence, max_bytes) + log.debug("Splitting %s (%.0fs) at %s", src.name, duration, [round(e) for _, e in boundaries]) chunks: list[tuple[Path, float]] = [] for start, end in boundaries: - tmp = Path(tempfile.mktemp(suffix=src.suffix)) + fd, name = tempfile.mkstemp(suffix=src.suffix, prefix="chunk-", dir=work_dir) + os.close(fd) + tmp = Path(name) subprocess.run( - [ - "ffmpeg", - "-y", - "-ss", - str(start), - "-i", - str(src), - "-t", - str(end - start), - "-c", - "copy", - str(tmp), - ], + ["ffmpeg", "-y", "-ss", str(start), "-i", str(src), "-t", str(end - start), "-c", "copy", str(tmp)], check=True, capture_output=True, ) - if tmp.stat().st_size > MAX_BYTES: - sub_chunks = split_at_silence(tmp, _offset=_offset + start) + if tmp.stat().st_size > max_bytes and end - start > 1: + chunks.extend(split_at_silence(tmp, work_dir, _offset=_offset + start, max_bytes=max_bytes)) tmp.unlink(missing_ok=True) - chunks.extend(sub_chunks) else: chunks.append((tmp, _offset + start)) return chunks -def parse_vtt_timestamp(ts: str) -> float: - parts = ts.strip().split(":") - if len(parts) == 3: - return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2]) - return int(parts[0]) * 60 + float(parts[1]) +# --------------------------------------------------------------------------- +# Transcribing +# --------------------------------------------------------------------------- -def format_vtt_timestamp(seconds: float) -> str: - h = int(seconds // 3600) - m = int((seconds % 3600) // 60) - s = seconds % 60 - return f"{h:02d}:{m:02d}:{s:06.3f}" - - -def parse_vtt(content: str) -> list[tuple[float, float, str]]: - cues: list[tuple[float, float, str]] = [] - for block in re.split(r"\n{2,}", content.strip()): - lines = block.strip().splitlines() - arrow = next((i for i, line in enumerate(lines) if "-->" in line), None) - if arrow is None: - continue - m = re.match(r"([\d:.]+)\s+-->\s+([\d:.]+)", lines[arrow]) - if not m: - continue - start = parse_vtt_timestamp(m.group(1)) - end = parse_vtt_timestamp(m.group(2)) - text = "\n".join(lines[arrow + 1 :]).strip() - if text: - cues.append((start, end, text)) - return cues +def merge_vtt(parts: list[tuple[str, float]]) -> str: + cues = [ + (start + offset, end + offset, text) + for vtt_content, offset in parts + for start, end, text in videolib.parse_vtt(vtt_content) + ] + return videolib.render_vtt(cues) -def merge_vtt(parts: list[tuple[str, float]]) -> str: - lines = ["WEBVTT", ""] - for vtt_content, offset in parts: - for start, end, text in parse_vtt(vtt_content): - lines.append( - f"{format_vtt_timestamp(start + offset)} --> " - f"{format_vtt_timestamp(end + offset)}" - ) - lines.append(text) - lines.append("") - return "\n".join(lines) +def build_prompt(frontmatter: dict[str, Any]) -> str: + parts = [] + title = frontmatter.get("title", "") + if title: + parts.append(str(title)) + parts.extend(str(s) for s in frontmatter.get("software", []) or []) + return ", ".join(parts) -class RateLimiter: - """Ensures at least `min_interval` seconds between API requests.""" +class TranscriptError(Exception): + def __init__(self, reason: str, detail: str): + super().__init__(detail) + self.reason = reason - def __init__(self, min_interval: float = 1.0) -> None: - self._lock = asyncio.Lock() - self._last: float = 0.0 - self._interval = min_interval - async def wait(self) -> None: - async with self._lock: - loop = asyncio.get_running_loop() - elapsed = loop.time() - self._last - if elapsed < self._interval: - await asyncio.sleep(self._interval - elapsed) - self._last = asyncio.get_running_loop().time() +def check_transcript(vtt: str, duration: int) -> tuple[list[videolib.Cue], list[str]]: + """Return ``(cues, warnings)``; raise TranscriptError if unusable.""" + cues = videolib.parse_vtt(vtt) + if not cues: + raise TranscriptError("empty_transcript", f"Whisper returned no cues ({len(vtt)} characters)") + warnings = [] + last_end = cues[-1][1] + if duration > 0 and last_end < TRUNCATION_RATIO * duration: + warnings.append( + f"suspect_truncated: last cue ends at {videolib.format_vtt_timestamp(last_end)} " + f"of {videolib.format_vtt_timestamp(duration)}" + ) + return cues, warnings -async def transcribe_audio( +async def transcribe_file( + client: Any, audio_file: Path, prompt: str, - name: str, - client: openai.AsyncOpenAI, - rate_limiter: RateLimiter, + *, + label: str, + rate_limiter: videolib.RateLimiter, + sleep: Callable[[float], Awaitable[Any]], ) -> str: - kwargs: dict[str, Any] = { - "model": "whisper-1", - "response_format": "vtt", - } + kwargs: dict[str, Any] = {"model": WHISPER_MODEL, "response_format": "vtt"} if prompt: kwargs["prompt"] = prompt - for attempt in range(MAX_RETRIES): - if attempt > 0: - delay = BASE_BACKOFF * (2 ** (attempt - 1)) + random.uniform(0, 1) - console.print( - f" [yellow]↻[/] {name} retry {attempt + 1}/{MAX_RETRIES} in {delay:.1f}s" - ) - await asyncio.sleep(delay) - + async def call() -> str: await rate_limiter.wait() + with open(audio_file, "rb") as f: + return await client.audio.transcriptions.create(file=f, **kwargs) + + return await videolib.retry_async(call, label=label, logger=log, sleep=sleep) + + +@dataclass +class Context: + client: Any + args: argparse.Namespace + semaphore: asyncio.Semaphore + rate_limiter: videolib.RateLimiter + breaker: CircuitBreaker + run: Callable[..., subprocess.CompletedProcess] + sleep: Callable[[float], Awaitable[Any]] + max_bytes: int = MAX_BYTES + + +async def process_video(c: Candidate, ctx: Context) -> dict[str, Any]: + async with ctx.semaphore: + label = f"{c.slug} ({c.video_id})" + item: dict[str, Any] = {"slug": c.slug, "video_id": c.video_id, "duration_s": c.duration} + if ctx.breaker.tripped: + log.info("%s: skipped, downloads are blocked this run", label) + return {**item, "status": "skipped", "reason": "download_blocked"} + + timings: dict[str, float] = {} + item["timings"] = timings + legacy_audio = find_audio_file(c.dir) try: - with open(audio_file, "rb") as f: - return await client.audio.transcriptions.create(file=f, **kwargs) - except openai.RateLimitError: - if attempt == MAX_RETRIES - 1: - raise - except openai.APIStatusError as e: - if e.status_code < 500 or attempt == MAX_RETRIES - 1: - raise - - raise RuntimeError("Exceeded max retries") - - -async def process_video_async( - video_dir: Path, - client: openai.AsyncOpenAI, - semaphore: asyncio.Semaphore, - rate_limiter: RateLimiter, - progress: Progress, - overall_task: TaskID, - browser: str | None, -) -> str: - async with semaphore: - name = video_dir.name - job_task = progress.add_task(f"[dim]{name}", total=None) - try: - transcription_file = video_dir / "transcription.vtt" - if transcription_file.exists(): - return "skipped" - - audio_file = find_audio_file(video_dir) - - if audio_file is None: - url = get_video_url(video_dir) - if not url: - return "no_url" - progress.update(job_task, description=f"[cyan]{name} downloading") - await asyncio.to_thread(download_audio, url, video_dir, browser) - audio_file = find_audio_file(video_dir) - if audio_file is None: - raise RuntimeError("Download succeeded but audio file not found") - - index_file = video_dir / "_index.md" - frontmatter = ( - parse_frontmatter(index_file.read_text(encoding="utf-8")) - if index_file.exists() - else {} - ) - prompt = build_prompt(frontmatter) - - if audio_file.stat().st_size > MAX_BYTES: - size_mb = audio_file.stat().st_size / 1024 / 1024 - progress.update( - job_task, - description=f"[yellow]{name} chunking audio ({size_mb:.1f} MB)", + with tempfile.TemporaryDirectory(prefix="transcribe-") as tmp: + work_dir = Path(tmp) + if legacy_audio: + audio = legacy_audio + log.info("%s: using existing %s", label, audio.name) + else: + log.info("%s: downloading audio (%s)", label, videolib.format_vtt_timestamp(c.duration)[:8]) + t0 = time.monotonic() + try: + audio = await download_with_retry(c, work_dir, ctx.args, ctx.run, ctx.sleep) + except DownloadError as e: + ctx.breaker.record(e.reason) + log.error("%s: download failed: %s: %s", label, e.reason, e) + for line in e.stderr.splitlines(): + log.error("%s: yt-dlp: %s", label, line) + return { + **item, + "status": "error", + "reason": e.reason, + "stage": "download", + "detail": str(e), + "attempts": e.attempts, + "stderr": e.stderr[-MAX_STDERR_CHARS:], + } + ctx.breaker.record(None) + timings["download_s"] = round(time.monotonic() - t0, 1) + + size = audio.stat().st_size + item["audio_bytes"] = size + item["audio_format"] = audio.suffix.lstrip(".") + log.info( + "%s: audio %s, %.1f MB%s", + label, + item["audio_format"], + size / 1024 / 1024, + f" in {timings['download_s']}s" if "download_s" in timings else "", ) - chunks: list[tuple[Path, float]] = await asyncio.to_thread( - split_at_silence, audio_file - ) - is_temp = True - else: - chunks = [(audio_file, 0.0)] - is_temp = False - - try: - vtt_parts: list[tuple[str, float]] = [] - total_chunks = len(chunks) - for i, (chunk_path, offset) in enumerate(chunks): - label = f" {i + 1}/{total_chunks}" if total_chunks > 1 else "" - progress.update( - job_task, description=f"[cyan]{name} transcribing{label}" + + if size > ctx.max_bytes: + t0 = time.monotonic() + chunks = await asyncio.to_thread(split_at_silence, audio, work_dir, 0.0, ctx.max_bytes) + timings["split_s"] = round(time.monotonic() - t0, 1) + log.info("%s: split into %d chunks at %s", label, len(chunks), ", ".join(f"{o:.0f}s" for _, o in chunks)) + else: + chunks = [(audio, 0.0)] + item["chunks"] = len(chunks) + + prompt = build_prompt(c.frontmatter) + parts: list[tuple[str, float]] = [] + t0 = time.monotonic() + for i, (chunk, offset) in enumerate(chunks, start=1): + chunk_label = f"{label} chunk {i}/{len(chunks)}" if len(chunks) > 1 else label + tc = time.monotonic() + vtt = await transcribe_file( + ctx.client, chunk, prompt, label=chunk_label, rate_limiter=ctx.rate_limiter, sleep=ctx.sleep ) - vtt = await transcribe_audio( - chunk_path, prompt, name, client, rate_limiter + log.info( + "%s: transcribed %.1f MB (offset %.0fs) in %.1fs", + chunk_label, + chunk.stat().st_size / 1024 / 1024, + offset, + time.monotonic() - tc, ) - vtt_parts.append((vtt, offset)) - finally: - if is_temp: - for chunk_path, _ in chunks: - chunk_path.unlink(missing_ok=True) - - if len(vtt_parts) == 1: - data = vtt_parts[0][0] - else: - progress.update( - job_task, description=f"[cyan]{name} merging transcript" - ) - data = merge_vtt(vtt_parts) - transcription_file.write_text(data, encoding="utf-8") - audio_file.unlink() - return "transcribed" + parts.append((vtt, offset)) + timings["transcribe_s"] = round(time.monotonic() - t0, 1) + + data = parts[0][0] if len(parts) == 1 else merge_vtt(parts) + cues, warnings = check_transcript(data, c.duration) + for w in warnings: + log.warning("%s: %s", label, w) + videolib.atomic_write_text(c.dir / "transcription.vtt", data) + if legacy_audio: + legacy_audio.unlink(missing_ok=True) + except TranscriptError as e: + log.error("%s: %s: %s", label, e.reason, e) + return {**item, "status": "error", "reason": e.reason, "stage": "check", "detail": str(e)} + except Exception as e: # noqa: BLE001 - every failure goes in the summary + log.exception("%s: transcription failed", label) + return { + **item, + "status": "error", + "reason": "transcription_failed", + "stage": "transcribe", + "detail": videolib.describe_api_error(e), + } + + cost = round(c.duration / 60 * WHISPER_USD_PER_MINUTE, 4) + log.info( + "%s: wrote transcription.vtt (%d cues, last at %s, ~$%.3f)", + label, + len(cues), + videolib.format_vtt_timestamp(cues[-1][1])[:8], + cost, + ) + return { + **item, + "status": "transcribed", + "cues": len(cues), + "estimated_cost_usd": cost, + "warnings": warnings or None, + } + - except Exception as e: - console.print(f" [red]✗[/] {name}: {e}") - return "error" - finally: - progress.remove_task(job_task) - progress.advance(overall_task) +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- -async def main_async(jobs: int, browser: str | None) -> None: - console.print("\n[bold blue]Video Transcriber[/]\n") +def build_client(project_root: Path) -> Any: + import openai + from dotenv import load_dotenv - load_dotenv() + load_dotenv(project_root / ".env") api_key = os.getenv("OPENAI_API_KEY") if not api_key: - console.print("[bold red]Error:[/] OPENAI_API_KEY not set in .env") - sys.exit(1) - - client = openai.AsyncOpenAI(api_key=api_key) - - videos_dir = Path(__file__).parent.parent / "content" / "resources" / "videos" - if not videos_dir.exists(): - console.print(f"[bold red]Error:[/] Videos directory not found: {videos_dir}") - sys.exit(1) - - dirs = sorted(d for d in videos_dir.iterdir() if d.is_dir()) - console.print(f"[dim]Found {len(dirs)} video directories (--jobs {jobs})[/]\n") - - semaphore = asyncio.Semaphore(jobs) - rate_limiter = RateLimiter(min_interval=1.0) - - with Progress( - SpinnerColumn(), - TextColumn("{task.description}"), - BarColumn(), - MofNCompleteColumn(), - TimeElapsedColumn(), - console=console, - ) as progress: - overall_task = progress.add_task("[bold cyan]Transcribing", total=len(dirs)) - - results: list[str] = await asyncio.gather( - *[ - process_video_async( - d, client, semaphore, rate_limiter, progress, overall_task, browser - ) - for d in dirs - ] - ) - - transcribed = results.count("transcribed") - skipped = results.count("skipped") - no_url = results.count("no_url") - errors = results.count("error") - - console.print("\n[bold]Summary:[/]") - console.print(f" [green]✓[/] Transcribed: {transcribed}") - console.print(f" [dim]○[/] Skipped: {skipped}") - console.print(f" [dim]○[/] No URL: {no_url}") - if errors: - console.print(f" [red]✗[/] Errors: {errors}") + raise RuntimeError("OPENAI_API_KEY not set in .env or environment") + return openai.AsyncOpenAI(api_key=api_key, max_retries=0) # we retry ourselves, with logging - console.print("\n[bold green]✓ Done![/]\n") - -def main() -> None: - parser = argparse.ArgumentParser( - description="Download and transcribe video audio files using OpenAI Whisper" - ) +def parse_args(argv: list[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Download and transcribe video audio with OpenAI Whisper") + parser.add_argument("--jobs", type=int, default=5, metavar="N", help="parallel jobs (default: 5)") + parser.add_argument("--limit", type=int, metavar="N", help="transcribe at most N videos (newest first)") parser.add_argument( - "--jobs", - type=int, - default=5, - metavar="N", - help="number of parallel transcription jobs (default: 5)", + "--max-audio-minutes", type=float, metavar="M", help="stop selecting videos after M minutes of audio" ) + parser.add_argument("--video", metavar="NAME", help="only this video directory (name, prefix, or path)") parser.add_argument( "--cookies-from-browser", metavar="BROWSER", - help="browser to read cookies from for yt-dlp (e.g. chrome, firefox, safari)", + help="browser to read YouTube cookies from for yt-dlp (e.g. chrome, firefox, safari)", ) - args = parser.parse_args() - asyncio.run(main_async(args.jobs, args.cookies_from_browser)) + parser.add_argument("--cookies", type=Path, metavar="FILE", help="cookies.txt file for yt-dlp") + parser.add_argument("--dry-run", action="store_true", help="list what would be transcribed") + videolib.add_common_args(parser) + return parser.parse_args(argv) + + +def main( + argv: list[str] | None = None, + client: Any = None, + run: Callable[..., subprocess.CompletedProcess] = subprocess.run, + sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, + max_bytes: int = MAX_BYTES, + min_request_interval: float = 1.0, +) -> int: + args = parse_args(argv) + videolib.setup_logging("transcribe", args.verbose) + summary = videolib.RunSummary("transcribe") + summary.extra.update(dry_run=args.dry_run, limit=args.limit, max_audio_minutes=args.max_audio_minutes) + videos_dir = args.project_root / "content" / "resources" / "videos" + + def body() -> None: + nonlocal client + summary.environment = videolib.log_environment( + log, packages=["yt-dlp", "yt-dlp-ejs", "openai", "bgutil-ytdlp-pot-provider"], tools=["deno", "node", "ffmpeg", "ffprobe"] + ) + if not videos_dir.exists(): + summary.fatal = f"Videos directory not found: {videos_dir}" + return + if not args.dry_run: + missing = [t for t in ("ffmpeg", "ffprobe") if not shutil.which(t)] + if missing: + summary.fatal = f"Required tools not found: {', '.join(missing)}" + return + if not (shutil.which("deno") or shutil.which("node")): + log.warning("No JavaScript runtime (deno or node) found; YouTube downloads will probably fail") + + dirs = [videolib.resolve_video_dir(args.video, videos_dir)] if args.video else videolib.video_dirs(videos_dir) + candidates = select_videos(dirs, summary, limit=args.limit, max_audio_minutes=args.max_audio_minutes) + log.info( + "Selected %d of %d video directories (%.0f minutes of audio, --jobs %d)", + len(candidates), + len(dirs), + summary.extra["selected_audio_minutes"], + args.jobs, + ) + if args.dry_run: + for c in candidates: + log.info("Would transcribe %s (%s, %ds)", c.slug, c.video_id, c.duration) + summary.add("would_transcribe", slug=c.slug, video_id=c.video_id, duration_s=c.duration) + return + if not candidates: + return + if client is None: + client = build_client(args.project_root) + + ctx = Context( + client=client, + args=args, + semaphore=asyncio.Semaphore(args.jobs), + rate_limiter=videolib.RateLimiter(min_interval=min_request_interval), + breaker=CircuitBreaker(), + run=run, + sleep=sleep, + max_bytes=max_bytes, + ) + + async def go() -> list[dict[str, Any]]: + return await asyncio.gather(*(process_video(c, ctx) for c in candidates)) + + for result in asyncio.run(go()): + summary.add(result.pop("status"), **result) + + transcribed = summary.by_status("transcribed") + summary.extra["breaker_tripped"] = ctx.breaker.tripped + summary.extra["estimated_cost_usd"] = round(sum(i["estimated_cost_usd"] for i in transcribed), 3) + summary.extra["transcribed_audio_minutes"] = round(sum(i["duration_s"] for i in transcribed) / 60, 1) + log.info( + "Transcribed %d videos (%.0f minutes, ~$%.2f)", + len(transcribed), + summary.extra["transcribed_audio_minutes"], + summary.extra["estimated_cost_usd"], + ) + + return videolib.run_main(summary, log, args.summary_json, body) if __name__ == "__main__": - main() + sys.exit(main()) From 1175f0ea808ea55b2f6f1ac5fe20c4702a0b18e5 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 13:13:31 +0100 Subject: [PATCH 05/20] Add video-sync-report.py to turn run summaries into a PR description Summarizes the four pipeline steps in a table, lists new, renamed, and changed videos and the new transcripts, and collects problems (blocked or failed downloads, failed formatting, crashed steps) with the command to fix each one locally. With --annotations it also prints ::warning:: and ::error:: workflow commands for the Actions run page. --- scripts/test-video-sync-report.py | 255 ++++++++++++++++++ scripts/video-sync-report.py | 432 ++++++++++++++++++++++++++++++ 2 files changed, 687 insertions(+) create mode 100755 scripts/test-video-sync-report.py create mode 100755 scripts/video-sync-report.py diff --git a/scripts/test-video-sync-report.py b/scripts/test-video-sync-report.py new file mode 100755 index 000000000..e57a20734 --- /dev/null +++ b/scripts/test-video-sync-report.py @@ -0,0 +1,255 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for video-sync-report.py.""" + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 +from videotesting import load_script # noqa: E402 + +r = load_script("video-sync-report.py") + +ALL = list(r.STEPS) +RUN_URL = "https://github.com/posit-dev/open-source-website/actions/runs/1" + + +def summary(script, items=(), tallies=None, extra=None, fatal=None): + s = videolib.RunSummary(script) + for item in items: + item = dict(item) + s.add(item.pop("status"), **item) + for status, n in (tallies or {}).items(): + s.tally(status, n) + s.extra.update(extra or {}) + s.fatal = fatal + return s.to_dict() + + +def quiet_night(): + return { + "update-youtube-videos": summary("update-youtube-videos", tallies={"unchanged": 1750}, extra={"refresh_stats": False}), + "sync-videos": summary("sync-videos", tallies={"unchanged": 1750}), + "transcribe": summary("transcribe", tallies={"already_transcribed": 1750}), + "format-transcriptions": summary("format-transcriptions", tallies={"already_formatted": 1750}), + } + + +def typical_night(): + s = quiet_night() + s["update-youtube-videos"] = summary( + "update-youtube-videos", + items=[ + {"status": "new", "url": "https://www.youtube.com/watch?v=aaaaaaaaaaa", "slug": "2026-11-11_new-talk", "title": "New | talk"}, + {"status": "changed", "slug": "2026-10-01_renamed", "changed_fields": ["title", "slug"], "old_slug": "2026-10-01_old"}, + {"status": "missing", "slug": "2025-01-01_gone", "url": "https://www.youtube.com/watch?v=ggggggggggg"}, + ], + tallies={"unchanged": 1748}, + extra={"refresh_stats": False}, + ) + s["sync-videos"] = summary( + "sync-videos", + items=[ + {"status": "created", "slug": "2026-11-11_new-talk"}, + { + "status": "renamed", + "action": "renamed", + "old_slug": "2026-10-01_old", + "slug": "2026-10-01_renamed", + "updated_references": ["content/blog/post/index.md"], + }, + ], + tallies={"unchanged": 1749}, + ) + s["transcribe"] = summary( + "transcribe", + items=[{"status": "transcribed", "slug": "2026-11-11_new-talk", "estimated_cost_usd": 0.12, "duration_s": 1200}], + tallies={"already_transcribed": 1750}, + extra={"estimated_cost_usd": 0.12, "breaker_tripped": False}, + ) + s["format-transcriptions"] = summary( + "format-transcriptions", + items=[{"status": "formatted", "slug": "2026-11-11_new-talk", "estimated_cost_usd": 0.3, "warnings": None}], + tallies={"already_formatted": 1750}, + extra={"estimated_cost_usd": 0.3}, + ) + return s + + +def blocked_night(): + s = typical_night() + s["transcribe"] = summary( + "transcribe", + items=[ + {"status": "error", "slug": "2026-11-11_new-talk", "stage": "download", "reason": "bot_check", "detail": "ERROR: Sign in to confirm you’re not a bot"}, + {"status": "error", "slug": "2026-11-10_b", "stage": "download", "reason": "bot_check", "detail": "ERROR: Sign in"}, + {"status": "error", "slug": "2026-11-09_c", "stage": "download", "reason": "private", "detail": "ERROR: Private video"}, + {"status": "skipped", "slug": "2026-11-08_d", "reason": "download_blocked"}, + ], + extra={"breaker_tripped": True}, + ) + s["format-transcriptions"] = summary("format-transcriptions", tallies={"already_formatted": 1750}) + return s + + +def build(summaries, expected=ALL): + return r.build_report(summaries, expected=expected, run_url=RUN_URL) + + +def test_quiet_night(): + report, annotations = build(quiet_night()) + assert report == ( + "Nightly sync of [Posit's YouTube videos](https://www.youtube.com/@PositPBC) into " + f"`content/resources/videos/` ([workflow run]({RUN_URL})).\n" + "\n" + "Review the changes below, then approve and merge to publish them.\n" + "\n" + "| Step | Result |\n" + "| --- | --- |\n" + "| YouTube metadata | 1,750 unchanged |\n" + "| Video pages | 1,750 unchanged |\n" + "| Transcription | nothing to do |\n" + "| Formatting | nothing to do |\n" + "\n" + "---\n" + "\n" + "Adjust a video with `include`, `exclude`, or `override` blocks in its `_index.md`, hide it with " + "`publish = false` in `data/videos.toml`, or skip its transcript with `transcribe: false`. " + f"Logs and run summaries are attached to the [workflow run]({RUN_URL}).\n" + ) + assert annotations == [] + + +def test_typical_night(): + report, annotations = build(typical_night()) + assert "| YouTube metadata | 1 new, 1 changed, 1,748 unchanged |" in report + assert "| Video pages | 1 created, 1 renamed, 1,749 unchanged |" in report + assert "| Transcription | 1 transcribed (~$0.12) |" in report + assert "| Formatting | 1 formatted (~$0.30) |" in report + assert "### New videos (1)" in report + assert "- [New \\| talk](https://www.youtube.com/watch?v=aaaaaaaaaaa) `2026-11-11_new-talk`" in report + assert "- `2026-10-01_old` → `2026-10-01_renamed`; references updated in `content/blog/post/index.md`" in report + assert "- `2026-10-01_renamed`: title, slug" in report + assert "- `2026-11-11_new-talk`: transcribed and formatted" in report + assert "### ⚠️ Problems" not in report + assert "### Notes" in report and "no longer on the channel" in report + assert annotations == [] + + +def test_blocked_night(): + report, annotations = build(blocked_night()) + assert "| Transcription | ⚠️ 3 failed, downloads blocked |" in report + assert "**YouTube blocked the downloads**" in report + assert "1 more video skipped after repeated bot_check errors" in report + assert "`just update-videos 15 chrome`" in report + assert "`2026-11-10_b`: **bot_check**" in report + assert "Fix: `just transcribe --video 2026-11-10_b --cookies-from-browser chrome`\n" in report + assert "`2026-11-09_c`: **private**" in report and "or set `transcribe: false`" in report + assert annotations == [ + "::warning title=YouTube blocked the downloads::1 item(s); see the step summary or PR description", + "::warning title=Audio downloads failed::3 item(s); see the step summary or PR description", + ] + + +def test_stats_refresh(): + s = quiet_night() + s["update-youtube-videos"] = summary( + "update-youtube-videos", tallies={"stats_refreshed": 1700, "unchanged": 50}, extra={"refresh_stats": True} + ) + report, _ = build(s) + assert "| YouTube metadata | 1,750 unchanged, stats refreshed |" in report + + +def test_formatting_errors_and_warnings(): + s = typical_night() + s["format-transcriptions"] = summary( + "format-transcriptions", + items=[ + {"status": "error", "slug": "2026-11-11_new-talk", "reason": "low_coverage", "detail": "Only 80% of the words"}, + {"status": "formatted", "slug": "2026-11-01_x", "warnings": ["timestamp_overshoot: 44s"]}, + ], + ) + report, annotations = build(s) + assert "`2026-11-11_new-talk`: transcribed (not formatted yet)" in report + assert "`2026-11-01_x`: ⚠️ timestamp_overshoot: 44s" in report + assert "**Formatting failed**" in report + assert "`just format-transcriptions --video 2026-11-11_new-talk`" in report + assert annotations == ["::warning title=Formatting failed::1 item(s); see the step summary or PR description"] + + +def test_fatal_and_missing_summaries(): + s = quiet_night() + s["update-youtube-videos"] = summary("update-youtube-videos", fatal="YOUTUBE_API_KEY not found") + del s["format-transcriptions"] + report, annotations = build(s) + assert "| YouTube metadata | ❌ failed: YOUTUBE_API_KEY not found |" in report + assert "| Formatting | ⚠️ no summary (crashed?) |" in report + assert "**YouTube metadata failed**" in report + assert annotations[0].startswith("::error title=YouTube metadata failed::") + assert annotations[1].startswith("::error title=Formatting produced no summary::") + + +def test_skipped_steps_not_expected(): + s = quiet_night() + del s["transcribe"], s["format-transcriptions"] + report, annotations = build(s, expected=["update-youtube-videos", "sync-videos"]) + assert "Transcription" not in report and annotations == [] + + +def test_long_lists_are_truncated(): + s = quiet_night() + s["update-youtube-videos"] = summary( + "update-youtube-videos", + items=[ + {"status": "new", "url": f"https://www.youtube.com/watch?v={i:011d}", "slug": f"s{i}", "title": f"T{i}"} + for i in range(100) + ], + ) + report, _ = build(s) + assert "…and 60 more" in report + assert len(report) < 65_536 + + +def test_main_reads_directory(tmp_path, capsys): + for script, data in typical_night().items(): + (tmp_path / f"{script}.json").write_text(json.dumps(data)) + (tmp_path / "junk.json").write_text("{not json") + out = tmp_path / "body.md" + assert r.main([str(tmp_path), "--output", str(out), "--annotations", "--run-url", RUN_URL]) == 0 + assert "### New videos (1)" in out.read_text() + captured = capsys.readouterr() + assert captured.out == "" # no annotations on a typical night + assert "could not read" in captured.err + + +def test_main_missing_directory(tmp_path, capsys): + assert r.main([str(tmp_path / "nope"), "--annotations"]) == 0 + out = capsys.readouterr().out + assert "⚠️ no summary (crashed?)" in out + assert "::error title=YouTube metadata produced no summary::" in out + + +def test_default_run_url(monkeypatch): + monkeypatch.setenv("GITHUB_SERVER_URL", "https://github.com") + monkeypatch.setenv("GITHUB_REPOSITORY", "posit-dev/open-source-website") + monkeypatch.setenv("GITHUB_RUN_ID", "1") + assert r.default_run_url() == RUN_URL + monkeypatch.delenv("GITHUB_RUN_ID") + assert r.default_run_url() is None + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/video-sync-report.py b/scripts/video-sync-report.py new file mode 100755 index 000000000..0e58fd2eb --- /dev/null +++ b/scripts/video-sync-report.py @@ -0,0 +1,432 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// + +""" +Turn the run summaries of the video pipeline into a Markdown report. + +Reads the --summary-json files written by update-youtube-videos.py, +sync-videos.py, transcribe.py, and format-transcriptions.py, and writes the +pull request description of the nightly sync (also used for the GitHub +Actions step summary). With --annotations, also prints ::warning:: and +::error:: workflow commands so problems show up on the run page. + +Usage: + ./scripts/video-sync-report.py SUMMARY_DIR [--output FILE] [--annotations] + [--expect update-youtube-videos,sync-videos,...] [--run-url URL] +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +STEPS = { + "update-youtube-videos": "YouTube metadata", + "sync-videos": "Video pages", + "transcribe": "Transcription", + "format-transcriptions": "Formatting", +} + +# Download failures that won't go away by trying again. +PERMANENT_REASONS = {"private", "members_only", "unavailable", "age_restricted"} +# Download failures that mean YouTube is blocking the runner (see transcribe.py). +BLOCKING_REASONS = {"bot_check", "po_token", "http_429"} + +MAX_LIST = 40 # items per list, to stay well under GitHub's 65,536-character limit + + +def load_summaries(directory: Path) -> dict[str, dict[str, Any]]: + summaries = {} + for path in sorted(directory.glob("*.json")): + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + print(f"warning: could not read {path}: {e}", file=sys.stderr) + continue + if isinstance(data, dict) and data.get("script") in STEPS: + summaries[data["script"]] = data + return summaries + + +def items(summary: dict[str, Any] | None, *statuses: str) -> list[dict[str, Any]]: + if not summary: + return [] + return [i for i in summary.get("items", []) if i.get("status") in statuses] + + +def count(summary: dict[str, Any] | None, status: str) -> int: + return (summary or {}).get("counts", {}).get(status, 0) + + +def plural(n: int, word: str, plural_word: str | None = None) -> str: + return f"{n:,} {word if n == 1 else (plural_word or word + 's')}" + + +def bullet_list(lines: list[str]) -> list[str]: + out = [f"- {line}" for line in lines[:MAX_LIST]] + if len(lines) > MAX_LIST: + out.append(f"- …and {len(lines) - MAX_LIST:,} more (see the run summaries)") + return out + + +def details(summary_text: str, lines: list[str]) -> list[str]: + return ["
" + summary_text + "", "", *bullet_list(lines), "", "
", ""] + + +def money(value: float | None) -> str: + return f" (~${value:.2f})" if value else "" + + +def code(text: str | None) -> str: + return f"`{text}`" if text else "(unknown)" + + +def one_line(text: str | None, limit: int = 200) -> str: + text = " ".join((text or "").split()) + text = text.replace("|", "\\|") + return text if len(text) <= limit else text[: limit - 1] + "…" + + +# --------------------------------------------------------------------------- +# Report sections +# --------------------------------------------------------------------------- + + +def results_table(summaries: dict[str, dict[str, Any]], expected: list[str]) -> list[str]: + update = summaries.get("update-youtube-videos") + sync = summaries.get("sync-videos") + transcribe = summaries.get("transcribe") + fmt = summaries.get("format-transcriptions") + + rows: list[tuple[str, str]] = [] + + def describe(script: str, parts: list[str]) -> str: + s = summaries.get(script) + if s is None: + return "⚠️ no summary (crashed?)" if script in expected else "skipped" + if s.get("fatal"): + return f"❌ failed: {one_line(s['fatal'], 120)}" + text = ", ".join(p for p in parts if p) or "nothing to do" + return ("⚠️ " if s.get("exit_code") == 2 else "") + text + + if "update-youtube-videos" in expected or update: + stats = (update or {}).get("extra", {}).get("refresh_stats") + rows.append( + ( + STEPS["update-youtube-videos"], + describe( + "update-youtube-videos", + [ + f"{count(update, 'new'):,} new" if count(update, "new") else "", + f"{count(update, 'changed'):,} changed" if count(update, "changed") else "", + f"{count(update, 'unchanged') + count(update, 'stats_refreshed'):,} unchanged", + "stats refreshed" if stats else "", + ], + ), + ) + ) + if "sync-videos" in expected or sync: + rows.append( + ( + STEPS["sync-videos"], + describe( + "sync-videos", + [ + f"{count(sync, 'created'):,} created" if count(sync, "created") else "", + f"{count(sync, 'updated'):,} updated" if count(sync, "updated") else "", + f"{count(sync, 'renamed'):,} renamed" if count(sync, "renamed") else "", + f"{count(sync, 'error'):,} failed" if count(sync, "error") else "", + f"{count(sync, 'unchanged'):,} unchanged" if count(sync, "unchanged") else "", + ], + ), + ) + ) + if "transcribe" in expected or transcribe: + extra = (transcribe or {}).get("extra", {}) + rows.append( + ( + STEPS["transcribe"], + describe( + "transcribe", + [ + f"{count(transcribe, 'transcribed'):,} transcribed{money(extra.get('estimated_cost_usd'))}" + if count(transcribe, "transcribed") + else "", + f"{count(transcribe, 'error'):,} failed" if count(transcribe, "error") else "", + "downloads blocked" if extra.get("breaker_tripped") else "", + f"{count(transcribe, 'deferred'):,} deferred" if count(transcribe, "deferred") else "", + ], + ), + ) + ) + if "format-transcriptions" in expected or fmt: + extra = (fmt or {}).get("extra", {}) + rows.append( + ( + STEPS["format-transcriptions"], + describe( + "format-transcriptions", + [ + f"{count(fmt, 'formatted'):,} formatted{money(extra.get('estimated_cost_usd'))}" + if count(fmt, "formatted") + else "", + f"{count(fmt, 'error'):,} failed" if count(fmt, "error") else "", + f"{count(fmt, 'deferred'):,} deferred" if count(fmt, "deferred") else "", + ], + ), + ) + ) + if not rows: + return [] + return ["| Step | Result |", "| --- | --- |", *(f"| {step} | {result} |" for step, result in rows), ""] + + +def changes_sections(summaries: dict[str, dict[str, Any]]) -> list[str]: + update = summaries.get("update-youtube-videos") + sync = summaries.get("sync-videos") + transcribe = summaries.get("transcribe") + fmt = summaries.get("format-transcriptions") + out: list[str] = [] + + new = items(update, "new") + if new: + out += [f"### New videos ({len(new)})", ""] + out += bullet_list([f"[{one_line(i.get('title'), 120)}]({i['url']}) {code(i.get('slug'))}" for i in new]) + out.append("") + + renamed = items(sync, "renamed") + if renamed: + out += [f"### Renamed ({len(renamed)})", "", "The URLs of these pages change; links in the repo were updated.", ""] + lines = [] + for i in renamed: + line = f"{code(i.get('old_slug'))} → {code(i.get('slug'))}" + if i.get("action") == "merged": + line += " (merged into an existing directory)" + refs = i.get("updated_references") or [] + if refs: + line += f"; references updated in {', '.join(code(r) for r in refs)}" + lines.append(line) + out += bullet_list(lines) + [""] + + changed = items(update, "changed") + if changed: + out += ["### Changed metadata", ""] + out += details( + plural(len(changed), "video"), + [f"{code(i.get('slug'))}: {', '.join(i.get('changed_fields', []))}" for i in changed], + ) + + transcribed = items(transcribe, "transcribed") + formatted = items(fmt, "formatted") + if transcribed or formatted: + out += ["### Transcripts", ""] + lines = [] + formatted_slugs = {i["slug"] for i in formatted} + for i in transcribed: + note = " and formatted" if i["slug"] in formatted_slugs else " (not formatted yet)" + warn = f" ⚠️ {'; '.join(i['warnings'])}" if i.get("warnings") else "" + lines.append(f"{code(i['slug'])}: transcribed{note}{warn}") + transcribed_slugs = {i["slug"] for i in transcribed} + for i in formatted: + if i["slug"] not in transcribed_slugs: + lines.append(f"{code(i['slug'])}: formatted") + for i in formatted: + if i.get("warnings"): + lines.append(f"{code(i['slug'])}: ⚠️ {'; '.join(i['warnings'])}") + out += details(plural(len(lines), "video"), lines) + return out + + +def problems_section(summaries: dict[str, dict[str, Any]], expected: list[str]) -> tuple[list[str], list[str]]: + """Return (markdown lines, annotations).""" + update = summaries.get("update-youtube-videos") + sync = summaries.get("sync-videos") + transcribe = summaries.get("transcribe") + fmt = summaries.get("format-transcriptions") + out: list[str] = [] + annotations: list[str] = [] + + def block(title: str, lines: list[str], hint: str | None = None, level: str = "warning") -> None: + if not lines: + return + nonlocal out + out += [f"**{title}**", ""] + if hint: + out += [hint, ""] + out += bullet_list(lines) + [""] + annotations.append(f"::{level} title={title}::{len(lines)} item(s); see the step summary or PR description") + + for script in STEPS: + s = summaries.get(script) + if s and s.get("fatal"): + block(f"{STEPS[script]} failed", [one_line(s["fatal"], 500)], level="error") + elif s is None and script in expected: + block(f"{STEPS[script]} produced no summary", ["The step crashed before writing it; check the job log."], level="error") + + block( + "Could not fetch some YouTube sources", + [f"{code(i.get('source'))}: {one_line(i.get('detail'))}" for i in items(update, "error")], + ) + + extra = (transcribe or {}).get("extra", {}) + download_errors = [i for i in items(transcribe, "error") if i.get("stage") == "download"] + blocked = items(transcribe, "skipped") + blocked = [i for i in blocked if i.get("reason") == "download_blocked"] + if extra.get("breaker_tripped"): + block( + "YouTube blocked the downloads", + [ + f"{plural(len(blocked), 'more video')} skipped after repeated " + f"{', '.join(sorted({i['reason'] for i in download_errors} & BLOCKING_REASONS)) or 'blocking'} errors" + ], + hint="Transcribe them locally with `just update-videos 15 chrome` on this branch and push, " + "or wait for the next run.", + ) + if download_errors: + lines = [] + for i in download_errors: + fix = f"`just transcribe --video {i['slug']} --cookies-from-browser chrome`" + if i.get("reason") in PERMANENT_REASONS: + fix += " or set `transcribe: false` in its `_index.md`" + lines.append(f"{code(i['slug'])}: **{i.get('reason')}**: {one_line(i.get('detail'))}. Fix: {fix}") + block("Audio downloads failed", lines) + + block( + "Transcription failed", + [ + f"{code(i['slug'])}: **{i.get('reason')}**: {one_line(i.get('detail'))}. " + f"Fix: `just transcribe --video {i['slug']}`" + for i in items(transcribe, "error") + if i.get("stage") != "download" + ], + ) + block( + "Formatting failed", + [ + f"{code(i['slug'])}: **{i.get('reason')}**: {one_line(i.get('detail'))}. " + f"It's retried in the next run, or run `just format-transcriptions --video {i['slug']}`" + for i in items(fmt, "error") + ], + ) + block( + "Video pages failed", + [f"{code(i.get('slug'))}: **{i.get('reason')}**: {one_line(i.get('detail'))}" for i in items(sync, "error")], + ) + + # Informational: not errors, but worth a look. + info: list[str] = [] + missing = items(update, "missing") + if missing: + info += details( + f"{plural(len(missing), 'video')} in data/videos.toml no longer on the channel or playlists " + "(set publish = false to unpublish)", + [f"{code(i.get('slug'))} ({i.get('url')})" for i in missing], + ) + unavailable = items(update, "unavailable") + if unavailable: + info += details( + f"{plural(len(unavailable), 'video')} listed but not returned by the API (private or deleted?)", + [f"{code(i.get('slug') or i.get('video_id'))} ({i.get('url')})" for i in unavailable], + ) + duplicates = items(update, "duplicate_slug") + if duplicates: + info += details( + f"{plural(len(duplicates), 'video')} skipped because another video has the same slug", + [f"{code(i.get('slug'))}: skipped {i.get('url')}, kept {i.get('kept')}" for i in duplicates], + ) + orphans = items(sync, "orphan") + if orphans: + info += details( + f"{plural(len(orphans), 'directory', 'directories')} without an entry in data/videos.toml", + [code(i.get("slug")) for i in orphans], + ) + skipped = [i for i in items(transcribe, "skipped") if i.get("reason") != "download_blocked"] + if skipped: + info += details( + f"{plural(len(skipped), 'video')} not transcribed", + [f"{code(i.get('slug'))}: {i.get('reason')}" for i in skipped], + ) + + if out: + out = ["### ⚠️ Problems", ""] + out + if info: + out += ["### Notes", ""] + info + return out, annotations + + +def build_report( + summaries: dict[str, dict[str, Any]], + *, + expected: list[str], + run_url: str | None = None, +) -> tuple[str, list[str]]: + out = [ + "Nightly sync of [Posit's YouTube videos](https://www.youtube.com/@PositPBC) into " + "`content/resources/videos/`" + + (f" ([workflow run]({run_url}))" if run_url else "") + + ".", + "", + "Review the changes below, then approve and merge to publish them.", + "", + ] + out += results_table(summaries, expected) + out += changes_sections(summaries) + problems, annotations = problems_section(summaries, expected) + out += problems + + out += [ + "---", + "", + "Adjust a video with `include`, `exclude`, or `override` blocks in its `_index.md`, " + "hide it with `publish = false` in `data/videos.toml`, or skip its transcript with " + "`transcribe: false`." + + (f" Logs and run summaries are attached to the [workflow run]({run_url})." if run_url else ""), + ] + return "\n".join(out).rstrip() + "\n", annotations + + +def default_run_url() -> str | None: + server = os.getenv("GITHUB_SERVER_URL") + repo = os.getenv("GITHUB_REPOSITORY") + run_id = os.getenv("GITHUB_RUN_ID") + if server and repo and run_id: + return f"{server}/{repo}/actions/runs/{run_id}" + return None + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.strip().split("\n\n")[0]) + parser.add_argument("summary_dir", type=Path, help="directory with the *.json run summaries") + parser.add_argument("--output", "-o", type=Path, help="write the report here instead of stdout") + parser.add_argument("--annotations", action="store_true", help="print GitHub Actions annotations to stdout") + parser.add_argument( + "--expect", + default=",".join(STEPS), + help="comma-separated steps that should have a summary (default: all four)", + ) + parser.add_argument("--run-url", default=default_run_url(), help="link to the workflow run") + args = parser.parse_args(argv) + + expected = [s for s in args.expect.split(",") if s] + summaries = load_summaries(args.summary_dir) if args.summary_dir.exists() else {} + report, annotations = build_report(summaries, expected=expected, run_url=args.run_url) + + if args.output: + args.output.write_text(report, encoding="utf-8") + else: + sys.stdout.write(report) + if args.annotations: + for line in annotations: + print(line) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 7eb48fea900ee5f2b2ac17e9ee94666bac776c3a Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 13:15:49 +0100 Subject: [PATCH 06/20] Add nightly Sync Videos workflow and a workflow for the script tests sync-videos.yml runs the four video scripts at 05:00 UTC (stats refreshed on Mondays) and opens or updates an automation/sync-videos pull request with the posit-open-source-website-sync app, like sync-courses.yml. It first restores the files added by a still-open sync PR, so videos aren't transcribed (and paid for) twice. Transcription and formatting never fail the job; video-sync-report.py writes the PR description and step summary, and the logs and run summaries are uploaded as an artifact. A temporary pull_request trigger runs dry runs (no PR, diff attached) on the PR that adds the workflow; it has to be removed before merging. test-scripts.yml runs scripts/run-tests.sh when scripts change. --- .github/scripts/restore-pending-videos.sh | 54 +++++ .github/workflows/sync-videos.yml | 240 ++++++++++++++++++++++ .github/workflows/test-scripts.yml | 41 ++++ scripts/run-tests.sh | 2 +- 4 files changed, 336 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/restore-pending-videos.sh create mode 100644 .github/workflows/sync-videos.yml create mode 100644 .github/workflows/test-scripts.yml diff --git a/.github/scripts/restore-pending-videos.sh b/.github/scripts/restore-pending-videos.sh new file mode 100755 index 000000000..4db823a52 --- /dev/null +++ b/.github/scripts/restore-pending-videos.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Carry over the work of an unmerged sync PR into this run. +# +# The nightly sync starts from main. If last night's PR hasn't been merged +# yet, its new video directories and transcripts would otherwise be created +# (and paid for) again, and the nondeterministic formatting would rewrite +# the PR every night. So: take every file the PR branch *added* under +# content/resources/videos/ (relative to its merge base with main) and put +# it in the working tree, unless main has a file at that path by now. +# Files the PR modified are not restored; the scripts regenerate them. +# +# Usage: restore-pending-videos.sh [BRANCH] [BASE] +# BRANCH defaults to automation/sync-videos, BASE to main. +# Needs a checkout with full history (actions/checkout fetch-depth: 0). + +set -euo pipefail + +branch="${1:-automation/sync-videos}" +base="${2:-main}" +prefix="content/resources/videos/" + +if ! git ls-remote --exit-code --heads origin "$branch" >/dev/null 2>&1; then + echo "No pending branch $branch; nothing to restore." + exit 0 +fi + +git fetch --quiet --no-tags origin "$branch" "$base" +remote_branch="origin/$branch" +remote_base="origin/$base" + +added=() +while IFS= read -r -d '' path; do + added+=("$path") +done < <(git diff -z --name-only --diff-filter=A "$remote_base...$remote_branch" -- "$prefix") + +restored=() +skipped=0 +for path in ${added[@]+"${added[@]}"}; do + if [[ -e "$path" ]]; then + echo " keep main's version: $path" + skipped=$((skipped + 1)) + continue + fi + restored+=("$path") +done + +if ((${#restored[@]})); then + # --worktree only: leave the index alone, create-pull-request picks the + # files up as changes. + git restore --source="$remote_branch" --worktree -- "${restored[@]}" + printf ' restored: %s\n' "${restored[@]}" +fi + +echo "Restored ${#restored[@]} file(s) from $branch (${#added[@]} added there, $skipped already on $base)." diff --git a/.github/workflows/sync-videos.yml b/.github/workflows/sync-videos.yml new file mode 100644 index 000000000..3872db7cf --- /dev/null +++ b/.github/workflows/sync-videos.yml @@ -0,0 +1,240 @@ +name: Sync Videos + +# Syncs Posit's YouTube videos into content/resources/videos/ and opens (or +# updates) a pull request with the changes, like sync-courses.yml: +# +# 1. update-youtube-videos.py YouTube metadata -> data/videos.toml +# (view counts are only refreshed on Mondays) +# 2. sync-videos.py data/videos.toml -> video pages, thumbnails +# 3. transcribe.py new videos -> transcription.vtt (yt-dlp, Whisper) +# 4. format-transcriptions.py transcription.vtt -> transcription.txt (Claude) +# +# YouTube may block audio downloads from GitHub's runners. Steps 3 and 4 never +# fail the job; the PR description lists every video that still needs a +# transcript, with the command to make one locally (`just update-videos`). +# Logs and machine-readable run summaries of every step are attached to the +# workflow run as an artifact. + +on: + schedule: + - cron: "0 5 * * *" # 5 AM UTC: before sync-courses (6 AM) and the nightly production deploy (8 AM) + workflow_dispatch: + inputs: + refresh_stats: + description: Refresh the view, like, and comment counts of all videos + type: boolean + default: false + skip_transcription: + description: Skip transcribing and formatting + type: boolean + default: false + limit: + description: Maximum number of videos to transcribe and format + type: number + default: 15 + dry_run: + description: Don't open a pull request; attach the diff to the run instead + type: boolean + default: false + # TEMPORARY: dry runs on the pull request that adds this workflow, because a + # new workflow can't be dispatched before it's on main. Remove before merging. + pull_request: + paths: + - .github/workflows/sync-videos.yml + - .github/scripts/restore-pending-videos.sh + - scripts/videolib.py + - scripts/update-youtube-videos.py + - scripts/sync-videos.py + - scripts/transcribe.py + - scripts/format-transcriptions.py + - scripts/video-sync-report.py + +concurrency: + group: sync-videos-${{ github.event_name == 'pull_request' && github.head_ref || 'main' }} + cancel-in-progress: false + +# Pushing the branch and opening the PR use the app token below, not +# GITHUB_TOKEN, so that deploy-preview.yml runs on the sync PRs. +permissions: + contents: read + +jobs: + sync-videos: + # Pull requests from forks don't get secrets. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + # Only scheduled runs and manual runs on main open a pull request. + DRY_RUN: ${{ github.event_name == 'pull_request' || inputs.dry_run == true || github.ref != 'refs/heads/main' }} + LIMIT: ${{ github.event_name == 'pull_request' && 2 || inputs.limit || 15 }} + SKIP_TRANSCRIPTION: ${{ inputs.skip_transcription == true }} + REFRESH_STATS_INPUT: ${{ inputs.refresh_stats == true }} + TRANSCRIPT_MODEL: claude-sonnet-4-6 + # Keep log lines unwrapped in the job log + COLUMNS: "200" + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # restore-pending-videos.sh diffs against the merge base + + - name: Prepare + id: prepare + shell: bash + run: | + sync_dir="$RUNNER_TEMP/video-sync" + mkdir -p "$sync_dir/logs" "$sync_dir/summaries" + echo "SYNC_DIR=$sync_dir" >> "$GITHUB_ENV" + + if [ "$REFRESH_STATS_INPUT" = "true" ] || [ "$(date -u +%u)" = "1" ]; then + refresh_stats=true + else + refresh_stats=false + fi + echo "refresh_stats=$refresh_stats" >> "$GITHUB_OUTPUT" + + if [ "$SKIP_TRANSCRIPTION" = "true" ]; then + echo "expect=update-youtube-videos,sync-videos" >> "$GITHUB_OUTPUT" + else + echo "expect=update-youtube-videos,sync-videos,transcribe,format-transcriptions" >> "$GITHUB_OUTPUT" + fi + echo "event=$GITHUB_EVENT_NAME ref=$GITHUB_REF dry_run=$DRY_RUN limit=$LIMIT" + echo "skip_transcription=$SKIP_TRANSCRIPTION refresh_stats=$refresh_stats model=$TRANSCRIPT_MODEL" + + - name: Restore files from the open sync pull request + shell: bash + run: ./.github/scripts/restore-pending-videos.sh automation/sync-videos main 2>&1 | tee "$SYNC_DIR/logs/0-restore-pending-videos.log" + + - name: Setup uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + + - name: Setup Deno (JavaScript runtime for yt-dlp) + if: env.SKIP_TRANSCRIPTION != 'true' + uses: denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed # v2.0.5 + + - name: Install ffmpeg + if: env.SKIP_TRANSCRIPTION != 'true' + shell: bash + run: | + if ! command -v ffmpeg >/dev/null || ! command -v ffprobe >/dev/null; then + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends ffmpeg + fi + ffmpeg -version | head -1 + deno --version | head -1 + + - name: 1. Update YouTube metadata + id: update + shell: bash + env: + YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} + REFRESH_STATS: ${{ steps.prepare.outputs.refresh_stats }} + run: | + set +e + args=() + if [ "$REFRESH_STATS" = "true" ]; then args+=(--refresh-stats); fi + ./scripts/update-youtube-videos.py "${args[@]}" \ + --summary-json "$SYNC_DIR/summaries/1-update-youtube-videos.json" \ + 2>&1 | tee "$SYNC_DIR/logs/1-update-youtube-videos.log" + status=${PIPESTATUS[0]} + echo "exit_code=$status" >> "$GITHUB_OUTPUT" + # Exit code 2 (a source failed) is partial success: keep going. + [ "$status" -eq 0 ] || [ "$status" -eq 2 ] + + - name: 2. Sync video pages + id: sync + shell: bash + run: | + set +e + ./scripts/sync-videos.py \ + --summary-json "$SYNC_DIR/summaries/2-sync-videos.json" \ + 2>&1 | tee "$SYNC_DIR/logs/2-sync-videos.log" + status=${PIPESTATUS[0]} + echo "exit_code=$status" >> "$GITHUB_OUTPUT" + [ "$status" -eq 0 ] || [ "$status" -eq 2 ] + + - name: 3. Transcribe new videos + id: transcribe + if: env.SKIP_TRANSCRIPTION != 'true' + shell: bash + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + set +e + ./scripts/transcribe.py --limit "$LIMIT" --jobs 3 \ + --summary-json "$SYNC_DIR/summaries/3-transcribe.json" \ + 2>&1 | tee "$SYNC_DIR/logs/3-transcribe.log" + status=${PIPESTATUS[0]} + echo "exit_code=$status" >> "$GITHUB_OUTPUT" + # Never fail the job here: failures are listed in the PR description. + if [ "$status" -ne 0 ]; then echo "::warning title=Transcription::transcribe.py exited with $status"; fi + + - name: 4. Format transcriptions + id: format + if: env.SKIP_TRANSCRIPTION != 'true' + shell: bash + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + set +e + ./scripts/format-transcriptions.py --limit "$LIMIT" \ + --summary-json "$SYNC_DIR/summaries/4-format-transcriptions.json" \ + 2>&1 | tee "$SYNC_DIR/logs/4-format-transcriptions.log" + status=${PIPESTATUS[0]} + echo "exit_code=$status" >> "$GITHUB_OUTPUT" + if [ "$status" -ne 0 ]; then echo "::warning title=Formatting::format-transcriptions.py exited with $status"; fi + + - name: Write report + if: always() + shell: bash + env: + EXPECT: ${{ steps.prepare.outputs.expect }} + run: | + ./scripts/video-sync-report.py "$SYNC_DIR/summaries" \ + --expect "$EXPECT" --output "$SYNC_DIR/pr-body.md" --annotations + cat "$SYNC_DIR/pr-body.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Save the diff (dry run) + if: always() && env.DRY_RUN == 'true' + shell: bash + run: | + git add --intent-to-add -- data/videos.toml content + git diff --stat -- data/videos.toml content > "$SYNC_DIR/diff-stat.txt" + git diff -- data/videos.toml content > "$SYNC_DIR/diff.patch" + echo "Dry run: no pull request. $(tail -1 "$SYNC_DIR/diff-stat.txt")" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**Dry run:** no pull request was opened. Changes: $(tail -1 "$SYNC_DIR/diff-stat.txt")" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload logs and run summaries + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: video-sync-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/video-sync + retention-days: 30 + if-no-files-found: warn + + - name: Generate app token + if: env.DRY_RUN != 'true' + id: app-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ secrets.POSIT_OPEN_SOURCE_WEBSITE_SYNC_CLIENT_ID }} + private-key: ${{ secrets.POSIT_OPEN_SOURCE_WEBSITE_SYNC_PEM }} + + - name: Create or update pull request + if: env.DRY_RUN != 'true' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.app-token.outputs.token }} + sign-commits: true + branch: automation/sync-videos + delete-branch: true + # All of content/: renamed videos also update links in other pages. + add-paths: | + data/videos.toml + content + commit-message: Sync videos from YouTube + title: Sync videos from YouTube + body-path: ${{ runner.temp }}/video-sync/pr-body.md diff --git a/.github/workflows/test-scripts.yml b/.github/workflows/test-scripts.yml new file mode 100644 index 000000000..81ea0c3fa --- /dev/null +++ b/.github/workflows/test-scripts.yml @@ -0,0 +1,41 @@ +name: Test Scripts + +# Runs the tests of the Python scripts (scripts/test-*.py) when they change. + +on: + pull_request: + paths: + - "scripts/**" + - ".github/workflows/test-scripts.yml" + - "justfile" + push: + branches: ["main"] + paths: + - "scripts/**" + - ".github/workflows/test-scripts.yml" + - "justfile" + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + + - name: Install ffmpeg + run: | + if ! command -v ffmpeg >/dev/null || ! command -v ffprobe >/dev/null; then + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends ffmpeg + fi + + - name: Run tests + run: ./scripts/run-tests.sh -ra diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 0130c4a7e..8b0f18882 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -4,7 +4,7 @@ # Extra arguments are passed on to pytest, e.g. `./scripts/run-tests.sh -q -x`. set -uo pipefail -cd "$(dirname "$0")/.." +cd "$(dirname "$0")/.." || exit 1 failed=() for test_file in scripts/test-*.py; do From 7045c5fe546f735116fd690a4f8b0b267e175084 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 13:17:30 +0100 Subject: [PATCH 07/20] Validate formatted transcripts and chunk videos longer than 20 minutes Checking the 1,710 published transcripts showed that 84-93% of the videos of 45-90 minutes lost more than 10% of their words: they were formatted in one request, which ran into the output token limit or got condensed, and a response cut off at max_tokens was written anyway. - Reject responses that stop at max_tokens - Validate the HTML: only h3/p/a/blockquote, only data-t attributes, data-t within the video (a small overshoot is a warning), and at least 90% of the spoken words outside pull quotes; invalid output is never written - Chunk videos longer than 20 minutes into 15-minute segments (was 90 and 30 minutes) and request the chunks concurrently - Add --check-existing (report invalid published transcripts, no API calls) and --reformat-invalid - Log model, tokens, stop reason, latency, and request ID per call; record estimated cost; add --limit, --dry-run, $TRANSCRIPT_MODEL, exit codes 0/1/2, and atomic writes --- scripts/format-transcriptions.py | 888 ++++++++++++++------------ scripts/test-format-transcriptions.py | 390 +++++++++++ 2 files changed, 857 insertions(+), 421 deletions(-) create mode 100755 scripts/test-format-transcriptions.py diff --git a/scripts/format-transcriptions.py b/scripts/format-transcriptions.py index 12a783cae..55873cca6 100755 --- a/scripts/format-transcriptions.py +++ b/scripts/format-transcriptions.py @@ -1,60 +1,80 @@ -#!/usr/bin/env -S uv run +#!/usr/bin/env -S uv run --script # /// script +# requires-python = ">=3.11" # dependencies = [ # "pyyaml>=6.0", # "rich>=13.0.0", # "anthropic>=0.40.0", # "python-dotenv>=1.0.0", -# "tomli>=2.0.0; python_version < '3.11'", # ] # /// """ Format VTT transcription files into HTML fragments using the Anthropic API. -For each subdirectory under content/resources/videos/: +For each subdirectory under content/resources/videos/, newest first: - Skip if transcription.txt already exists (unless --force). - Skip if _index.md frontmatter has transcribe: false. - Skip if transcription.vtt does not exist. - For long videos (> 90 min), split the VTT into chunks and process each segment separately, then concatenate the results. -- After the API call, inject links for known people and software. +- Validate the HTML: only

,

, , and

; every + data-t within the video; at least 90% of the spoken words present; and a + response that hit the max_tokens limit is rejected. Invalid output is + never written. +- Inject links for known people and software. + +Requires ANTHROPIC_API_KEY in the environment or .env. The model can be set +with --model or $TRANSCRIPT_MODEL. + +Exit codes: 0 = success, 1 = fatal, 2 = partial (some videos failed). """ +from __future__ import annotations + import argparse import asyncio +import html as html_lib +import logging import os -import random import re import sys +import time +import tomllib +from collections import Counter +from dataclasses import dataclass from pathlib import Path -from typing import Any - -import anthropic -import yaml -from dotenv import load_dotenv -from rich.console import Console -from rich.progress import ( - BarColumn, - MofNCompleteColumn, - Progress, - SpinnerColumn, - TaskID, - TextColumn, - TimeElapsedColumn, -) - -try: - import tomllib -except ImportError: - import tomli as tomllib # type: ignore[no-redef] - -console = Console(stderr=True) - -MAX_RETRIES = 5 -BASE_BACKOFF = 1.0 -CHUNK_DURATION_S = 1_800 # 30 min per chunk (default) -LONG_VIDEO_THRESHOLD_S = 5_400 # 90 min — chunk automatically +from typing import Any, Awaitable, Callable + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 + +log = logging.getLogger("format-transcriptions") + +DEFAULT_MODEL = "claude-sonnet-4-6" +# Long transcripts hit the output token limit or get condensed by the model: +# before chunking at 20 minutes, 84-93% of the 45-90 minute videos lost more +# than 10% of their words. 15-minute chunks behave like short videos. +CHUNK_DURATION_S = 900 # 15 min per chunk (default) +LONG_VIDEO_THRESHOLD_S = 1_200 # 20 min — chunk automatically + +ALLOWED_TAGS = {"h3", "p", "a", "blockquote"} +# Share of the VTT's words that must appear in the HTML (outside pull quotes). +MIN_COVERAGE = 0.90 +# data-t values beyond the end of the video: a warning beyond the slack, an +# error beyond max(TOLERANCE_S, TOLERANCE_RATIO * duration). The model's +# timestamp arithmetic is often off by tens of seconds (median 44 s in the +# existing transcripts), which is annoying but not worth throwing away. +DATA_T_SLACK_S = 5 +DATA_T_TOLERANCE_S = 60 +DATA_T_TOLERANCE_RATIO = 0.10 + +# Estimated USD per million (input, output) tokens, matched by model prefix. +PRICES_PER_MTOK = { + "claude-sonnet": (3.0, 15.0), + "claude-haiku": (1.0, 5.0), +} PROMPT = """\ You convert WebVTT transcription files into clean HTML fragments for video pages. @@ -211,74 +231,8 @@

Now let's actually build the app. I'll start with a minimal ui definition.

""" - -# --------------------------------------------------------------------------- -# Helpers copied from transcribe.py -# --------------------------------------------------------------------------- - - -def parse_frontmatter(content: str) -> dict[str, Any]: - lines = content.split("\n") - if not lines or lines[0].strip() != "---": - return {} - end_idx = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) - if end_idx is None: - return {} - try: - return yaml.safe_load("\n".join(lines[1:end_idx])) or {} - except yaml.YAMLError: - return {} - - -def parse_vtt_timestamp(ts: str) -> float: - parts = ts.strip().split(":") - if len(parts) == 3: - return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2]) - return int(parts[0]) * 60 + float(parts[1]) - - -def format_vtt_timestamp(seconds: float) -> str: - h = int(seconds // 3600) - m = int((seconds % 3600) // 60) - s = seconds % 60 - return f"{h:02d}:{m:02d}:{s:06.3f}" - - -def parse_vtt(content: str) -> list[tuple[float, float, str]]: - cues: list[tuple[float, float, str]] = [] - for block in re.split(r"\n{2,}", content.strip()): - lines = block.strip().splitlines() - arrow = next((i for i, line in enumerate(lines) if "-->" in line), None) - if arrow is None: - continue - m = re.match(r"([\d:.]+)\s+-->\s+([\d:.]+)", lines[arrow]) - if not m: - continue - start = parse_vtt_timestamp(m.group(1)) - end = parse_vtt_timestamp(m.group(2)) - text = "\n".join(lines[arrow + 1 :]).strip() - if text: - cues.append((start, end, text)) - return cues - - -class RateLimiter: - def __init__(self, min_interval: float = 1.0) -> None: - self._lock = asyncio.Lock() - self._last: float = 0.0 - self._interval = min_interval - - async def wait(self) -> None: - async with self._lock: - loop = asyncio.get_running_loop() - elapsed = loop.time() - self._last - if elapsed < self._interval: - await asyncio.sleep(self._interval - elapsed) - self._last = asyncio.get_running_loop().time() - - # --------------------------------------------------------------------------- -# Phase 1 — Prompt helpers +# Prompt helpers # --------------------------------------------------------------------------- @@ -294,9 +248,7 @@ def build_user_message(vtt_content: str, frontmatter: dict[str, Any]) -> str: if people: header_parts.append(f"People featured: {', '.join(str(p) for p in people)}") if software: - header_parts.append( - f"Technologies mentioned: {', '.join(str(s) for s in software)}" - ) + header_parts.append(f"Technologies mentioned: {', '.join(str(s) for s in software)}") if description: header_parts.append(f"Description: {description[:600]}") @@ -304,11 +256,46 @@ def build_user_message(vtt_content: str, frontmatter: dict[str, Any]) -> str: return f"{header}\n\n---\n\n{vtt_content}" if header else vtt_content +def build_chunk_system(segment_index: int, total_segments: int) -> str: + return ( + PROMPT.rstrip() + + "\n\n## Segment context\n" + + f"This is segment {segment_index + 1} of {total_segments}.\n" + + "Start each paragraph cleanly — do not begin mid-sentence.\n" + + "End each paragraph cleanly — do not leave a sentence dangling.\n" + ) + + +def split_vtt_into_chunks(vtt_content: str, chunk_duration: int) -> list[tuple[str, int]]: + cues = videolib.parse_vtt(vtt_content) + chunks: list[tuple[str, int]] = [] + seg_start = 0.0 + seg_cues: list[videolib.Cue] = [] + + for start, end, text in cues: + if seg_cues and (start - seg_start) >= chunk_duration: + chunks.append((videolib.render_vtt(seg_cues), int(seg_start))) + seg_start = start + seg_cues = [] + seg_cues.append((start, end, text)) + + if seg_cues: + chunks.append((videolib.render_vtt(seg_cues), int(seg_start))) + + return chunks + + # --------------------------------------------------------------------------- -# Phase 2 — Core script +# Validation # --------------------------------------------------------------------------- +class FormatError(Exception): + def __init__(self, reason: str, detail: str): + super().__init__(detail) + self.reason = reason + + def strip_markdown_fence(text: str) -> str: text = text.strip() if text.startswith("```"): @@ -317,232 +304,158 @@ def strip_markdown_fence(text: str) -> str: return text.strip() -def validate_html(html: str) -> None: - if not html.lstrip().startswith("<"): - raise ValueError("Response does not start with '<' — not HTML") - for bad in ("" not in html and "
" not in html: - raise ValueError("Response contains no

or

elements") +WORD_RE = re.compile(r"[\w']+") -async def call_api( - client: anthropic.AsyncAnthropic, - user_message: str, - model: str, - max_tokens: int, - rate_limiter: RateLimiter, - name: str, - system: str = PROMPT, -) -> str: - for attempt in range(MAX_RETRIES): - if attempt > 0: - delay = BASE_BACKOFF * (2 ** (attempt - 1)) + random.uniform(0, 1) - console.print( - f" [yellow]↻[/] {name} retry {attempt + 1}/{MAX_RETRIES} in {delay:.1f}s" - ) - await asyncio.sleep(delay) +def word_counts(text: str) -> Counter: + return Counter(w.lower().strip("'") for w in WORD_RE.findall(text)) - await rate_limiter.wait() - try: - response = await client.messages.create( - model=model, - max_tokens=max_tokens, - system=system, - messages=[{"role": "user", "content": user_message}], - ) - return response.content[0].text - except anthropic.RateLimitError: - if attempt == MAX_RETRIES - 1: - raise - except anthropic.APIStatusError as e: - if e.status_code < 500 or attempt == MAX_RETRIES - 1: - raise - - raise RuntimeError("Exceeded max retries") - - -def resolve_video_dir(name_or_path: str, videos_dir: Path) -> Path: - p = Path(name_or_path) - if p.is_absolute() and p.is_dir(): - return p - candidate = videos_dir / name_or_path - if candidate.is_dir(): - return candidate - matches = [ - d - for d in videos_dir.iterdir() - if d.is_dir() and d.name.startswith(name_or_path) - ] - if len(matches) == 1: - return matches[0] - if len(matches) > 1: - listed = "\n ".join(m.name for m in sorted(matches)) - raise ValueError(f"Ambiguous --video value, matches:\n {listed}") - raise ValueError(f"Video directory not found: {name_or_path}") - - -async def process_video_async( - video_dir: Path, - client: anthropic.AsyncAnthropic, - semaphore: asyncio.Semaphore, - rate_limiter: RateLimiter, - progress: Progress, - overall_task: TaskID, - args: argparse.Namespace, - people: dict[str, str], - software: dict[str, str], -) -> str: - async with semaphore: - name = video_dir.name - job_task = progress.add_task(f"[dim]{name}", total=None) - try: - html_file = video_dir / "transcription.txt" - vtt_file = video_dir / "transcription.vtt" - index_file = video_dir / "_index.md" - if html_file.exists() and not args.force: - return "skipped" +def html_to_text(html: str, *, drop_blockquotes: bool = True) -> str: + if drop_blockquotes: + html = re.sub(r"", " ", html, flags=re.DOTALL | re.IGNORECASE) + html = re.sub(r"", " ", html, flags=re.DOTALL | re.IGNORECASE) + return html_lib.unescape(re.sub(r"<[^>]+>", " ", html)) - frontmatter = ( - parse_frontmatter(index_file.read_text(encoding="utf-8")) - if index_file.exists() - else {} - ) - if frontmatter.get("transcribe", True) is False: - return "excluded" +def coverage(cues: list[videolib.Cue], html: str) -> float: + """Share of the spoken words (with multiplicity) found in the HTML prose.""" + spoken = word_counts(" ".join(text for _, _, text in cues)) + total = sum(spoken.values()) + if not total: + return 1.0 + written = word_counts(html_to_text(html)) + return sum(min(n, written[w]) for w, n in spoken.items()) / total - if not vtt_file.exists(): - return "no_vtt" - vtt_content = vtt_file.read_text(encoding="utf-8") - duration = (frontmatter.get("external") or {}).get("duration", 0) +def validate_html(html: str, cues: list[videolib.Cue], duration: int) -> dict[str, Any]: + """Raise FormatError when the model output can't be used; return metrics.""" + if not html.lstrip().startswith("<"): + raise FormatError("not_html", f"Response does not start with '<': {html[:80]!r}") + + tags = {t.lower() for t in re.findall(r"]*)?>", html, flags=re.IGNORECASE): + if m.group(2) and m.group(2).strip(): + raise FormatError("disallowed_attribute", f"Attributes on <{m.group(1)}>: {m.group(2).strip()}") + + end = max([duration] + [end for _, end, _ in cues]) + max_t = 0 + anchors = 0 + for m in re.finditer(r"]*)?>", html, flags=re.IGNORECASE): + attrs = (m.group(1) or "").strip() + am = re.fullmatch(r'data-t="(\d+)"', attrs) + if not am: + raise FormatError("disallowed_attribute", f"Bad attributes: {attrs!r}") + max_t = max(max_t, int(am.group(1))) + anchors += 1 + + warnings: list[str] = [] + overshoot = max_t - end + if overshoot > max(DATA_T_TOLERANCE_S, DATA_T_TOLERANCE_RATIO * end): + raise FormatError("bad_timestamp", f'data-t="{max_t}" is {overshoot:.0f}s beyond the end of the video ({end:.0f}s)') + if overshoot > DATA_T_SLACK_S: + warnings.append(f"timestamp_overshoot: data-t=\"{max_t}\" is {overshoot:.0f}s beyond the end of the video") - if duration > LONG_VIDEO_THRESHOLD_S: - progress.update(job_task, description=f"[cyan]{name} chunking") - html = await format_chunked( - vtt_content, - frontmatter, - client, - rate_limiter, - args, - name, - progress, - job_task, - ) - else: - user_message = build_user_message(vtt_content, frontmatter) - progress.update(job_task, description=f"[cyan]{name} formatting") - raw = await call_api( - client, - user_message, - args.model, - args.max_tokens, - rate_limiter, - name, - ) - html = strip_markdown_fence(raw) + if "

" not in html and "

" not in html: + raise FormatError("no_paragraphs", "Response contains no

or

elements") - validate_html(html) - html = inject_links(html, people, software) - html_file.write_text(html, encoding="utf-8") - return "formatted" + cov = coverage(cues, html) + if cov < MIN_COVERAGE: + raise FormatError("low_coverage", f"Only {cov:.1%} of the spoken words are in the HTML (minimum {MIN_COVERAGE:.0%})") + return {"coverage": round(cov, 4), "anchors": anchors, "warnings": warnings or None} - except Exception as e: - console.print(f" [red]✗[/] {name}: {e}") - return "error" - finally: - progress.remove_task(job_task) - progress.advance(overall_task) +def unlink_entities(html: str) -> str: + """Undo inject_links, so a published transcript can be validated again.""" + return re.sub(r'(.*?)', r"\1", html, flags=re.DOTALL) -# --------------------------------------------------------------------------- -# Phase 3 — Chunked processing -# --------------------------------------------------------------------------- +def check_existing(video_dir: Path) -> tuple[bool, str | None, str | None]: + """Validate a published transcription.txt: ``(ok, reason, detail)``.""" + fm = videolib.read_frontmatter(video_dir / "_index.md") + duration = int((fm.get("external") or {}).get("duration") or 0) + cues = videolib.parse_vtt((video_dir / "transcription.vtt").read_text(encoding="utf-8")) + html = unlink_entities((video_dir / "transcription.txt").read_text(encoding="utf-8")) + try: + validate_html(html, cues, duration) + except FormatError as e: + return False, e.reason, str(e) + return True, None, None -def _render_vtt(cues: list[tuple[float, float, str]]) -> str: - lines = ["WEBVTT", ""] - for start, end, text in cues: - lines.append(f"{format_vtt_timestamp(start)} --> {format_vtt_timestamp(end)}") - lines.append(text) - lines.append("") - return "\n".join(lines) +# --------------------------------------------------------------------------- +# API +# --------------------------------------------------------------------------- -def split_vtt_into_chunks( - vtt_content: str, chunk_duration: int -) -> list[tuple[str, int]]: - cues = parse_vtt(vtt_content) - chunks: list[tuple[str, int]] = [] - seg_start = 0.0 - seg_cues: list[tuple[float, float, str]] = [] - for start, end, text in cues: - if seg_cues and (start - seg_start) >= chunk_duration: - chunks.append((_render_vtt(seg_cues), int(seg_start))) - seg_start = start - seg_cues = [] - seg_cues.append((start, end, text)) +def estimate_cost(model: str, input_tokens: int, output_tokens: int) -> float | None: + for prefix, (price_in, price_out) in PRICES_PER_MTOK.items(): + if model.startswith(prefix): + return round((input_tokens * price_in + output_tokens * price_out) / 1_000_000, 4) + return None - if seg_cues: - chunks.append((_render_vtt(seg_cues), int(seg_start))) - return chunks +async def call_api( + client: Any, + user_message: str, + *, + model: str, + max_tokens: int, + rate_limiter: videolib.RateLimiter, + label: str, + system: str = PROMPT, + sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, +) -> tuple[str, dict[str, Any]]: + """Return ``(html, usage)``; raise FormatError if the output was cut off.""" + async def call(): + await rate_limiter.wait() + return await client.messages.create( + model=model, + max_tokens=max_tokens, + system=system, + messages=[{"role": "user", "content": user_message}], + ) -def build_chunk_system(segment_index: int, total_segments: int) -> str: - return ( - PROMPT.rstrip() - + "\n\n## Segment context\n" - + f"This is segment {segment_index + 1} of {total_segments}.\n" - + "Start each paragraph cleanly — do not begin mid-sentence.\n" - + "End each paragraph cleanly — do not leave a sentence dangling.\n" + t0 = time.monotonic() + response = await videolib.retry_async(call, label=label, logger=log, sleep=sleep) + usage = { + "input_tokens": getattr(response.usage, "input_tokens", 0), + "output_tokens": getattr(response.usage, "output_tokens", 0), + "stop_reason": response.stop_reason, + "request_id": getattr(response, "_request_id", None), + "latency_s": round(time.monotonic() - t0, 1), + } + log.info( + "%s: %s responded in %.1fs (%d in / %d out tokens, stop_reason=%s, request_id=%s)", + label, + model, + usage["latency_s"], + usage["input_tokens"], + usage["output_tokens"], + usage["stop_reason"], + usage["request_id"], ) - - -async def format_chunked( - vtt_content: str, - frontmatter: dict[str, Any], - client: anthropic.AsyncAnthropic, - rate_limiter: RateLimiter, - args: argparse.Namespace, - name: str, - progress: Progress, - job_task: TaskID, -) -> str: - chunks = split_vtt_into_chunks(vtt_content, args.chunk_duration) - total = len(chunks) - html_parts: list[str] = [] - - for i, (chunk_vtt, _seg_start) in enumerate(chunks): - progress.update(job_task, description=f"[cyan]{name} chunk {i + 1}/{total}") - user_message = build_user_message(chunk_vtt, frontmatter) - chunk_system = build_chunk_system(i, total) - raw = await call_api( - client, - user_message, - args.model, - args.max_tokens, - rate_limiter, - name, - system=chunk_system, + if response.stop_reason == "max_tokens": + raise FormatError( + "truncated", + f"Response hit max_tokens={max_tokens} after {usage['output_tokens']} tokens " + f"(request_id={usage['request_id']}); try a smaller --chunk-duration", ) - html_parts.append(strip_markdown_fence(raw)) - - return "\n\n".join(html_parts) + text = "".join(getattr(block, "text", "") for block in response.content) + return strip_markdown_fence(text), usage # --------------------------------------------------------------------------- -# Phase 4 — Entity linking +# Entity linking # --------------------------------------------------------------------------- -def build_entity_dicts( - repo_root: Path, -) -> tuple[dict[str, str], dict[str, str]]: +def build_entity_dicts(repo_root: Path) -> tuple[dict[str, str], dict[str, str]]: ignore_file = repo_root / "data" / "software-match-ignore.toml" ignored: set[str] = set() if ignore_file.exists(): @@ -552,13 +465,10 @@ def build_entity_dicts( people: dict[str, str] = {} people_dir = repo_root / "content" / "people" if people_dir.exists(): - for d in people_dir.iterdir(): + for d in sorted(people_dir.iterdir()): if not d.is_dir(): continue - index = d / "_index.md" - if not index.exists(): - continue - fm = parse_frontmatter(index.read_text(encoding="utf-8")) + fm = videolib.read_frontmatter(d / "_index.md") title = (fm.get("title") or "").strip() if title: people[title] = f"/people/{d.name}/" @@ -566,13 +476,10 @@ def build_entity_dicts( software: dict[str, str] = {} software_dir = repo_root / "content" / "software" if software_dir.exists(): - for d in software_dir.iterdir(): + for d in sorted(software_dir.iterdir()): if not d.is_dir() or d.name in ignored: continue - index = d / "_index.md" - if not index.exists(): - continue - fm = parse_frontmatter(index.read_text(encoding="utf-8")) + fm = videolib.read_frontmatter(d / "_index.md") override = (fm.get("override") or {}).get("title") title = (override or fm.get("title") or "").strip() if title: @@ -581,11 +488,7 @@ def build_entity_dicts( return people, software -def _inject_into_text( - text: str, - entities: list[tuple[str, str, bool]], - linked: set[str], -) -> str: +def _inject_into_text(text: str, entities: list[tuple[str, str, bool]], linked: set[str]) -> str: for name, url, is_software in entities: if name in linked: continue @@ -604,10 +507,7 @@ def _inject_into_text( def _inject_into_anchor( - content: str, - attrs: str, - entities: list[tuple[str, str, bool]], - linked: set[str], + content: str, attrs: str, entities: list[tuple[str, str, bool]], linked: set[str] ) -> str: for name, url, is_software in entities: if name in linked: @@ -630,11 +530,7 @@ def _inject_into_anchor( return f"{content}" -def inject_links( - html: str, - people: dict[str, str], - software: dict[str, str], -) -> str: +def inject_links(html: str, people: dict[str, str], software: dict[str, str]) -> str: linked: set[str] = set() entities: list[tuple[str, str, bool]] = sorted( @@ -658,116 +554,184 @@ def inject_links( # --------------------------------------------------------------------------- -# Entry points +# Per-video processing # --------------------------------------------------------------------------- -async def main_async(args: argparse.Namespace) -> None: - console.print("\n[bold blue]Transcription Formatter[/]\n") +@dataclass +class Context: + client: Any + args: argparse.Namespace + semaphore: asyncio.Semaphore + rate_limiter: videolib.RateLimiter + people: dict[str, str] + software: dict[str, str] + sleep: Callable[[float], Awaitable[Any]] + + +def select_videos( + dirs: list[Path], + summary: videolib.RunSummary, + *, + force: bool, + limit: int | None, + reformat_invalid: bool = False, +) -> list[Path]: + selected: list[Path] = [] + deferred: list[str] = [] + for d in dirs: + fm = videolib.read_frontmatter(d / "_index.md") + if fm.get("transcribe", True) is False: + summary.tally("excluded") + continue + if not (d / "transcription.vtt").exists(): + summary.tally("no_vtt") + continue + if (d / "transcription.txt").exists() and not force: + if not reformat_invalid: + summary.tally("already_formatted") + continue + ok, reason, _ = check_existing(d) + if ok: + summary.tally("already_formatted") + continue + log.info("%s: existing transcript is invalid (%s); reformatting", d.name, reason) + if limit is not None and len(selected) >= limit: + deferred.append(d.name) + continue + selected.append(d) + if deferred: + summary.tally("deferred", len(deferred)) + summary.extra["deferred"] = deferred + log.info("Deferred %d videos to a later run (--limit)", len(deferred)) + return selected + + +async def process_video(video_dir: Path, ctx: Context) -> dict[str, Any]: + async with ctx.semaphore: + args = ctx.args + fm = videolib.read_frontmatter(video_dir / "_index.md") + external = fm.get("external") or {} + video_id = videolib.youtube_id(external.get("url")) + label = f"{video_dir.name} ({video_id})" + duration = int(external.get("duration") or 0) + item: dict[str, Any] = {"slug": video_dir.name, "video_id": video_id, "duration_s": duration, "model": args.model} + try: + vtt_content = (video_dir / "transcription.vtt").read_text(encoding="utf-8") + cues = videolib.parse_vtt(vtt_content) + if not cues: + raise FormatError("empty_vtt", "transcription.vtt has no cues") - load_dotenv() - api_key = os.getenv("ANTHROPIC_API_KEY") - if not api_key: - console.print("[bold red]Error:[/] ANTHROPIC_API_KEY not set in .env") - sys.exit(1) + if duration > LONG_VIDEO_THRESHOLD_S: + chunks = split_vtt_into_chunks(vtt_content, args.chunk_duration) + else: + chunks = [(vtt_content, 0)] + item["chunks"] = len(chunks) + log.info("%s: formatting %d cues in %d chunk(s)", label, len(cues), len(chunks)) + + # Chunks are independent, so request them concurrently (the rate + # limiter still spaces out the requests). + results = await asyncio.gather( + *( + call_api( + ctx.client, + build_user_message(chunk_vtt, fm), + model=args.model, + max_tokens=args.max_tokens, + rate_limiter=ctx.rate_limiter, + label=f"{label} chunk {i + 1}/{len(chunks)}" if len(chunks) > 1 else label, + system=build_chunk_system(i, len(chunks)) if len(chunks) > 1 else PROMPT, + sleep=ctx.sleep, + ) + for i, (chunk_vtt, _seg_start) in enumerate(chunks) + ) + ) + parts = [html_part for html_part, _ in results] + usages = [usage for _, usage in results] + html = "\n\n".join(parts) + + item["input_tokens"] = sum(u["input_tokens"] for u in usages) + item["output_tokens"] = sum(u["output_tokens"] for u in usages) + item["request_ids"] = [u["request_id"] for u in usages if u["request_id"]] or None + item["estimated_cost_usd"] = estimate_cost(args.model, item["input_tokens"], item["output_tokens"]) + + metrics = validate_html(html, cues, duration) + item.update(metrics) + for w in metrics["warnings"] or []: + log.warning("%s: %s", label, w) + html = inject_links(html, ctx.people, ctx.software) + videolib.atomic_write_text(video_dir / "transcription.txt", html) + except FormatError as e: + log.error("%s: %s: %s", label, e.reason, e) + return {**item, "status": "error", "reason": e.reason, "detail": str(e)} + except Exception as e: # noqa: BLE001 - every failure goes in the summary + log.exception("%s: formatting failed", label) + return {**item, "status": "error", "reason": "api_error", "detail": videolib.describe_api_error(e)} + + log.info( + "%s: wrote transcription.txt (%d anchors, %.1f%% coverage%s)", + label, + item["anchors"], + item["coverage"] * 100, + f", ~${item['estimated_cost_usd']:.3f}" if item["estimated_cost_usd"] is not None else "", + ) + return {**item, "status": "formatted"} - client = anthropic.AsyncAnthropic(api_key=api_key) - repo_root = Path(__file__).parent.parent - videos_dir = repo_root / "content" / "resources" / "videos" - if not videos_dir.exists(): - console.print(f"[bold red]Error:[/] Videos directory not found: {videos_dir}") - sys.exit(1) +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- - people, software = build_entity_dicts(repo_root) - console.print( - f"[dim]Loaded {len(people)} people and {len(software)} software entries[/]" + +def run_check_existing(dirs: list[Path], summary: videolib.RunSummary) -> None: + """Report published transcripts that fail validation (no API calls).""" + for d in dirs: + if not ((d / "transcription.txt").exists() and (d / "transcription.vtt").exists()): + continue + ok, reason, detail = check_existing(d) + if ok: + summary.tally("valid") + else: + summary.add("invalid", slug=d.name, reason=reason, detail=detail) + log.warning("Invalid: %s: %s: %s", d.name, reason, detail) + invalid = summary.by_status("invalid") + reasons: dict[str, int] = {} + for i in invalid: + reasons[i["reason"]] = reasons.get(i["reason"], 0) + 1 + summary.extra["invalid_by_reason"] = reasons + log.info( + "%d of %d transcripts are invalid %s; reformat them with --reformat-invalid", + len(invalid), + len(invalid) + summary.tallies.get("valid", 0), + reasons, ) - if args.video: - dirs = [resolve_video_dir(args.video, videos_dir)] - else: - dirs = sorted(d for d in videos_dir.iterdir() if d.is_dir()) - console.print( - f"[dim]Found {len(dirs)} director(ies) to process (--jobs {args.jobs})[/]\n" - ) +def build_client(project_root: Path) -> Any: + import anthropic + from dotenv import load_dotenv - semaphore = asyncio.Semaphore(args.jobs) - rate_limiter = RateLimiter(min_interval=1.0) - - with Progress( - SpinnerColumn(), - TextColumn("{task.description}"), - BarColumn(), - MofNCompleteColumn(), - TimeElapsedColumn(), - console=console, - ) as progress: - overall_task = progress.add_task("[bold cyan]Formatting", total=len(dirs)) - results: list[str] = await asyncio.gather( - *[ - process_video_async( - d, - client, - semaphore, - rate_limiter, - progress, - overall_task, - args, - people, - software, - ) - for d in dirs - ] - ) + load_dotenv(project_root / ".env") + api_key = os.getenv("ANTHROPIC_API_KEY") + if not api_key: + raise RuntimeError("ANTHROPIC_API_KEY not set in .env or environment") + return anthropic.AsyncAnthropic(api_key=api_key, max_retries=0) # we retry ourselves, with logging - formatted = results.count("formatted") - skipped = results.count("skipped") - excluded = results.count("excluded") - no_vtt = results.count("no_vtt") - errors = results.count("error") - - console.print("\n[bold]Summary:[/]") - console.print(f" [green]✓[/] Formatted: {formatted}") - console.print(f" [dim]○[/] Skipped: {skipped}") - console.print(f" [dim]○[/] Excluded: {excluded}") - console.print(f" [dim]○[/] No VTT: {no_vtt}") - if errors: - console.print(f" [red]✗[/] Errors: {errors}") - console.print("\n[bold green]✓ Done![/]\n") - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Format VTT transcriptions into HTML using the Anthropic API" - ) - parser.add_argument( - "--jobs", - type=int, - default=5, - metavar="N", - help="parallel formatting jobs (default: 5)", - ) - parser.add_argument( - "--force", - action="store_true", - help="overwrite existing transcription.txt files", - ) - parser.add_argument( - "--video", - metavar="NAME", - help="process only this video directory (name or path)", - ) + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Format VTT transcriptions into HTML using the Anthropic API") + parser.add_argument("--jobs", type=int, default=5, metavar="N", help="parallel formatting jobs (default: 5)") + parser.add_argument("--force", action="store_true", help="overwrite existing transcription.txt files") + parser.add_argument("--video", metavar="NAME", help="process only this video directory (name, prefix, or path)") + parser.add_argument("--limit", type=int, metavar="N", help="format at most N videos (newest first)") parser.add_argument( - "--model", default="claude-sonnet-4-6", metavar="ID", help="Anthropic model ID" + "--model", + default=os.getenv("TRANSCRIPT_MODEL") or DEFAULT_MODEL, + metavar="ID", + help=f"Anthropic model ID (default: $TRANSCRIPT_MODEL or {DEFAULT_MODEL})", ) parser.add_argument( - "--max-tokens", - type=int, - default=16384, - metavar="N", - help="max output tokens per API call (default: 16384)", + "--max-tokens", type=int, default=16384, metavar="N", help="max output tokens per API call (default: 16384)" ) parser.add_argument( "--chunk-duration", @@ -776,9 +740,91 @@ def main() -> None: metavar="SECS", help=f"VTT seconds per chunk for long videos (default: {CHUNK_DURATION_S})", ) - args = parser.parse_args() - asyncio.run(main_async(args)) + parser.add_argument( + "--reformat-invalid", + action="store_true", + help="also reformat existing transcripts that fail validation (see --check-existing)", + ) + parser.add_argument( + "--check-existing", + action="store_true", + help="only validate existing transcription.txt files and report the invalid ones (no API calls)", + ) + parser.add_argument("--dry-run", action="store_true", help="list what would be formatted") + videolib.add_common_args(parser) + return parser.parse_args(argv) + + +def main( + argv: list[str] | None = None, + client: Any = None, + sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, + min_request_interval: float = 1.0, +) -> int: + args = parse_args(argv) + videolib.setup_logging("format-transcriptions", args.verbose) + summary = videolib.RunSummary("format-transcriptions") + summary.extra.update(model=args.model, dry_run=args.dry_run, limit=args.limit) + videos_dir = args.project_root / "content" / "resources" / "videos" + + def body() -> None: + nonlocal client + summary.environment = videolib.log_environment(log, packages=["anthropic"]) + if not videos_dir.exists(): + summary.fatal = f"Videos directory not found: {videos_dir}" + return + people, software = build_entity_dicts(args.project_root) + log.info("Model: %s; loaded %d people and %d software entries for linking", args.model, len(people), len(software)) + + dirs = [videolib.resolve_video_dir(args.video, videos_dir)] if args.video else videolib.video_dirs(videos_dir) + if args.check_existing: + run_check_existing(dirs, summary) + return + selected = select_videos( + dirs, summary, force=args.force, limit=args.limit, reformat_invalid=args.reformat_invalid + ) + log.info("Selected %d of %d video directories (--jobs %d)", len(selected), len(dirs), args.jobs) + if args.dry_run: + for d in selected: + log.info("Would format %s", d.name) + summary.add("would_format", slug=d.name) + return + if not selected: + return + if client is None: + client = build_client(args.project_root) + + ctx = Context( + client=client, + args=args, + semaphore=asyncio.Semaphore(args.jobs), + rate_limiter=videolib.RateLimiter(min_interval=min_request_interval), + people=people, + software=software, + sleep=sleep, + ) + + async def go() -> list[dict[str, Any]]: + return await asyncio.gather(*(process_video(d, ctx) for d in selected)) + + for result in asyncio.run(go()): + summary.add(result.pop("status"), **result) + + formatted = summary.by_status("formatted", "error") + costs = [i.get("estimated_cost_usd") for i in formatted if i.get("estimated_cost_usd") is not None] + summary.extra["estimated_cost_usd"] = round(sum(costs), 3) + summary.extra["input_tokens"] = sum(i.get("input_tokens", 0) for i in formatted) + summary.extra["output_tokens"] = sum(i.get("output_tokens", 0) for i in formatted) + log.info( + "Formatted %d videos (%d in / %d out tokens, ~$%.2f)", + len(summary.by_status("formatted")), + summary.extra["input_tokens"], + summary.extra["output_tokens"], + summary.extra["estimated_cost_usd"], + ) + + return videolib.run_main(summary, log, args.summary_json, body) if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/scripts/test-format-transcriptions.py b/scripts/test-format-transcriptions.py new file mode 100755 index 000000000..69d2a1e64 --- /dev/null +++ b/scripts/test-format-transcriptions.py @@ -0,0 +1,390 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "anthropic>=0.40.0", +# "python-dotenv>=1.0.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for format-transcriptions.py. The Anthropic API is faked.""" + +import json +import re +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 +from videotesting import ( # noqa: E402 + SAMPLE_VTT, + FakeAnthropic, + RateLimitError, + StatusError, + load_script, + make_repo, + no_sleep, + vtt_to_html, + write_video_dir, +) + +f = load_script("format-transcriptions.py") + +CUES = videolib.parse_vtt(SAMPLE_VTT) +GOOD_HTML = vtt_to_html(SAMPLE_VTT) + + +# --- Pure helpers --- + + +@pytest.mark.parametrize( + "raw, clean", + [ + ("```html\n

x

\n```", "

x

"), + ("```\n

x

```", "

x

"), + ("

x

", "

x

"), + ], +) +def test_strip_markdown_fence(raw, clean): + assert f.strip_markdown_fence(raw) == clean + + +def test_build_user_message(): + fm = {"title": "T", "people": ["A"], "software": ["quarto"], "description": "d" * 700} + msg = f.build_user_message("WEBVTT", fm) + header, vtt = msg.split("\n\n---\n\n") + assert vtt == "WEBVTT" + assert header.splitlines() == [ + "Video title: T", + "People featured: A", + "Technologies mentioned: quarto", + "Description: " + "d" * 600, + ] + assert f.build_user_message("WEBVTT", {}) == "WEBVTT" + + +def test_split_vtt_into_chunks(): + cues = [(float(t), float(t) + 4, f"cue {t}") for t in range(0, 100, 10)] + chunks = f.split_vtt_into_chunks(videolib.render_vtt(cues), 30) + assert [start for _, start in chunks] == [0, 30, 60, 90] + assert [len(videolib.parse_vtt(vtt)) for vtt, _ in chunks] == [3, 3, 3, 1] + + +def test_chunk_system_prompt(): + system = f.build_chunk_system(1, 4) + assert system.startswith(f.PROMPT.rstrip()) + assert "This is segment 2 of 4." in system + + +class TestCoverage: + def test_full(self): + assert f.coverage(CUES, GOOD_HTML) == 1.0 + + def test_blockquotes_and_headings_dont_count(self): + quote = "
Hello everyone, today we talk about Quarto.
" + html = "

It was built by the team that also made Shiny

" + quote + assert f.coverage(CUES, html) == 0.0 + + def test_entities_and_case(self): + cues = [(0.0, 1.0, "R & Python, it's GREAT")] + assert f.coverage(cues, '

r & python, it's great

') == 1.0 + + def test_multiplicity(self): + cues = [(0.0, 1.0, "the the the the")] + assert f.coverage(cues, '

the the

') == 0.5 + + +class TestValidateHtml: + def test_valid(self): + metrics = f.validate_html(GOOD_HTML, CUES, 26) + assert metrics == {"coverage": 1.0, "anchors": 3, "warnings": None} + + def test_allowed_structure(self): + html = ( + GOOD_HTML + + "\n\n

More

Hadley Wickham joins us later.
" + ) + assert f.validate_html(html, CUES, 26)["anchors"] == 4 + + @pytest.mark.parametrize( + "html, reason", + [ + ("Here is your HTML:

", "not_html"), + ("
" + GOOD_HTML + "
", "disallowed_tag"), + (GOOD_HTML + "

x

", "disallowed_tag"), + (GOOD_HTML.replace("

", "").replace("

", ""), "disallowed_tag"), + (GOOD_HTML.replace("

", '

'), "disallowed_attribute"), + (GOOD_HTML.replace('', ''), "disallowed_attribute"), + (GOOD_HTML.replace('', ''), "disallowed_attribute"), + (GOOD_HTML.replace('', ''), "disallowed_attribute"), + (GOOD_HTML.replace('data-t="20"', 'data-t="120"'), "bad_timestamp"), + ('Hello everyone, today we talk about Quarto.', "no_paragraphs"), + ('

Hello everyone.

', "low_coverage"), + ], + ) + def test_invalid(self, html, reason): + with pytest.raises(f.FormatError) as e: + f.validate_html(html, CUES, 26) + assert e.value.reason == reason + + def test_small_overshoot_is_a_warning(self): + html = GOOD_HTML.replace('data-t="20"', 'data-t="70"') # 44 s beyond 26 s, under 60 s + metrics = f.validate_html(html, CUES, 26) + assert metrics["warnings"] == ['timestamp_overshoot: data-t="70" is 44s beyond the end of the video'] + + def test_overshoot_tolerance_scales_with_duration(self): + cues = [(0.0, 3600.0, "word " * 10)] + html = '

' + "word " * 10 + "

" # 300 s over, under 10 % + assert f.validate_html(html, cues, 3600)["warnings"] + html = '

' + "word " * 10 + "

" # 400 s over + with pytest.raises(f.FormatError): + f.validate_html(html, cues, 3600) + + def test_coverage_threshold(self): + cues = [(0.0, 1.0, " ".join(f"w{i}" for i in range(100)))] + + def html(n): + return '

' + " ".join(f"w{i}" for i in range(n)) + "

" + + assert f.validate_html(html(95), cues, 1)["coverage"] == 0.95 + assert f.validate_html(html(90), cues, 1)["coverage"] == 0.90 + with pytest.raises(f.FormatError, match="80.0%"): + f.validate_html(html(80), cues, 1) + + +def test_estimate_cost(): + assert f.estimate_cost("claude-sonnet-4-6", 1_000_000, 100_000) == 4.5 + assert f.estimate_cost("claude-haiku-4-5", 1_000_000, 0) == 1.0 + assert f.estimate_cost("some-other-model", 1, 1) is None + + +# --- Entity linking --- + + +class TestInjectLinks: + people = {"Hadley Wickham": "/people/hadley-wickham/"} + software = {"Quarto": "/software/quarto/", "Shiny": "/software/shiny/", "Shiny for Python": "/software/shiny-python/"} + + def test_splits_anchor_and_links_first_occurrence_only(self): + html = '

I love quarto a lot. Quarto again.

' + assert f.inject_links(html, {}, self.software) == ( + '

I love quarto a lot. ' + 'Quarto again.

' + ) + + def test_match_at_anchor_start_and_end(self): + html = '

Shiny

' + assert f.inject_links(html, {}, self.software) == '

Shiny

' + + def test_longest_name_first(self): + html = '

We use Shiny for Python here.

' + out = f.inject_links(html, {}, self.software) + assert 'Shiny for Python' in out + assert '"/software/shiny/"' not in out + + def test_people_are_case_sensitive(self): + html = '

hadley wickham and Hadley Wickham

' + out = f.inject_links(html, self.people, {}) + assert out == ( + '

hadley wickham and Hadley Wickham

' + ) + + def test_never_nested(self): + out = f.inject_links(GOOD_HTML, self.people, self.software) + assert out.count(" opens inside another . + depth = 0 + for tag in re.findall(r"Quarto' in out + assert 'Hadley Wickham' in out + (call,) = client.calls + assert call["model"] == "claude-sonnet-4-6" + assert call["system"] == f.PROMPT + assert call["messages"][0]["content"].startswith("Video title: Quarto talk\n") + (done,) = items(data, "formatted") + assert done["coverage"] == 1.0 + assert done["input_tokens"] == 1000 and done["output_tokens"] == 500 + assert done["request_ids"] == ["req_fake"] + assert done["estimated_cost_usd"] == pytest.approx(0.0105) + assert data["extra"]["model"] == "claude-sonnet-4-6" + + def test_selection(self, repo): + write_video_dir(repo, "2026-01-04_done", url=URL, vtt=SAMPLE_VTT, html="

x

") + write_video_dir(repo, "2026-01-03_excluded", url=URL, vtt=SAMPLE_VTT, extra="transcribe: false\n") + write_video_dir(repo, "2026-01-02_novtt", url=URL) + write_video_dir(repo, "2026-01-01_todo", url=URL, duration=26, vtt=SAMPLE_VTT) + code, data = run(repo) + assert [i["slug"] for i in items(data, "formatted")] == ["2026-01-01_todo"] + assert data["counts"] == {"already_formatted": 1, "excluded": 1, "formatted": 1, "no_vtt": 1} + + def test_limit_newest_first(self, repo): + for i in range(1, 4): + write_video_dir(repo, f"2026-01-0{i}_v", url=URL, duration=26, vtt=SAMPLE_VTT) + code, data = run(repo, "--limit", "2") + assert [i["slug"] for i in items(data, "formatted")] == ["2026-01-03_v", "2026-01-02_v"] + assert data["counts"]["deferred"] == 1 + + def test_force(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT, html="

old

") + run(repo, "--force") + assert "old" not in (d / "transcription.txt").read_text() + + def test_truncated_response_rejected(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + code, data = run(repo, client=FakeAnthropic([(GOOD_HTML, "max_tokens")])) + assert code == 2 + (error,) = items(data, "error") + assert error["reason"] == "truncated" + assert "max_tokens=16384" in error["detail"] + assert not (d / "transcription.txt").exists() + + def test_invalid_html_not_written(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + code, data = run(repo, client=FakeAnthropic(["
nope
"])) + assert code == 2 + assert items(data, "error")[0]["reason"] == "disallowed_tag" + assert not (d / "transcription.txt").exists() + + def test_markdown_fence_stripped(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + code, _ = run(repo, client=FakeAnthropic([f"```html\n{GOOD_HTML}\n```"])) + assert code == 0 and (d / "transcription.txt").read_text().startswith("

") + + def test_overloaded_api_retried(self, repo): + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + client = FakeAnthropic([StatusError(529, "overloaded"), RateLimitError(), GOOD_HTML]) + code, _ = run(repo, client=client) + assert code == 0 and len(client.calls) == 3 + + def test_auth_error_not_retried(self, repo): + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + client = FakeAnthropic([StatusError(401, "invalid x-api-key")]) + code, data = run(repo, client=client) + assert code == 2 and len(client.calls) == 1 + assert items(data, "error")[0]["detail"] == "StatusError status=401 request_id=req_401: invalid x-api-key" + + def test_long_video_is_chunked(self, repo): + cues = [(float(t), float(t) + 5, f"sentence number {t}.") for t in range(0, 3000, 10)] + d = write_video_dir(repo, "2026-01-01_long", url=URL, duration=3000, vtt=videolib.render_vtt(cues)) + client = FakeAnthropic() + code, data = run(repo, client=client) + assert code == 0 + assert len(client.calls) == 4 # 50 minutes in 15-minute chunks + assert "This is segment 1 of 4." in client.calls[0]["system"] + assert "This is segment 4 of 4." in client.calls[3]["system"] + out = (d / "transcription.txt").read_text() + assert out.count("") == 3 + (done,) = items(data, "formatted") + assert done["chunks"] == 4 and done["input_tokens"] == 4000 + + def test_short_video_not_chunked(self, repo): + cues = [(float(t), float(t) + 5, f"sentence number {t}.") for t in range(0, 1100, 10)] + write_video_dir(repo, "2026-01-01_a", url=URL, duration=1100, vtt=videolib.render_vtt(cues)) + client = FakeAnthropic() + run(repo, client=client) + assert len(client.calls) == 1 + + def test_model_from_env(self, repo, monkeypatch): + monkeypatch.setenv("TRANSCRIPT_MODEL", "claude-haiku-4-5") + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + client = FakeAnthropic() + code, data = run(repo, client=client) + assert client.calls[0]["model"] == "claude-haiku-4-5" + code, data = run(repo, "--force", "--model", "claude-sonnet-4-6", client=client) + assert client.calls[1]["model"] == "claude-sonnet-4-6" + + def test_check_existing_and_reformat_invalid(self, repo): + linked = f.inject_links(GOOD_HTML, {"Hadley Wickham": "/people/hadley-wickham/"}, {}) + write_video_dir(repo, "2026-01-02_ok", url=URL, duration=26, vtt=SAMPLE_VTT, html=linked) + bad = write_video_dir( + repo, "2026-01-01_bad", url=URL, duration=26, vtt=SAMPLE_VTT, html='

Hello.

' + ) + client = FakeAnthropic() + code, data = run(repo, "--check-existing", client=client) + assert code == 0 and client.calls == [] + (invalid,) = items(data, "invalid") + assert invalid["slug"] == "2026-01-01_bad" and invalid["reason"] == "low_coverage" + assert data["counts"]["valid"] == 1 + + code, data = run(repo, "--reformat-invalid", client=client) + assert [i["slug"] for i in items(data, "formatted")] == ["2026-01-01_bad"] + assert len(client.calls) == 1 + assert f.check_existing(bad)[0] + + def test_dry_run(self, repo): + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + client = FakeAnthropic() + code, data = run(repo, "--dry-run", client=client) + assert code == 0 and client.calls == [] + assert items(data, "would_format")[0]["slug"] == "2026-01-01_a" + + def test_missing_api_key_is_fatal(self, repo, monkeypatch): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + summary = repo / "s.json" + code = f.main(["--project-root", str(repo), "--summary-json", str(summary)]) + assert code == 1 + assert "ANTHROPIC_API_KEY" in json.loads(summary.read_text())["fatal"] + + def test_logging(self, repo, capsys, monkeypatch): + monkeypatch.setenv("CI", "true") + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + run(repo) + err = capsys.readouterr().err + assert "2026-01-01_a (aaaaaaaaaaa): claude-sonnet-4-6 responded in" in err + assert "1000 in / 500 out tokens, stop_reason=end_turn, request_id=req_fake" in err + assert "wrote transcription.txt (3 anchors, 100.0% coverage, ~$0.011)" in err + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) From 08c0a91b264f06a48eea76190585eac5290a4680 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 13:17:30 +0100 Subject: [PATCH 08/20] Add an end-to-end test of the video pipeline and document the video sync The end-to-end test runs the four scripts and the report against a fixture repo with all services faked, and checks that a second run changes nothing, that a renamed video keeps its transcripts, that stats only change on a refresh, and that blocked downloads are retried the next run. --- README.md | 81 ++++++++++++- scripts/test-video-pipeline.py | 214 +++++++++++++++++++++++++++++++++ 2 files changed, 294 insertions(+), 1 deletion(-) create mode 100755 scripts/test-video-pipeline.py diff --git a/README.md b/README.md index b3eb1d8fa..4349fd821 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,15 @@ open-source-website/ │ ├── download-software-readmes.py # Fetch README files │ ├── summarize-software-readmes.py # Generate AI summaries │ ├── validate-blog-posts.py # Validate blog post frontmatter and placement -│ └── test-validate-blog-posts.py # Tests for validation script +│ ├── update-youtube-videos.py # Fetch video metadata from YouTube (see Video Sync) +│ ├── sync-videos.py # Create/update video pages from data/videos.toml +│ ├── transcribe.py # Download audio and transcribe it with Whisper +│ ├── format-transcriptions.py # Turn VTT transcripts into HTML with Claude +│ ├── video-sync-report.py # PR description for the nightly video sync +│ ├── videolib.py # Shared helpers of the video scripts +│ ├── videotesting.py # Fakes and fixtures for the video script tests +│ ├── run-tests.sh # Run all scripts/test-*.py (`just test`) +│ └── test-*.py # Tests for the scripts │ ├── archetypes/ # Content templates for hugo new │ ├── default.md @@ -194,6 +202,8 @@ just build-tailwind # Build Tailwind CSS only just build-search # Generate Pagefind search index just update-github-repos # Sync GitHub repository metadata to data/github-repos.toml just update-software-frontmatter # Update software frontmatter from github-repos.toml +just update-videos # Run the whole video pipeline locally (see Video Sync) +just test # Run the tests of the Python scripts just quarto-preview # Preview Quarto documents just hugo-serve # Start Hugo dev server with drafts ``` @@ -511,6 +521,70 @@ just update-software-frontmatter - Contributors (top contributors by commit count) - README image paths +### Video Sync + +Videos from Posit's YouTube channel and playlists (configured in +`data/youtube.toml`) are synced by four scripts, in this order: + +| Step | Script | Writes | Needs | +| --- | --- | --- | --- | +| 1 | `update-youtube-videos.py` | `data/videos.toml` | `YOUTUBE_API_KEY` | +| 2 | `sync-videos.py` | `content/resources/videos//_index.md`, `thumbnail.jpg` | | +| 3 | `transcribe.py` | `transcription.vtt` | `OPENAI_API_KEY`, ffmpeg, Deno | +| 4 | `format-transcriptions.py` | `transcription.txt` (the transcript shown on the page) | `ANTHROPIC_API_KEY` | + +**Nightly workflow.** `.github/workflows/sync-videos.yml` runs all four steps +at 05:00 UTC and opens (or updates) a pull request on the +`automation/sync-videos` branch, using the same GitHub App as the course sync. +Review the PR (it gets a deploy preview) and merge it to publish the changes. +The PR description lists new, renamed, and changed videos, the new transcripts, +and any problems, each with the command to fix it locally. If last night's PR +is still open, the next run builds on it instead of transcribing the same +videos again. You can also run the workflow by hand (Actions → Sync Videos → +Run workflow), with options to refresh view counts, skip transcription, change +how many videos get transcribed, or do a dry run. + +**Only real changes.** Every run fetches the metadata of all videos, but an +entry (and its `last_updated`) only changes when a field changed. View, like, +and comment counts are refreshed once a week, on Mondays, or with +`--refresh-stats`. Pages and transcripts are only written when their content +changes, so a quiet night produces no PR at all. + +**Transcripts.** Transcribing downloads the audio with yt-dlp, which YouTube +sometimes blocks on GitHub's runners. Download failures never fail the +workflow; the PR lists them, and the next run tries again. To transcribe +locally instead, check out the sync branch (or any branch) and run: + +```bash +just update-videos # all four steps, at most 15 new videos +just update-videos 5 chrome # at most 5, with YouTube cookies from Chrome +just transcribe --video 2026-11-11_some-talk --cookies-from-browser chrome +just format-transcriptions --video 2026-11-11_some-talk +``` + +Set `transcribe: false` in a video's `_index.md` to skip its transcript (e.g. +for music or members-only videos), or `publish = false` in +`data/videos.toml` to remove a video from the site. + +Formatted transcripts are validated before they're written: only `

`, +`

`, ``, and `

`; timestamps within the video; at least +90% of the spoken words; and no responses cut off at the token limit. Videos +longer than 20 minutes are formatted in 15-minute chunks. To check the +transcripts on the site, and to redo the ones that fail: + +```bash +just format-transcriptions --check-existing # report only, no API calls +just format-transcriptions --reformat-invalid --limit 10 +``` + +**Logs.** Every script logs what it does per video (with the slug and YouTube +ID), and with `--summary-json PATH` writes a machine-readable summary. The +workflow attaches the logs and summaries of every run as an artifact, kept for +30 days. All scripts exit with 0 on success, 1 on a fatal error (nothing +useful written), and 2 when some videos failed but the rest were written. +Pass `--dry-run` to see what a script would do, and `--verbose` for debug +logging. + ### Environment Variables Create a `.env` file in the project root (gitignored): @@ -520,6 +594,11 @@ Create a `.env` file in the project root (gitignored): # Get one at: https://github.com/settings/tokens GH_TOKEN="ghp_your_token_here" +# For the video sync (update-youtube-videos.py, transcribe.py, format-transcriptions.py) +YOUTUBE_API_KEY="..." +OPENAI_API_KEY="..." +ANTHROPIC_API_KEY="..." + # Plausible Analytics API key (optional, for analytics integration) PLAUSIBLE_KEY="your_key_here" ``` diff --git a/scripts/test-video-pipeline.py b/scripts/test-video-pipeline.py new file mode 100755 index 000000000..7fa592172 --- /dev/null +++ b/scripts/test-video-pipeline.py @@ -0,0 +1,214 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "google-api-python-client>=2.100.0", +# "tomli-w>=1.0.0", +# "python-dotenv>=1.0.0", +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "requests>=2.28.0", +# "openai>=2.0.0", +# "anthropic>=0.40.0", +# "yt-dlp>=2025.0", +# "pytest>=8.0", +# ] +# /// + +""" +End-to-end test of the video pipeline: the four scripts and the report, run +in order against a fixture repository, with every external service faked. + +The key property: a second run without changes on YouTube changes nothing, +which is what keeps the nightly pull requests small. +""" + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from videotesting import ( # noqa: E402 + FakeAnthropic, + FakeHttp, + FakeOpenAI, + FakeYouTube, + FakeYtDlp, + load_script, + make_repo, + no_sleep, + yt_item, +) + +update = load_script("update-youtube-videos.py") +sync = load_script("sync-videos.py") +transcribe = load_script("transcribe.py") +fmt = load_script("format-transcriptions.py") +report = load_script("video-sync-report.py") + + +def youtube(**titles): + titles = {"video000001": "Quarto with Hadley Wickham", "video000002": "Shiny tips", **titles} + return FakeYouTube( + channels={"PositPBC": ("UUposit", "Posit PBC")}, + playlists={"UUposit": list(titles), "PLtest": []}, + videos={ + vid: yt_item(vid, title, published=f"2026-11-0{i + 1}T10:00:00Z", duration="PT26S", views=10) + for i, (vid, title) in enumerate(titles.items()) + }, + ) + + +class Pipeline: + def __init__(self, root: Path): + self.root = root + self.ytdlp = FakeYtDlp() + self.openai = FakeOpenAI() + self.anthropic = FakeAnthropic() + self.http = FakeHttp() + + def run(self, yt: FakeYouTube, *update_args: str) -> dict[str, int]: + summaries = self.root.parent / "summaries" + common = ["--project-root", str(self.root)] + codes = { + "update": update.main( + [*common, *update_args, "--summary-json", str(summaries / "1.json")], youtube=yt + ), + "sync": sync.main([*common, "--summary-json", str(summaries / "2.json")], http_get=self.http), + "transcribe": transcribe.main( + [*common, "--summary-json", str(summaries / "3.json")], + client=self.openai, + run=self.ytdlp, + sleep=no_sleep, + min_request_interval=0, + ), + "format": fmt.main( + [*common, "--summary-json", str(summaries / "4.json")], + client=self.anthropic, + sleep=no_sleep, + min_request_interval=0, + ), + } + self.report, self.annotations = report.build_report( + report.load_summaries(summaries), expected=list(report.STEPS) + ) + return codes + + def snapshot(self) -> dict[str, tuple[bytes, int]]: + return { + str(p.relative_to(self.root)): (p.read_bytes(), p.stat().st_mtime_ns) + for p in sorted(self.root.rglob("*")) + if p.is_file() + } + + +@pytest.fixture +def pipeline(tmp_path): + return Pipeline(make_repo(tmp_path / "repo")) + + +def changed(before, after): + return sorted( + path + for path in set(before) | set(after) + if before.get(path, (None,))[0] != after.get(path, (None,))[0] + ) + + +def test_first_run_creates_everything(pipeline): + codes = pipeline.run(youtube()) + assert codes == {"update": 0, "sync": 0, "transcribe": 0, "format": 0} + d = pipeline.root / "content/resources/videos/2026-11-01_quarto-with-hadley-wickham" + assert sorted(p.name for p in d.iterdir()) == ["_index.md", "thumbnail.jpg", "transcription.txt", "transcription.vtt"] + html = (d / "transcription.txt").read_text() + assert 'Hadley Wickham' in html + fm = sync.parse_frontmatter((d / "_index.md").read_text())[0] + assert fm["people"] == ["Hadley Wickham"] and fm["software"] == ["quarto"] + # The transcription prompt used the synced frontmatter. + assert pipeline.openai.calls[-1]["prompt"].startswith("Quarto with Hadley Wickham") + assert "### New videos (2)" in pipeline.report + assert "| Transcription | 2 transcribed" in pipeline.report + assert "| Formatting | 2 formatted" in pipeline.report + assert pipeline.annotations == [] + + +def test_second_run_changes_nothing(pipeline): + pipeline.run(youtube()) + before = pipeline.snapshot() + calls = (len(pipeline.ytdlp.calls), len(pipeline.openai.calls), len(pipeline.anthropic.calls)) + codes = pipeline.run(youtube()) + after = pipeline.snapshot() + assert codes == {"update": 0, "sync": 0, "transcribe": 0, "format": 0} + assert before == after # contents and modification times + assert (len(pipeline.ytdlp.calls), len(pipeline.openai.calls), len(pipeline.anthropic.calls)) == calls + assert "| YouTube metadata | 2 unchanged |" in pipeline.report + + +def test_title_change_renames_without_new_transcripts(pipeline): + pipeline.run(youtube()) + before = pipeline.snapshot() + calls = (len(pipeline.ytdlp.calls), len(pipeline.openai.calls), len(pipeline.anthropic.calls)) + pipeline.run(youtube(video000002="Shiny tips and tricks")) + after = pipeline.snapshot() + old, new = "content/resources/videos/2026-11-02_shiny-tips", "content/resources/videos/2026-11-02_shiny-tips-and-tricks" + assert changed(before, after) == sorted([ + f"{old}/_index.md", + f"{old}/thumbnail.jpg", + f"{old}/transcription.txt", + f"{old}/transcription.vtt", + f"{new}/_index.md", + f"{new}/thumbnail.jpg", + f"{new}/transcription.txt", + f"{new}/transcription.vtt", + "data/videos.toml", + ]) + # The transcripts moved along; nothing was transcribed or formatted again. + assert after[f"{new}/transcription.vtt"][0] == before[f"{old}/transcription.vtt"][0] + assert (len(pipeline.ytdlp.calls), len(pipeline.openai.calls), len(pipeline.anthropic.calls)) == calls + assert "`2026-11-02_shiny-tips` → `2026-11-02_shiny-tips-and-tricks`" in pipeline.report + + +def test_stats_refresh_only_touches_metadata(pipeline): + pipeline.run(youtube()) + before = pipeline.snapshot() + yt = youtube() + for item in yt.videos_data.values(): + item["statistics"]["viewCount"] = "5000" + pipeline.run(yt) + assert changed(before, pipeline.snapshot()) == [] # not a stats day + pipeline.run(yt, "--refresh-stats") + assert changed(before, pipeline.snapshot()) == [ + "content/resources/videos/2026-11-01_quarto-with-hadley-wickham/_index.md", + "content/resources/videos/2026-11-02_shiny-tips/_index.md", + "data/videos.toml", + ] + assert "stats refreshed" in pipeline.report + + +def test_blocked_downloads_are_retried_next_run(pipeline): + blocked = "ERROR: [youtube] video000001: Sign in to confirm you’re not a bot." + pipeline.ytdlp = FakeYtDlp({"https://www.youtube.com/watch?v=video000001": [blocked]}) + codes = pipeline.run(youtube()) + assert codes["transcribe"] == 2 and codes["format"] == 0 + assert "**Audio downloads failed**" in pipeline.report + assert "::warning title=Audio downloads failed::" in pipeline.annotations[0] + d = pipeline.root / "content/resources/videos/2026-11-01_quarto-with-hadley-wickham" + assert not (d / "transcription.vtt").exists() + # Next night the download works and the video catches up. + codes = pipeline.run(youtube()) + assert codes == {"update": 0, "sync": 0, "transcribe": 0, "format": 0} + assert (d / "transcription.txt").exists() + + +def test_summaries_are_valid_json(pipeline): + pipeline.run(youtube()) + for path in sorted((pipeline.root.parent / "summaries").glob("*.json")): + data = json.loads(path.read_text()) + assert data["exit_code"] == 0 and data["finished_at"] + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) From 78e70f76f010e911f75fdd4c19c2eb53faac9219 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 13:23:40 +0100 Subject: [PATCH 09/20] Repair transcript timestamps from the VTT instead of trusting the model The model copies each cue's text verbatim but often gets its timestamp wrong. Across the 1,706 published transcripts only 64% of the data-t values are within a second of their cue (p90: 60 s off); 271 transcripts have timestamps beyond the end of the video. repair_timestamps() walks the anchors in order, finds each one's first words in the VTT (searching forward from the previous match), and sets data-t to that cue's start. Pull quotes reuse the value of their prose anchor, and anchors split by inject_links stay together. Unmatched anchors keep the model's value, capped at the last cue. On the published transcripts this brings exact timestamps from 64% to 97% (p90: 0 s) without changing anything but data-t values. New transcripts are repaired before validation, and --repair-timestamps fixes existing ones without API calls. --- README.md | 7 +- scripts/format-transcriptions.py | 139 ++++++++++++++++++++++++++ scripts/test-format-transcriptions.py | 106 ++++++++++++++++++++ 3 files changed, 250 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4349fd821..73f2a6825 100644 --- a/README.md +++ b/README.md @@ -569,11 +569,14 @@ for music or members-only videos), or `publish = false` in Formatted transcripts are validated before they're written: only `

`, `

`, ``, and `

`; timestamps within the video; at least 90% of the spoken words; and no responses cut off at the token limit. Videos -longer than 20 minutes are formatted in 15-minute chunks. To check the -transcripts on the site, and to redo the ones that fail: +longer than 20 minutes are formatted in 15-minute chunks. The model's +timestamps are often off by tens of seconds, so every `` is matched +back to its VTT cue and gets that cue's start time. To check the transcripts on +the site, and to fix the ones that fail: ```bash just format-transcriptions --check-existing # report only, no API calls +just format-transcriptions --repair-timestamps # fix timestamps only, no API calls just format-transcriptions --reformat-invalid --limit 10 ``` diff --git a/scripts/format-transcriptions.py b/scripts/format-transcriptions.py index 55873cca6..607799bb0 100755 --- a/scripts/format-transcriptions.py +++ b/scripts/format-transcriptions.py @@ -18,6 +18,8 @@ - Skip if transcription.vtt does not exist. - For long videos (> 90 min), split the VTT into chunks and process each segment separately, then concatenate the results. +- Repair the data-t timestamps: the model copies the cue text verbatim but + often gets the seconds wrong, so each is matched back to its VTT cue. - Validate the HTML: only

,

, , and

; every data-t within the video; at least 90% of the spoken words present; and a response that hit the max_tokens limit is rejected. Invalid output is @@ -369,6 +371,97 @@ def validate_html(html: str, cues: list[videolib.Cue], duration: int) -> dict[st return {"coverage": round(cov, 4), "anchors": anchors, "warnings": warnings or None} +# --------------------------------------------------------------------------- +# Timestamp repair +# --------------------------------------------------------------------------- + +# How far ahead (in words) to look for the next anchor's text in the VTT. +REPAIR_WINDOW_WORDS = 800 +# Words of an anchor used to find it in the VTT. +REPAIR_KEY_WORDS = 4 + +ANCHOR_RE = re.compile(r'(.*?)', flags=re.DOTALL) +BLOCKQUOTE_RE = re.compile(r"()", flags=re.DOTALL | re.IGNORECASE) + + +def _norm_words(text: str) -> list[str]: + return [w.lower().strip("'") for w in WORD_RE.findall(html_lib.unescape(re.sub(r"<[^>]+>", " ", text)))] + + +def repair_timestamps(html: str, cues: list[videolib.Cue]) -> tuple[str, dict[str, int]]: + """Set every ``data-t`` to the start of the VTT cue its text comes from. + + The model copies the cue text verbatim but often gets the seconds wrong + (in the existing transcripts typically by 5-20 s, sometimes by minutes). + Walk the anchors in document order and find each one's first words in + the VTT, searching forward from the previous match. Anchors that can't + be found keep the model's value. Pull quotes reuse the value of the + matching prose anchor. Consecutive anchors with the same original value + (split by inject_links) stay together. + """ + words: list[tuple[str, int]] = [] # (word, cue start in whole seconds) + for start, _end, text in cues: + words.extend((w, int(start)) for w in _norm_words(text)) + vocab = [w for w, _ in words] + + def find(key: list[str], pos: int) -> int | None: + limit = min(len(vocab) - len(key) + 1, pos + REPAIR_WINDOW_WORDS) + for i in range(pos, max(pos, limit)): + if vocab[i] == key[0] and vocab[i : i + len(key)] == key: + return i + return None + + stats = {"anchors": 0, "matched": 0, "changed": 0, "unmatched": 0} + by_text: dict[str, int] = {} + pos = 0 + prev: tuple[int, int] | None = None # (original, repaired) + + def repair_prose(m: re.Match) -> str: + nonlocal pos, prev + original, inner = int(m.group(1)), m.group(2) + stats["anchors"] += 1 + ws = _norm_words(inner) + new = original + found = None + # Skip a leading word or two in case the model corrected it + # (e.g. "Korto" -> "Quarto"). + for skip in (0, 1, 2): + key = ws[skip : skip + REPAIR_KEY_WORDS] + if not key: + break + i = find(key, pos) + if i is not None: + pos = i + len(ws) - skip + found = words[i][1] + break + if found is not None: + stats["matched"] += 1 + new = found + else: + stats["unmatched"] += 1 + if prev is not None and prev[0] == original: + new = prev[1] # a piece of an anchor that inject_links split + elif words: + # Keep the model's value, but not beyond the last cue. + # (Clamping to the neighbours was measured to make things + # worse: one wrong match then drags the next anchors along.) + new = min(original, words[-1][1]) + prev = (original, new) + if new != original: + stats["changed"] += 1 + by_text.setdefault(" ".join(ws), new) + return f'{inner}' + + def repair_quote(m: re.Match) -> str: + new = by_text.get(" ".join(_norm_words(m.group(2))), int(m.group(1))) + return f'{m.group(2)}' + + parts = BLOCKQUOTE_RE.split(html) + parts = [p if i % 2 else ANCHOR_RE.sub(repair_prose, p) for i, p in enumerate(parts)] + parts = [ANCHOR_RE.sub(repair_quote, p) if i % 2 else p for i, p in enumerate(parts)] + return "".join(parts), stats + + def unlink_entities(html: str) -> str: """Undo inject_links, so a published transcript can be validated again.""" return re.sub(r'(.*?)', r"\1", html, flags=re.DOTALL) @@ -655,6 +748,16 @@ async def process_video(video_dir: Path, ctx: Context) -> dict[str, Any]: item["request_ids"] = [u["request_id"] for u in usages if u["request_id"]] or None item["estimated_cost_usd"] = estimate_cost(args.model, item["input_tokens"], item["output_tokens"]) + html, repair = repair_timestamps(html, cues) + item["timestamps_repaired"] = repair["changed"] + item["timestamps_unmatched"] = repair["unmatched"] + log.info( + "%s: repaired %d of %d timestamps (%d anchors not found in the VTT)", + label, + repair["changed"], + repair["anchors"], + repair["unmatched"], + ) metrics = validate_html(html, cues, duration) item.update(metrics) for w in metrics["warnings"] or []: @@ -707,6 +810,34 @@ def run_check_existing(dirs: list[Path], summary: videolib.RunSummary) -> None: ) +def run_repair_timestamps(dirs: list[Path], summary: videolib.RunSummary, *, dry_run: bool) -> None: + """Fix the data-t values of published transcripts (no API calls).""" + for d in dirs: + txt, vtt = d / "transcription.txt", d / "transcription.vtt" + if not (txt.exists() and vtt.exists()): + continue + cues = videolib.parse_vtt(vtt.read_text(encoding="utf-8")) + if not cues: + summary.tally("no_cues") + continue + html = txt.read_text(encoding="utf-8") + repaired, stats = repair_timestamps(html, cues) + if repaired == html: + summary.tally("timestamps_ok") + continue + if not dry_run: + videolib.atomic_write_text(txt, repaired) + summary.add("timestamps_repaired", slug=d.name, **stats) + log.info( + "%s: %s %d of %d timestamps (%d anchors not found in the VTT)", + d.name, + "would repair" if dry_run else "repaired", + stats["changed"], + stats["anchors"], + stats["unmatched"], + ) + + def build_client(project_root: Path) -> Any: import anthropic from dotenv import load_dotenv @@ -750,6 +881,11 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: action="store_true", help="only validate existing transcription.txt files and report the invalid ones (no API calls)", ) + parser.add_argument( + "--repair-timestamps", + action="store_true", + help="only fix the data-t values of existing transcription.txt files from their VTT (no API calls)", + ) parser.add_argument("--dry-run", action="store_true", help="list what would be formatted") videolib.add_common_args(parser) return parser.parse_args(argv) @@ -780,6 +916,9 @@ def body() -> None: if args.check_existing: run_check_existing(dirs, summary) return + if args.repair_timestamps: + run_repair_timestamps(dirs, summary, dry_run=args.dry_run) + return selected = select_videos( dirs, summary, force=args.force, limit=args.limit, reformat_invalid=args.reformat_invalid ) diff --git a/scripts/test-format-transcriptions.py b/scripts/test-format-transcriptions.py index 69d2a1e64..4ddd5713f 100755 --- a/scripts/test-format-transcriptions.py +++ b/scripts/test-format-transcriptions.py @@ -158,6 +158,88 @@ def html(n): f.validate_html(html(80), cues, 1) +# --- Timestamp repair --- + + +def cues_from(*texts, step=10): + return [(float(i * step) + 0.7, float(i * step) + 9, t) for i, t in enumerate(texts)] + + +class TestRepairTimestamps: + cues = cues_from( + "Hello everyone and welcome.", + "Today we talk about Quarto,", + "the publishing system.", + "And then about Shiny.", + "Thanks for watching.", + ) + + def test_wrong_values_are_fixed(self): + html = ( + '

Hello everyone and welcome. Today we talk about Quarto, ' + 'the publishing system.

\n\n

And then about Shiny.

' + ) + out, stats = f.repair_timestamps(html, self.cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "10", "20", "30"] + assert stats == {"anchors": 4, "matched": 4, "changed": 3, "unmatched": 0} + assert re.sub(r'data-t="\d+"', "", out) == re.sub(r'data-t="\d+"', "", html) # only data-t changed + + def test_merged_cues_get_the_first_start(self): + html = '

Today we talk about Quarto, the publishing system.

' + out, _ = f.repair_timestamps(html, self.cues) + assert 'data-t="10"' in out + + def test_anchor_starting_mid_cue(self): + html = '

Hello everyone and welcome.

' + out, _ = f.repair_timestamps(html, self.cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "0"] + + def test_corrected_leading_word(self): + # Whisper heard "Korto"; the model corrected it to "Quarto". + cues = cues_from("Hello everyone and welcome.", "Korto is great for documents.") + html = '

Hello everyone and welcome. Quarto is great for documents.

' + out, stats = f.repair_timestamps(html, cues) + assert 'data-t="10"' in out and stats["unmatched"] == 0 + + def test_repeated_phrase_searches_forward(self): + cues = cues_from("Thank you.", "Something in between.", "Thank you.") + html = '

Thank you. Something in between. Thank you.

' + out, _ = f.repair_timestamps(html, cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "10", "20"] + + def test_blockquote_reuses_prose_value(self): + html = ( + '

Hello everyone and welcome. And then about Shiny.

\n\n' + '
And then about Shiny.
\n\n' + '

Thanks for watching.

' + ) + out, stats = f.repair_timestamps(html, self.cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "30", "30", "40"] + assert stats["anchors"] == 3 # pull quotes aren't counted twice + + def test_pieces_split_by_inject_links_stay_together(self): + linked = f.inject_links( + '

Today we talk about Quarto, the publishing system.

', + {}, + {"Quarto": "/software/quarto/"}, + ) + assert 'Quarto' in linked + out, _ = f.repair_timestamps(linked, self.cues) + # "Today we talk about ", "," (the pieces around the link), "the publishing system." + assert re.findall(r'data-t="(\d+)"', out) == ["10", "10", "20"] + assert 'Quarto' in out + + def test_unmatched_keeps_value_but_capped_at_last_cue(self): + html = '

Words that were never said. Neither were these.

' + out, stats = f.repair_timestamps(html, self.cues) + assert re.findall(r'data-t="(\d+)"', out) == ["5", "40"] + assert stats["unmatched"] == 2 + + def test_no_cues(self): + html = '

x

' + assert f.repair_timestamps(html, [])[0] == html + + def test_estimate_cost(): assert f.estimate_cost("claude-sonnet-4-6", 1_000_000, 100_000) == 4.5 assert f.estimate_cost("claude-haiku-4-5", 1_000_000, 0) == 1.0 @@ -361,6 +443,30 @@ def test_check_existing_and_reformat_invalid(self, repo): assert len(client.calls) == 1 assert f.check_existing(bad)[0] + def test_timestamps_repaired_before_writing(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + shifted = GOOD_HTML.replace('data-t="4"', 'data-t="9"').replace('data-t="20"', 'data-t="24"') + code, data = run(repo, client=FakeAnthropic([shifted])) + assert code == 0 + values = re.findall(r'data-t="(\d+)"', (d / "transcription.txt").read_text()) + assert set(values) == {"0", "4", "20"} # pieces split by inject_links repeat a value + (done,) = items(data, "formatted") + assert done["timestamps_repaired"] == 2 and done["timestamps_unmatched"] == 0 + + def test_repair_timestamps_mode(self, repo): + shifted = GOOD_HTML.replace('data-t="20"', 'data-t="23"') + d = write_video_dir(repo, "2026-01-02_a", url=URL, duration=26, vtt=SAMPLE_VTT, html=shifted) + write_video_dir(repo, "2026-01-01_ok", url=URL, duration=26, vtt=SAMPLE_VTT, html=GOOD_HTML) + client = FakeAnthropic() + code, data = run(repo, "--repair-timestamps", "--dry-run", client=client) + assert (d / "transcription.txt").read_text() == shifted + code, data = run(repo, "--repair-timestamps", client=client) + assert code == 0 and client.calls == [] + assert 'data-t="20"' in (d / "transcription.txt").read_text() + (fixed,) = items(data, "timestamps_repaired") + assert fixed["slug"] == "2026-01-02_a" and fixed["changed"] == 1 + assert data["counts"]["timestamps_ok"] == 1 + def test_dry_run(self, repo): write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) client = FakeAnthropic() From 9ba278165f03caee733667680d3aec970fdc1930 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 13:27:23 +0100 Subject: [PATCH 10/20] Catch degenerate Whisper output and keep rejected transcripts for debugging Found in a local end-to-end run: Whisper returned only "." cues for the last 22 minutes of a video, and the transcript passed every check. One of the published transcripts has the same problem. - transcribe.py: retry a chunk once when it has more than 2 minutes of cues without words, and reject the transcript (degenerate_transcript) when it still does - transcribe.py: aim audio chunks at 90% of the API limit, so a chunk that comes out slightly larger isn't split again into a few seconds - format-transcriptions.py: log and record the coverage per chunk, and save rejected output with --rejected-dir (the workflow keeps it in the artifact) - videolib.py: also quiet the httpx2 logger --- .github/workflows/sync-videos.yml | 1 + scripts/format-transcriptions.py | 20 +++++++++++ scripts/test-format-transcriptions.py | 9 +++++ scripts/test-transcribe.py | 45 +++++++++++++++++++++--- scripts/transcribe.py | 49 ++++++++++++++++++++++++--- scripts/videolib.py | 1 + 6 files changed, 117 insertions(+), 8 deletions(-) diff --git a/.github/workflows/sync-videos.yml b/.github/workflows/sync-videos.yml index 3872db7cf..970b7e926 100644 --- a/.github/workflows/sync-videos.yml +++ b/.github/workflows/sync-videos.yml @@ -179,6 +179,7 @@ jobs: run: | set +e ./scripts/format-transcriptions.py --limit "$LIMIT" \ + --rejected-dir "$SYNC_DIR/rejected" \ --summary-json "$SYNC_DIR/summaries/4-format-transcriptions.json" \ 2>&1 | tee "$SYNC_DIR/logs/4-format-transcriptions.log" status=${PIPESTATUS[0]} diff --git a/scripts/format-transcriptions.py b/scripts/format-transcriptions.py index 607799bb0..9a1c3d0a5 100755 --- a/scripts/format-transcriptions.py +++ b/scripts/format-transcriptions.py @@ -709,6 +709,7 @@ async def process_video(video_dir: Path, ctx: Context) -> dict[str, Any]: label = f"{video_dir.name} ({video_id})" duration = int(external.get("duration") or 0) item: dict[str, Any] = {"slug": video_dir.name, "video_id": video_id, "duration_s": duration, "model": args.model} + html: str | None = None try: vtt_content = (video_dir / "transcription.vtt").read_text(encoding="utf-8") cues = videolib.parse_vtt(vtt_content) @@ -742,6 +743,13 @@ async def process_video(video_dir: Path, ctx: Context) -> dict[str, Any]: parts = [html_part for html_part, _ in results] usages = [usage for _, usage in results] html = "\n\n".join(parts) + if len(chunks) > 1: + # Which chunk lost words, if any: useful when coverage is low. + item["chunk_coverage"] = [ + round(coverage(videolib.parse_vtt(chunk_vtt), part), 3) + for (chunk_vtt, _), part in zip(chunks, parts) + ] + log.info("%s: coverage per chunk: %s", label, ", ".join(f"{c:.1%}" for c in item["chunk_coverage"])) item["input_tokens"] = sum(u["input_tokens"] for u in usages) item["output_tokens"] = sum(u["output_tokens"] for u in usages) @@ -766,6 +774,12 @@ async def process_video(video_dir: Path, ctx: Context) -> dict[str, Any]: videolib.atomic_write_text(video_dir / "transcription.txt", html) except FormatError as e: log.error("%s: %s: %s", label, e.reason, e) + if args.rejected_dir and html is not None: + args.rejected_dir.mkdir(parents=True, exist_ok=True) + rejected = args.rejected_dir / f"{video_dir.name}.html" + videolib.atomic_write_text(rejected, html) + item["rejected_output"] = str(rejected) + log.info("%s: saved the rejected output to %s", label, rejected) return {**item, "status": "error", "reason": e.reason, "detail": str(e)} except Exception as e: # noqa: BLE001 - every failure goes in the summary log.exception("%s: formatting failed", label) @@ -886,6 +900,12 @@ def parse_args(argv: list[str] | None) -> argparse.Namespace: action="store_true", help="only fix the data-t values of existing transcription.txt files from their VTT (no API calls)", ) + parser.add_argument( + "--rejected-dir", + type=Path, + metavar="DIR", + help="save model output that fails validation to DIR/.html, for debugging", + ) parser.add_argument("--dry-run", action="store_true", help="list what would be formatted") videolib.add_common_args(parser) return parser.parse_args(argv) diff --git a/scripts/test-format-transcriptions.py b/scripts/test-format-transcriptions.py index 4ddd5713f..21b502f92 100755 --- a/scripts/test-format-transcriptions.py +++ b/scripts/test-format-transcriptions.py @@ -377,6 +377,14 @@ def test_invalid_html_not_written(self, repo): assert items(data, "error")[0]["reason"] == "disallowed_tag" assert not (d / "transcription.txt").exists() + def test_rejected_output_saved(self, repo, tmp_path): + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + rejected = tmp_path / "rejected" + code, data = run(repo, "--rejected-dir", str(rejected), client=FakeAnthropic(['

Hello.

'])) + assert code == 2 + assert (rejected / "2026-01-01_a.html").read_text() == '

Hello.

' + assert items(data, "error")[0]["rejected_output"] == str(rejected / "2026-01-01_a.html") + def test_markdown_fence_stripped(self, repo): d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) code, _ = run(repo, client=FakeAnthropic([f"```html\n{GOOD_HTML}\n```"])) @@ -408,6 +416,7 @@ def test_long_video_is_chunked(self, repo): assert out.count("") == 3 (done,) = items(data, "formatted") assert done["chunks"] == 4 and done["input_tokens"] == 4000 + assert done["chunk_coverage"] == [1.0, 1.0, 1.0, 1.0] def test_short_video_not_chunked(self, repo): cues = [(float(t), float(t) + 5, f"sentence number {t}.") for t in range(0, 1100, 10)] diff --git a/scripts/test-transcribe.py b/scripts/test-transcribe.py index 2f6b4c642..0f1f4e27c 100755 --- a/scripts/test-transcribe.py +++ b/scripts/test-transcribe.py @@ -141,15 +141,15 @@ def test_single_chunk(self): assert t.compute_chunk_boundaries(100, 10, [50], max_bytes=100) == [(0.0, 100)] def test_cuts_at_last_silence_before_limit(self): - # 100 s at 1 byte/s, 40 bytes per chunk -> at most 40 s per chunk. + # 100 s at 1 byte/s, 40 bytes per chunk, aiming at 90% -> at most 36 s per chunk. assert t.compute_chunk_boundaries(100, 100, [10, 35, 38, 70, 79], max_bytes=40) == [ - (0.0, 38), - (38, 70), + (0.0, 35), + (35, 70), (70, 100), ] def test_hard_cut_without_silence(self): - assert t.compute_chunk_boundaries(100, 100, [], max_bytes=40) == [(0.0, 40.0), (40.0, 80.0), (80.0, 100)] + assert t.compute_chunk_boundaries(100, 100, [], max_bytes=40) == [(0.0, 36.0), (36.0, 72.0), (72.0, 100)] def test_merge_vtt_offsets(self): part = "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nhi\n" @@ -157,7 +157,27 @@ def test_merge_vtt_offsets(self): assert videolib.parse_vtt(merged) == [(1.0, 2.0, "hi"), (601.5, 602.5, "hi")] +def degenerate_vtt(start=0, words_until=30, dots_until=300): + cues = [(float(t), float(t) + 2, f"word {t}") for t in range(start, words_until, 2)] + cues += [(float(t), float(t) + 2, ".") for t in range(words_until, dots_until, 2)] + return videolib.render_vtt(cues) + + class TestCheckTranscript: + def test_wordless_stretch(self): + cues = videolib.parse_vtt(degenerate_vtt()) + assert t.longest_wordless_stretch(cues) == (270.0, 30.0) + assert t.longest_wordless_stretch(videolib.parse_vtt(SAMPLE_VTT)) == (0.0, 0.0) + + def test_degenerate(self): + with pytest.raises(t.TranscriptError) as e: + t.check_transcript(degenerate_vtt(), 300) + assert e.value.reason == "degenerate_transcript" + assert str(e.value) == "270s of cues without words from 00:00:30" + + def test_short_wordless_stretch_is_fine(self): + t.check_transcript(degenerate_vtt(dots_until=100), 100) + def test_empty(self): with pytest.raises(t.TranscriptError) as e: t.check_transcript("WEBVTT\n\n", 100) @@ -342,6 +362,23 @@ def test_whisper_client_error_not_retried(self, repo): assert error["detail"] == "StatusError status=400 request_id=req_400: Invalid file format" assert not (a / "transcription.vtt").exists() + def test_degenerate_chunk_retried(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + client = FakeOpenAI([degenerate_vtt()]) # then SAMPLE_VTT + code, data = run(repo, client=client) + assert code == 0 and len(client.calls) == 2 + assert (a / "transcription.vtt").read_text() == SAMPLE_VTT + assert items(data, "transcribed")[0]["whisper_retries"] == 1 + + def test_degenerate_twice_is_an_error(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=300) + client = FakeOpenAI([degenerate_vtt(), degenerate_vtt()]) + code, data = run(repo, client=client) + assert code == 2 and len(client.calls) == 2 + (error,) = items(data, "error") + assert error["reason"] == "degenerate_transcript" + assert not (a / "transcription.vtt").exists() + def test_empty_transcript_not_written(self, repo): a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) code, data = run(repo, client=FakeOpenAI(["WEBVTT\n\n"])) diff --git a/scripts/transcribe.py b/scripts/transcribe.py index 4ac2770a7..9bc7823be 100755 --- a/scripts/transcribe.py +++ b/scripts/transcribe.py @@ -72,6 +72,13 @@ DOWNLOAD_RETRY_DELAY = 5.0 # Warn when the transcript ends before this fraction of the video. TRUNCATION_RATIO = 0.9 +# Whisper sometimes returns a long run of cues without words (just ".") +# for part of the audio. A chunk with a stretch this long is retried once; +# a transcript that still has one is rejected. +MAX_WORDLESS_S = 120 +# Aim chunks at this share of the API limit: bitrates vary, and a chunk +# that ends up just over the limit gets split again into a tiny piece. +CHUNK_TARGET_RATIO = 0.9 MAX_STDERR_CHARS = 4000 @@ -368,7 +375,7 @@ def compute_chunk_boundaries( silence_points: list[float], max_bytes: int = MAX_BYTES, ) -> list[tuple[float, float]]: - max_chunk_secs = max_bytes / (file_size / duration) + max_chunk_secs = CHUNK_TARGET_RATIO * max_bytes / (file_size / duration) boundaries: list[tuple[float, float]] = [] start = 0.0 while start < duration: @@ -438,11 +445,32 @@ def __init__(self, reason: str, detail: str): self.reason = reason +def longest_wordless_stretch(cues: list[videolib.Cue]) -> tuple[float, float]: + """``(seconds, start)`` of the longest run of consecutive cues without words.""" + longest, longest_start = 0.0, 0.0 + run_start = None + for start, end, text in cues: + if re.search(r"\w", text): + run_start = None + continue + if run_start is None: + run_start = start + if end - run_start > longest: + longest, longest_start = end - run_start, run_start + return longest, longest_start + + def check_transcript(vtt: str, duration: int) -> tuple[list[videolib.Cue], list[str]]: """Return ``(cues, warnings)``; raise TranscriptError if unusable.""" cues = videolib.parse_vtt(vtt) if not cues: raise TranscriptError("empty_transcript", f"Whisper returned no cues ({len(vtt)} characters)") + stretch, at = longest_wordless_stretch(cues) + if stretch > MAX_WORDLESS_S: + raise TranscriptError( + "degenerate_transcript", + f"{stretch:.0f}s of cues without words from {videolib.format_vtt_timestamp(at)[:8]}", + ) warnings = [] last_end = cues[-1][1] if duration > 0 and last_end < TRUNCATION_RATIO * duration: @@ -551,9 +579,22 @@ async def process_video(c: Candidate, ctx: Context) -> dict[str, Any]: for i, (chunk, offset) in enumerate(chunks, start=1): chunk_label = f"{label} chunk {i}/{len(chunks)}" if len(chunks) > 1 else label tc = time.monotonic() - vtt = await transcribe_file( - ctx.client, chunk, prompt, label=chunk_label, rate_limiter=ctx.rate_limiter, sleep=ctx.sleep - ) + for attempt in (1, 2): + vtt = await transcribe_file( + ctx.client, chunk, prompt, label=chunk_label, rate_limiter=ctx.rate_limiter, sleep=ctx.sleep + ) + stretch, at = longest_wordless_stretch(videolib.parse_vtt(vtt)) + if stretch <= MAX_WORDLESS_S: + break + log.warning( + "%s: Whisper returned %.0fs of cues without words from %s (attempt %d/2)%s", + chunk_label, + stretch, + videolib.format_vtt_timestamp(at + offset)[:8], + attempt, + "; retrying" if attempt == 1 else "", + ) + item["whisper_retries"] = item.get("whisper_retries", 0) + (attempt == 1) log.info( "%s: transcribed %.1f MB (offset %.0fs) in %.1fs", chunk_label, diff --git a/scripts/videolib.py b/scripts/videolib.py index 9f6483426..78c5bfd9e 100644 --- a/scripts/videolib.py +++ b/scripts/videolib.py @@ -80,6 +80,7 @@ def formatTime(self, record, datefmt=None): # noqa: N802 (logging API) NOISY_LOGGERS = [ "httpx", + "httpx2", "httpcore", "openai", "anthropic", From fd04cc15de7d99e6c4cd9883eda575e922e46464 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 13:33:33 +0100 Subject: [PATCH 11/20] Tell the model not to skip anything, and retry chunks that lost words In a local run the model skipped small talk, logistics ("can you see my slides?"), and parts of the Q&A, leaving chunks at 78-82% of the spoken words. The prompt said to keep the text verbatim but never said not to leave things out. With an explicit instruction to include every cue, the same video came back at 97-100% per chunk. Chunks that still lose more than 10% of their words are requested once more, and the better answer is kept; both requests count toward the usage and cost in the summary. --- scripts/format-transcriptions.py | 62 ++++++++++++++++++--------- scripts/test-format-transcriptions.py | 32 +++++++++++++- 2 files changed, 72 insertions(+), 22 deletions(-) diff --git a/scripts/format-transcriptions.py b/scripts/format-transcriptions.py index 9a1c3d0a5..ecbe5e080 100755 --- a/scripts/format-transcriptions.py +++ b/scripts/format-transcriptions.py @@ -109,6 +109,11 @@ Copy the spoken words exactly as they appear in the VTT, including filler words ("um", "uh", "sort of", "you know") and false starts. +Include EVERY cue, from the first to the last. Do NOT skip, shorten, or +summarize any part — not small talk, not logistics ("can you see my +screen?", "next slide"), not jokes, not audience questions, not the closing +remarks. The output must contain all the words of the VTT. + ## Correcting technology-name mishearings The metadata lists technologies used in this video. If a word is clearly a phonetic mishearing of one of those technology names, silently correct it. @@ -723,33 +728,50 @@ async def process_video(video_dir: Path, ctx: Context) -> dict[str, Any]: item["chunks"] = len(chunks) log.info("%s: formatting %d cues in %d chunk(s)", label, len(cues), len(chunks)) + def request(i: int): + return call_api( + ctx.client, + build_user_message(chunks[i][0], fm), + model=args.model, + max_tokens=args.max_tokens, + rate_limiter=ctx.rate_limiter, + label=f"{label} chunk {i + 1}/{len(chunks)}" if len(chunks) > 1 else label, + system=build_chunk_system(i, len(chunks)) if len(chunks) > 1 else PROMPT, + sleep=ctx.sleep, + ) + # Chunks are independent, so request them concurrently (the rate # limiter still spaces out the requests). - results = await asyncio.gather( - *( - call_api( - ctx.client, - build_user_message(chunk_vtt, fm), - model=args.model, - max_tokens=args.max_tokens, - rate_limiter=ctx.rate_limiter, - label=f"{label} chunk {i + 1}/{len(chunks)}" if len(chunks) > 1 else label, - system=build_chunk_system(i, len(chunks)) if len(chunks) > 1 else PROMPT, - sleep=ctx.sleep, - ) - for i, (chunk_vtt, _seg_start) in enumerate(chunks) - ) - ) + results = list(await asyncio.gather(*(request(i) for i in range(len(chunks))))) parts = [html_part for html_part, _ in results] usages = [usage for _, usage in results] + chunk_cues = [videolib.parse_vtt(chunk_vtt) for chunk_vtt, _ in chunks] + chunk_coverage = [coverage(cs, part) for cs, part in zip(chunk_cues, parts)] + + # The model sometimes skips parts of a chunk. Ask once more for + # the chunks that lost too many words and keep the better answer. + retry = [i for i, c in enumerate(chunk_coverage) if c < MIN_COVERAGE] + if retry: + log.warning( + "%s: %s lost words (%s); asking once more", + label, + "chunks " + ", ".join(str(i + 1) for i in retry) if len(chunks) > 1 else "the response", + ", ".join(f"{chunk_coverage[i]:.1%}" for i in retry), + ) + second = await asyncio.gather(*(request(i) for i in retry)) + for i, (html_part, usage) in zip(retry, second): + usages.append(usage) + c = coverage(chunk_cues[i], html_part) + log.info("%s: retry of chunk %d: %.1f%% coverage (was %.1f%%)", label, i + 1, c * 100, chunk_coverage[i] * 100) + if c > chunk_coverage[i]: + parts[i], chunk_coverage[i] = html_part, c + item["coverage_retries"] = len(retry) + html = "\n\n".join(parts) if len(chunks) > 1: # Which chunk lost words, if any: useful when coverage is low. - item["chunk_coverage"] = [ - round(coverage(videolib.parse_vtt(chunk_vtt), part), 3) - for (chunk_vtt, _), part in zip(chunks, parts) - ] - log.info("%s: coverage per chunk: %s", label, ", ".join(f"{c:.1%}" for c in item["chunk_coverage"])) + item["chunk_coverage"] = [round(c, 3) for c in chunk_coverage] + log.info("%s: coverage per chunk: %s", label, ", ".join(f"{c:.1%}" for c in chunk_coverage)) item["input_tokens"] = sum(u["input_tokens"] for u in usages) item["output_tokens"] = sum(u["output_tokens"] for u in usages) diff --git a/scripts/test-format-transcriptions.py b/scripts/test-format-transcriptions.py index 21b502f92..dac0f124a 100755 --- a/scripts/test-format-transcriptions.py +++ b/scripts/test-format-transcriptions.py @@ -372,15 +372,43 @@ def test_truncated_response_rejected(self, repo): def test_invalid_html_not_written(self, repo): d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) - code, data = run(repo, client=FakeAnthropic(["
nope
"])) + code, data = run(repo, client=FakeAnthropic(["
nope
", "
nope
"])) assert code == 2 assert items(data, "error")[0]["reason"] == "disallowed_tag" assert not (d / "transcription.txt").exists() + def test_low_coverage_retried(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + client = FakeAnthropic(['

Hello everyone.

']) # then a complete answer + code, data = run(repo, client=client) + assert code == 0 and len(client.calls) == 2 + (done,) = items(data, "formatted") + assert done["coverage"] == 1.0 and done["coverage_retries"] == 1 + assert done["input_tokens"] == 2000 # both requests are counted + assert (d / "transcription.txt").exists() + + def test_retry_keeps_the_better_answer(self, repo): + cues = [(float(t), float(t) + 5, f"sentence number {t}.") for t in range(0, 1500, 10)] + vtt = videolib.render_vtt(cues) + write_video_dir(repo, "2026-01-01_long", url=URL, duration=1500, vtt=vtt) + chunks = f.split_vtt_into_chunks(vtt, f.CHUNK_DURATION_S) + assert len(chunks) == 2 + good = [vtt_to_html(c) for c, _ in chunks] + half = vtt_to_html(videolib.render_vtt(videolib.parse_vtt(chunks[1][0])[:40])) + worse = vtt_to_html(videolib.render_vtt(videolib.parse_vtt(chunks[1][0])[:10])) + client = FakeAnthropic([good[0], half, worse]) + code, data = run(repo, client=client) + assert len(client.calls) == 3 # only chunk 2 is asked again + (error,) = items(data, "error") + assert error["reason"] == "low_coverage" + assert error["coverage_retries"] == 1 + assert error["chunk_coverage"][0] == 1.0 + assert 0.6 < error["chunk_coverage"][1] < 0.7 # the first, better answer was kept + def test_rejected_output_saved(self, repo, tmp_path): write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) rejected = tmp_path / "rejected" - code, data = run(repo, "--rejected-dir", str(rejected), client=FakeAnthropic(['

Hello.

'])) + code, data = run(repo, "--rejected-dir", str(rejected), client=FakeAnthropic(['

Hello.

'] * 2)) assert code == 2 assert (rejected / "2026-01-01_a.html").read_text() == '

Hello.

' assert items(data, "error")[0]["rejected_output"] == str(rejected / "2026-01-01_a.html") From 680cdb2e08620b5377386a72cd19c0b692284a18 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 13:35:32 +0100 Subject: [PATCH 12/20] Keep short anchors from derailing the timestamp repair A short anchor such as "For us," could match a later occurrence hundreds of words ahead, after which every following anchor was behind the search position: in one real transcript 565 of 1,298 anchors went unmatched. Anchors shorter than the 4-word key now only match within the next 40 words. That video now has 2 unmatched anchors, and exact timestamps across the published transcripts go from 97.1% to 97.4%. --- scripts/format-transcriptions.py | 7 ++++++- scripts/test-format-transcriptions.py | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/scripts/format-transcriptions.py b/scripts/format-transcriptions.py index ecbe5e080..d63426908 100755 --- a/scripts/format-transcriptions.py +++ b/scripts/format-transcriptions.py @@ -382,6 +382,10 @@ def validate_html(html: str, cues: list[videolib.Cue], duration: int) -> dict[st # How far ahead (in words) to look for the next anchor's text in the VTT. REPAIR_WINDOW_WORDS = 800 +# Anchors shorter than REPAIR_KEY_WORDS ("For us,", "I came from") occur all +# over a transcript, so they may only match this close to the previous one; +# otherwise one false match far ahead derails all the anchors after it. +REPAIR_SHORT_WINDOW_WORDS = 40 # Words of an anchor used to find it in the VTT. REPAIR_KEY_WORDS = 4 @@ -410,7 +414,8 @@ def repair_timestamps(html: str, cues: list[videolib.Cue]) -> tuple[str, dict[st vocab = [w for w, _ in words] def find(key: list[str], pos: int) -> int | None: - limit = min(len(vocab) - len(key) + 1, pos + REPAIR_WINDOW_WORDS) + window = REPAIR_WINDOW_WORDS if len(key) >= REPAIR_KEY_WORDS else REPAIR_SHORT_WINDOW_WORDS + limit = min(len(vocab) - len(key) + 1, pos + window) for i in range(pos, max(pos, limit)): if vocab[i] == key[0] and vocab[i : i + len(key)] == key: return i diff --git a/scripts/test-format-transcriptions.py b/scripts/test-format-transcriptions.py index dac0f124a..ed47ba5c0 100755 --- a/scripts/test-format-transcriptions.py +++ b/scripts/test-format-transcriptions.py @@ -207,6 +207,19 @@ def test_repeated_phrase_searches_forward(self): out, _ = f.repair_timestamps(html, cues) assert re.findall(r'data-t="(\d+)"', out) == ["0", "10", "20"] + def test_short_anchor_does_not_jump_far_ahead(self): + # "For us," occurs again much later; matching it there would leave + # every following anchor behind the search position. + filler = [f"filler sentence number {i} goes here." for i in range(30)] + cues = cues_from("Something else entirely.", "and then more words.", *filler, "For us, it was later.") + html = ( + '

For us, Something else entirely. ' + 'and then more words.

' + ) + out, stats = f.repair_timestamps(html, cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "0", "10"] + assert stats["unmatched"] == 1 + def test_blockquote_reuses_prose_value(self): html = ( '

Hello everyone and welcome. And then about Shiny.

\n\n' From 197621e516af9d347c50aa56a4079bf6506ad95a Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 14:24:17 +0100 Subject: [PATCH 13/20] Try the bgutil PO token provider for yt-dlp on GitHub's runners The first dry run on the runners failed both downloads with "Sign in to confirm you're not a bot". Run the bgutil provider as a service container and add its yt-dlp plugin, with debug logging for transcribe.py, to see whether PO tokens get the downloads through. --- .github/workflows/sync-videos.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sync-videos.yml b/.github/workflows/sync-videos.yml index 970b7e926..dbeb4184f 100644 --- a/.github/workflows/sync-videos.yml +++ b/.github/workflows/sync-videos.yml @@ -64,6 +64,14 @@ jobs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest timeout-minutes: 120 + services: + # PO token provider for yt-dlp, which YouTube wants from datacenter IPs + # (https://github.com/Brainicism/bgutil-ytdlp-pot-provider). Keep the + # image and the bgutil-ytdlp-pot-provider plugin below on the same version. + bgutil: + image: brainicism/bgutil-ytdlp-pot-provider:2.0.2@sha256:d7d34108b76e52c7d3b575fe9e797e4bf41c1b2a08f80c5292eff6f851ce8472 + ports: + - 4416:4416 env: # Only scheduled runs and manual runs on main open a pull request. DRY_RUN: ${{ github.event_name == 'pull_request' || inputs.dry_run == true || github.ref != 'refs/heads/main' }} @@ -162,7 +170,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | set +e - ./scripts/transcribe.py --limit "$LIMIT" --jobs 3 \ + uv run --with bgutil-ytdlp-pot-provider==2.0.2 --script ./scripts/transcribe.py --limit "$LIMIT" --jobs 3 --verbose \ --summary-json "$SYNC_DIR/summaries/3-transcribe.json" \ 2>&1 | tee "$SYNC_DIR/logs/3-transcribe.log" status=${PIPESTATUS[0]} From 0adff3ba01c836fcde26eb32e0eca51624cadb84 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 14:24:51 +0100 Subject: [PATCH 14/20] Make yt-dlp verbose when transcribe.py runs with --verbose yt-dlp's verbose output shows the JavaScript runtime, PO token providers, and player clients it uses, which is what we need to see when downloads are blocked. --- scripts/test-transcribe.py | 3 +++ scripts/transcribe.py | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/test-transcribe.py b/scripts/test-transcribe.py index 0f1f4e27c..1928a89f5 100755 --- a/scripts/test-transcribe.py +++ b/scripts/test-transcribe.py @@ -133,7 +133,10 @@ def test_build_ytdlp_cmd(tmp_path): assert cmd[cmd.index("--cookies-from-browser") + 1] == "chrome" assert cmd[cmd.index("--cookies") + 1] == "c.txt" assert "--no-warnings" not in cmd # warnings explain failures + assert "--quiet" in cmd and "--verbose" not in cmd assert cmd[-1] == URL_A + verbose = t.build_ytdlp_cmd(URL_A, tmp_path, None, None, verbose=True) + assert "--verbose" in verbose and "--quiet" not in verbose class TestChunking: diff --git a/scripts/transcribe.py b/scripts/transcribe.py index 9bc7823be..6201f4b02 100755 --- a/scripts/transcribe.py +++ b/scripts/transcribe.py @@ -256,7 +256,9 @@ def record(self, reason: str | None) -> None: self.consecutive = 0 -def build_ytdlp_cmd(url: str, dest_dir: Path, browser: str | None, cookies: Path | None) -> list[str]: +def build_ytdlp_cmd( + url: str, dest_dir: Path, browser: str | None, cookies: Path | None, verbose: bool = False +) -> list[str]: cmd = [ sys.executable, "-m", @@ -269,7 +271,8 @@ def build_ytdlp_cmd(url: str, dest_dir: Path, browser: str | None, cookies: Path str(dest_dir / "audio.%(ext)s"), "--no-playlist", "--no-progress", - "--quiet", + # --verbose shows the JS runtime, PO token providers, and clients used. + "--verbose" if verbose else "--quiet", ] if browser: cmd += ["--cookies-from-browser", browser] @@ -295,7 +298,7 @@ def download_audio( run: Callable[..., subprocess.CompletedProcess], label: str, ) -> Path: - cmd = build_ytdlp_cmd(url, dest_dir, browser, cookies) + cmd = build_ytdlp_cmd(url, dest_dir, browser, cookies, verbose=log.isEnabledFor(logging.DEBUG)) log.debug("%s: running %s", label, " ".join(cmd)) result = run(cmd, capture_output=True, text=True) output = "\n".join(s for s in (result.stderr, result.stdout) if s and s.strip()) From 0fa12cf9819ca29928d421db5a515e633ad79185 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 14:31:25 +0100 Subject: [PATCH 15/20] Transcribe locally: YouTube blocks downloads from GitHub's runners Two dry runs on the runners, the second with the bgutil PO token provider (loaded and working, with Deno for the JS challenges), both got "Sign in to confirm you're not a bot": the block is based on the runner's IP. So: - The nightly run no longer transcribes (a manual run can still try, with the new transcribe input); the PO token provider is removed again - It still formats transcripts with Claude, which works fine on the runners - format-transcriptions.py records the videos without a transcription.vtt, and the PR description lists them with the commands to transcribe them on the PR branch; the next run formats them and updates the PR --- .github/workflows/sync-videos.yml | 40 ++++++++++++--------------- README.md | 22 +++++++++++---- scripts/format-transcriptions.py | 6 ++++ scripts/test-format-transcriptions.py | 1 + scripts/test-video-sync-report.py | 17 ++++++++++++ scripts/video-sync-report.py | 15 ++++++++++ 6 files changed, 72 insertions(+), 29 deletions(-) diff --git a/.github/workflows/sync-videos.yml b/.github/workflows/sync-videos.yml index dbeb4184f..4b24b2504 100644 --- a/.github/workflows/sync-videos.yml +++ b/.github/workflows/sync-videos.yml @@ -9,9 +9,12 @@ name: Sync Videos # 3. transcribe.py new videos -> transcription.vtt (yt-dlp, Whisper) # 4. format-transcriptions.py transcription.vtt -> transcription.txt (Claude) # -# YouTube may block audio downloads from GitHub's runners. Steps 3 and 4 never -# fail the job; the PR description lists every video that still needs a -# transcript, with the command to make one locally (`just update-videos`). +# Step 3 only runs when asked for (the `transcribe` input): YouTube blocks +# audio downloads from GitHub's runners ("Sign in to confirm you're not a +# bot"), also with a PO token provider (tested in November 2026). So +# transcripts are made locally: the PR description lists the videos that need +# one, you run `just transcribe` on the PR branch and push, and the next run +# formats them (step 4) and updates the PR. Steps 3 and 4 never fail the job. # Logs and machine-readable run summaries of every step are attached to the # workflow run as an artifact. @@ -24,8 +27,8 @@ on: description: Refresh the view, like, and comment counts of all videos type: boolean default: false - skip_transcription: - description: Skip transcribing and formatting + transcribe: + description: Try to download audio and transcribe new videos (YouTube usually blocks GitHub's runners) type: boolean default: false limit: @@ -64,19 +67,11 @@ jobs: if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest timeout-minutes: 120 - services: - # PO token provider for yt-dlp, which YouTube wants from datacenter IPs - # (https://github.com/Brainicism/bgutil-ytdlp-pot-provider). Keep the - # image and the bgutil-ytdlp-pot-provider plugin below on the same version. - bgutil: - image: brainicism/bgutil-ytdlp-pot-provider:2.0.2@sha256:d7d34108b76e52c7d3b575fe9e797e4bf41c1b2a08f80c5292eff6f851ce8472 - ports: - - 4416:4416 env: # Only scheduled runs and manual runs on main open a pull request. DRY_RUN: ${{ github.event_name == 'pull_request' || inputs.dry_run == true || github.ref != 'refs/heads/main' }} LIMIT: ${{ github.event_name == 'pull_request' && 2 || inputs.limit || 15 }} - SKIP_TRANSCRIPTION: ${{ inputs.skip_transcription == true }} + TRANSCRIBE: ${{ inputs.transcribe == true }} REFRESH_STATS_INPUT: ${{ inputs.refresh_stats == true }} TRANSCRIPT_MODEL: claude-sonnet-4-6 # Keep log lines unwrapped in the job log @@ -102,13 +97,13 @@ jobs: fi echo "refresh_stats=$refresh_stats" >> "$GITHUB_OUTPUT" - if [ "$SKIP_TRANSCRIPTION" = "true" ]; then - echo "expect=update-youtube-videos,sync-videos" >> "$GITHUB_OUTPUT" - else + if [ "$TRANSCRIBE" = "true" ]; then echo "expect=update-youtube-videos,sync-videos,transcribe,format-transcriptions" >> "$GITHUB_OUTPUT" + else + echo "expect=update-youtube-videos,sync-videos,format-transcriptions" >> "$GITHUB_OUTPUT" fi echo "event=$GITHUB_EVENT_NAME ref=$GITHUB_REF dry_run=$DRY_RUN limit=$LIMIT" - echo "skip_transcription=$SKIP_TRANSCRIPTION refresh_stats=$refresh_stats model=$TRANSCRIPT_MODEL" + echo "transcribe=$TRANSCRIBE refresh_stats=$refresh_stats model=$TRANSCRIPT_MODEL" - name: Restore files from the open sync pull request shell: bash @@ -118,11 +113,11 @@ jobs: uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 - name: Setup Deno (JavaScript runtime for yt-dlp) - if: env.SKIP_TRANSCRIPTION != 'true' + if: env.TRANSCRIBE == 'true' uses: denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed # v2.0.5 - name: Install ffmpeg - if: env.SKIP_TRANSCRIPTION != 'true' + if: env.TRANSCRIBE == 'true' shell: bash run: | if ! command -v ffmpeg >/dev/null || ! command -v ffprobe >/dev/null; then @@ -164,13 +159,13 @@ jobs: - name: 3. Transcribe new videos id: transcribe - if: env.SKIP_TRANSCRIPTION != 'true' + if: env.TRANSCRIBE == 'true' shell: bash env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} run: | set +e - uv run --with bgutil-ytdlp-pot-provider==2.0.2 --script ./scripts/transcribe.py --limit "$LIMIT" --jobs 3 --verbose \ + ./scripts/transcribe.py --limit "$LIMIT" --jobs 3 \ --summary-json "$SYNC_DIR/summaries/3-transcribe.json" \ 2>&1 | tee "$SYNC_DIR/logs/3-transcribe.log" status=${PIPESTATUS[0]} @@ -180,7 +175,6 @@ jobs: - name: 4. Format transcriptions id: format - if: env.SKIP_TRANSCRIPTION != 'true' shell: bash env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/README.md b/README.md index 73f2a6825..e1d4c4995 100644 --- a/README.md +++ b/README.md @@ -541,8 +541,8 @@ The PR description lists new, renamed, and changed videos, the new transcripts, and any problems, each with the command to fix it locally. If last night's PR is still open, the next run builds on it instead of transcribing the same videos again. You can also run the workflow by hand (Actions → Sync Videos → -Run workflow), with options to refresh view counts, skip transcription, change -how many videos get transcribed, or do a dry run. +Run workflow), with options to refresh view counts, try transcription, change +how many videos get formatted, or do a dry run. **Only real changes.** Every run fetches the metadata of all videos, but an entry (and its `last_updated`) only changes when a field changed. View, like, @@ -550,10 +550,20 @@ and comment counts are refreshed once a week, on Mondays, or with `--refresh-stats`. Pages and transcripts are only written when their content changes, so a quiet night produces no PR at all. -**Transcripts.** Transcribing downloads the audio with yt-dlp, which YouTube -sometimes blocks on GitHub's runners. Download failures never fail the -workflow; the PR lists them, and the next run tries again. To transcribe -locally instead, check out the sync branch (or any branch) and run: +**Transcripts are made locally.** Transcribing downloads the audio with +yt-dlp, and YouTube blocks that on GitHub's runners ("Sign in to confirm +you're not a bot"), even with a PO token provider. So the nightly run skips +step 3. The PR lists the videos waiting for a transcript; transcribe them on +the PR branch and push, and the next run formats them (step 4, with Claude) +and updates the PR: + +```bash +git fetch origin && git switch -C automation/sync-videos origin/automation/sync-videos +just transcribe --limit 15 # add --cookies-from-browser chrome if YouTube asks you to sign in +git add content/resources/videos && git commit -m "Add transcripts" && git push +``` + +To do everything locally instead (on any branch): ```bash just update-videos # all four steps, at most 15 new videos diff --git a/scripts/format-transcriptions.py b/scripts/format-transcriptions.py index d63426908..9ae1e2d45 100755 --- a/scripts/format-transcriptions.py +++ b/scripts/format-transcriptions.py @@ -682,6 +682,7 @@ def select_videos( ) -> list[Path]: selected: list[Path] = [] deferred: list[str] = [] + needs_transcript: list[str] = [] for d in dirs: fm = videolib.read_frontmatter(d / "_index.md") if fm.get("transcribe", True) is False: @@ -689,6 +690,7 @@ def select_videos( continue if not (d / "transcription.vtt").exists(): summary.tally("no_vtt") + needs_transcript.append(d.name) continue if (d / "transcription.txt").exists() and not force: if not reformat_invalid: @@ -707,6 +709,10 @@ def select_videos( summary.tally("deferred", len(deferred)) summary.extra["deferred"] = deferred log.info("Deferred %d videos to a later run (--limit)", len(deferred)) + if needs_transcript: + # Newest first; video-sync-report.py lists them in the PR. + summary.extra["needs_transcript"] = needs_transcript + log.info("%d videos have no transcription.vtt yet (run transcribe.py)", len(needs_transcript)) return selected diff --git a/scripts/test-format-transcriptions.py b/scripts/test-format-transcriptions.py index ed47ba5c0..ac2efe4dd 100755 --- a/scripts/test-format-transcriptions.py +++ b/scripts/test-format-transcriptions.py @@ -361,6 +361,7 @@ def test_selection(self, repo): code, data = run(repo) assert [i["slug"] for i in items(data, "formatted")] == ["2026-01-01_todo"] assert data["counts"] == {"already_formatted": 1, "excluded": 1, "formatted": 1, "no_vtt": 1} + assert data["extra"]["needs_transcript"] == ["2026-01-02_novtt"] def test_limit_newest_first(self, repo): for i in range(1, 4): diff --git a/scripts/test-video-sync-report.py b/scripts/test-video-sync-report.py index e57a20734..d20196080 100755 --- a/scripts/test-video-sync-report.py +++ b/scripts/test-video-sync-report.py @@ -164,6 +164,23 @@ def test_blocked_night(): ] +def test_waiting_for_transcripts(): + s = quiet_night() + del s["transcribe"] + s["format-transcriptions"] = summary( + "format-transcriptions", + tallies={"already_formatted": 1700, "no_vtt": 2}, + extra={"needs_transcript": ["2026-11-11_b", "2026-11-10_a"]}, + ) + report, annotations = build(s, expected=["update-youtube-videos", "sync-videos", "format-transcriptions"]) + assert "Transcription" not in report # the step didn't run and wasn't expected + assert "**2 videos waiting for a transcript.**" in report + assert "git switch -C automation/sync-videos origin/automation/sync-videos" in report + assert "just transcribe --limit 15" in report + assert "- `2026-11-11_b`\n- `2026-11-10_a`" in report + assert annotations == [] + + def test_stats_refresh(): s = quiet_night() s["update-youtube-videos"] = summary( diff --git a/scripts/video-sync-report.py b/scripts/video-sync-report.py index 0e58fd2eb..41a4ed529 100755 --- a/scripts/video-sync-report.py +++ b/scripts/video-sync-report.py @@ -354,6 +354,21 @@ def block(title: str, lines: list[str], hint: str | None = None, level: str = "w [f"{code(i.get('slug'))}: {i.get('reason')}" for i in skipped], ) + waiting = (fmt or {}).get("extra", {}).get("needs_transcript") or [] + if waiting: + info += [ + f"**{plural(len(waiting), 'video')} waiting for a transcript.** YouTube blocks audio downloads from " + "GitHub's runners, so transcribe them locally on this branch and push; the next run formats them:", + "", + "```bash", + "git fetch origin && git switch -C automation/sync-videos origin/automation/sync-videos", + "just transcribe --limit 15 # add --cookies-from-browser chrome if YouTube asks you to sign in", + 'git add content/resources/videos && git commit -m "Add transcripts" && git push', + "```", + "", + ] + info += details("Videos without a transcript (newest first)", [code(slug) for slug in waiting]) + if out: out = ["### ⚠️ Problems", ""] + out if info: From 9aa747608651a5c36c26cdee30d05de792e6914a Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 14:42:30 +0100 Subject: [PATCH 16/20] Fix the model's known tag typos before validating The model occasionally writes '' or '' (4 of the published transcripts, and again during the re-format). Repair those instead of rejecting an otherwise complete transcript. --- scripts/format-transcriptions.py | 21 +++++++++++++++++++++ scripts/test-format-transcriptions.py | 21 +++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/scripts/format-transcriptions.py b/scripts/format-transcriptions.py index 9ae1e2d45..de2b61070 100755 --- a/scripts/format-transcriptions.py +++ b/scripts/format-transcriptions.py @@ -311,6 +311,23 @@ def strip_markdown_fence(text: str) -> str: return text.strip() +# Typos the model makes in otherwise fine output, seen in real transcripts: +# '' and ''. +ANCHOR_TYPOS = [ + (re.compile(r" tuple[str, int]: + """Repair known harmless typos in tags; return (html, number fixed).""" + fixed = 0 + for pattern, replacement in ANCHOR_TYPOS: + html, n = pattern.subn(replacement, html) + fixed += n + return html, fixed + + WORD_RE = re.compile(r"[\w']+") @@ -789,6 +806,10 @@ def request(i: int): item["request_ids"] = [u["request_id"] for u in usages if u["request_id"]] or None item["estimated_cost_usd"] = estimate_cost(args.model, item["input_tokens"], item["output_tokens"]) + html, typos = fix_anchor_typos(html) + if typos: + item["typos_fixed"] = typos + log.info("%s: fixed %d malformed tags", label, typos) html, repair = repair_timestamps(html, cues) item["timestamps_repaired"] = repair["changed"] item["timestamps_unmatched"] = repair["unmatched"] diff --git a/scripts/test-format-transcriptions.py b/scripts/test-format-transcriptions.py index ac2efe4dd..3012807a8 100755 --- a/scripts/test-format-transcriptions.py +++ b/scripts/test-format-transcriptions.py @@ -253,6 +253,20 @@ def test_no_cues(self): assert f.repair_timestamps(html, [])[0] == html +@pytest.mark.parametrize( + "html, fixed, count", + [ + ('

x

', '

x

', 1), + ('

x

', '

x

', 1), + ('

x

', '

x

', 1), + ('

a a data-t="6"

', '

a a data-t="6"

', 0), + (GOOD_HTML, GOOD_HTML, 0), + ], +) +def test_fix_anchor_typos(html, fixed, count): + assert f.fix_anchor_typos(html) == (fixed, count) + + def test_estimate_cost(): assert f.estimate_cost("claude-sonnet-4-6", 1_000_000, 100_000) == 4.5 assert f.estimate_cost("claude-haiku-4-5", 1_000_000, 0) == 1.0 @@ -427,6 +441,13 @@ def test_rejected_output_saved(self, repo, tmp_path): assert (rejected / "2026-01-01_a.html").read_text() == '

Hello.

' assert items(data, "error")[0]["rejected_output"] == str(rejected / "2026-01-01_a.html") + def test_anchor_typos_fixed_before_validation(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + code, data = run(repo, client=FakeAnthropic([GOOD_HTML.replace('', '')])) + assert code == 0 + assert " Date: Thu, 12 Nov 2026 16:02:24 +0100 Subject: [PATCH 17/20] Remove the temporary pull_request trigger from Sync Videos It ran dry runs on the PR that adds the workflow, which settled that YouTube blocks downloads from GitHub's runners. From now on the workflow runs on its schedule and by hand from main. --- .github/workflows/sync-videos.yml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/sync-videos.yml b/.github/workflows/sync-videos.yml index 4b24b2504..d8092a04a 100644 --- a/.github/workflows/sync-videos.yml +++ b/.github/workflows/sync-videos.yml @@ -39,21 +39,9 @@ on: description: Don't open a pull request; attach the diff to the run instead type: boolean default: false - # TEMPORARY: dry runs on the pull request that adds this workflow, because a - # new workflow can't be dispatched before it's on main. Remove before merging. - pull_request: - paths: - - .github/workflows/sync-videos.yml - - .github/scripts/restore-pending-videos.sh - - scripts/videolib.py - - scripts/update-youtube-videos.py - - scripts/sync-videos.py - - scripts/transcribe.py - - scripts/format-transcriptions.py - - scripts/video-sync-report.py concurrency: - group: sync-videos-${{ github.event_name == 'pull_request' && github.head_ref || 'main' }} + group: sync-videos cancel-in-progress: false # Pushing the branch and opening the PR use the app token below, not @@ -63,14 +51,12 @@ permissions: jobs: sync-videos: - # Pull requests from forks don't get secrets. - if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest timeout-minutes: 120 env: # Only scheduled runs and manual runs on main open a pull request. - DRY_RUN: ${{ github.event_name == 'pull_request' || inputs.dry_run == true || github.ref != 'refs/heads/main' }} - LIMIT: ${{ github.event_name == 'pull_request' && 2 || inputs.limit || 15 }} + DRY_RUN: ${{ inputs.dry_run == true || github.ref != 'refs/heads/main' }} + LIMIT: ${{ inputs.limit || 15 }} TRANSCRIBE: ${{ inputs.transcribe == true }} REFRESH_STATS_INPUT: ${{ inputs.refresh_stats == true }} TRANSCRIPT_MODEL: claude-sonnet-4-6 From 816edb6aff07efebbfc9c08f7a7c93147bc26060 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 16:10:04 +0100 Subject: [PATCH 18/20] Cap unmatched pull-quote timestamps at the last cue too A pull quote whose text matches no prose anchor kept the model's value unchecked; one re-formatted transcript had data-t="46039" in a 1-hour video. --- scripts/format-transcriptions.py | 8 +++++++- scripts/test-format-transcriptions.py | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/format-transcriptions.py b/scripts/format-transcriptions.py index de2b61070..5920783bb 100755 --- a/scripts/format-transcriptions.py +++ b/scripts/format-transcriptions.py @@ -479,8 +479,14 @@ def repair_prose(m: re.Match) -> str: by_text.setdefault(" ".join(ws), new) return f'{inner}' + last_start = words[-1][1] if words else None + def repair_quote(m: re.Match) -> str: - new = by_text.get(" ".join(_norm_words(m.group(2))), int(m.group(1))) + new = by_text.get(" ".join(_norm_words(m.group(2)))) + if new is None: + new = int(m.group(1)) + if last_start is not None: + new = min(new, last_start) return f'{m.group(2)}' parts = BLOCKQUOTE_RE.split(html) diff --git a/scripts/test-format-transcriptions.py b/scripts/test-format-transcriptions.py index 3012807a8..850a6e450 100755 --- a/scripts/test-format-transcriptions.py +++ b/scripts/test-format-transcriptions.py @@ -230,6 +230,14 @@ def test_blockquote_reuses_prose_value(self): assert re.findall(r'data-t="(\d+)"', out) == ["0", "30", "30", "40"] assert stats["anchors"] == 3 # pull quotes aren't counted twice + def test_unmatched_blockquote_capped(self): + html = ( + '

Hello everyone and welcome.

\n\n' + '
A quote that never appeared.
' + ) + out, _ = f.repair_timestamps(html, self.cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "40"] + def test_pieces_split_by_inject_links_stay_together(self): linked = f.inject_links( '

Today we talk about Quarto, the publishing system.

', From 7cdffc6898ca469d8d69d02e74c52567183478be Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 18:56:29 +0100 Subject: [PATCH 19/20] Retry API calls for about a minute instead of 15 seconds A ten-second network interruption failed 15 videos in a row: the backoff (1, 2, 4, 8 s) gave up before the connection was back. Now 6 attempts starting at 2 s, each wait capped at 60 s. --- scripts/test-videolib.py | 36 ++++++++++++++++++++++++++++++++++-- scripts/videolib.py | 11 +++++++---- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/scripts/test-videolib.py b/scripts/test-videolib.py index f3fff98fc..e14c8ad48 100755 --- a/scripts/test-videolib.py +++ b/scripts/test-videolib.py @@ -298,8 +298,40 @@ async def fn(): assert result == "ok" assert len(sleeps) == 2 assert 1.0 <= sleeps[0] < 2.0 and 2.0 <= sleeps[1] < 3.0 # exponential + jitter - assert "video-x: attempt 1/5 failed: RateLimitError status=429" in caplog.text - assert "attempt 2/5 failed: StatusError status=503" in caplog.text + assert "video-x: attempt 1/6 failed: RateLimitError status=429" in caplog.text + assert "attempt 2/6 failed: StatusError status=503" in caplog.text + + def test_default_backoff_rides_out_a_minute(self): + async def fn(): + raise TimeoutError() + + sleeps = [] + + async def fake_sleep(s): + sleeps.append(s) + + with pytest.raises(TimeoutError): + asyncio.run(videolib.retry_async(fn, label="x", logger=logging.getLogger("t"), sleep=fake_sleep)) + assert len(sleeps) == 5 + assert 60 <= sum(sleeps) <= 75 + assert max(sleeps) <= 62 + + def test_delay_capped(self): + sleeps = [] + + async def fake_sleep(s): + sleeps.append(s) + + async def fn(): + raise TimeoutError() + + with pytest.raises(TimeoutError): + asyncio.run( + videolib.retry_async( + fn, label="x", logger=logging.getLogger("t"), sleep=fake_sleep, attempts=10, max_delay=5 + ) + ) + assert max(sleeps) <= 5 + 2.0 def test_non_retryable_raises_immediately(self, caplog): calls = [] diff --git a/scripts/videolib.py b/scripts/videolib.py index 78c5bfd9e..f1e252582 100644 --- a/scripts/videolib.py +++ b/scripts/videolib.py @@ -544,8 +544,9 @@ async def retry_async( *, label: str, logger: logging.Logger, - attempts: int = 5, - base_delay: float = 1.0, + attempts: int = 6, + base_delay: float = 2.0, + max_delay: float = 60.0, is_retryable: Callable[[BaseException], bool] = is_retryable_api_error, describe: Callable[[BaseException], str] = describe_api_error, sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, @@ -553,7 +554,9 @@ async def retry_async( """Call ``fn`` with exponential backoff (plus jitter) on retryable errors. Every failed attempt is logged with the error description, so a flaky - API is visible in the logs even when the call eventually succeeds. + API is visible in the logs even when the call eventually succeeds. The + defaults wait about a minute in total, enough to ride out a short + network outage (with 1 s steps, a ten-second outage failed 15 videos). """ for attempt in range(1, attempts + 1): try: @@ -565,7 +568,7 @@ async def retry_async( if attempt == attempts: logger.error("%s: %s (giving up after %d attempts)", label, describe(e), attempts) raise - delay = base_delay * (2 ** (attempt - 1)) + random.uniform(0, base_delay) + delay = min(max_delay, base_delay * (2 ** (attempt - 1))) + random.uniform(0, base_delay) logger.warning( "%s: attempt %d/%d failed: %s; retrying in %.1fs", label, From b40a0eddecf987e97e290063930149295815e052 Mon Sep 17 00:00:00 2001 From: Jeroen Janssens Date: Thu, 12 Nov 2026 19:05:01 +0100 Subject: [PATCH 20/20] Fix a link inserted inside another link, and Whisper repeats in coverage - inject_links: in plain text (such as an

), a shorter name could match inside the URL of a link inserted just before, e.g. "Shiny" in /software/shiny-python/, producing broken HTML (once on the live site). Insert placeholders first and the links at the end. - coverage: count a cue whose text repeats an earlier cue once. Whisper sometimes loops on a line, and the model rightly leaves the copies out; three short videos were rejected for that. --- scripts/format-transcriptions.py | 35 +++++++++++++++++++-------- scripts/test-format-transcriptions.py | 14 +++++++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/scripts/format-transcriptions.py b/scripts/format-transcriptions.py index 5920783bb..2ef2df96e 100755 --- a/scripts/format-transcriptions.py +++ b/scripts/format-transcriptions.py @@ -343,8 +343,20 @@ def html_to_text(html: str, *, drop_blockquotes: bool = True) -> str: def coverage(cues: list[videolib.Cue], html: str) -> float: - """Share of the spoken words (with multiplicity) found in the HTML prose.""" - spoken = word_counts(" ".join(text for _, _, text in cues)) + """Share of the spoken words (with multiplicity) found in the HTML prose. + + A cue whose text repeats an earlier cue counts once: Whisper sometimes + loops on a line (one VTT has a speaker's name nine times), and the model + rightly leaves the copies out. + """ + seen: set[str] = set() + unique = [] + for _, _, text in cues: + key = " ".join(_norm_words(text)) + if key not in seen: + seen.add(key) + unique.append(text) + spoken = word_counts(" ".join(unique)) total = sum(spoken.values()) if not total: return 1.0 @@ -615,6 +627,10 @@ def build_entity_dicts(repo_root: Path) -> tuple[dict[str, str], dict[str, str]] def _inject_into_text(text: str, entities: list[tuple[str, str, bool]], linked: set[str]) -> str: + # Insert placeholders first and the links at the end, so a later, shorter + # name can't match inside an inserted link: "Shiny" in the URL of + # "Shiny for Python" once produced . + links: list[str] = [] for name, url, is_software in entities: if name in linked: continue @@ -622,14 +638,13 @@ def _inject_into_text(text: str, entities: list[tuple[str, str, bool]], linked: pattern = r"\b" + re.escape(name) + r"\b" if re.search(pattern, text, flags=flags): linked.add(name) - text = re.sub( - pattern, - lambda m, u=url: f'{m.group(0)}', - text, - count=1, - flags=flags, - ) - return text + + def placeholder(m: re.Match, u: str = url) -> str: + links.append(f'{m.group(0)}') + return f"\ue000{len(links) - 1}\ue001" + + text = re.sub(pattern, placeholder, text, count=1, flags=flags) + return re.sub("\ue000(\\d+)\ue001", lambda m: links[int(m.group(1))], text) def _inject_into_anchor( diff --git a/scripts/test-format-transcriptions.py b/scripts/test-format-transcriptions.py index 850a6e450..77d930d68 100755 --- a/scripts/test-format-transcriptions.py +++ b/scripts/test-format-transcriptions.py @@ -95,6 +95,11 @@ def test_entities_and_case(self): cues = [(0.0, 1.0, "R & Python, it's GREAT")] assert f.coverage(cues, '

r & python, it's great

') == 1.0 + def test_repeated_cues_count_once(self): + cues = [(0.0, 1.0, "Rishabh Sharma Vemuri & Abiye Berhanu")] * 9 + [(9.0, 10.0, "And now the talk.")] + html = '

Rishabh Sharma Vemuri & Abiye Berhanu And now the talk.

' + assert f.coverage(cues, html) == 1.0 + def test_multiplicity(self): cues = [(0.0, 1.0, "the the the the")] assert f.coverage(cues, '

the the

') == 0.5 @@ -305,6 +310,15 @@ def test_longest_name_first(self): assert 'Shiny for Python' in out assert '"/software/shiny/"' not in out + def test_no_link_inside_an_inserted_link(self): + # A heading is plain text: "Shiny" must not match inside the URL of + # the "Shiny for Python" link inserted just before. + html = "

Building the Shiny for Python app

" + software = {"Shiny for Python": "/software/shiny-python/", "Shiny": "/software/shiny-r/"} + assert f.inject_links(html, {}, software) == ( + '

Building the Shiny for Python app

' + ) + def test_people_are_case_sensitive(self): html = '

hadley wickham and Hadley Wickham

' out = f.inject_links(html, self.people, {})