diff --git a/.github/scripts/restore-pending-videos.sh b/.github/scripts/restore-pending-videos.sh new file mode 100755 index 000000000..4db823a52 --- /dev/null +++ b/.github/scripts/restore-pending-videos.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Carry over the work of an unmerged sync PR into this run. +# +# The nightly sync starts from main. If last night's PR hasn't been merged +# yet, its new video directories and transcripts would otherwise be created +# (and paid for) again, and the nondeterministic formatting would rewrite +# the PR every night. So: take every file the PR branch *added* under +# content/resources/videos/ (relative to its merge base with main) and put +# it in the working tree, unless main has a file at that path by now. +# Files the PR modified are not restored; the scripts regenerate them. +# +# Usage: restore-pending-videos.sh [BRANCH] [BASE] +# BRANCH defaults to automation/sync-videos, BASE to main. +# Needs a checkout with full history (actions/checkout fetch-depth: 0). + +set -euo pipefail + +branch="${1:-automation/sync-videos}" +base="${2:-main}" +prefix="content/resources/videos/" + +if ! git ls-remote --exit-code --heads origin "$branch" >/dev/null 2>&1; then + echo "No pending branch $branch; nothing to restore." + exit 0 +fi + +git fetch --quiet --no-tags origin "$branch" "$base" +remote_branch="origin/$branch" +remote_base="origin/$base" + +added=() +while IFS= read -r -d '' path; do + added+=("$path") +done < <(git diff -z --name-only --diff-filter=A "$remote_base...$remote_branch" -- "$prefix") + +restored=() +skipped=0 +for path in ${added[@]+"${added[@]}"}; do + if [[ -e "$path" ]]; then + echo " keep main's version: $path" + skipped=$((skipped + 1)) + continue + fi + restored+=("$path") +done + +if ((${#restored[@]})); then + # --worktree only: leave the index alone, create-pull-request picks the + # files up as changes. + git restore --source="$remote_branch" --worktree -- "${restored[@]}" + printf ' restored: %s\n' "${restored[@]}" +fi + +echo "Restored ${#restored[@]} file(s) from $branch (${#added[@]} added there, $skipped already on $base)." diff --git a/.github/workflows/sync-videos.yml b/.github/workflows/sync-videos.yml new file mode 100644 index 000000000..d8092a04a --- /dev/null +++ b/.github/workflows/sync-videos.yml @@ -0,0 +1,229 @@ +name: Sync Videos + +# Syncs Posit's YouTube videos into content/resources/videos/ and opens (or +# updates) a pull request with the changes, like sync-courses.yml: +# +# 1. update-youtube-videos.py YouTube metadata -> data/videos.toml +# (view counts are only refreshed on Mondays) +# 2. sync-videos.py data/videos.toml -> video pages, thumbnails +# 3. transcribe.py new videos -> transcription.vtt (yt-dlp, Whisper) +# 4. format-transcriptions.py transcription.vtt -> transcription.txt (Claude) +# +# Step 3 only runs when asked for (the `transcribe` input): YouTube blocks +# audio downloads from GitHub's runners ("Sign in to confirm you're not a +# bot"), also with a PO token provider (tested in November 2026). So +# transcripts are made locally: the PR description lists the videos that need +# one, you run `just transcribe` on the PR branch and push, and the next run +# formats them (step 4) and updates the PR. Steps 3 and 4 never fail the job. +# Logs and machine-readable run summaries of every step are attached to the +# workflow run as an artifact. + +on: + schedule: + - cron: "0 5 * * *" # 5 AM UTC: before sync-courses (6 AM) and the nightly production deploy (8 AM) + workflow_dispatch: + inputs: + refresh_stats: + description: Refresh the view, like, and comment counts of all videos + type: boolean + default: false + transcribe: + description: Try to download audio and transcribe new videos (YouTube usually blocks GitHub's runners) + type: boolean + default: false + limit: + description: Maximum number of videos to transcribe and format + type: number + default: 15 + dry_run: + description: Don't open a pull request; attach the diff to the run instead + type: boolean + default: false + +concurrency: + group: sync-videos + cancel-in-progress: false + +# Pushing the branch and opening the PR use the app token below, not +# GITHUB_TOKEN, so that deploy-preview.yml runs on the sync PRs. +permissions: + contents: read + +jobs: + sync-videos: + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + # Only scheduled runs and manual runs on main open a pull request. + DRY_RUN: ${{ inputs.dry_run == true || github.ref != 'refs/heads/main' }} + LIMIT: ${{ inputs.limit || 15 }} + TRANSCRIBE: ${{ inputs.transcribe == true }} + REFRESH_STATS_INPUT: ${{ inputs.refresh_stats == true }} + TRANSCRIPT_MODEL: claude-sonnet-4-6 + # Keep log lines unwrapped in the job log + COLUMNS: "200" + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 # restore-pending-videos.sh diffs against the merge base + + - name: Prepare + id: prepare + shell: bash + run: | + sync_dir="$RUNNER_TEMP/video-sync" + mkdir -p "$sync_dir/logs" "$sync_dir/summaries" + echo "SYNC_DIR=$sync_dir" >> "$GITHUB_ENV" + + if [ "$REFRESH_STATS_INPUT" = "true" ] || [ "$(date -u +%u)" = "1" ]; then + refresh_stats=true + else + refresh_stats=false + fi + echo "refresh_stats=$refresh_stats" >> "$GITHUB_OUTPUT" + + if [ "$TRANSCRIBE" = "true" ]; then + echo "expect=update-youtube-videos,sync-videos,transcribe,format-transcriptions" >> "$GITHUB_OUTPUT" + else + echo "expect=update-youtube-videos,sync-videos,format-transcriptions" >> "$GITHUB_OUTPUT" + fi + echo "event=$GITHUB_EVENT_NAME ref=$GITHUB_REF dry_run=$DRY_RUN limit=$LIMIT" + echo "transcribe=$TRANSCRIBE refresh_stats=$refresh_stats model=$TRANSCRIPT_MODEL" + + - name: Restore files from the open sync pull request + shell: bash + run: ./.github/scripts/restore-pending-videos.sh automation/sync-videos main 2>&1 | tee "$SYNC_DIR/logs/0-restore-pending-videos.log" + + - name: Setup uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + + - name: Setup Deno (JavaScript runtime for yt-dlp) + if: env.TRANSCRIBE == 'true' + uses: denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed # v2.0.5 + + - name: Install ffmpeg + if: env.TRANSCRIBE == 'true' + shell: bash + run: | + if ! command -v ffmpeg >/dev/null || ! command -v ffprobe >/dev/null; then + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends ffmpeg + fi + ffmpeg -version | head -1 + deno --version | head -1 + + - name: 1. Update YouTube metadata + id: update + shell: bash + env: + YOUTUBE_API_KEY: ${{ secrets.YOUTUBE_API_KEY }} + REFRESH_STATS: ${{ steps.prepare.outputs.refresh_stats }} + run: | + set +e + args=() + if [ "$REFRESH_STATS" = "true" ]; then args+=(--refresh-stats); fi + ./scripts/update-youtube-videos.py "${args[@]}" \ + --summary-json "$SYNC_DIR/summaries/1-update-youtube-videos.json" \ + 2>&1 | tee "$SYNC_DIR/logs/1-update-youtube-videos.log" + status=${PIPESTATUS[0]} + echo "exit_code=$status" >> "$GITHUB_OUTPUT" + # Exit code 2 (a source failed) is partial success: keep going. + [ "$status" -eq 0 ] || [ "$status" -eq 2 ] + + - name: 2. Sync video pages + id: sync + shell: bash + run: | + set +e + ./scripts/sync-videos.py \ + --summary-json "$SYNC_DIR/summaries/2-sync-videos.json" \ + 2>&1 | tee "$SYNC_DIR/logs/2-sync-videos.log" + status=${PIPESTATUS[0]} + echo "exit_code=$status" >> "$GITHUB_OUTPUT" + [ "$status" -eq 0 ] || [ "$status" -eq 2 ] + + - name: 3. Transcribe new videos + id: transcribe + if: env.TRANSCRIBE == 'true' + shell: bash + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + set +e + ./scripts/transcribe.py --limit "$LIMIT" --jobs 3 \ + --summary-json "$SYNC_DIR/summaries/3-transcribe.json" \ + 2>&1 | tee "$SYNC_DIR/logs/3-transcribe.log" + status=${PIPESTATUS[0]} + echo "exit_code=$status" >> "$GITHUB_OUTPUT" + # Never fail the job here: failures are listed in the PR description. + if [ "$status" -ne 0 ]; then echo "::warning title=Transcription::transcribe.py exited with $status"; fi + + - name: 4. Format transcriptions + id: format + shell: bash + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + run: | + set +e + ./scripts/format-transcriptions.py --limit "$LIMIT" \ + --rejected-dir "$SYNC_DIR/rejected" \ + --summary-json "$SYNC_DIR/summaries/4-format-transcriptions.json" \ + 2>&1 | tee "$SYNC_DIR/logs/4-format-transcriptions.log" + status=${PIPESTATUS[0]} + echo "exit_code=$status" >> "$GITHUB_OUTPUT" + if [ "$status" -ne 0 ]; then echo "::warning title=Formatting::format-transcriptions.py exited with $status"; fi + + - name: Write report + if: always() + shell: bash + env: + EXPECT: ${{ steps.prepare.outputs.expect }} + run: | + ./scripts/video-sync-report.py "$SYNC_DIR/summaries" \ + --expect "$EXPECT" --output "$SYNC_DIR/pr-body.md" --annotations + cat "$SYNC_DIR/pr-body.md" >> "$GITHUB_STEP_SUMMARY" + + - name: Save the diff (dry run) + if: always() && env.DRY_RUN == 'true' + shell: bash + run: | + git add --intent-to-add -- data/videos.toml content + git diff --stat -- data/videos.toml content > "$SYNC_DIR/diff-stat.txt" + git diff -- data/videos.toml content > "$SYNC_DIR/diff.patch" + echo "Dry run: no pull request. $(tail -1 "$SYNC_DIR/diff-stat.txt")" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**Dry run:** no pull request was opened. Changes: $(tail -1 "$SYNC_DIR/diff-stat.txt")" >> "$GITHUB_STEP_SUMMARY" + + - name: Upload logs and run summaries + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: video-sync-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/video-sync + retention-days: 30 + if-no-files-found: warn + + - name: Generate app token + if: env.DRY_RUN != 'true' + id: app-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ secrets.POSIT_OPEN_SOURCE_WEBSITE_SYNC_CLIENT_ID }} + private-key: ${{ secrets.POSIT_OPEN_SOURCE_WEBSITE_SYNC_PEM }} + + - name: Create or update pull request + if: env.DRY_RUN != 'true' + uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 + with: + token: ${{ steps.app-token.outputs.token }} + sign-commits: true + branch: automation/sync-videos + delete-branch: true + # All of content/: renamed videos also update links in other pages. + add-paths: | + data/videos.toml + content + commit-message: Sync videos from YouTube + title: Sync videos from YouTube + body-path: ${{ runner.temp }}/video-sync/pr-body.md diff --git a/.github/workflows/test-scripts.yml b/.github/workflows/test-scripts.yml new file mode 100644 index 000000000..81ea0c3fa --- /dev/null +++ b/.github/workflows/test-scripts.yml @@ -0,0 +1,41 @@ +name: Test Scripts + +# Runs the tests of the Python scripts (scripts/test-*.py) when they change. + +on: + pull_request: + paths: + - "scripts/**" + - ".github/workflows/test-scripts.yml" + - "justfile" + push: + branches: ["main"] + paths: + - "scripts/**" + - ".github/workflows/test-scripts.yml" + - "justfile" + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Setup uv + uses: astral-sh/setup-uv@07a047e608db84b3d8c06cd9a7c7dcc9c355c18e # v10.3.2 + + - name: Install ffmpeg + run: | + if ! command -v ffmpeg >/dev/null || ! command -v ffprobe >/dev/null; then + sudo apt-get update -q + sudo apt-get install -y -q --no-install-recommends ffmpeg + fi + + - name: Run tests + run: ./scripts/run-tests.sh -ra diff --git a/README.md b/README.md index b3eb1d8fa..e1d4c4995 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,15 @@ open-source-website/ │ ├── download-software-readmes.py # Fetch README files │ ├── summarize-software-readmes.py # Generate AI summaries │ ├── validate-blog-posts.py # Validate blog post frontmatter and placement -│ └── test-validate-blog-posts.py # Tests for validation script +│ ├── update-youtube-videos.py # Fetch video metadata from YouTube (see Video Sync) +│ ├── sync-videos.py # Create/update video pages from data/videos.toml +│ ├── transcribe.py # Download audio and transcribe it with Whisper +│ ├── format-transcriptions.py # Turn VTT transcripts into HTML with Claude +│ ├── video-sync-report.py # PR description for the nightly video sync +│ ├── videolib.py # Shared helpers of the video scripts +│ ├── videotesting.py # Fakes and fixtures for the video script tests +│ ├── run-tests.sh # Run all scripts/test-*.py (`just test`) +│ └── test-*.py # Tests for the scripts │ ├── archetypes/ # Content templates for hugo new │ ├── default.md @@ -194,6 +202,8 @@ just build-tailwind # Build Tailwind CSS only just build-search # Generate Pagefind search index just update-github-repos # Sync GitHub repository metadata to data/github-repos.toml just update-software-frontmatter # Update software frontmatter from github-repos.toml +just update-videos # Run the whole video pipeline locally (see Video Sync) +just test # Run the tests of the Python scripts just quarto-preview # Preview Quarto documents just hugo-serve # Start Hugo dev server with drafts ``` @@ -511,6 +521,83 @@ just update-software-frontmatter - Contributors (top contributors by commit count) - README image paths +### Video Sync + +Videos from Posit's YouTube channel and playlists (configured in +`data/youtube.toml`) are synced by four scripts, in this order: + +| Step | Script | Writes | Needs | +| --- | --- | --- | --- | +| 1 | `update-youtube-videos.py` | `data/videos.toml` | `YOUTUBE_API_KEY` | +| 2 | `sync-videos.py` | `content/resources/videos//_index.md`, `thumbnail.jpg` | | +| 3 | `transcribe.py` | `transcription.vtt` | `OPENAI_API_KEY`, ffmpeg, Deno | +| 4 | `format-transcriptions.py` | `transcription.txt` (the transcript shown on the page) | `ANTHROPIC_API_KEY` | + +**Nightly workflow.** `.github/workflows/sync-videos.yml` runs all four steps +at 05:00 UTC and opens (or updates) a pull request on the +`automation/sync-videos` branch, using the same GitHub App as the course sync. +Review the PR (it gets a deploy preview) and merge it to publish the changes. +The PR description lists new, renamed, and changed videos, the new transcripts, +and any problems, each with the command to fix it locally. If last night's PR +is still open, the next run builds on it instead of transcribing the same +videos again. You can also run the workflow by hand (Actions → Sync Videos → +Run workflow), with options to refresh view counts, try transcription, change +how many videos get formatted, or do a dry run. + +**Only real changes.** Every run fetches the metadata of all videos, but an +entry (and its `last_updated`) only changes when a field changed. View, like, +and comment counts are refreshed once a week, on Mondays, or with +`--refresh-stats`. Pages and transcripts are only written when their content +changes, so a quiet night produces no PR at all. + +**Transcripts are made locally.** Transcribing downloads the audio with +yt-dlp, and YouTube blocks that on GitHub's runners ("Sign in to confirm +you're not a bot"), even with a PO token provider. So the nightly run skips +step 3. The PR lists the videos waiting for a transcript; transcribe them on +the PR branch and push, and the next run formats them (step 4, with Claude) +and updates the PR: + +```bash +git fetch origin && git switch -C automation/sync-videos origin/automation/sync-videos +just transcribe --limit 15 # add --cookies-from-browser chrome if YouTube asks you to sign in +git add content/resources/videos && git commit -m "Add transcripts" && git push +``` + +To do everything locally instead (on any branch): + +```bash +just update-videos # all four steps, at most 15 new videos +just update-videos 5 chrome # at most 5, with YouTube cookies from Chrome +just transcribe --video 2026-11-11_some-talk --cookies-from-browser chrome +just format-transcriptions --video 2026-11-11_some-talk +``` + +Set `transcribe: false` in a video's `_index.md` to skip its transcript (e.g. +for music or members-only videos), or `publish = false` in +`data/videos.toml` to remove a video from the site. + +Formatted transcripts are validated before they're written: only `

`, +`

`, ``, and `

`; timestamps within the video; at least +90% of the spoken words; and no responses cut off at the token limit. Videos +longer than 20 minutes are formatted in 15-minute chunks. The model's +timestamps are often off by tens of seconds, so every `` is matched +back to its VTT cue and gets that cue's start time. To check the transcripts on +the site, and to fix the ones that fail: + +```bash +just format-transcriptions --check-existing # report only, no API calls +just format-transcriptions --repair-timestamps # fix timestamps only, no API calls +just format-transcriptions --reformat-invalid --limit 10 +``` + +**Logs.** Every script logs what it does per video (with the slug and YouTube +ID), and with `--summary-json PATH` writes a machine-readable summary. The +workflow attaches the logs and summaries of every run as an artifact, kept for +30 days. All scripts exit with 0 on success, 1 on a fatal error (nothing +useful written), and 2 when some videos failed but the rest were written. +Pass `--dry-run` to see what a script would do, and `--verbose` for debug +logging. + ### Environment Variables Create a `.env` file in the project root (gitignored): @@ -520,6 +607,11 @@ Create a `.env` file in the project root (gitignored): # Get one at: https://github.com/settings/tokens GH_TOKEN="ghp_your_token_here" +# For the video sync (update-youtube-videos.py, transcribe.py, format-transcriptions.py) +YOUTUBE_API_KEY="..." +OPENAI_API_KEY="..." +ANTHROPIC_API_KEY="..." + # Plausible Analytics API key (optional, for analytics integration) PLAUSIBLE_KEY="your_key_here" ``` diff --git a/justfile b/justfile index abc229f8c..f1029c13d 100644 --- a/justfile +++ b/justfile @@ -70,27 +70,51 @@ extract-software-colors: create-hex-logos: ./scripts/create-hex-logos.py -# Update videos -update-youtube-videos: - ./scripts/update-youtube-videos.py +# Fetch video metadata from YouTube into data/videos.toml (pass --refresh-stats to update view counts) +update-youtube-videos *args: + ./scripts/update-youtube-videos.py "$@" # Sync video entries from data/videos.toml into content -sync-videos: - ./scripts/sync-videos.py +sync-videos *args: + ./scripts/sync-videos.py "$@" # Sync courses and learning paths from Posit Academy (needs SKILLJAR_API_KEY) sync-courses *args: ./scripts/sync-courses.py {{args}} -download-youtube-audio *args: - ./scripts/download-youtube-audio.py {{args}} - -transcribe: - ./scripts/transcribe.py +# Download audio and transcribe new videos with Whisper (needs OPENAI_API_KEY) +transcribe *args: + ./scripts/transcribe.py "$@" # Format VTT transcriptions into HTML using the Anthropic API format-transcriptions *args: - ./scripts/format-transcriptions.py {{args}} + ./scripts/format-transcriptions.py "$@" + +# Run the whole video pipeline locally (fallback for the nightly workflow), e.g. `just update-videos 5 chrome` +update-videos limit="15" browser="": + #!/usr/bin/env bash + set -euo pipefail + # Exit code 2 means some videos failed; keep going and fix them later. + step() { + "$@" && return 0 + status=$? + if [ "$status" -eq 2 ]; then + echo "⚠ $1 finished with errors (exit code 2); continuing" >&2 + return 0 + fi + echo "✗ $1 failed (exit code $status); stopping" >&2 + exit "$status" + } + cookies=() + if [ -n "{{ browser }}" ]; then cookies=(--cookies-from-browser "{{ browser }}"); fi + step ./scripts/update-youtube-videos.py + step ./scripts/sync-videos.py + step ./scripts/transcribe.py --limit "{{ limit }}" ${cookies[@]+"${cookies[@]}"} + step ./scripts/format-transcriptions.py --limit "{{ limit }}" + +# Run the tests of the Python scripts (extra arguments go to pytest) +test *args: + ./scripts/run-tests.sh "$@" quarto-preview: uv run quarto preview diff --git a/scripts/format-transcriptions.py b/scripts/format-transcriptions.py index 12a783cae..2ef2df96e 100755 --- a/scripts/format-transcriptions.py +++ b/scripts/format-transcriptions.py @@ -1,60 +1,82 @@ -#!/usr/bin/env -S uv run +#!/usr/bin/env -S uv run --script # /// script +# requires-python = ">=3.11" # dependencies = [ # "pyyaml>=6.0", # "rich>=13.0.0", # "anthropic>=0.40.0", # "python-dotenv>=1.0.0", -# "tomli>=2.0.0; python_version < '3.11'", # ] # /// """ Format VTT transcription files into HTML fragments using the Anthropic API. -For each subdirectory under content/resources/videos/: +For each subdirectory under content/resources/videos/, newest first: - Skip if transcription.txt already exists (unless --force). - Skip if _index.md frontmatter has transcribe: false. - Skip if transcription.vtt does not exist. - For long videos (> 90 min), split the VTT into chunks and process each segment separately, then concatenate the results. -- After the API call, inject links for known people and software. +- Repair the data-t timestamps: the model copies the cue text verbatim but + often gets the seconds wrong, so each is matched back to its VTT cue. +- Validate the HTML: only

,

, , and

; every + data-t within the video; at least 90% of the spoken words present; and a + response that hit the max_tokens limit is rejected. Invalid output is + never written. +- Inject links for known people and software. + +Requires ANTHROPIC_API_KEY in the environment or .env. The model can be set +with --model or $TRANSCRIPT_MODEL. + +Exit codes: 0 = success, 1 = fatal, 2 = partial (some videos failed). """ +from __future__ import annotations + import argparse import asyncio +import html as html_lib +import logging import os -import random import re import sys +import time +import tomllib +from collections import Counter +from dataclasses import dataclass from pathlib import Path -from typing import Any - -import anthropic -import yaml -from dotenv import load_dotenv -from rich.console import Console -from rich.progress import ( - BarColumn, - MofNCompleteColumn, - Progress, - SpinnerColumn, - TaskID, - TextColumn, - TimeElapsedColumn, -) - -try: - import tomllib -except ImportError: - import tomli as tomllib # type: ignore[no-redef] - -console = Console(stderr=True) - -MAX_RETRIES = 5 -BASE_BACKOFF = 1.0 -CHUNK_DURATION_S = 1_800 # 30 min per chunk (default) -LONG_VIDEO_THRESHOLD_S = 5_400 # 90 min — chunk automatically +from typing import Any, Awaitable, Callable + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 + +log = logging.getLogger("format-transcriptions") + +DEFAULT_MODEL = "claude-sonnet-4-6" +# Long transcripts hit the output token limit or get condensed by the model: +# before chunking at 20 minutes, 84-93% of the 45-90 minute videos lost more +# than 10% of their words. 15-minute chunks behave like short videos. +CHUNK_DURATION_S = 900 # 15 min per chunk (default) +LONG_VIDEO_THRESHOLD_S = 1_200 # 20 min — chunk automatically + +ALLOWED_TAGS = {"h3", "p", "a", "blockquote"} +# Share of the VTT's words that must appear in the HTML (outside pull quotes). +MIN_COVERAGE = 0.90 +# data-t values beyond the end of the video: a warning beyond the slack, an +# error beyond max(TOLERANCE_S, TOLERANCE_RATIO * duration). The model's +# timestamp arithmetic is often off by tens of seconds (median 44 s in the +# existing transcripts), which is annoying but not worth throwing away. +DATA_T_SLACK_S = 5 +DATA_T_TOLERANCE_S = 60 +DATA_T_TOLERANCE_RATIO = 0.10 + +# Estimated USD per million (input, output) tokens, matched by model prefix. +PRICES_PER_MTOK = { + "claude-sonnet": (3.0, 15.0), + "claude-haiku": (1.0, 5.0), +} PROMPT = """\ You convert WebVTT transcription files into clean HTML fragments for video pages. @@ -87,6 +109,11 @@ Copy the spoken words exactly as they appear in the VTT, including filler words ("um", "uh", "sort of", "you know") and false starts. +Include EVERY cue, from the first to the last. Do NOT skip, shorten, or +summarize any part — not small talk, not logistics ("can you see my +screen?", "next slide"), not jokes, not audience questions, not the closing +remarks. The output must contain all the words of the VTT. + ## Correcting technology-name mishearings The metadata lists technologies used in this video. If a word is clearly a phonetic mishearing of one of those technology names, silently correct it. @@ -211,74 +238,8 @@

Now let's actually build the app. I'll start with a minimal ui definition.

""" - -# --------------------------------------------------------------------------- -# Helpers copied from transcribe.py -# --------------------------------------------------------------------------- - - -def parse_frontmatter(content: str) -> dict[str, Any]: - lines = content.split("\n") - if not lines or lines[0].strip() != "---": - return {} - end_idx = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) - if end_idx is None: - return {} - try: - return yaml.safe_load("\n".join(lines[1:end_idx])) or {} - except yaml.YAMLError: - return {} - - -def parse_vtt_timestamp(ts: str) -> float: - parts = ts.strip().split(":") - if len(parts) == 3: - return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2]) - return int(parts[0]) * 60 + float(parts[1]) - - -def format_vtt_timestamp(seconds: float) -> str: - h = int(seconds // 3600) - m = int((seconds % 3600) // 60) - s = seconds % 60 - return f"{h:02d}:{m:02d}:{s:06.3f}" - - -def parse_vtt(content: str) -> list[tuple[float, float, str]]: - cues: list[tuple[float, float, str]] = [] - for block in re.split(r"\n{2,}", content.strip()): - lines = block.strip().splitlines() - arrow = next((i for i, line in enumerate(lines) if "-->" in line), None) - if arrow is None: - continue - m = re.match(r"([\d:.]+)\s+-->\s+([\d:.]+)", lines[arrow]) - if not m: - continue - start = parse_vtt_timestamp(m.group(1)) - end = parse_vtt_timestamp(m.group(2)) - text = "\n".join(lines[arrow + 1 :]).strip() - if text: - cues.append((start, end, text)) - return cues - - -class RateLimiter: - def __init__(self, min_interval: float = 1.0) -> None: - self._lock = asyncio.Lock() - self._last: float = 0.0 - self._interval = min_interval - - async def wait(self) -> None: - async with self._lock: - loop = asyncio.get_running_loop() - elapsed = loop.time() - self._last - if elapsed < self._interval: - await asyncio.sleep(self._interval - elapsed) - self._last = asyncio.get_running_loop().time() - - # --------------------------------------------------------------------------- -# Phase 1 — Prompt helpers +# Prompt helpers # --------------------------------------------------------------------------- @@ -294,9 +255,7 @@ def build_user_message(vtt_content: str, frontmatter: dict[str, Any]) -> str: if people: header_parts.append(f"People featured: {', '.join(str(p) for p in people)}") if software: - header_parts.append( - f"Technologies mentioned: {', '.join(str(s) for s in software)}" - ) + header_parts.append(f"Technologies mentioned: {', '.join(str(s) for s in software)}") if description: header_parts.append(f"Description: {description[:600]}") @@ -304,11 +263,46 @@ def build_user_message(vtt_content: str, frontmatter: dict[str, Any]) -> str: return f"{header}\n\n---\n\n{vtt_content}" if header else vtt_content +def build_chunk_system(segment_index: int, total_segments: int) -> str: + return ( + PROMPT.rstrip() + + "\n\n## Segment context\n" + + f"This is segment {segment_index + 1} of {total_segments}.\n" + + "Start each paragraph cleanly — do not begin mid-sentence.\n" + + "End each paragraph cleanly — do not leave a sentence dangling.\n" + ) + + +def split_vtt_into_chunks(vtt_content: str, chunk_duration: int) -> list[tuple[str, int]]: + cues = videolib.parse_vtt(vtt_content) + chunks: list[tuple[str, int]] = [] + seg_start = 0.0 + seg_cues: list[videolib.Cue] = [] + + for start, end, text in cues: + if seg_cues and (start - seg_start) >= chunk_duration: + chunks.append((videolib.render_vtt(seg_cues), int(seg_start))) + seg_start = start + seg_cues = [] + seg_cues.append((start, end, text)) + + if seg_cues: + chunks.append((videolib.render_vtt(seg_cues), int(seg_start))) + + return chunks + + # --------------------------------------------------------------------------- -# Phase 2 — Core script +# Validation # --------------------------------------------------------------------------- +class FormatError(Exception): + def __init__(self, reason: str, detail: str): + super().__init__(detail) + self.reason = reason + + def strip_markdown_fence(text: str) -> str: text = text.strip() if text.startswith("```"): @@ -317,232 +311,289 @@ def strip_markdown_fence(text: str) -> str: return text.strip() -def validate_html(html: str) -> None: - if not html.lstrip().startswith("<"): - raise ValueError("Response does not start with '<' — not HTML") - for bad in ("" not in html and "
" not in html: - raise ValueError("Response contains no

or

elements") +# Typos the model makes in otherwise fine output, seen in real transcripts: +# '' and ''. +ANCHOR_TYPOS = [ + (re.compile(r" str: - for attempt in range(MAX_RETRIES): - if attempt > 0: - delay = BASE_BACKOFF * (2 ** (attempt - 1)) + random.uniform(0, 1) - console.print( - f" [yellow]↻[/] {name} retry {attempt + 1}/{MAX_RETRIES} in {delay:.1f}s" - ) - await asyncio.sleep(delay) +def fix_anchor_typos(html: str) -> tuple[str, int]: + """Repair known harmless typos in tags; return (html, number fixed).""" + fixed = 0 + for pattern, replacement in ANCHOR_TYPOS: + html, n = pattern.subn(replacement, html) + fixed += n + return html, fixed - await rate_limiter.wait() - try: - response = await client.messages.create( - model=model, - max_tokens=max_tokens, - system=system, - messages=[{"role": "user", "content": user_message}], - ) - return response.content[0].text - except anthropic.RateLimitError: - if attempt == MAX_RETRIES - 1: - raise - except anthropic.APIStatusError as e: - if e.status_code < 500 or attempt == MAX_RETRIES - 1: - raise - - raise RuntimeError("Exceeded max retries") - - -def resolve_video_dir(name_or_path: str, videos_dir: Path) -> Path: - p = Path(name_or_path) - if p.is_absolute() and p.is_dir(): - return p - candidate = videos_dir / name_or_path - if candidate.is_dir(): - return candidate - matches = [ - d - for d in videos_dir.iterdir() - if d.is_dir() and d.name.startswith(name_or_path) - ] - if len(matches) == 1: - return matches[0] - if len(matches) > 1: - listed = "\n ".join(m.name for m in sorted(matches)) - raise ValueError(f"Ambiguous --video value, matches:\n {listed}") - raise ValueError(f"Video directory not found: {name_or_path}") - - -async def process_video_async( - video_dir: Path, - client: anthropic.AsyncAnthropic, - semaphore: asyncio.Semaphore, - rate_limiter: RateLimiter, - progress: Progress, - overall_task: TaskID, - args: argparse.Namespace, - people: dict[str, str], - software: dict[str, str], -) -> str: - async with semaphore: - name = video_dir.name - job_task = progress.add_task(f"[dim]{name}", total=None) - try: - html_file = video_dir / "transcription.txt" - vtt_file = video_dir / "transcription.vtt" - index_file = video_dir / "_index.md" - if html_file.exists() and not args.force: - return "skipped" +WORD_RE = re.compile(r"[\w']+") - frontmatter = ( - parse_frontmatter(index_file.read_text(encoding="utf-8")) - if index_file.exists() - else {} - ) - if frontmatter.get("transcribe", True) is False: - return "excluded" +def word_counts(text: str) -> Counter: + return Counter(w.lower().strip("'") for w in WORD_RE.findall(text)) - if not vtt_file.exists(): - return "no_vtt" - vtt_content = vtt_file.read_text(encoding="utf-8") - duration = (frontmatter.get("external") or {}).get("duration", 0) +def html_to_text(html: str, *, drop_blockquotes: bool = True) -> str: + if drop_blockquotes: + html = re.sub(r"", " ", html, flags=re.DOTALL | re.IGNORECASE) + html = re.sub(r"", " ", html, flags=re.DOTALL | re.IGNORECASE) + return html_lib.unescape(re.sub(r"<[^>]+>", " ", html)) - if duration > LONG_VIDEO_THRESHOLD_S: - progress.update(job_task, description=f"[cyan]{name} chunking") - html = await format_chunked( - vtt_content, - frontmatter, - client, - rate_limiter, - args, - name, - progress, - job_task, - ) - else: - user_message = build_user_message(vtt_content, frontmatter) - progress.update(job_task, description=f"[cyan]{name} formatting") - raw = await call_api( - client, - user_message, - args.model, - args.max_tokens, - rate_limiter, - name, - ) - html = strip_markdown_fence(raw) - validate_html(html) - html = inject_links(html, people, software) - html_file.write_text(html, encoding="utf-8") - return "formatted" +def coverage(cues: list[videolib.Cue], html: str) -> float: + """Share of the spoken words (with multiplicity) found in the HTML prose. - except Exception as e: - console.print(f" [red]✗[/] {name}: {e}") - return "error" - finally: - progress.remove_task(job_task) - progress.advance(overall_task) + A cue whose text repeats an earlier cue counts once: Whisper sometimes + loops on a line (one VTT has a speaker's name nine times), and the model + rightly leaves the copies out. + """ + seen: set[str] = set() + unique = [] + for _, _, text in cues: + key = " ".join(_norm_words(text)) + if key not in seen: + seen.add(key) + unique.append(text) + spoken = word_counts(" ".join(unique)) + total = sum(spoken.values()) + if not total: + return 1.0 + written = word_counts(html_to_text(html)) + return sum(min(n, written[w]) for w, n in spoken.items()) / total -# --------------------------------------------------------------------------- -# Phase 3 — Chunked processing -# --------------------------------------------------------------------------- +def validate_html(html: str, cues: list[videolib.Cue], duration: int) -> dict[str, Any]: + """Raise FormatError when the model output can't be used; return metrics.""" + if not html.lstrip().startswith("<"): + raise FormatError("not_html", f"Response does not start with '<': {html[:80]!r}") + + tags = {t.lower() for t in re.findall(r"]*)?>", html, flags=re.IGNORECASE): + if m.group(2) and m.group(2).strip(): + raise FormatError("disallowed_attribute", f"Attributes on <{m.group(1)}>: {m.group(2).strip()}") + + end = max([duration] + [end for _, end, _ in cues]) + max_t = 0 + anchors = 0 + for m in re.finditer(r"]*)?>", html, flags=re.IGNORECASE): + attrs = (m.group(1) or "").strip() + am = re.fullmatch(r'data-t="(\d+)"', attrs) + if not am: + raise FormatError("disallowed_attribute", f"Bad attributes: {attrs!r}") + max_t = max(max_t, int(am.group(1))) + anchors += 1 + + warnings: list[str] = [] + overshoot = max_t - end + if overshoot > max(DATA_T_TOLERANCE_S, DATA_T_TOLERANCE_RATIO * end): + raise FormatError("bad_timestamp", f'data-t="{max_t}" is {overshoot:.0f}s beyond the end of the video ({end:.0f}s)') + if overshoot > DATA_T_SLACK_S: + warnings.append(f"timestamp_overshoot: data-t=\"{max_t}\" is {overshoot:.0f}s beyond the end of the video") + if "

" not in html and "

" not in html: + raise FormatError("no_paragraphs", "Response contains no

or

elements") -def _render_vtt(cues: list[tuple[float, float, str]]) -> str: - lines = ["WEBVTT", ""] - for start, end, text in cues: - lines.append(f"{format_vtt_timestamp(start)} --> {format_vtt_timestamp(end)}") - lines.append(text) - lines.append("") - return "\n".join(lines) + cov = coverage(cues, html) + if cov < MIN_COVERAGE: + raise FormatError("low_coverage", f"Only {cov:.1%} of the spoken words are in the HTML (minimum {MIN_COVERAGE:.0%})") + return {"coverage": round(cov, 4), "anchors": anchors, "warnings": warnings or None} -def split_vtt_into_chunks( - vtt_content: str, chunk_duration: int -) -> list[tuple[str, int]]: - cues = parse_vtt(vtt_content) - chunks: list[tuple[str, int]] = [] - seg_start = 0.0 - seg_cues: list[tuple[float, float, str]] = [] +# --------------------------------------------------------------------------- +# Timestamp repair +# --------------------------------------------------------------------------- - for start, end, text in cues: - if seg_cues and (start - seg_start) >= chunk_duration: - chunks.append((_render_vtt(seg_cues), int(seg_start))) - seg_start = start - seg_cues = [] - seg_cues.append((start, end, text)) +# How far ahead (in words) to look for the next anchor's text in the VTT. +REPAIR_WINDOW_WORDS = 800 +# Anchors shorter than REPAIR_KEY_WORDS ("For us,", "I came from") occur all +# over a transcript, so they may only match this close to the previous one; +# otherwise one false match far ahead derails all the anchors after it. +REPAIR_SHORT_WINDOW_WORDS = 40 +# Words of an anchor used to find it in the VTT. +REPAIR_KEY_WORDS = 4 + +ANCHOR_RE = re.compile(r'(.*?)', flags=re.DOTALL) +BLOCKQUOTE_RE = re.compile(r"()", flags=re.DOTALL | re.IGNORECASE) + + +def _norm_words(text: str) -> list[str]: + return [w.lower().strip("'") for w in WORD_RE.findall(html_lib.unescape(re.sub(r"<[^>]+>", " ", text)))] + + +def repair_timestamps(html: str, cues: list[videolib.Cue]) -> tuple[str, dict[str, int]]: + """Set every ``data-t`` to the start of the VTT cue its text comes from. + + The model copies the cue text verbatim but often gets the seconds wrong + (in the existing transcripts typically by 5-20 s, sometimes by minutes). + Walk the anchors in document order and find each one's first words in + the VTT, searching forward from the previous match. Anchors that can't + be found keep the model's value. Pull quotes reuse the value of the + matching prose anchor. Consecutive anchors with the same original value + (split by inject_links) stay together. + """ + words: list[tuple[str, int]] = [] # (word, cue start in whole seconds) + for start, _end, text in cues: + words.extend((w, int(start)) for w in _norm_words(text)) + vocab = [w for w, _ in words] + + def find(key: list[str], pos: int) -> int | None: + window = REPAIR_WINDOW_WORDS if len(key) >= REPAIR_KEY_WORDS else REPAIR_SHORT_WINDOW_WORDS + limit = min(len(vocab) - len(key) + 1, pos + window) + for i in range(pos, max(pos, limit)): + if vocab[i] == key[0] and vocab[i : i + len(key)] == key: + return i + return None + + stats = {"anchors": 0, "matched": 0, "changed": 0, "unmatched": 0} + by_text: dict[str, int] = {} + pos = 0 + prev: tuple[int, int] | None = None # (original, repaired) + + def repair_prose(m: re.Match) -> str: + nonlocal pos, prev + original, inner = int(m.group(1)), m.group(2) + stats["anchors"] += 1 + ws = _norm_words(inner) + new = original + found = None + # Skip a leading word or two in case the model corrected it + # (e.g. "Korto" -> "Quarto"). + for skip in (0, 1, 2): + key = ws[skip : skip + REPAIR_KEY_WORDS] + if not key: + break + i = find(key, pos) + if i is not None: + pos = i + len(ws) - skip + found = words[i][1] + break + if found is not None: + stats["matched"] += 1 + new = found + else: + stats["unmatched"] += 1 + if prev is not None and prev[0] == original: + new = prev[1] # a piece of an anchor that inject_links split + elif words: + # Keep the model's value, but not beyond the last cue. + # (Clamping to the neighbours was measured to make things + # worse: one wrong match then drags the next anchors along.) + new = min(original, words[-1][1]) + prev = (original, new) + if new != original: + stats["changed"] += 1 + by_text.setdefault(" ".join(ws), new) + return f'{inner}' + + last_start = words[-1][1] if words else None + + def repair_quote(m: re.Match) -> str: + new = by_text.get(" ".join(_norm_words(m.group(2)))) + if new is None: + new = int(m.group(1)) + if last_start is not None: + new = min(new, last_start) + return f'{m.group(2)}' + + parts = BLOCKQUOTE_RE.split(html) + parts = [p if i % 2 else ANCHOR_RE.sub(repair_prose, p) for i, p in enumerate(parts)] + parts = [ANCHOR_RE.sub(repair_quote, p) if i % 2 else p for i, p in enumerate(parts)] + return "".join(parts), stats + + +def unlink_entities(html: str) -> str: + """Undo inject_links, so a published transcript can be validated again.""" + return re.sub(r'(.*?)', r"\1", html, flags=re.DOTALL) + + +def check_existing(video_dir: Path) -> tuple[bool, str | None, str | None]: + """Validate a published transcription.txt: ``(ok, reason, detail)``.""" + fm = videolib.read_frontmatter(video_dir / "_index.md") + duration = int((fm.get("external") or {}).get("duration") or 0) + cues = videolib.parse_vtt((video_dir / "transcription.vtt").read_text(encoding="utf-8")) + html = unlink_entities((video_dir / "transcription.txt").read_text(encoding="utf-8")) + try: + validate_html(html, cues, duration) + except FormatError as e: + return False, e.reason, str(e) + return True, None, None - if seg_cues: - chunks.append((_render_vtt(seg_cues), int(seg_start))) - return chunks +# --------------------------------------------------------------------------- +# API +# --------------------------------------------------------------------------- -def build_chunk_system(segment_index: int, total_segments: int) -> str: - return ( - PROMPT.rstrip() - + "\n\n## Segment context\n" - + f"This is segment {segment_index + 1} of {total_segments}.\n" - + "Start each paragraph cleanly — do not begin mid-sentence.\n" - + "End each paragraph cleanly — do not leave a sentence dangling.\n" - ) +def estimate_cost(model: str, input_tokens: int, output_tokens: int) -> float | None: + for prefix, (price_in, price_out) in PRICES_PER_MTOK.items(): + if model.startswith(prefix): + return round((input_tokens * price_in + output_tokens * price_out) / 1_000_000, 4) + return None -async def format_chunked( - vtt_content: str, - frontmatter: dict[str, Any], - client: anthropic.AsyncAnthropic, - rate_limiter: RateLimiter, - args: argparse.Namespace, - name: str, - progress: Progress, - job_task: TaskID, -) -> str: - chunks = split_vtt_into_chunks(vtt_content, args.chunk_duration) - total = len(chunks) - html_parts: list[str] = [] - - for i, (chunk_vtt, _seg_start) in enumerate(chunks): - progress.update(job_task, description=f"[cyan]{name} chunk {i + 1}/{total}") - user_message = build_user_message(chunk_vtt, frontmatter) - chunk_system = build_chunk_system(i, total) - raw = await call_api( - client, - user_message, - args.model, - args.max_tokens, - rate_limiter, - name, - system=chunk_system, +async def call_api( + client: Any, + user_message: str, + *, + model: str, + max_tokens: int, + rate_limiter: videolib.RateLimiter, + label: str, + system: str = PROMPT, + sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, +) -> tuple[str, dict[str, Any]]: + """Return ``(html, usage)``; raise FormatError if the output was cut off.""" + + async def call(): + await rate_limiter.wait() + return await client.messages.create( + model=model, + max_tokens=max_tokens, + system=system, + messages=[{"role": "user", "content": user_message}], ) - html_parts.append(strip_markdown_fence(raw)) - return "\n\n".join(html_parts) + t0 = time.monotonic() + response = await videolib.retry_async(call, label=label, logger=log, sleep=sleep) + usage = { + "input_tokens": getattr(response.usage, "input_tokens", 0), + "output_tokens": getattr(response.usage, "output_tokens", 0), + "stop_reason": response.stop_reason, + "request_id": getattr(response, "_request_id", None), + "latency_s": round(time.monotonic() - t0, 1), + } + log.info( + "%s: %s responded in %.1fs (%d in / %d out tokens, stop_reason=%s, request_id=%s)", + label, + model, + usage["latency_s"], + usage["input_tokens"], + usage["output_tokens"], + usage["stop_reason"], + usage["request_id"], + ) + if response.stop_reason == "max_tokens": + raise FormatError( + "truncated", + f"Response hit max_tokens={max_tokens} after {usage['output_tokens']} tokens " + f"(request_id={usage['request_id']}); try a smaller --chunk-duration", + ) + text = "".join(getattr(block, "text", "") for block in response.content) + return strip_markdown_fence(text), usage # --------------------------------------------------------------------------- -# Phase 4 — Entity linking +# Entity linking # --------------------------------------------------------------------------- -def build_entity_dicts( - repo_root: Path, -) -> tuple[dict[str, str], dict[str, str]]: +def build_entity_dicts(repo_root: Path) -> tuple[dict[str, str], dict[str, str]]: ignore_file = repo_root / "data" / "software-match-ignore.toml" ignored: set[str] = set() if ignore_file.exists(): @@ -552,13 +603,10 @@ def build_entity_dicts( people: dict[str, str] = {} people_dir = repo_root / "content" / "people" if people_dir.exists(): - for d in people_dir.iterdir(): + for d in sorted(people_dir.iterdir()): if not d.is_dir(): continue - index = d / "_index.md" - if not index.exists(): - continue - fm = parse_frontmatter(index.read_text(encoding="utf-8")) + fm = videolib.read_frontmatter(d / "_index.md") title = (fm.get("title") or "").strip() if title: people[title] = f"/people/{d.name}/" @@ -566,13 +614,10 @@ def build_entity_dicts( software: dict[str, str] = {} software_dir = repo_root / "content" / "software" if software_dir.exists(): - for d in software_dir.iterdir(): + for d in sorted(software_dir.iterdir()): if not d.is_dir() or d.name in ignored: continue - index = d / "_index.md" - if not index.exists(): - continue - fm = parse_frontmatter(index.read_text(encoding="utf-8")) + fm = videolib.read_frontmatter(d / "_index.md") override = (fm.get("override") or {}).get("title") title = (override or fm.get("title") or "").strip() if title: @@ -581,11 +626,11 @@ def build_entity_dicts( return people, software -def _inject_into_text( - text: str, - entities: list[tuple[str, str, bool]], - linked: set[str], -) -> str: +def _inject_into_text(text: str, entities: list[tuple[str, str, bool]], linked: set[str]) -> str: + # Insert placeholders first and the links at the end, so a later, shorter + # name can't match inside an inserted link: "Shiny" in the URL of + # "Shiny for Python" once produced . + links: list[str] = [] for name, url, is_software in entities: if name in linked: continue @@ -593,21 +638,17 @@ def _inject_into_text( pattern = r"\b" + re.escape(name) + r"\b" if re.search(pattern, text, flags=flags): linked.add(name) - text = re.sub( - pattern, - lambda m, u=url: f'{m.group(0)}', - text, - count=1, - flags=flags, - ) - return text + + def placeholder(m: re.Match, u: str = url) -> str: + links.append(f'{m.group(0)}') + return f"\ue000{len(links) - 1}\ue001" + + text = re.sub(pattern, placeholder, text, count=1, flags=flags) + return re.sub("\ue000(\\d+)\ue001", lambda m: links[int(m.group(1))], text) def _inject_into_anchor( - content: str, - attrs: str, - entities: list[tuple[str, str, bool]], - linked: set[str], + content: str, attrs: str, entities: list[tuple[str, str, bool]], linked: set[str] ) -> str: for name, url, is_software in entities: if name in linked: @@ -630,11 +671,7 @@ def _inject_into_anchor( return f"{content}" -def inject_links( - html: str, - people: dict[str, str], - software: dict[str, str], -) -> str: +def inject_links(html: str, people: dict[str, str], software: dict[str, str]) -> str: linked: set[str] = set() entities: list[tuple[str, str, bool]] = sorted( @@ -658,127 +695,370 @@ def inject_links( # --------------------------------------------------------------------------- -# Entry points +# Per-video processing # --------------------------------------------------------------------------- -async def main_async(args: argparse.Namespace) -> None: - console.print("\n[bold blue]Transcription Formatter[/]\n") +@dataclass +class Context: + client: Any + args: argparse.Namespace + semaphore: asyncio.Semaphore + rate_limiter: videolib.RateLimiter + people: dict[str, str] + software: dict[str, str] + sleep: Callable[[float], Awaitable[Any]] + + +def select_videos( + dirs: list[Path], + summary: videolib.RunSummary, + *, + force: bool, + limit: int | None, + reformat_invalid: bool = False, +) -> list[Path]: + selected: list[Path] = [] + deferred: list[str] = [] + needs_transcript: list[str] = [] + for d in dirs: + fm = videolib.read_frontmatter(d / "_index.md") + if fm.get("transcribe", True) is False: + summary.tally("excluded") + continue + if not (d / "transcription.vtt").exists(): + summary.tally("no_vtt") + needs_transcript.append(d.name) + continue + if (d / "transcription.txt").exists() and not force: + if not reformat_invalid: + summary.tally("already_formatted") + continue + ok, reason, _ = check_existing(d) + if ok: + summary.tally("already_formatted") + continue + log.info("%s: existing transcript is invalid (%s); reformatting", d.name, reason) + if limit is not None and len(selected) >= limit: + deferred.append(d.name) + continue + selected.append(d) + if deferred: + summary.tally("deferred", len(deferred)) + summary.extra["deferred"] = deferred + log.info("Deferred %d videos to a later run (--limit)", len(deferred)) + if needs_transcript: + # Newest first; video-sync-report.py lists them in the PR. + summary.extra["needs_transcript"] = needs_transcript + log.info("%d videos have no transcription.vtt yet (run transcribe.py)", len(needs_transcript)) + return selected + + +async def process_video(video_dir: Path, ctx: Context) -> dict[str, Any]: + async with ctx.semaphore: + args = ctx.args + fm = videolib.read_frontmatter(video_dir / "_index.md") + external = fm.get("external") or {} + video_id = videolib.youtube_id(external.get("url")) + label = f"{video_dir.name} ({video_id})" + duration = int(external.get("duration") or 0) + item: dict[str, Any] = {"slug": video_dir.name, "video_id": video_id, "duration_s": duration, "model": args.model} + html: str | None = None + try: + vtt_content = (video_dir / "transcription.vtt").read_text(encoding="utf-8") + cues = videolib.parse_vtt(vtt_content) + if not cues: + raise FormatError("empty_vtt", "transcription.vtt has no cues") - load_dotenv() - api_key = os.getenv("ANTHROPIC_API_KEY") - if not api_key: - console.print("[bold red]Error:[/] ANTHROPIC_API_KEY not set in .env") - sys.exit(1) + if duration > LONG_VIDEO_THRESHOLD_S: + chunks = split_vtt_into_chunks(vtt_content, args.chunk_duration) + else: + chunks = [(vtt_content, 0)] + item["chunks"] = len(chunks) + log.info("%s: formatting %d cues in %d chunk(s)", label, len(cues), len(chunks)) + + def request(i: int): + return call_api( + ctx.client, + build_user_message(chunks[i][0], fm), + model=args.model, + max_tokens=args.max_tokens, + rate_limiter=ctx.rate_limiter, + label=f"{label} chunk {i + 1}/{len(chunks)}" if len(chunks) > 1 else label, + system=build_chunk_system(i, len(chunks)) if len(chunks) > 1 else PROMPT, + sleep=ctx.sleep, + ) - client = anthropic.AsyncAnthropic(api_key=api_key) - repo_root = Path(__file__).parent.parent - videos_dir = repo_root / "content" / "resources" / "videos" + # Chunks are independent, so request them concurrently (the rate + # limiter still spaces out the requests). + results = list(await asyncio.gather(*(request(i) for i in range(len(chunks))))) + parts = [html_part for html_part, _ in results] + usages = [usage for _, usage in results] + chunk_cues = [videolib.parse_vtt(chunk_vtt) for chunk_vtt, _ in chunks] + chunk_coverage = [coverage(cs, part) for cs, part in zip(chunk_cues, parts)] + + # The model sometimes skips parts of a chunk. Ask once more for + # the chunks that lost too many words and keep the better answer. + retry = [i for i, c in enumerate(chunk_coverage) if c < MIN_COVERAGE] + if retry: + log.warning( + "%s: %s lost words (%s); asking once more", + label, + "chunks " + ", ".join(str(i + 1) for i in retry) if len(chunks) > 1 else "the response", + ", ".join(f"{chunk_coverage[i]:.1%}" for i in retry), + ) + second = await asyncio.gather(*(request(i) for i in retry)) + for i, (html_part, usage) in zip(retry, second): + usages.append(usage) + c = coverage(chunk_cues[i], html_part) + log.info("%s: retry of chunk %d: %.1f%% coverage (was %.1f%%)", label, i + 1, c * 100, chunk_coverage[i] * 100) + if c > chunk_coverage[i]: + parts[i], chunk_coverage[i] = html_part, c + item["coverage_retries"] = len(retry) + + html = "\n\n".join(parts) + if len(chunks) > 1: + # Which chunk lost words, if any: useful when coverage is low. + item["chunk_coverage"] = [round(c, 3) for c in chunk_coverage] + log.info("%s: coverage per chunk: %s", label, ", ".join(f"{c:.1%}" for c in chunk_coverage)) + + item["input_tokens"] = sum(u["input_tokens"] for u in usages) + item["output_tokens"] = sum(u["output_tokens"] for u in usages) + item["request_ids"] = [u["request_id"] for u in usages if u["request_id"]] or None + item["estimated_cost_usd"] = estimate_cost(args.model, item["input_tokens"], item["output_tokens"]) + + html, typos = fix_anchor_typos(html) + if typos: + item["typos_fixed"] = typos + log.info("%s: fixed %d malformed tags", label, typos) + html, repair = repair_timestamps(html, cues) + item["timestamps_repaired"] = repair["changed"] + item["timestamps_unmatched"] = repair["unmatched"] + log.info( + "%s: repaired %d of %d timestamps (%d anchors not found in the VTT)", + label, + repair["changed"], + repair["anchors"], + repair["unmatched"], + ) + metrics = validate_html(html, cues, duration) + item.update(metrics) + for w in metrics["warnings"] or []: + log.warning("%s: %s", label, w) + html = inject_links(html, ctx.people, ctx.software) + videolib.atomic_write_text(video_dir / "transcription.txt", html) + except FormatError as e: + log.error("%s: %s: %s", label, e.reason, e) + if args.rejected_dir and html is not None: + args.rejected_dir.mkdir(parents=True, exist_ok=True) + rejected = args.rejected_dir / f"{video_dir.name}.html" + videolib.atomic_write_text(rejected, html) + item["rejected_output"] = str(rejected) + log.info("%s: saved the rejected output to %s", label, rejected) + return {**item, "status": "error", "reason": e.reason, "detail": str(e)} + except Exception as e: # noqa: BLE001 - every failure goes in the summary + log.exception("%s: formatting failed", label) + return {**item, "status": "error", "reason": "api_error", "detail": videolib.describe_api_error(e)} + + log.info( + "%s: wrote transcription.txt (%d anchors, %.1f%% coverage%s)", + label, + item["anchors"], + item["coverage"] * 100, + f", ~${item['estimated_cost_usd']:.3f}" if item["estimated_cost_usd"] is not None else "", + ) + return {**item, "status": "formatted"} - if not videos_dir.exists(): - console.print(f"[bold red]Error:[/] Videos directory not found: {videos_dir}") - sys.exit(1) - people, software = build_entity_dicts(repo_root) - console.print( - f"[dim]Loaded {len(people)} people and {len(software)} software entries[/]" - ) +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- - if args.video: - dirs = [resolve_video_dir(args.video, videos_dir)] - else: - dirs = sorted(d for d in videos_dir.iterdir() if d.is_dir()) - console.print( - f"[dim]Found {len(dirs)} director(ies) to process (--jobs {args.jobs})[/]\n" +def run_check_existing(dirs: list[Path], summary: videolib.RunSummary) -> None: + """Report published transcripts that fail validation (no API calls).""" + for d in dirs: + if not ((d / "transcription.txt").exists() and (d / "transcription.vtt").exists()): + continue + ok, reason, detail = check_existing(d) + if ok: + summary.tally("valid") + else: + summary.add("invalid", slug=d.name, reason=reason, detail=detail) + log.warning("Invalid: %s: %s: %s", d.name, reason, detail) + invalid = summary.by_status("invalid") + reasons: dict[str, int] = {} + for i in invalid: + reasons[i["reason"]] = reasons.get(i["reason"], 0) + 1 + summary.extra["invalid_by_reason"] = reasons + log.info( + "%d of %d transcripts are invalid %s; reformat them with --reformat-invalid", + len(invalid), + len(invalid) + summary.tallies.get("valid", 0), + reasons, ) - semaphore = asyncio.Semaphore(args.jobs) - rate_limiter = RateLimiter(min_interval=1.0) - - with Progress( - SpinnerColumn(), - TextColumn("{task.description}"), - BarColumn(), - MofNCompleteColumn(), - TimeElapsedColumn(), - console=console, - ) as progress: - overall_task = progress.add_task("[bold cyan]Formatting", total=len(dirs)) - results: list[str] = await asyncio.gather( - *[ - process_video_async( - d, - client, - semaphore, - rate_limiter, - progress, - overall_task, - args, - people, - software, - ) - for d in dirs - ] + +def run_repair_timestamps(dirs: list[Path], summary: videolib.RunSummary, *, dry_run: bool) -> None: + """Fix the data-t values of published transcripts (no API calls).""" + for d in dirs: + txt, vtt = d / "transcription.txt", d / "transcription.vtt" + if not (txt.exists() and vtt.exists()): + continue + cues = videolib.parse_vtt(vtt.read_text(encoding="utf-8")) + if not cues: + summary.tally("no_cues") + continue + html = txt.read_text(encoding="utf-8") + repaired, stats = repair_timestamps(html, cues) + if repaired == html: + summary.tally("timestamps_ok") + continue + if not dry_run: + videolib.atomic_write_text(txt, repaired) + summary.add("timestamps_repaired", slug=d.name, **stats) + log.info( + "%s: %s %d of %d timestamps (%d anchors not found in the VTT)", + d.name, + "would repair" if dry_run else "repaired", + stats["changed"], + stats["anchors"], + stats["unmatched"], ) - formatted = results.count("formatted") - skipped = results.count("skipped") - excluded = results.count("excluded") - no_vtt = results.count("no_vtt") - errors = results.count("error") - - console.print("\n[bold]Summary:[/]") - console.print(f" [green]✓[/] Formatted: {formatted}") - console.print(f" [dim]○[/] Skipped: {skipped}") - console.print(f" [dim]○[/] Excluded: {excluded}") - console.print(f" [dim]○[/] No VTT: {no_vtt}") - if errors: - console.print(f" [red]✗[/] Errors: {errors}") - console.print("\n[bold green]✓ Done![/]\n") - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Format VTT transcriptions into HTML using the Anthropic API" + +def build_client(project_root: Path) -> Any: + import anthropic + from dotenv import load_dotenv + + load_dotenv(project_root / ".env") + api_key = os.getenv("ANTHROPIC_API_KEY") + if not api_key: + raise RuntimeError("ANTHROPIC_API_KEY not set in .env or environment") + return anthropic.AsyncAnthropic(api_key=api_key, max_retries=0) # we retry ourselves, with logging + + +def parse_args(argv: list[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Format VTT transcriptions into HTML using the Anthropic API") + parser.add_argument("--jobs", type=int, default=5, metavar="N", help="parallel formatting jobs (default: 5)") + parser.add_argument("--force", action="store_true", help="overwrite existing transcription.txt files") + parser.add_argument("--video", metavar="NAME", help="process only this video directory (name, prefix, or path)") + parser.add_argument("--limit", type=int, metavar="N", help="format at most N videos (newest first)") + parser.add_argument( + "--model", + default=os.getenv("TRANSCRIPT_MODEL") or DEFAULT_MODEL, + metavar="ID", + help=f"Anthropic model ID (default: $TRANSCRIPT_MODEL or {DEFAULT_MODEL})", ) parser.add_argument( - "--jobs", - type=int, - default=5, - metavar="N", - help="parallel formatting jobs (default: 5)", + "--max-tokens", type=int, default=16384, metavar="N", help="max output tokens per API call (default: 16384)" ) parser.add_argument( - "--force", - action="store_true", - help="overwrite existing transcription.txt files", + "--chunk-duration", + type=int, + default=CHUNK_DURATION_S, + metavar="SECS", + help=f"VTT seconds per chunk for long videos (default: {CHUNK_DURATION_S})", ) parser.add_argument( - "--video", - metavar="NAME", - help="process only this video directory (name or path)", + "--reformat-invalid", + action="store_true", + help="also reformat existing transcripts that fail validation (see --check-existing)", ) parser.add_argument( - "--model", default="claude-sonnet-4-6", metavar="ID", help="Anthropic model ID" + "--check-existing", + action="store_true", + help="only validate existing transcription.txt files and report the invalid ones (no API calls)", ) parser.add_argument( - "--max-tokens", - type=int, - default=16384, - metavar="N", - help="max output tokens per API call (default: 16384)", + "--repair-timestamps", + action="store_true", + help="only fix the data-t values of existing transcription.txt files from their VTT (no API calls)", ) parser.add_argument( - "--chunk-duration", - type=int, - default=CHUNK_DURATION_S, - metavar="SECS", - help=f"VTT seconds per chunk for long videos (default: {CHUNK_DURATION_S})", + "--rejected-dir", + type=Path, + metavar="DIR", + help="save model output that fails validation to DIR/.html, for debugging", ) - args = parser.parse_args() - asyncio.run(main_async(args)) + parser.add_argument("--dry-run", action="store_true", help="list what would be formatted") + videolib.add_common_args(parser) + return parser.parse_args(argv) + + +def main( + argv: list[str] | None = None, + client: Any = None, + sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, + min_request_interval: float = 1.0, +) -> int: + args = parse_args(argv) + videolib.setup_logging("format-transcriptions", args.verbose) + summary = videolib.RunSummary("format-transcriptions") + summary.extra.update(model=args.model, dry_run=args.dry_run, limit=args.limit) + videos_dir = args.project_root / "content" / "resources" / "videos" + + def body() -> None: + nonlocal client + summary.environment = videolib.log_environment(log, packages=["anthropic"]) + if not videos_dir.exists(): + summary.fatal = f"Videos directory not found: {videos_dir}" + return + people, software = build_entity_dicts(args.project_root) + log.info("Model: %s; loaded %d people and %d software entries for linking", args.model, len(people), len(software)) + + dirs = [videolib.resolve_video_dir(args.video, videos_dir)] if args.video else videolib.video_dirs(videos_dir) + if args.check_existing: + run_check_existing(dirs, summary) + return + if args.repair_timestamps: + run_repair_timestamps(dirs, summary, dry_run=args.dry_run) + return + selected = select_videos( + dirs, summary, force=args.force, limit=args.limit, reformat_invalid=args.reformat_invalid + ) + log.info("Selected %d of %d video directories (--jobs %d)", len(selected), len(dirs), args.jobs) + if args.dry_run: + for d in selected: + log.info("Would format %s", d.name) + summary.add("would_format", slug=d.name) + return + if not selected: + return + if client is None: + client = build_client(args.project_root) + + ctx = Context( + client=client, + args=args, + semaphore=asyncio.Semaphore(args.jobs), + rate_limiter=videolib.RateLimiter(min_interval=min_request_interval), + people=people, + software=software, + sleep=sleep, + ) + + async def go() -> list[dict[str, Any]]: + return await asyncio.gather(*(process_video(d, ctx) for d in selected)) + + for result in asyncio.run(go()): + summary.add(result.pop("status"), **result) + + formatted = summary.by_status("formatted", "error") + costs = [i.get("estimated_cost_usd") for i in formatted if i.get("estimated_cost_usd") is not None] + summary.extra["estimated_cost_usd"] = round(sum(costs), 3) + summary.extra["input_tokens"] = sum(i.get("input_tokens", 0) for i in formatted) + summary.extra["output_tokens"] = sum(i.get("output_tokens", 0) for i in formatted) + log.info( + "Formatted %d videos (%d in / %d out tokens, ~$%.2f)", + len(summary.by_status("formatted")), + summary.extra["input_tokens"], + summary.extra["output_tokens"], + summary.extra["estimated_cost_usd"], + ) + + return videolib.run_main(summary, log, args.summary_json, body) if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh new file mode 100755 index 000000000..8b0f18882 --- /dev/null +++ b/scripts/run-tests.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Run every scripts/test-*.py. Each test file is a PEP 723 uv script that +# declares its own dependencies, so each one runs in its own environment. +# Extra arguments are passed on to pytest, e.g. `./scripts/run-tests.sh -q -x`. + +set -uo pipefail +cd "$(dirname "$0")/.." || exit 1 + +failed=() +for test_file in scripts/test-*.py; do + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then echo "::group::$test_file"; else echo "==> $test_file"; fi + if ! uv run --quiet --script "$test_file" "$@"; then + failed+=("$test_file") + fi + if [[ -n "${GITHUB_ACTIONS:-}" ]]; then echo "::endgroup::"; fi +done + +if ((${#failed[@]})); then + echo + echo "Failed test files:" + printf ' %s\n' "${failed[@]}" + exit 1 +fi +echo +echo "All test files passed." diff --git a/scripts/sync-videos.py b/scripts/sync-videos.py index 9e87070ac..aa99addca 100755 --- a/scripts/sync-videos.py +++ b/scripts/sync-videos.py @@ -1,5 +1,6 @@ -#!/usr/bin/env -S uv run +#!/usr/bin/env -S uv run --script # /// script +# requires-python = ">=3.11" # dependencies = [ # "pyyaml>=6.0", # "rich>=13.0.0", @@ -16,31 +17,43 @@ and can be adjusted with include/exclude/override blocks in the _index.md. When a video's slug changes (e.g., because its title changed), the existing directory is renamed, or merged into the new one, and references are updated. + +Files are only written when their content changes, so a run without changes +on YouTube leaves the working tree untouched. The thumbnail is downloaded +again when its URL changes. + +Exit codes: 0 = success, 1 = fatal (nothing written), 2 = partial (some +videos failed; the rest was written). """ +from __future__ import annotations + +import argparse +import logging import re import shutil import sys +import time import tomllib from pathlib import Path -from typing import Any +from typing import Any, Callable -import requests import yaml -from rich.console import Console -from rich.progress import ( - BarColumn, - Progress, - SpinnerColumn, - TaskProgressColumn, - TextColumn, -) -console = Console(stderr=True) +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 + +log = logging.getLogger("sync-videos") + +THUMBNAIL_ATTEMPTS = 2 + +parse_frontmatter = videolib.split_frontmatter # --------------------------------------------------------------------------- -# Shared utilities (mirrored from update-software-frontmatter.py) +# Frontmatter formatting (mirrored from update-software-frontmatter.py; the +# output must stay byte-for-byte stable, or every page shows up as changed) # --------------------------------------------------------------------------- @@ -49,34 +62,6 @@ def ignore_aliases(self, data): return True -def parse_frontmatter(content: str) -> tuple[dict[str, Any], str, str]: - lines = content.split("\n") - - if not lines or lines[0].strip() != "---": - return {}, "", content - - end_idx = None - for i in range(1, len(lines)): - if lines[i].strip() == "---": - end_idx = i - break - - if end_idx is None: - return {}, "", content - - yaml_lines = lines[1:end_idx] - yaml_section = "\n".join(yaml_lines) - - try: - frontmatter = yaml.safe_load(yaml_section) or {} - except yaml.YAMLError as e: - console.print(f"[yellow]Warning:[/] Failed to parse YAML: {e}") - return {}, "", content - - remaining_content = "\n".join(lines[end_idx + 1 :]) - return frontmatter, yaml_section, remaining_content - - def sort_dict_with_keys_at_end(data: Any, keys_at_end: list[str]) -> Any: if isinstance(data, dict): regular_keys = [] @@ -147,19 +132,17 @@ def format_frontmatter(frontmatter: dict[str, Any]) -> str: ) yaml_str = add_blank_lines_before_keys(yaml_str, keys_at_end) - yaml_str = add_comment_after_key( - yaml_str, "external", "updated automatically, do not edit" - ) + yaml_str = add_comment_after_key(yaml_str, "external", "updated automatically, do not edit") return yaml_str.strip() -def write_frontmatter( - file_path: Path, frontmatter: dict[str, Any], remaining_content: str -) -> None: - yaml_content = format_frontmatter(frontmatter) - new_content = f"---\n{yaml_content}\n---\n{remaining_content}" - file_path.write_text(new_content, encoding="utf-8") +def render_index(frontmatter: dict[str, Any], remaining_content: str) -> str: + return f"---\n{format_frontmatter(frontmatter)}\n---\n{remaining_content}" + + +def write_frontmatter(file_path: Path, frontmatter: dict[str, Any], remaining_content: str) -> None: + videolib.atomic_write_text(file_path, render_index(frontmatter, remaining_content)) def compute_top_level_keys( @@ -198,6 +181,23 @@ def compute_top_level_keys( return result +def changed_keys(old: dict[str, Any], new: dict[str, Any]) -> list[str]: + """Top-level keys that differ, with ``external.`` for the synced block.""" + changes = [] + for key in sorted(set(old) | set(new)): + if key == "external": + old_ext = old.get("external") or {} + new_ext = new.get("external") or {} + changes += [ + f"external.{k}" + for k in sorted(set(old_ext) | set(new_ext)) + if old_ext.get(k) != new_ext.get(k) + ] + elif old.get(key) != new.get(key): + changes.append(key) + return changes + + # --------------------------------------------------------------------------- # Data loading # --------------------------------------------------------------------------- @@ -209,16 +209,13 @@ def load_videos(videos_toml: Path) -> list[dict[str, Any]]: return data.get("videos", []) -def load_titles_by_slug( - directory: Path, ignore_slugs: set[str] | None = None -) -> dict[str, str]: +def load_titles_by_slug(directory: Path, ignore_slugs: set[str] | None = None) -> dict[str, str]: titles = {} for index_file in sorted(directory.glob("*/_index.md")): slug = index_file.parent.name if ignore_slugs and slug in ignore_slugs: continue - content = index_file.read_text(encoding="utf-8") - frontmatter, _, _ = parse_frontmatter(content) + frontmatter, _, _ = parse_frontmatter(index_file.read_text(encoding="utf-8")) title = str(frontmatter.get("title", "")).strip() if title: titles[slug] = title @@ -233,11 +230,6 @@ def load_software_match_ignore(ignore_toml: Path) -> set[str]: return set(data.get("names", [])) -# --------------------------------------------------------------------------- -# Filtering -# --------------------------------------------------------------------------- - - def should_process(video: dict[str, Any]) -> tuple[bool, str]: if video.get("publish") is False: return False, "publish=false" @@ -263,52 +255,49 @@ def detect_people(video: dict[str, Any], people_names: list[str]) -> list[str]: return find_matches(combined, people_names) -def detect_software( - video: dict[str, Any], software_titles: dict[str, str] -) -> list[str]: +def detect_software(video: dict[str, Any], software_titles: dict[str, str]) -> list[str]: # Match on display titles but return slugs, since Hugo builds # /software// from the stored value. combined = video.get("title", "") + " " + video.get("description", "") - return [ - slug - for slug, title in software_titles.items() - if find_matches(combined, [title]) - ] + return [slug for slug, title in software_titles.items() if find_matches(combined, [title])] # --------------------------------------------------------------------------- # Building external # --------------------------------------------------------------------------- +EXTERNAL_FIELDS = [ + "url", + "title", + "date", + "description", + "duration", + "channel", + "playlist", + "tags", + "view_count", + "like_count", + "comment_count", + "thumbnail", + "language", + "has_captions", + "definition", + "last_updated", +] + def build_external( video: dict[str, Any], detected_people: list[str], detected_software: list[str], ) -> dict[str, Any]: - fields = [ - "url", - "title", - "date", - "description", - "duration", - "channel", - "playlist", - "tags", - "view_count", - "like_count", - "comment_count", - "thumbnail", - "language", - "has_captions", - "definition", - "last_updated", - ] external: dict[str, Any] = { - f: video[f] for f in fields if f in video and video[f] is not None + f: video[f] for f in EXTERNAL_FIELDS if f in video and video[f] is not None } if "tags" in external and isinstance(external["tags"], list): - external["tags"] = [re.sub(r"[#?/]", "", t) for t in external["tags"] if re.sub(r"[#?/]", "", t)] + external["tags"] = [ + re.sub(r"[#?/]", "", t) for t in external["tags"] if re.sub(r"[#?/]", "", t) + ] if detected_people: external["people"] = detected_people if detected_software: @@ -316,24 +305,61 @@ def build_external( return external +def build_frontmatter( + video: dict[str, Any], + frontmatter: dict[str, Any], + people_names: list[str], + software_titles: dict[str, str], +) -> dict[str, Any]: + """Return the new frontmatter for ``video``, given the current one.""" + frontmatter = dict(frontmatter) + external = build_external( + video, detect_people(video, people_names), detect_software(video, software_titles) + ) + frontmatter["external"] = external + + include = frontmatter.get("include", {}) + exclude = frontmatter.get("exclude", {}) + override = frontmatter.get("override", {}) + top_level = compute_top_level_keys(external, include, exclude, override) + + if "date" in override: + date_str = str(override["date"])[:10] + else: + date_str = str(video["date"])[:10] + + frontmatter["title"] = top_level.get("title", video["title"]) + frontmatter["resource_type"] = "video" + frontmatter["date"] = date_str + frontmatter["description"] = top_level.get("description", video["description"]) + frontmatter["people"] = top_level.get("people", []) + frontmatter["software"] = top_level.get("software", []) + frontmatter["tags"] = top_level.get("tags", []) + frontmatter["image"] = "thumbnail.jpg" + if "resources" not in frontmatter: + frontmatter["resources"] = [] + return frontmatter + + # --------------------------------------------------------------------------- # Renamed videos # --------------------------------------------------------------------------- -def merge_video_dirs(old_dir: Path, new_dir: Path) -> None: - # Keep files and manual blocks from the old directory that the new one lacks. - for item in old_dir.iterdir(): +def merge_video_dirs(old_dir: Path, new_dir: Path) -> list[str]: + """Move files and manual blocks the new directory lacks; remove the old one. + + Returns the names of the files that were moved. + """ + moved = [] + for item in sorted(old_dir.iterdir()): if item.name != "_index.md" and not (new_dir / item.name).exists(): shutil.move(str(item), new_dir / item.name) + moved.append(item.name) - old_frontmatter, _, _ = parse_frontmatter( - (old_dir / "_index.md").read_text(encoding="utf-8") - ) + old_frontmatter, _, _ = parse_frontmatter((old_dir / "_index.md").read_text(encoding="utf-8")) new_index = new_dir / "_index.md" - new_frontmatter, _, remaining_content = parse_frontmatter( - new_index.read_text(encoding="utf-8") - ) + new_frontmatter, _, remaining_content = parse_frontmatter(new_index.read_text(encoding="utf-8")) merged = False for key in ["include", "exclude", "override", "resources"]: if old_frontmatter.get(key) and not new_frontmatter.get(key): @@ -343,22 +369,28 @@ def merge_video_dirs(old_dir: Path, new_dir: Path) -> None: write_frontmatter(new_index, new_frontmatter, remaining_content) shutil.rmtree(old_dir) + return moved def update_video_references(content_dir: Path, old_slug: str, new_slug: str) -> list[Path]: pattern = re.compile(r"videos/" + re.escape(old_slug) + r"(?![\w-])") updated = [] - for md_file in content_dir.rglob("*.md"): + for md_file in sorted(content_dir.rglob("*.md")): text = md_file.read_text(encoding="utf-8") new_text = pattern.sub(f"videos/{new_slug}", text) if new_text != text: - md_file.write_text(new_text, encoding="utf-8") + videolib.atomic_write_text(md_file, new_text) updated.append(md_file) return updated def handle_renamed_videos( - videos: list[dict[str, Any]], videos_dir: Path, content_dir: Path + videos: list[dict[str, Any]], + videos_dir: Path, + content_dir: Path, + summary: videolib.RunSummary, + *, + dry_run: bool = False, ) -> None: # Slugs are derived from YouTube titles, so a title change produces a new # slug. Match directories to videos by URL and move or merge stale ones. @@ -375,21 +407,37 @@ def handle_renamed_videos( url = (frontmatter.get("external") or {}).get("url") new_slug = slug_by_url.get(url) if not new_slug: - console.print( - f" [yellow]Warning:[/] {old_slug} is not in videos.toml, leaving as is" - ) + log.warning("Orphan: %s is not in videos.toml (or not published); leaving it as is", old_slug) + summary.add("orphan", slug=old_slug, url=url, video_id=videolib.youtube_id(url)) continue new_dir = videos_dir / new_slug - if new_dir.exists(): - merge_video_dirs(old_dir, new_dir) - console.print(f" [magenta]⇢[/] Merged {old_slug} into {new_slug}") + action = "merged" if new_dir.exists() else "renamed" + log.info("%s %s -> %s (%s)", "Merging" if action == "merged" else "Renaming", old_slug, new_slug, videolib.youtube_id(url)) + if dry_run: + summary.add("renamed", action=action, old_slug=old_slug, slug=new_slug, url=url, video_id=videolib.youtube_id(url), dry_run=True) + continue + + moved = None + if action == "merged": + moved = merge_video_dirs(old_dir, new_dir) + log.info(" Moved files: %s", ", ".join(moved) or "(none)") else: old_dir.rename(new_dir) - console.print(f" [magenta]⇢[/] Renamed {old_slug} to {new_slug}") - for md_file in update_video_references(content_dir, old_slug, new_slug): - console.print(f" [dim]Updated reference in {md_file}[/]") + updated = update_video_references(content_dir, old_slug, new_slug) + for md_file in updated: + log.info(" Updated reference in %s", md_file.relative_to(content_dir.parent)) + summary.add( + "renamed", + action=action, + old_slug=old_slug, + slug=new_slug, + url=url, + video_id=videolib.youtube_id(url), + moved_files=moved, + updated_references=[str(p.relative_to(content_dir.parent)) for p in updated], + ) # --------------------------------------------------------------------------- @@ -397,19 +445,26 @@ def handle_renamed_videos( # --------------------------------------------------------------------------- -def download_thumbnail(url: str, dest_dir: Path) -> bool: - dest = dest_dir / "thumbnail.jpg" - if dest.exists(): - return False - - try: - response = requests.get(url, timeout=15) - response.raise_for_status() - dest.write_bytes(response.content) - return True - except requests.RequestException as e: - console.print(f" [yellow]Warning:[/] Could not download thumbnail: {e}") - return False +def download_thumbnail( + url: str, + dest: Path, + http_get: Callable[..., Any], + *, + attempts: int = THUMBNAIL_ATTEMPTS, + retry_delay: float = 2.0, +) -> None: + """Download ``url`` to ``dest`` atomically; raise after ``attempts`` tries.""" + for attempt in range(1, attempts + 1): + try: + response = http_get(url, timeout=15) + response.raise_for_status() + videolib.atomic_write_bytes(dest, response.content) + return + except Exception as e: + if attempt == attempts: + raise + log.warning(" Thumbnail download failed (attempt %d/%d): %s; retrying", attempt, attempts, e) + time.sleep(retry_delay) # --------------------------------------------------------------------------- @@ -422,60 +477,60 @@ def process_video( videos_dir: Path, people_names: list[str], software_titles: dict[str, str], -) -> str: - try: - slug = video["slug"] - video_dir = videos_dir / slug - index_file = video_dir / "_index.md" - is_new = not video_dir.exists() - - if is_new: - video_dir.mkdir(parents=True, exist_ok=True) - frontmatter: dict[str, Any] = {} - remaining_content = "\n" - else: - content = index_file.read_text(encoding="utf-8") - frontmatter, _, remaining_content = parse_frontmatter(content) - - detected_people = detect_people(video, people_names) - detected_software = detect_software(video, software_titles) - - external = build_external(video, detected_people, detected_software) - frontmatter["external"] = external - - include = frontmatter.get("include", {}) - exclude = frontmatter.get("exclude", {}) - override = frontmatter.get("override", {}) - - top_level = compute_top_level_keys(external, include, exclude, override) - - if "date" in override: - date_str = str(override["date"])[:10] + *, + http_get: Callable[..., Any], + dry_run: bool = False, + retry_delay: float = 2.0, +) -> dict[str, Any]: + """Create or update one video directory; return a summary item.""" + slug = video["slug"] + video_dir = videos_dir / slug + index_file = video_dir / "_index.md" + is_new = not index_file.exists() + + if is_new: + old_frontmatter: dict[str, Any] = {} + remaining_content = "\n" + old_text = None + else: + old_text = index_file.read_text(encoding="utf-8") + old_frontmatter, _, remaining_content = parse_frontmatter(old_text) + + frontmatter = build_frontmatter(video, old_frontmatter, people_names, software_titles) + new_text = render_index(frontmatter, remaining_content) + + result: dict[str, Any] = {"slug": slug, "url": video.get("url"), "video_id": videolib.youtube_id(video.get("url"))} + if is_new: + result["status"] = "created" + result["title"] = frontmatter["title"] + elif new_text != old_text: + result["status"] = "updated" + result["changes"] = changed_keys(old_frontmatter, frontmatter) + else: + result["status"] = "unchanged" + + if new_text != old_text and not dry_run: + video_dir.mkdir(parents=True, exist_ok=True) + videolib.atomic_write_text(index_file, new_text) + + thumbnail = video_dir / "thumbnail.jpg" + old_thumbnail_url = (old_frontmatter.get("external") or {}).get("thumbnail") + thumbnail_url = video.get("thumbnail") + if thumbnail_url and (not thumbnail.exists() or (old_thumbnail_url and old_thumbnail_url != thumbnail_url)): + reason = "missing" if not thumbnail.exists() else "url changed" + if dry_run: + result["thumbnail"] = f"would download ({reason})" else: - date_val = video["date"] - date_str = str(date_val)[:10] - - frontmatter["title"] = top_level.get("title", video["title"]) - frontmatter["resource_type"] = "video" - frontmatter["date"] = date_str - frontmatter["description"] = top_level.get("description", video["description"]) - frontmatter["people"] = top_level.get("people", []) - frontmatter["software"] = top_level.get("software", []) - frontmatter["tags"] = top_level.get("tags", []) - frontmatter["image"] = "thumbnail.jpg" - if "resources" not in frontmatter: - frontmatter["resources"] = [] - - write_frontmatter(index_file, frontmatter, remaining_content) - download_thumbnail(video["thumbnail"], video_dir) - - return "created" if is_new else "updated" - - except Exception as e: - console.print( - f" [bold red]Error:[/] Failed to process {video.get('slug', '?')}: {e}" - ) - return "error" + try: + video_dir.mkdir(parents=True, exist_ok=True) + download_thumbnail(thumbnail_url, thumbnail, http_get, retry_delay=retry_delay) + result["thumbnail"] = f"downloaded ({reason})" + log.debug(" Downloaded thumbnail for %s (%s)", slug, reason) + except Exception as e: # noqa: BLE001 - recorded as an item error + result["status"] = "error" + result["reason"] = "thumbnail_failed" + result["detail"] = f"{type(e).__name__}: {e} ({thumbnail_url})" + return result # --------------------------------------------------------------------------- @@ -483,89 +538,92 @@ def process_video( # --------------------------------------------------------------------------- -def main() -> None: - console.print("\n[bold blue]Video Syncer[/]\n") - - script_dir = Path(__file__).parent - project_root = script_dir.parent - videos_toml = project_root / "data" / "videos.toml" - videos_dir = project_root / "content" / "resources" / "videos" - people_dir = project_root / "content" / "people" - software_dir = project_root / "content" / "software" - ignore_toml = project_root / "data" / "software-match-ignore.toml" - - for path, label in [ - (videos_toml, "videos TOML"), - (people_dir, "people directory"), - (software_dir, "software directory"), - ]: - if not path.exists(): - console.print(f"[bold red]Error:[/] {label} not found: {path}") - sys.exit(1) - - all_videos = load_videos(videos_toml) - people_names = list(load_titles_by_slug(people_dir).values()) - ignore_slugs = load_software_match_ignore(ignore_toml) - software_titles = load_titles_by_slug(software_dir, ignore_slugs=ignore_slugs) - - console.print( - f"[dim]Loaded {len(all_videos)} videos, " - f"{len(people_names)} people, " - f"{len(software_titles)} software entries " - f"({len(ignore_slugs)} ignored)[/]\n" - ) - - videos_dir.mkdir(parents=True, exist_ok=True) - - handle_renamed_videos( - [v for v in all_videos if should_process(v)[0]], - videos_dir, - project_root / "content", - ) - - created = updated = skipped = errors = 0 - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TaskProgressColumn(), - console=console, - ) as progress: - task = progress.add_task("[cyan]Syncing...", total=len(all_videos)) - - for video in all_videos: - slug = video.get("slug", "?") - progress.update(task, description=f"[cyan]{slug}") +def main(argv: list[str] | None = None, http_get: Callable[..., Any] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.strip().split("\n\n")[0]) + parser.add_argument("--dry-run", action="store_true", help="show what would change; write nothing") + videolib.add_common_args(parser) + args = parser.parse_args(argv) + + videolib.setup_logging("sync-videos", args.verbose) + summary = videolib.RunSummary("sync-videos") + summary.extra["dry_run"] = args.dry_run + + root = args.project_root + videos_toml = root / "data" / "videos.toml" + videos_dir = root / "content" / "resources" / "videos" + people_dir = root / "content" / "people" + software_dir = root / "content" / "software" + ignore_toml = root / "data" / "software-match-ignore.toml" + + def body() -> None: + nonlocal http_get + summary.environment = videolib.log_environment(log, packages=["pyyaml", "requests"]) + for path, label in [ + (videos_toml, "videos TOML"), + (people_dir, "people directory"), + (software_dir, "software directory"), + ]: + if not path.exists(): + summary.fatal = f"{label} not found: {path}" + return + if http_get is None: + import requests + + http_get = requests.get + + all_videos = load_videos(videos_toml) + people_names = list(load_titles_by_slug(people_dir).values()) + ignore_slugs = load_software_match_ignore(ignore_toml) + software_titles = load_titles_by_slug(software_dir, ignore_slugs=ignore_slugs) + log.info( + "Loaded %d videos, %d people, %d software entries (%d ignored)", + len(all_videos), + len(people_names), + len(software_titles), + len(ignore_slugs), + ) - ok, _ = should_process(video) - if not ok: - skipped += 1 + published = [v for v in all_videos if should_process(v)[0]] + summary.tally("unpublished", len(all_videos) - len(published)) + + videos_dir.mkdir(parents=True, exist_ok=True) + handle_renamed_videos(published, videos_dir, root / "content", summary, dry_run=args.dry_run) + + with videolib.make_progress() as progress: + task = progress.add_task("Syncing", total=len(published)) + for video in published: + progress.update(task, description=video.get("slug", "?")) + try: + result = process_video( + video, videos_dir, people_names, software_titles, http_get=http_get, dry_run=args.dry_run + ) + except Exception as e: # noqa: BLE001 - recorded as an item error + log.exception("Failed to process %s", video.get("slug", "?")) + result = { + "status": "error", + "slug": video.get("slug"), + "video_id": videolib.youtube_id(video.get("url")), + "reason": "exception", + "detail": f"{type(e).__name__}: {e}", + } + status = result.pop("status") + if status == "created": + log.info("Created %s (%s)", result["slug"], result["video_id"]) + elif status == "updated": + log.info("Updated %s (%s): %s", result["slug"], result["video_id"], ", ".join(result["changes"])) + elif status == "error": + log.error("Error %s (%s): %s", result["slug"], result.get("video_id"), result.get("detail")) + if "thumbnail" in result: + log.info(" Thumbnail %s: %s", result["slug"], result["thumbnail"]) + + if status == "unchanged" and "thumbnail" not in result: + summary.tally("unchanged") + else: + summary.add(status, **result) progress.advance(task) - continue - - result = process_video(video, videos_dir, people_names, software_titles) - if result == "created": - console.print(f" [green]✓[/] Created {slug}") - created += 1 - elif result == "updated": - console.print(f" [cyan]↑[/] Updated {slug}") - updated += 1 - elif result == "error": - errors += 1 - - progress.advance(task) - - console.print("\n[bold]Summary:[/]") - console.print(f" [green]✓[/] Created: {created}") - console.print(f" [cyan]↑[/] Updated: {updated}") - console.print(f" [dim]○[/] Skipped: {skipped}") - if errors: - console.print(f" [red]✗[/] Errors: {errors}") - sys.exit(1) - console.print("\n[bold green]✓ Done![/]\n") + return videolib.run_main(summary, log, args.summary_json, body) if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/scripts/test-format-transcriptions.py b/scripts/test-format-transcriptions.py new file mode 100755 index 000000000..77d930d68 --- /dev/null +++ b/scripts/test-format-transcriptions.py @@ -0,0 +1,590 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "anthropic>=0.40.0", +# "python-dotenv>=1.0.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for format-transcriptions.py. The Anthropic API is faked.""" + +import json +import re +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 +from videotesting import ( # noqa: E402 + SAMPLE_VTT, + FakeAnthropic, + RateLimitError, + StatusError, + load_script, + make_repo, + no_sleep, + vtt_to_html, + write_video_dir, +) + +f = load_script("format-transcriptions.py") + +CUES = videolib.parse_vtt(SAMPLE_VTT) +GOOD_HTML = vtt_to_html(SAMPLE_VTT) + + +# --- Pure helpers --- + + +@pytest.mark.parametrize( + "raw, clean", + [ + ("```html\n

x

\n```", "

x

"), + ("```\n

x

```", "

x

"), + ("

x

", "

x

"), + ], +) +def test_strip_markdown_fence(raw, clean): + assert f.strip_markdown_fence(raw) == clean + + +def test_build_user_message(): + fm = {"title": "T", "people": ["A"], "software": ["quarto"], "description": "d" * 700} + msg = f.build_user_message("WEBVTT", fm) + header, vtt = msg.split("\n\n---\n\n") + assert vtt == "WEBVTT" + assert header.splitlines() == [ + "Video title: T", + "People featured: A", + "Technologies mentioned: quarto", + "Description: " + "d" * 600, + ] + assert f.build_user_message("WEBVTT", {}) == "WEBVTT" + + +def test_split_vtt_into_chunks(): + cues = [(float(t), float(t) + 4, f"cue {t}") for t in range(0, 100, 10)] + chunks = f.split_vtt_into_chunks(videolib.render_vtt(cues), 30) + assert [start for _, start in chunks] == [0, 30, 60, 90] + assert [len(videolib.parse_vtt(vtt)) for vtt, _ in chunks] == [3, 3, 3, 1] + + +def test_chunk_system_prompt(): + system = f.build_chunk_system(1, 4) + assert system.startswith(f.PROMPT.rstrip()) + assert "This is segment 2 of 4." in system + + +class TestCoverage: + def test_full(self): + assert f.coverage(CUES, GOOD_HTML) == 1.0 + + def test_blockquotes_and_headings_dont_count(self): + quote = "
Hello everyone, today we talk about Quarto.
" + html = "

It was built by the team that also made Shiny

" + quote + assert f.coverage(CUES, html) == 0.0 + + def test_entities_and_case(self): + cues = [(0.0, 1.0, "R & Python, it's GREAT")] + assert f.coverage(cues, '

r & python, it's great

') == 1.0 + + def test_repeated_cues_count_once(self): + cues = [(0.0, 1.0, "Rishabh Sharma Vemuri & Abiye Berhanu")] * 9 + [(9.0, 10.0, "And now the talk.")] + html = '

Rishabh Sharma Vemuri & Abiye Berhanu And now the talk.

' + assert f.coverage(cues, html) == 1.0 + + def test_multiplicity(self): + cues = [(0.0, 1.0, "the the the the")] + assert f.coverage(cues, '

the the

') == 0.5 + + +class TestValidateHtml: + def test_valid(self): + metrics = f.validate_html(GOOD_HTML, CUES, 26) + assert metrics == {"coverage": 1.0, "anchors": 3, "warnings": None} + + def test_allowed_structure(self): + html = ( + GOOD_HTML + + "\n\n

More

Hadley Wickham joins us later.
" + ) + assert f.validate_html(html, CUES, 26)["anchors"] == 4 + + @pytest.mark.parametrize( + "html, reason", + [ + ("Here is your HTML:

", "not_html"), + ("
" + GOOD_HTML + "
", "disallowed_tag"), + (GOOD_HTML + "

x

", "disallowed_tag"), + (GOOD_HTML.replace("

", "").replace("

", ""), "disallowed_tag"), + (GOOD_HTML.replace("

", '

'), "disallowed_attribute"), + (GOOD_HTML.replace('', ''), "disallowed_attribute"), + (GOOD_HTML.replace('', ''), "disallowed_attribute"), + (GOOD_HTML.replace('', ''), "disallowed_attribute"), + (GOOD_HTML.replace('data-t="20"', 'data-t="120"'), "bad_timestamp"), + ('Hello everyone, today we talk about Quarto.', "no_paragraphs"), + ('

Hello everyone.

', "low_coverage"), + ], + ) + def test_invalid(self, html, reason): + with pytest.raises(f.FormatError) as e: + f.validate_html(html, CUES, 26) + assert e.value.reason == reason + + def test_small_overshoot_is_a_warning(self): + html = GOOD_HTML.replace('data-t="20"', 'data-t="70"') # 44 s beyond 26 s, under 60 s + metrics = f.validate_html(html, CUES, 26) + assert metrics["warnings"] == ['timestamp_overshoot: data-t="70" is 44s beyond the end of the video'] + + def test_overshoot_tolerance_scales_with_duration(self): + cues = [(0.0, 3600.0, "word " * 10)] + html = '

' + "word " * 10 + "

" # 300 s over, under 10 % + assert f.validate_html(html, cues, 3600)["warnings"] + html = '

' + "word " * 10 + "

" # 400 s over + with pytest.raises(f.FormatError): + f.validate_html(html, cues, 3600) + + def test_coverage_threshold(self): + cues = [(0.0, 1.0, " ".join(f"w{i}" for i in range(100)))] + + def html(n): + return '

' + " ".join(f"w{i}" for i in range(n)) + "

" + + assert f.validate_html(html(95), cues, 1)["coverage"] == 0.95 + assert f.validate_html(html(90), cues, 1)["coverage"] == 0.90 + with pytest.raises(f.FormatError, match="80.0%"): + f.validate_html(html(80), cues, 1) + + +# --- Timestamp repair --- + + +def cues_from(*texts, step=10): + return [(float(i * step) + 0.7, float(i * step) + 9, t) for i, t in enumerate(texts)] + + +class TestRepairTimestamps: + cues = cues_from( + "Hello everyone and welcome.", + "Today we talk about Quarto,", + "the publishing system.", + "And then about Shiny.", + "Thanks for watching.", + ) + + def test_wrong_values_are_fixed(self): + html = ( + '

Hello everyone and welcome. Today we talk about Quarto, ' + 'the publishing system.

\n\n

And then about Shiny.

' + ) + out, stats = f.repair_timestamps(html, self.cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "10", "20", "30"] + assert stats == {"anchors": 4, "matched": 4, "changed": 3, "unmatched": 0} + assert re.sub(r'data-t="\d+"', "", out) == re.sub(r'data-t="\d+"', "", html) # only data-t changed + + def test_merged_cues_get_the_first_start(self): + html = '

Today we talk about Quarto, the publishing system.

' + out, _ = f.repair_timestamps(html, self.cues) + assert 'data-t="10"' in out + + def test_anchor_starting_mid_cue(self): + html = '

Hello everyone and welcome.

' + out, _ = f.repair_timestamps(html, self.cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "0"] + + def test_corrected_leading_word(self): + # Whisper heard "Korto"; the model corrected it to "Quarto". + cues = cues_from("Hello everyone and welcome.", "Korto is great for documents.") + html = '

Hello everyone and welcome. Quarto is great for documents.

' + out, stats = f.repair_timestamps(html, cues) + assert 'data-t="10"' in out and stats["unmatched"] == 0 + + def test_repeated_phrase_searches_forward(self): + cues = cues_from("Thank you.", "Something in between.", "Thank you.") + html = '

Thank you. Something in between. Thank you.

' + out, _ = f.repair_timestamps(html, cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "10", "20"] + + def test_short_anchor_does_not_jump_far_ahead(self): + # "For us," occurs again much later; matching it there would leave + # every following anchor behind the search position. + filler = [f"filler sentence number {i} goes here." for i in range(30)] + cues = cues_from("Something else entirely.", "and then more words.", *filler, "For us, it was later.") + html = ( + '

For us, Something else entirely. ' + 'and then more words.

' + ) + out, stats = f.repair_timestamps(html, cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "0", "10"] + assert stats["unmatched"] == 1 + + def test_blockquote_reuses_prose_value(self): + html = ( + '

Hello everyone and welcome. And then about Shiny.

\n\n' + '
And then about Shiny.
\n\n' + '

Thanks for watching.

' + ) + out, stats = f.repair_timestamps(html, self.cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "30", "30", "40"] + assert stats["anchors"] == 3 # pull quotes aren't counted twice + + def test_unmatched_blockquote_capped(self): + html = ( + '

Hello everyone and welcome.

\n\n' + '
A quote that never appeared.
' + ) + out, _ = f.repair_timestamps(html, self.cues) + assert re.findall(r'data-t="(\d+)"', out) == ["0", "40"] + + def test_pieces_split_by_inject_links_stay_together(self): + linked = f.inject_links( + '

Today we talk about Quarto, the publishing system.

', + {}, + {"Quarto": "/software/quarto/"}, + ) + assert 'Quarto' in linked + out, _ = f.repair_timestamps(linked, self.cues) + # "Today we talk about ", "," (the pieces around the link), "the publishing system." + assert re.findall(r'data-t="(\d+)"', out) == ["10", "10", "20"] + assert 'Quarto' in out + + def test_unmatched_keeps_value_but_capped_at_last_cue(self): + html = '

Words that were never said. Neither were these.

' + out, stats = f.repair_timestamps(html, self.cues) + assert re.findall(r'data-t="(\d+)"', out) == ["5", "40"] + assert stats["unmatched"] == 2 + + def test_no_cues(self): + html = '

x

' + assert f.repair_timestamps(html, [])[0] == html + + +@pytest.mark.parametrize( + "html, fixed, count", + [ + ('

x

', '

x

', 1), + ('

x

', '

x

', 1), + ('

x

', '

x

', 1), + ('

a a data-t="6"

', '

a a data-t="6"

', 0), + (GOOD_HTML, GOOD_HTML, 0), + ], +) +def test_fix_anchor_typos(html, fixed, count): + assert f.fix_anchor_typos(html) == (fixed, count) + + +def test_estimate_cost(): + assert f.estimate_cost("claude-sonnet-4-6", 1_000_000, 100_000) == 4.5 + assert f.estimate_cost("claude-haiku-4-5", 1_000_000, 0) == 1.0 + assert f.estimate_cost("some-other-model", 1, 1) is None + + +# --- Entity linking --- + + +class TestInjectLinks: + people = {"Hadley Wickham": "/people/hadley-wickham/"} + software = {"Quarto": "/software/quarto/", "Shiny": "/software/shiny/", "Shiny for Python": "/software/shiny-python/"} + + def test_splits_anchor_and_links_first_occurrence_only(self): + html = '

I love quarto a lot. Quarto again.

' + assert f.inject_links(html, {}, self.software) == ( + '

I love quarto a lot. ' + 'Quarto again.

' + ) + + def test_match_at_anchor_start_and_end(self): + html = '

Shiny

' + assert f.inject_links(html, {}, self.software) == '

Shiny

' + + def test_longest_name_first(self): + html = '

We use Shiny for Python here.

' + out = f.inject_links(html, {}, self.software) + assert 'Shiny for Python' in out + assert '"/software/shiny/"' not in out + + def test_no_link_inside_an_inserted_link(self): + # A heading is plain text: "Shiny" must not match inside the URL of + # the "Shiny for Python" link inserted just before. + html = "

Building the Shiny for Python app

" + software = {"Shiny for Python": "/software/shiny-python/", "Shiny": "/software/shiny-r/"} + assert f.inject_links(html, {}, software) == ( + '

Building the Shiny for Python app

' + ) + + def test_people_are_case_sensitive(self): + html = '

hadley wickham and Hadley Wickham

' + out = f.inject_links(html, self.people, {}) + assert out == ( + '

hadley wickham and Hadley Wickham

' + ) + + def test_never_nested(self): + out = f.inject_links(GOOD_HTML, self.people, self.software) + assert out.count(" opens inside another . + depth = 0 + for tag in re.findall(r"Quarto' in out + assert 'Hadley Wickham' in out + (call,) = client.calls + assert call["model"] == "claude-sonnet-4-6" + assert call["system"] == f.PROMPT + assert call["messages"][0]["content"].startswith("Video title: Quarto talk\n") + (done,) = items(data, "formatted") + assert done["coverage"] == 1.0 + assert done["input_tokens"] == 1000 and done["output_tokens"] == 500 + assert done["request_ids"] == ["req_fake"] + assert done["estimated_cost_usd"] == pytest.approx(0.0105) + assert data["extra"]["model"] == "claude-sonnet-4-6" + + def test_selection(self, repo): + write_video_dir(repo, "2026-01-04_done", url=URL, vtt=SAMPLE_VTT, html="

x

") + write_video_dir(repo, "2026-01-03_excluded", url=URL, vtt=SAMPLE_VTT, extra="transcribe: false\n") + write_video_dir(repo, "2026-01-02_novtt", url=URL) + write_video_dir(repo, "2026-01-01_todo", url=URL, duration=26, vtt=SAMPLE_VTT) + code, data = run(repo) + assert [i["slug"] for i in items(data, "formatted")] == ["2026-01-01_todo"] + assert data["counts"] == {"already_formatted": 1, "excluded": 1, "formatted": 1, "no_vtt": 1} + assert data["extra"]["needs_transcript"] == ["2026-01-02_novtt"] + + def test_limit_newest_first(self, repo): + for i in range(1, 4): + write_video_dir(repo, f"2026-01-0{i}_v", url=URL, duration=26, vtt=SAMPLE_VTT) + code, data = run(repo, "--limit", "2") + assert [i["slug"] for i in items(data, "formatted")] == ["2026-01-03_v", "2026-01-02_v"] + assert data["counts"]["deferred"] == 1 + + def test_force(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT, html="

old

") + run(repo, "--force") + assert "old" not in (d / "transcription.txt").read_text() + + def test_truncated_response_rejected(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + code, data = run(repo, client=FakeAnthropic([(GOOD_HTML, "max_tokens")])) + assert code == 2 + (error,) = items(data, "error") + assert error["reason"] == "truncated" + assert "max_tokens=16384" in error["detail"] + assert not (d / "transcription.txt").exists() + + def test_invalid_html_not_written(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + code, data = run(repo, client=FakeAnthropic(["
nope
", "
nope
"])) + assert code == 2 + assert items(data, "error")[0]["reason"] == "disallowed_tag" + assert not (d / "transcription.txt").exists() + + def test_low_coverage_retried(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + client = FakeAnthropic(['

Hello everyone.

']) # then a complete answer + code, data = run(repo, client=client) + assert code == 0 and len(client.calls) == 2 + (done,) = items(data, "formatted") + assert done["coverage"] == 1.0 and done["coverage_retries"] == 1 + assert done["input_tokens"] == 2000 # both requests are counted + assert (d / "transcription.txt").exists() + + def test_retry_keeps_the_better_answer(self, repo): + cues = [(float(t), float(t) + 5, f"sentence number {t}.") for t in range(0, 1500, 10)] + vtt = videolib.render_vtt(cues) + write_video_dir(repo, "2026-01-01_long", url=URL, duration=1500, vtt=vtt) + chunks = f.split_vtt_into_chunks(vtt, f.CHUNK_DURATION_S) + assert len(chunks) == 2 + good = [vtt_to_html(c) for c, _ in chunks] + half = vtt_to_html(videolib.render_vtt(videolib.parse_vtt(chunks[1][0])[:40])) + worse = vtt_to_html(videolib.render_vtt(videolib.parse_vtt(chunks[1][0])[:10])) + client = FakeAnthropic([good[0], half, worse]) + code, data = run(repo, client=client) + assert len(client.calls) == 3 # only chunk 2 is asked again + (error,) = items(data, "error") + assert error["reason"] == "low_coverage" + assert error["coverage_retries"] == 1 + assert error["chunk_coverage"][0] == 1.0 + assert 0.6 < error["chunk_coverage"][1] < 0.7 # the first, better answer was kept + + def test_rejected_output_saved(self, repo, tmp_path): + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + rejected = tmp_path / "rejected" + code, data = run(repo, "--rejected-dir", str(rejected), client=FakeAnthropic(['

Hello.

'] * 2)) + assert code == 2 + assert (rejected / "2026-01-01_a.html").read_text() == '

Hello.

' + assert items(data, "error")[0]["rejected_output"] == str(rejected / "2026-01-01_a.html") + + def test_anchor_typos_fixed_before_validation(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + code, data = run(repo, client=FakeAnthropic([GOOD_HTML.replace('', '')])) + assert code == 0 + assert "") + + def test_overloaded_api_retried(self, repo): + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + client = FakeAnthropic([StatusError(529, "overloaded"), RateLimitError(), GOOD_HTML]) + code, _ = run(repo, client=client) + assert code == 0 and len(client.calls) == 3 + + def test_auth_error_not_retried(self, repo): + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + client = FakeAnthropic([StatusError(401, "invalid x-api-key")]) + code, data = run(repo, client=client) + assert code == 2 and len(client.calls) == 1 + assert items(data, "error")[0]["detail"] == "StatusError status=401 request_id=req_401: invalid x-api-key" + + def test_long_video_is_chunked(self, repo): + cues = [(float(t), float(t) + 5, f"sentence number {t}.") for t in range(0, 3000, 10)] + d = write_video_dir(repo, "2026-01-01_long", url=URL, duration=3000, vtt=videolib.render_vtt(cues)) + client = FakeAnthropic() + code, data = run(repo, client=client) + assert code == 0 + assert len(client.calls) == 4 # 50 minutes in 15-minute chunks + assert "This is segment 1 of 4." in client.calls[0]["system"] + assert "This is segment 4 of 4." in client.calls[3]["system"] + out = (d / "transcription.txt").read_text() + assert out.count("") == 3 + (done,) = items(data, "formatted") + assert done["chunks"] == 4 and done["input_tokens"] == 4000 + assert done["chunk_coverage"] == [1.0, 1.0, 1.0, 1.0] + + def test_short_video_not_chunked(self, repo): + cues = [(float(t), float(t) + 5, f"sentence number {t}.") for t in range(0, 1100, 10)] + write_video_dir(repo, "2026-01-01_a", url=URL, duration=1100, vtt=videolib.render_vtt(cues)) + client = FakeAnthropic() + run(repo, client=client) + assert len(client.calls) == 1 + + def test_model_from_env(self, repo, monkeypatch): + monkeypatch.setenv("TRANSCRIPT_MODEL", "claude-haiku-4-5") + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + client = FakeAnthropic() + code, data = run(repo, client=client) + assert client.calls[0]["model"] == "claude-haiku-4-5" + code, data = run(repo, "--force", "--model", "claude-sonnet-4-6", client=client) + assert client.calls[1]["model"] == "claude-sonnet-4-6" + + def test_check_existing_and_reformat_invalid(self, repo): + linked = f.inject_links(GOOD_HTML, {"Hadley Wickham": "/people/hadley-wickham/"}, {}) + write_video_dir(repo, "2026-01-02_ok", url=URL, duration=26, vtt=SAMPLE_VTT, html=linked) + bad = write_video_dir( + repo, "2026-01-01_bad", url=URL, duration=26, vtt=SAMPLE_VTT, html='

Hello.

' + ) + client = FakeAnthropic() + code, data = run(repo, "--check-existing", client=client) + assert code == 0 and client.calls == [] + (invalid,) = items(data, "invalid") + assert invalid["slug"] == "2026-01-01_bad" and invalid["reason"] == "low_coverage" + assert data["counts"]["valid"] == 1 + + code, data = run(repo, "--reformat-invalid", client=client) + assert [i["slug"] for i in items(data, "formatted")] == ["2026-01-01_bad"] + assert len(client.calls) == 1 + assert f.check_existing(bad)[0] + + def test_timestamps_repaired_before_writing(self, repo): + d = write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + shifted = GOOD_HTML.replace('data-t="4"', 'data-t="9"').replace('data-t="20"', 'data-t="24"') + code, data = run(repo, client=FakeAnthropic([shifted])) + assert code == 0 + values = re.findall(r'data-t="(\d+)"', (d / "transcription.txt").read_text()) + assert set(values) == {"0", "4", "20"} # pieces split by inject_links repeat a value + (done,) = items(data, "formatted") + assert done["timestamps_repaired"] == 2 and done["timestamps_unmatched"] == 0 + + def test_repair_timestamps_mode(self, repo): + shifted = GOOD_HTML.replace('data-t="20"', 'data-t="23"') + d = write_video_dir(repo, "2026-01-02_a", url=URL, duration=26, vtt=SAMPLE_VTT, html=shifted) + write_video_dir(repo, "2026-01-01_ok", url=URL, duration=26, vtt=SAMPLE_VTT, html=GOOD_HTML) + client = FakeAnthropic() + code, data = run(repo, "--repair-timestamps", "--dry-run", client=client) + assert (d / "transcription.txt").read_text() == shifted + code, data = run(repo, "--repair-timestamps", client=client) + assert code == 0 and client.calls == [] + assert 'data-t="20"' in (d / "transcription.txt").read_text() + (fixed,) = items(data, "timestamps_repaired") + assert fixed["slug"] == "2026-01-02_a" and fixed["changed"] == 1 + assert data["counts"]["timestamps_ok"] == 1 + + def test_dry_run(self, repo): + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + client = FakeAnthropic() + code, data = run(repo, "--dry-run", client=client) + assert code == 0 and client.calls == [] + assert items(data, "would_format")[0]["slug"] == "2026-01-01_a" + + def test_missing_api_key_is_fatal(self, repo, monkeypatch): + monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False) + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + summary = repo / "s.json" + code = f.main(["--project-root", str(repo), "--summary-json", str(summary)]) + assert code == 1 + assert "ANTHROPIC_API_KEY" in json.loads(summary.read_text())["fatal"] + + def test_logging(self, repo, capsys, monkeypatch): + monkeypatch.setenv("CI", "true") + write_video_dir(repo, "2026-01-01_a", url=URL, duration=26, vtt=SAMPLE_VTT) + run(repo) + err = capsys.readouterr().err + assert "2026-01-01_a (aaaaaaaaaaa): claude-sonnet-4-6 responded in" in err + assert "1000 in / 500 out tokens, stop_reason=end_turn, request_id=req_fake" in err + assert "wrote transcription.txt (3 anchors, 100.0% coverage, ~$0.011)" in err + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/test-sync-videos.py b/scripts/test-sync-videos.py new file mode 100755 index 000000000..3e226dd1a --- /dev/null +++ b/scripts/test-sync-videos.py @@ -0,0 +1,325 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "requests>=2.28.0", +# "tomli-w>=1.0.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for sync-videos.py.""" + +import json +import sys +from pathlib import Path + +import pytest +import tomli_w + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from videotesting import FakeHttp, load_script, make_repo, write # noqa: E402 + +s = load_script("sync-videos.py") + + +def video(vid="abcdefghijk", title="Intro to Quarto with Hadley Wickham", **overrides): + v = { + "url": f"https://www.youtube.com/watch?v={vid}", + "title": title, + "slug": f"2026-10-01_{title.lower().replace(' ', '-')}", + "date": "2026-10-01T12:00:00Z", + "description": "Learn Quarto and Shiny. #quarto", + "duration": 600, + "channel": "Posit PBC", + "playlist": "", + "tags": ["#quarto", "R/Python", "?", "data"], + "view_count": 1, + "like_count": 1, + "comment_count": 0, + "thumbnail": f"https://i.ytimg.com/vi/{vid}/hqdefault.jpg", + "language": "en", + "has_captions": False, + "definition": "hd", + "publish": True, + "last_updated": "2026-11-01T00:00:00+00:00", + } + v.update(overrides) + return v + + +def write_videos(repo, videos): + (repo / "data/videos.toml").write_text(tomli_w.dumps({"videos": videos})) + + +def run(repo, http=None, *args): + summary = repo / "summary.json" + code = s.main( + ["--project-root", str(repo), "--summary-json", str(summary), *args], + http_get=http or FakeHttp(), + ) + return code, json.loads(summary.read_text()) + + +def items(data, status): + return [i for i in data["items"] if i["status"] == status] + + +@pytest.fixture +def repo(tmp_path): + return make_repo(tmp_path) + + +# --- Pure helpers --- + + +class TestComputeTopLevelKeys: + def test_external_only(self): + assert s.compute_top_level_keys({"title": "T", "people": ["A"]}, {}, {}, {}) == { + "title": "T", + "people": ["A"], + } + + def test_include_dedupes(self): + result = s.compute_top_level_keys({"software": ["quarto"]}, {"software": ["shiny", "quarto"]}, {}, {}) + assert result["software"] == ["quarto", "shiny"] + + def test_include_without_external_value(self): + assert s.compute_top_level_keys({}, {"people": ["A"]}, {}, {})["people"] == ["A"] + + def test_exclude(self): + result = s.compute_top_level_keys({"people": ["A", "B"]}, {}, {"people": ["B"]}, {}) + assert result["people"] == ["A"] + + def test_override_wins(self): + result = s.compute_top_level_keys({"title": "T"}, {"title": ["x"]}, {}, {"title": "Better"}) + assert result["title"] == "Better" + + +class TestDetection: + def test_people_word_boundaries(self): + v = {"title": "Talk by Hadley Wickham", "description": "Not Hadley Wickhamson"} + assert s.detect_people(v, ["Hadley Wickham", "Wickham Hadley"]) == ["Hadley Wickham"] + + def test_people_case_insensitive_and_unicode(self): + v = {"title": "mine çetinkaya-rundel on teaching", "description": ""} + assert s.detect_people(v, ["Mine Çetinkaya-Rundel"]) == ["Mine Çetinkaya-Rundel"] + + def test_no_partial_word_match(self): + assert s.find_matches("Shinylive apps", ["Shiny"]) == [] + assert s.find_matches("Shiny, live", ["Shiny"]) == ["Shiny"] + + def test_software_returns_slugs(self): + v = {"title": "Quarto tips", "description": "with shiny"} + assert s.detect_software(v, {"quarto": "Quarto", "shiny": "Shiny", "gt": "gt"}) == ["quarto", "shiny"] + + +def test_build_external_cleans_tags(): + ext = s.build_external(video(), ["Hadley Wickham"], ["quarto"]) + assert ext["tags"] == ["quarto", "RPython", "data"] + assert ext["people"] == ["Hadley Wickham"] + assert ext["software"] == ["quarto"] + assert "publish" not in ext and "slug" not in ext + + +def test_changed_keys(): + old = {"title": "A", "external": {"url": "u", "view_count": 1}, "tags": []} + new = {"title": "B", "external": {"url": "u", "view_count": 2, "people": ["X"]}, "tags": []} + assert s.changed_keys(old, new) == ["external.people", "external.view_count", "title"] + + +# --- main --- + + +class TestMain: + def test_create(self, repo): + write_videos(repo, [video()]) + http = FakeHttp() + code, data = run(repo, http) + assert code == 0 + d = repo / "content/resources/videos/2026-10-01_intro-to-quarto-with-hadley-wickham" + fm, _, _ = s.parse_frontmatter((d / "_index.md").read_text()) + assert fm["title"] == "Intro to Quarto with Hadley Wickham" + assert fm["date"] == "2026-10-01" + assert fm["people"] == ["Hadley Wickham"] + assert fm["software"] == ["quarto", "shiny"] # "commons" is in the ignore list + assert fm["image"] == "thumbnail.jpg" + assert fm["resources"] == [] + assert fm["external"]["url"] == video()["url"] + assert (d / "thumbnail.jpg").read_bytes() == b"JPEG:" + video()["thumbnail"].encode() + (created,) = items(data, "created") + assert created["video_id"] == "abcdefghijk" + assert created["thumbnail"] == "downloaded (missing)" + + def test_second_run_unchanged_and_untouched(self, repo): + write_videos(repo, [video()]) + run(repo) + index = next((repo / "content/resources/videos").glob("*/_index.md")) + mtime = index.stat().st_mtime_ns + http = FakeHttp() + code, data = run(repo, http) + assert code == 0 + assert data["counts"] == {"unchanged": 1, "unpublished": 0} + assert data["items"] == [] + assert index.stat().st_mtime_ns == mtime + assert http.requested == [] # no thumbnail download + + def test_update_reports_changed_keys(self, repo): + write_videos(repo, [video()]) + run(repo) + write_videos(repo, [video(view_count=99, description="Nothing else here")]) + code, data = run(repo) + (updated,) = items(data, "updated") + assert updated["changes"] == [ + "description", + "external.description", + "external.software", + "external.view_count", + "software", + ] + + def test_manual_blocks_and_body_preserved(self, repo): + write_videos(repo, [video()]) + run(repo) + index = next((repo / "content/resources/videos").glob("*/_index.md")) + text = index.read_text().replace("---\n", "---\noverride:\n title: Custom\nexclude:\n software:\n - shiny\n", 1) + index.write_text(text + "Some body text.\n") + run(repo) + fm, _, body = s.parse_frontmatter(index.read_text()) + assert fm["title"] == "Custom" + assert fm["software"] == ["quarto"] + assert "Some body text." in body + + def test_override_date(self, repo): + write_videos(repo, [video()]) + run(repo) + index = next((repo / "content/resources/videos").glob("*/_index.md")) + index.write_text(index.read_text().replace("---\n", "---\noverride:\n date: 2025-09-17\n", 1)) + run(repo) + assert s.parse_frontmatter(index.read_text())[0]["date"] == "2025-09-17" + + def test_unpublished_skipped(self, repo): + write_videos(repo, [video(publish=False)]) + code, data = run(repo) + assert data["counts"] == {"unpublished": 1} + assert list((repo / "content/resources/videos").iterdir()) == [] + + def test_rename_moves_dir_and_updates_references(self, repo): + write_videos(repo, [video()]) + run(repo) + videos_dir = repo / "content/resources/videos" + old = videos_dir / "2026-10-01_intro-to-quarto-with-hadley-wickham" + write(old / "transcription.vtt", "WEBVTT\n") + post = write( + repo / "content/blog/post/index.md", + "See [talk](/resources/videos/2026-10-01_intro-to-quarto-with-hadley-wickham/) " + "but not videos/2026-10-01_intro-to-quarto-with-hadley-wickham-part-2.\n", + ) + write_videos(repo, [video(title="Quarto deep dive", slug="2026-10-01_quarto-deep-dive")]) + code, data = run(repo) + assert code == 0 + assert not old.exists() + assert (videos_dir / "2026-10-01_quarto-deep-dive/transcription.vtt").exists() + assert post.read_text() == ( + "See [talk](/resources/videos/2026-10-01_quarto-deep-dive/) " + "but not videos/2026-10-01_intro-to-quarto-with-hadley-wickham-part-2.\n" + ) + (renamed,) = items(data, "renamed") + assert renamed["action"] == "renamed" + assert renamed["old_slug"] == "2026-10-01_intro-to-quarto-with-hadley-wickham" + assert renamed["slug"] == "2026-10-01_quarto-deep-dive" + assert renamed["updated_references"] == ["content/blog/post/index.md"] + + def test_merge_into_existing_dir(self, repo): + videos_dir = repo / "content/resources/videos" + write_videos(repo, [video()]) + run(repo) + old = videos_dir / "2026-10-01_intro-to-quarto-with-hadley-wickham" + write(old / "transcription.vtt", "WEBVTT\n") + index = old / "_index.md" + index.write_text(index.read_text().replace("---\n", "---\ninclude:\n people:\n - Someone\n", 1)) + # A directory for the new slug already exists (e.g. created by hand). + write(videos_dir / "2026-10-01_new/_index.md", "---\ntitle: placeholder\n---\n") + write_videos(repo, [video(title="New", slug="2026-10-01_new")]) + code, data = run(repo) + assert not old.exists() + new = videos_dir / "2026-10-01_new" + assert (new / "transcription.vtt").exists() + fm, _, _ = s.parse_frontmatter((new / "_index.md").read_text()) + assert fm["include"] == {"people": ["Someone"]} + assert "Someone" in fm["people"] + (renamed,) = items(data, "renamed") + assert renamed["action"] == "merged" + assert "transcription.vtt" in renamed["moved_files"] + + def test_orphan_reported(self, repo): + write( + repo / "content/resources/videos/2020-01-01_gone/_index.md", + "---\ntitle: Gone\nexternal:\n url: https://www.youtube.com/watch?v=gonegonegon\n---\n", + ) + write_videos(repo, [video()]) + code, data = run(repo) + assert code == 0 + (orphan,) = items(data, "orphan") + assert orphan["slug"] == "2020-01-01_gone" + assert (repo / "content/resources/videos/2020-01-01_gone").exists() + + def test_thumbnail_redownloaded_when_url_changes(self, repo): + write_videos(repo, [video()]) + run(repo) + new_url = "https://i.ytimg.com/vi/abcdefghijk/maxresdefault.jpg" + write_videos(repo, [video(thumbnail=new_url)]) + http = FakeHttp() + code, data = run(repo, http) + assert http.requested == [new_url] + thumb = next((repo / "content/resources/videos").glob("*/thumbnail.jpg")) + assert thumb.read_bytes() == b"JPEG:" + new_url.encode() + (updated,) = items(data, "updated") + assert updated["thumbnail"] == "downloaded (url changed)" + + def test_thumbnail_failure_is_partial(self, repo, monkeypatch): + monkeypatch.setattr(s.time, "sleep", lambda _: None) + write_videos(repo, [video(), video("zyxwvutsrqp", "Second")]) + http = FakeHttp(fail_urls={video()["thumbnail"]}) + code, data = run(repo, http) + assert code == 2 + (error,) = items(data, "error") + assert error["reason"] == "thumbnail_failed" + assert "connection refused" in error["detail"] + assert http.requested.count(video()["thumbnail"]) == 2 # retried once + # The page itself and the other video were still written. + assert len(list((repo / "content/resources/videos").glob("*/_index.md"))) == 2 + # The next run tries again because the file is still missing. + http2 = FakeHttp() + code, _ = run(repo, http2) + assert code == 0 and http2.requested == [video()["thumbnail"]] + + def test_dry_run_writes_nothing(self, repo): + write_videos(repo, [video()]) + http = FakeHttp() + code, data = run(repo, http, "--dry-run") + assert code == 0 + assert list((repo / "content/resources/videos").iterdir()) == [] + assert http.requested == [] + (created,) = items(data, "created") + assert created["thumbnail"] == "would download (missing)" + + def test_missing_videos_toml_is_fatal(self, repo): + code, data = run(repo) + assert code == 1 + assert "videos TOML not found" in data["fatal"] + + def test_logging(self, repo, capsys, monkeypatch): + monkeypatch.setenv("CI", "true") + write_videos(repo, [video()]) + run(repo) + err = capsys.readouterr().err + assert "Created 2026-10-01_intro-to-quarto-with-hadley-wickham (abcdefghijk)" in err + assert "Summary: created=1, unpublished=0" in err + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/test-transcribe.py b/scripts/test-transcribe.py new file mode 100755 index 000000000..1928a89f5 --- /dev/null +++ b/scripts/test-transcribe.py @@ -0,0 +1,492 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "openai>=2.0.0", +# "python-dotenv>=1.0.0", +# "yt-dlp>=2025.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for transcribe.py. yt-dlp and Whisper are faked; ffmpeg is real when installed.""" + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 +from videotesting import ( # noqa: E402 + SAMPLE_VTT, + FakeOpenAI, + FakeYtDlp, + RateLimitError, + StatusError, + load_script, + make_repo, + no_sleep, + write_video_dir, +) + +t = load_script("transcribe.py") + +needs_ffmpeg = pytest.mark.skipif(not shutil.which("ffmpeg"), reason="ffmpeg not installed") + +# Real yt-dlp error output (shortened where it's long). +BOT_CHECK = ( + "ERROR: [youtube] dQw4w9WgXcQ: Sign in to confirm you’re not a bot. Use --cookies-from-browser " + "or --cookies for the authentication. See https://github.com/yt-dlp/yt-dlp/wiki/FAQ" +) +PO_TOKEN = ( + "WARNING: [youtube] dQw4w9WgXcQ: mweb client https formats require a GVS PO Token which was not " + "provided. They will be skipped as they may yield HTTP Error 403.\n" + "ERROR: [youtube] dQw4w9WgXcQ: Requested format is not available. Use --list-formats for a list of available formats" +) +NETWORK = ( + "ERROR: [youtube] dQw4w9WgXcQ: Unable to download API page: (caused by TransportError)" +) +HTTP_403 = "ERROR: unable to download video data: HTTP Error 403: Forbidden" + +URL_A = "https://www.youtube.com/watch?v=aaaaaaaaaaa" +URL_B = "https://www.youtube.com/watch?v=bbbbbbbbbbb" +URL_C = "https://www.youtube.com/watch?v=ccccccccccc" +URL_D = "https://www.youtube.com/watch?v=ddddddddddd" + + +@pytest.mark.parametrize( + "output, reason", + [ + (BOT_CHECK, "bot_check"), + ("ERROR: [youtube] x: Sign in to confirm you're not a bot. This helps protect our community.", "bot_check"), + (PO_TOKEN, "po_token"), + ( + "ERROR: [youtube] x: Sign in to confirm your age. This video may be inappropriate for some users.", + "age_restricted", + ), + ( + "ERROR: [youtube] x: Private video. Sign in if you've been granted access to this video", + "private", + ), + ( + "ERROR: [youtube] x: Join this channel to get access to members-only content like this video", + "members_only", + ), + ("ERROR: [youtube] x: This live event will begin in 3 hours.", "live_or_upcoming"), + ("ERROR: [youtube] x: Premieres in 2 hours", "live_or_upcoming"), + ("ERROR: [youtube] x: Video unavailable. This video has been removed by the uploader", "unavailable"), + (HTTP_403, "http_403"), + ("ERROR: unable to download video data: HTTP Error 429: Too Many Requests", "http_429"), + (NETWORK, "network"), + ("ERROR: [youtube] x: Read timed out.", "network"), + ( + "WARNING: [youtube] No supported JavaScript runtime could be found.\n" + "ERROR: [youtube] x: Requested format is not available.", + "js_runtime", + ), + ("ERROR: [youtube] x: Requested format is not available.", "format_unavailable"), + # An unrelated warning must not override a clear error. + ("WARNING: GVS PO Token which was not provided\nERROR: [youtube] x: Video unavailable", "unavailable"), + ("something else entirely", "unknown"), + ("", "unknown"), + ], +) +def test_classify_ytdlp_error(output, reason): + assert t.classify_ytdlp_error(output) == reason + + +def test_last_error_line(): + assert t.last_error_line(PO_TOKEN).startswith("ERROR: [youtube] dQw4w9WgXcQ: Requested format") + assert t.last_error_line("just a line\n\n") == "just a line" + assert t.last_error_line("") == "(no output)" + + +class TestCircuitBreaker: + def test_trips_after_consecutive_blocks(self): + b = t.CircuitBreaker(threshold=3) + for reason in ["bot_check", "po_token"]: + b.record(reason) + assert not b.tripped + b.record("http_429") + assert b.tripped + + def test_other_outcomes_reset(self): + b = t.CircuitBreaker(threshold=3) + for reason in ["bot_check", "bot_check", None, "bot_check", "private", "bot_check"]: + b.record(reason) + assert not b.tripped and b.consecutive == 1 + + +def test_build_ytdlp_cmd(tmp_path): + cmd = t.build_ytdlp_cmd(URL_A, tmp_path, "chrome", Path("c.txt")) + assert cmd[1:3] == ["-m", "yt_dlp"] + assert "--remote-components" in cmd and "ejs:github" in cmd + assert cmd[cmd.index("--output") + 1] == str(tmp_path / "audio.%(ext)s") + assert cmd[cmd.index("--cookies-from-browser") + 1] == "chrome" + assert cmd[cmd.index("--cookies") + 1] == "c.txt" + assert "--no-warnings" not in cmd # warnings explain failures + assert "--quiet" in cmd and "--verbose" not in cmd + assert cmd[-1] == URL_A + verbose = t.build_ytdlp_cmd(URL_A, tmp_path, None, None, verbose=True) + assert "--verbose" in verbose and "--quiet" not in verbose + + +class TestChunking: + def test_single_chunk(self): + assert t.compute_chunk_boundaries(100, 10, [50], max_bytes=100) == [(0.0, 100)] + + def test_cuts_at_last_silence_before_limit(self): + # 100 s at 1 byte/s, 40 bytes per chunk, aiming at 90% -> at most 36 s per chunk. + assert t.compute_chunk_boundaries(100, 100, [10, 35, 38, 70, 79], max_bytes=40) == [ + (0.0, 35), + (35, 70), + (70, 100), + ] + + def test_hard_cut_without_silence(self): + assert t.compute_chunk_boundaries(100, 100, [], max_bytes=40) == [(0.0, 36.0), (36.0, 72.0), (72.0, 100)] + + def test_merge_vtt_offsets(self): + part = "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\nhi\n" + merged = t.merge_vtt([(part, 0.0), (part, 600.5)]) + assert videolib.parse_vtt(merged) == [(1.0, 2.0, "hi"), (601.5, 602.5, "hi")] + + +def degenerate_vtt(start=0, words_until=30, dots_until=300): + cues = [(float(t), float(t) + 2, f"word {t}") for t in range(start, words_until, 2)] + cues += [(float(t), float(t) + 2, ".") for t in range(words_until, dots_until, 2)] + return videolib.render_vtt(cues) + + +class TestCheckTranscript: + def test_wordless_stretch(self): + cues = videolib.parse_vtt(degenerate_vtt()) + assert t.longest_wordless_stretch(cues) == (270.0, 30.0) + assert t.longest_wordless_stretch(videolib.parse_vtt(SAMPLE_VTT)) == (0.0, 0.0) + + def test_degenerate(self): + with pytest.raises(t.TranscriptError) as e: + t.check_transcript(degenerate_vtt(), 300) + assert e.value.reason == "degenerate_transcript" + assert str(e.value) == "270s of cues without words from 00:00:30" + + def test_short_wordless_stretch_is_fine(self): + t.check_transcript(degenerate_vtt(dots_until=100), 100) + + def test_empty(self): + with pytest.raises(t.TranscriptError) as e: + t.check_transcript("WEBVTT\n\n", 100) + assert e.value.reason == "empty_transcript" + + def test_ok(self): + cues, warnings = t.check_transcript(SAMPLE_VTT, 26) + assert len(cues) == 3 and warnings == [] + + def test_truncated(self): + _, warnings = t.check_transcript(SAMPLE_VTT, 600) + assert warnings == ["suspect_truncated: last cue ends at 00:00:25.000 of 00:10:00.000"] + + +def test_build_prompt(): + assert t.build_prompt({"title": "Quarto talk", "software": ["quarto", "shiny"]}) == "Quarto talk, quarto, shiny" + assert t.build_prompt({}) == "" + + +# --- Selection --- + + +@pytest.fixture +def repo(tmp_path): + return make_repo(tmp_path) + + +def videos_dir(repo): + return repo / "content/resources/videos" + + +class TestSelect: + def test_rules_and_order(self, repo): + write_video_dir(repo, "2026-01-01_old", url=URL_A) + write_video_dir(repo, "2026-03-01_new", url=URL_B) + write_video_dir(repo, "2026-02-01_done", url=URL_C, vtt=SAMPLE_VTT) + write_video_dir(repo, "2026-02-02_excluded", url=URL_D, extra="transcribe: false\n") + write_video_dir(repo, "2026-02-03_live", url=URL_D, duration=0) + (videos_dir(repo) / "2026-02-04_empty").mkdir() + summary = videolib.RunSummary("transcribe") + selected = t.select_videos(videolib.video_dirs(videos_dir(repo)), summary) + assert [c.slug for c in selected] == ["2026-03-01_new", "2026-01-01_old"] + assert summary.counts() == {"already_transcribed": 1, "excluded": 1, "skipped": 2} + reasons = {i["slug"]: i["reason"] for i in summary.items} + assert reasons == {"2026-02-04_empty": "no_index", "2026-02-03_live": "no_duration"} + + def test_limit(self, repo): + for i in range(5): + write_video_dir(repo, f"2026-01-0{i + 1}_v", url=URL_A) + summary = videolib.RunSummary("transcribe") + selected = t.select_videos(videolib.video_dirs(videos_dir(repo)), summary, limit=2) + assert [c.slug for c in selected] == ["2026-01-05_v", "2026-01-04_v"] + assert summary.tallies["deferred"] == 3 + assert summary.extra["deferred"] == ["2026-01-03_v", "2026-01-02_v", "2026-01-01_v"] + + def test_audio_budget(self, repo): + write_video_dir(repo, "2026-01-03_a", url=URL_A, duration=3600) + write_video_dir(repo, "2026-01-02_b", url=URL_B, duration=1800) + write_video_dir(repo, "2026-01-01_c", url=URL_C, duration=600) + summary = videolib.RunSummary("transcribe") + selected = t.select_videos(videolib.video_dirs(videos_dir(repo)), summary, max_audio_minutes=75) + # 60 min fits, 30 more doesn't, but 10 more does. + assert [c.slug for c in selected] == ["2026-01-03_a", "2026-01-01_c"] + assert summary.extra["selected_audio_minutes"] == 70.0 + + def test_first_video_over_budget_still_taken(self, repo): + write_video_dir(repo, "2026-01-03_long", url=URL_A, duration=4 * 3600) + summary = videolib.RunSummary("transcribe") + selected = t.select_videos(videolib.video_dirs(videos_dir(repo)), summary, max_audio_minutes=60) + assert [c.slug for c in selected] == ["2026-01-03_long"] + + +# --- main --- + + +def run(repo, *args, client=None, ytdlp=None, max_bytes=t.MAX_BYTES): + summary = repo / "summary.json" + code = t.main( + ["--project-root", str(repo), "--summary-json", str(summary), *args], + client=client if client is not None else FakeOpenAI(), + run=ytdlp or FakeYtDlp(), + sleep=no_sleep, + max_bytes=max_bytes, + min_request_interval=0, + ) + return code, json.loads(summary.read_text()) + + +def items(data, status): + return [i for i in data["items"] if i["status"] == status] + + +class TestMain: + def test_transcribes_and_cleans_up(self, repo, monkeypatch, tmp_path): + temp_root = tmp_path / "tmp" + temp_root.mkdir() + monkeypatch.setattr(t.tempfile, "tempdir", str(temp_root)) + a = write_video_dir(repo, "2026-01-02_a", url=URL_A, duration=26, title="Quarto talk") + b = write_video_dir(repo, "2026-01-01_b", url=URL_B, duration=26) + client = FakeOpenAI() + ytdlp = FakeYtDlp() + code, data = run(repo, client=client, ytdlp=ytdlp) + assert code == 0 + assert (a / "transcription.vtt").read_text() == SAMPLE_VTT + assert (b / "transcription.vtt").exists() + assert not list(a.glob("audio.*")) + assert list(temp_root.iterdir()) == [] # temporary download dirs removed + assert [c[-1] for c in ytdlp.calls] == [URL_A, URL_B] + assert client.calls[0]["model"] == "whisper-1" + assert client.calls[0]["response_format"] == "vtt" + assert client.calls[0]["prompt"] == "Quarto talk, quarto" + done = items(data, "transcribed") + assert [i["slug"] for i in done] == ["2026-01-02_a", "2026-01-01_b"] + assert done[0]["cues"] == 3 + assert done[0]["chunks"] == 1 + assert done[0]["estimated_cost_usd"] == pytest.approx(26 / 60 * 0.006, abs=1e-4) + assert "download_s" in done[0]["timings"] + assert data["extra"]["breaker_tripped"] is False + + def test_second_run_does_nothing(self, repo): + write_video_dir(repo, "2026-01-02_a", url=URL_A, duration=26) + run(repo) + ytdlp = FakeYtDlp() + code, data = run(repo, ytdlp=ytdlp) + assert code == 0 and ytdlp.calls == [] + assert data["counts"] == {"already_transcribed": 1} + + def test_circuit_breaker(self, repo): + for i, url in enumerate([URL_A, URL_B, URL_C, URL_D]): + write_video_dir(repo, f"2026-01-0{4 - i}_v{i}", url=url, duration=26) + ytdlp = FakeYtDlp({URL_A: [BOT_CHECK], URL_B: [PO_TOKEN], URL_C: [BOT_CHECK]}) + code, data = run(repo, "--jobs", "1", ytdlp=ytdlp) + assert code == 2 + assert [i["reason"] for i in items(data, "error")] == ["bot_check", "po_token", "bot_check"] + (skipped,) = items(data, "skipped") + assert skipped["reason"] == "download_blocked" and skipped["video_id"] == "ddddddddddd" + assert len(ytdlp.calls) == 3 + assert data["extra"]["breaker_tripped"] is True + + def test_download_error_details_in_summary_and_log(self, repo, capsys, monkeypatch): + monkeypatch.setenv("CI", "true") + write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + code, data = run(repo, ytdlp=FakeYtDlp({URL_A: [PO_TOKEN]})) + assert code == 2 + (error,) = items(data, "error") + assert error["stage"] == "download" + assert error["reason"] == "po_token" + assert error["detail"].startswith("ERROR: [youtube] dQw4w9WgXcQ: Requested format") + assert "GVS PO Token" in error["stderr"] + err = capsys.readouterr().err + assert "2026-01-01_a (aaaaaaaaaaa): download failed: po_token" in err + assert "yt-dlp: WARNING: [youtube] dQw4w9WgXcQ: mweb client" in err + + def test_network_error_retried(self, repo): + write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + ytdlp = FakeYtDlp({URL_A: [NETWORK, "ok"]}) + code, data = run(repo, ytdlp=ytdlp) + assert code == 0 and len(ytdlp.calls) == 2 + + def test_403_retried_once_then_error(self, repo): + write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + ytdlp = FakeYtDlp({URL_A: [HTTP_403, HTTP_403]}) + code, data = run(repo, ytdlp=ytdlp) + (error,) = items(data, "error") + assert error["reason"] == "http_403" and error["attempts"] == 2 + assert len(ytdlp.calls) == 2 + + def test_whisper_rate_limit_retried(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + client = FakeOpenAI([RateLimitError(), StatusError(503)]) + code, _ = run(repo, client=client) + assert code == 0 and len(client.calls) == 3 + assert (a / "transcription.vtt").exists() + + def test_whisper_client_error_not_retried(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + client = FakeOpenAI([StatusError(400, "Invalid file format")]) + code, data = run(repo, client=client) + assert code == 2 and len(client.calls) == 1 + (error,) = items(data, "error") + assert error["reason"] == "transcription_failed" + assert error["detail"] == "StatusError status=400 request_id=req_400: Invalid file format" + assert not (a / "transcription.vtt").exists() + + def test_degenerate_chunk_retried(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + client = FakeOpenAI([degenerate_vtt()]) # then SAMPLE_VTT + code, data = run(repo, client=client) + assert code == 0 and len(client.calls) == 2 + assert (a / "transcription.vtt").read_text() == SAMPLE_VTT + assert items(data, "transcribed")[0]["whisper_retries"] == 1 + + def test_degenerate_twice_is_an_error(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=300) + client = FakeOpenAI([degenerate_vtt(), degenerate_vtt()]) + code, data = run(repo, client=client) + assert code == 2 and len(client.calls) == 2 + (error,) = items(data, "error") + assert error["reason"] == "degenerate_transcript" + assert not (a / "transcription.vtt").exists() + + def test_empty_transcript_not_written(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + code, data = run(repo, client=FakeOpenAI(["WEBVTT\n\n"])) + assert code == 2 + assert items(data, "error")[0]["reason"] == "empty_transcript" + assert not (a / "transcription.vtt").exists() + + def test_truncated_is_a_warning(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=600) + code, data = run(repo) + assert code == 0 and (a / "transcription.vtt").exists() + assert items(data, "transcribed")[0]["warnings"][0].startswith("suspect_truncated") + + def test_existing_audio_used_and_deleted(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + (a / "audio.m4a").write_bytes(b"\0" * 100) + ytdlp = FakeYtDlp() + code, _ = run(repo, ytdlp=ytdlp) + assert code == 0 and ytdlp.calls == [] + assert not (a / "audio.m4a").exists() + + def test_existing_audio_kept_on_failure(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + (a / "audio.m4a").write_bytes(b"\0" * 100) + run(repo, client=FakeOpenAI([StatusError(400)])) + assert (a / "audio.m4a").exists() + + def test_video_option(self, repo): + write_video_dir(repo, "2026-01-02_a", url=URL_A, duration=26) + b = write_video_dir(repo, "2026-01-01_b", url=URL_B, duration=26) + code, data = run(repo, "--video", "2026-01-01") + assert [i["slug"] for i in items(data, "transcribed")] == ["2026-01-01_b"] + assert (b / "transcription.vtt").exists() + + def test_dry_run(self, repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + ytdlp, client = FakeYtDlp(), FakeOpenAI() + code, data = run(repo, "--dry-run", ytdlp=ytdlp, client=client) + assert code == 0 and ytdlp.calls == [] and client.calls == [] + assert items(data, "would_transcribe")[0]["slug"] == "2026-01-01_a" + assert not (a / "transcription.vtt").exists() + + def test_missing_api_key_is_fatal(self, repo, monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=26) + summary = repo / "s.json" + code = t.main(["--project-root", str(repo), "--summary-json", str(summary)], run=FakeYtDlp()) + assert code == 1 + assert "OPENAI_API_KEY" in json.loads(summary.read_text())["fatal"] + + def test_nothing_to_do_needs_no_api_key(self, repo, monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + code = t.main(["--project-root", str(repo)], run=FakeYtDlp()) + assert code == 0 + + def test_missing_ffmpeg_is_fatal(self, repo, monkeypatch): + real_which = shutil.which + monkeypatch.setattr(t.shutil, "which", lambda name: None if name == "ffprobe" else real_which(name)) + code, data = run(repo) + assert code == 1 and data["fatal"] == "Required tools not found: ffprobe" + + +# --- ffmpeg (real) --- + + +def make_tone_silence_tone(path: Path) -> None: + subprocess.run( + [ + "ffmpeg", "-y", "-v", "error", + "-f", "lavfi", "-i", "sine=frequency=440:duration=1", + "-f", "lavfi", "-i", "anullsrc=r=44100:cl=mono:d=1", + "-f", "lavfi", "-i", "sine=frequency=440:duration=1", + "-filter_complex", "[0][1][2]concat=n=3:v=0:a=1", + "-ac", "1", "-ar", "44100", str(path), + ], + check=True, + ) # fmt: skip + + +@needs_ffmpeg +def test_detect_silence(tmp_path): + audio = tmp_path / "audio.wav" + make_tone_silence_tone(audio) + (point,) = t.detect_silence(audio) + assert 1.2 < point < 1.8 + assert t.get_audio_duration(audio) == pytest.approx(3.0, abs=0.05) + + +@needs_ffmpeg +def test_large_audio_split_and_merged(repo): + a = write_video_dir(repo, "2026-01-01_a", url=URL_A, duration=3) + audio = a / "audio.wav" + make_tone_silence_tone(audio) + one_cue = "WEBVTT\n\n00:00:00.200 --> 00:00:00.900\npart\n" + client = FakeOpenAI(default=one_cue) + # Force a split: chunks may be at most ~60% of the file. + code, data = run(repo, client=client, max_bytes=int(audio.stat().st_size * 0.6)) + assert code == 0 + (done,) = items(data, "transcribed") + assert done["chunks"] == 2 + cues = videolib.parse_vtt((a / "transcription.vtt").read_text()) + assert len(cues) == 2 + assert cues[0][0] == pytest.approx(0.2) + assert 1.2 < cues[1][0] - 0.2 < 1.8 # second chunk starts at the silence + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/test-update-youtube-videos.py b/scripts/test-update-youtube-videos.py new file mode 100755 index 000000000..eae8029b7 --- /dev/null +++ b/scripts/test-update-youtube-videos.py @@ -0,0 +1,373 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "google-api-python-client>=2.100.0", +# "tomli-w>=1.0.0", +# "python-dotenv>=1.0.0", +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for update-youtube-videos.py.""" + +import json +import sys +import tomllib +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from videotesting import FakeYouTube, load_script, make_repo, yt_item # noqa: E402 + +u = load_script("update-youtube-videos.py") + +NOW = "2026-11-12T05:00:00+00:00" +EARLIER = "2026-10-01T00:00:00+00:00" + + +# --- Pure helpers --- + + +@pytest.mark.parametrize( + "title, slug", + [ + ("Hello, World!", "hello-world"), + ("Mine Çetinkaya-Rundel | Teaching R", "mine-cetinkaya-rundel-teaching-r"), + ("posit::conf(2025) — keynote", "positconf2025-keynote"), + (" spaces and--dashes ", "spaces-and-dashes"), + ], +) +def test_slugify(title, slug): + assert u.slugify(title) == slug + + +@pytest.mark.parametrize( + "duration, seconds", + [ + ("PT1H2M3S", 3723), + ("PT45S", 45), + ("PT10M", 600), + ("P1DT2H", 93600), + ("P0D", 0), # live streams + ("", 0), + ("garbage", 0), + ], +) +def test_parse_iso8601_duration(duration, seconds): + assert u.parse_iso8601_duration(duration) == seconds + + +def test_best_thumbnail(): + assert u.best_thumbnail({"high": {"url": "h"}, "maxres": {"url": "m"}}) == "m" + assert u.best_thumbnail({"default": {"url": "d"}}) == "d" + assert u.best_thumbnail({}) == "" + + +def test_transform_video(): + item = yt_item("abcdefghijk", "My Talk", tags=["R", "#quarto"], views=5, duration="PT1M5S") + v = u.transform_video(item, "Posit PBC", "PLx", NOW) + assert v == { + "url": "https://www.youtube.com/watch?v=abcdefghijk", + "title": "My Talk", + "slug": "2026-10-01_my-talk", + "date": "2026-10-01T12:00:00Z", + "description": "", + "duration": 65, + "channel": "Posit PBC", + "playlist": "PLx", + "tags": ["R", "#quarto"], + "view_count": 5, + "like_count": 10, + "comment_count": 1, + "thumbnail": "https://i.ytimg.com/vi/abcdefghijk/hqdefault.jpg", + "language": "en", + "has_captions": False, + "definition": "hd", + "publish": True, + "last_updated": NOW, + } + + +def test_transform_uses_channel_title_when_unknown(): + v = u.transform_video(yt_item("abcdefghijk", channel_title="Other"), "", "", NOW) + assert v["channel"] == "Other" + + +# --- merge_video --- + + +def stored(**overrides): + v = u.transform_video(yt_item("abcdefghijk", "My Talk"), "Posit PBC", "", EARLIER) + v.update(overrides) + return v + + +def fetched(**item_kwargs): + return u.transform_video(yt_item("abcdefghijk", **{"title": "My Talk", **item_kwargs}), "Posit PBC", "", NOW) + + +class TestMergeVideo: + def test_new(self): + video, changed, stats = u.merge_video(None, fetched(), refresh_stats=False, now=NOW) + assert video["last_updated"] == NOW and changed == [] and stats == [] + + def test_unchanged_keeps_timestamp(self): + video, changed, stats = u.merge_video(stored(), fetched(), refresh_stats=False, now=NOW) + assert changed == [] and stats == [] + assert video["last_updated"] == EARLIER + assert video == stored() + + def test_title_change_bumps_timestamp_and_slug(self): + video, changed, _ = u.merge_video(stored(), fetched(title="New Title"), refresh_stats=False, now=NOW) + assert changed == ["title", "slug"] + assert video["slug"] == "2026-10-01_new-title" + assert video["last_updated"] == NOW + + def test_stats_kept_without_refresh(self): + video, changed, stats = u.merge_video(stored(), fetched(views=999), refresh_stats=False, now=NOW) + assert changed == [] and stats == [] + assert video["view_count"] == 100 + assert video["last_updated"] == EARLIER + + def test_stats_refreshed(self): + video, changed, stats = u.merge_video(stored(), fetched(views=999, likes=11), refresh_stats=True, now=NOW) + assert changed == [] + assert stats == ["view_count", "like_count"] + assert video["view_count"] == 999 and video["last_updated"] == NOW + + def test_refresh_without_stat_changes_keeps_timestamp(self): + video, _, stats = u.merge_video(stored(), fetched(), refresh_stats=True, now=NOW) + assert stats == [] and video["last_updated"] == EARLIER + + def test_publish_false_preserved(self): + video, changed, _ = u.merge_video(stored(publish=False), fetched(), refresh_stats=False, now=NOW) + assert video["publish"] is False and changed == [] + + def test_manual_keys_preserved(self): + video, changed, _ = u.merge_video(stored(note="keep me"), fetched(), refresh_stats=False, now=NOW) + assert video["note"] == "keep me" and changed == [] + + +# --- deduplicate_videos --- + + +class TestDeduplicate: + def videos(self): + a = {"slug": "s", "date": "2026-01-01T10:00:00Z"} + b = {"slug": "s", "date": "2026-01-01T11:00:00Z"} + c = {"slug": "other", "date": "2026-01-01T10:00:00Z"} + return {"url-a": a, "url-b": b, "url-c": c} + + def test_prefers_existing_entry(self): + result, discarded = u.deduplicate_videos(self.videos(), existing_urls={"url-a"}) + assert set(result) == {"url-a", "url-c"} + assert discarded == [{"url": "url-b", "slug": "s", "kept": "url-a"}] + + def test_then_later_date(self): + result, _ = u.deduplicate_videos(self.videos(), existing_urls=set()) + assert set(result) == {"url-b", "url-c"} + + def test_stable_across_runs(self): + first, _ = u.deduplicate_videos(self.videos(), existing_urls={"url-a", "url-b"}) + second, _ = u.deduplicate_videos(self.videos(), existing_urls=set(first)) + assert set(first) == set(second) + + def test_preserves_order(self): + result, _ = u.deduplicate_videos(self.videos(), existing_urls={"url-a"}) + assert list(result) == ["url-a", "url-c"] + + +# --- Fake API --- + + +def fake_api(**overrides) -> FakeYouTube: + yt = FakeYouTube( + channels={"PositPBC": ("UUposit", "Posit PBC")}, + playlists={"UUposit": ["video000001", "video000002", "video000003"], "PLtest": ["video000003"]}, + videos={ + "video000001": yt_item("video000001", "First talk", published="2026-10-01T10:00:00Z"), + "video000002": yt_item("video000002", "Second talk", published="2026-10-02T10:00:00Z"), + "video000003": yt_item("video000003", "Third talk", published="2026-10-03T10:00:00Z"), + "indiv000001": yt_item("indiv000001", "Elsewhere", published="2025-01-01T10:00:00Z"), + }, + page_size=2, # exercise pagination + ) + for k, v in overrides.items(): + setattr(yt, k, v) + return yt + + +def run(repo, yt, *args): + summary = repo / "summary.json" + code = u.main(["--project-root", str(repo), "--summary-json", str(summary), *args], youtube=yt) + data = json.loads(summary.read_text()) + return code, data + + +def read_videos(repo): + with open(repo / "data/videos.toml", "rb") as f: + return {v["url"].rsplit("=", 1)[1]: v for v in tomllib.load(f)["videos"]} + + +@pytest.fixture +def repo(tmp_path): + return make_repo(tmp_path) + + +class TestMain: + def test_first_run(self, repo): + code, data = run(repo, fake_api()) + assert code == 0 + videos = read_videos(repo) + assert set(videos) == {"video000001", "video000002", "video000003", "indiv000001"} + assert videos["video000003"]["playlist"] == "PLtest" + assert videos["video000003"]["channel"] == "Posit PBC" + assert videos["indiv000001"]["channel"] == "Posit PBC" # from snippet.channelTitle + assert data["counts"] == {"new": 4} + new = {i["video_id"]: i for i in data["items"]} + assert new["video000001"]["slug"] == "2026-10-01_first-talk" + assert new["video000001"]["title"] == "First talk" + assert data["extra"]["quota_used"] > 0 + + def test_second_run_is_byte_identical_and_not_rewritten(self, repo): + run(repo, fake_api()) + path = repo / "data/videos.toml" + before = path.read_bytes() + mtime = path.stat().st_mtime_ns + code, data = run(repo, fake_api()) + assert code == 0 + assert path.read_bytes() == before + assert path.stat().st_mtime_ns == mtime + assert data["counts"] == {"unchanged": 4} + assert data["extra"]["written"] is False + + def test_stats_only_change_without_refresh(self, repo): + run(repo, fake_api()) + before = (repo / "data/videos.toml").read_bytes() + yt = fake_api() + yt.videos_data["video000001"] = yt_item("video000001", "First talk", published="2026-10-01T10:00:00Z", views=5000) + code, data = run(repo, yt) + assert (repo / "data/videos.toml").read_bytes() == before + assert data["counts"] == {"unchanged": 4} + + def test_stats_change_with_refresh(self, repo): + run(repo, fake_api()) + yt = fake_api() + yt.videos_data["video000001"] = yt_item("video000001", "First talk", published="2026-10-01T10:00:00Z", views=5000) + code, data = run(repo, yt, "--refresh-stats") + assert read_videos(repo)["video000001"]["view_count"] == 5000 + assert data["counts"] == {"stats_refreshed": 1, "unchanged": 3} + assert data["extra"]["refresh_stats"] is True + + def test_title_change(self, repo): + run(repo, fake_api()) + yt = fake_api() + yt.videos_data["video000002"] = yt_item("video000002", "Renamed talk", published="2026-10-02T10:00:00Z") + code, data = run(repo, yt) + (item,) = data["items"] + assert item["status"] == "changed" + assert item["changed_fields"] == ["title", "slug"] + assert item["old_slug"] == "2026-10-02_second-talk" + assert item["slug"] == "2026-10-02_renamed-talk" + + def test_missing_and_unavailable(self, repo): + run(repo, fake_api()) + yt = fake_api() + yt.playlists_data["UUposit"] = ["video000002", "video000003"] # 1 removed from channel + del yt.videos_data["video000003"] # 3 went private + code, data = run(repo, yt) + assert code == 0 + statuses = {i["video_id"]: i["status"] for i in data["items"]} + assert statuses == {"video000001": "missing", "video000003": "unavailable"} + # Nothing is removed automatically. + assert set(read_videos(repo)) == {"video000001", "video000002", "video000003", "indiv000001"} + + def test_publish_false_survives(self, repo): + run(repo, fake_api()) + path = repo / "data/videos.toml" + path.write_text(path.read_text().replace("publish = true", "publish = false", 1)) + before = path.read_bytes() + run(repo, fake_api()) + assert path.read_bytes() == before + + def test_one_source_fails_is_partial(self, repo): + yt = fake_api() + yt.fail_playlists = {"PLtest"} + code, data = run(repo, yt) + assert code == 2 + (error,) = [i for i in data["items"] if i["status"] == "error"] + assert error["source"] == "playlist:PLtest" + assert "PLtest" in error["detail"] + # Everything else is still written. + assert "video000001" in read_videos(repo) + + def test_all_sources_fail_is_fatal_and_writes_nothing(self, repo): + yt = fake_api(channels_data={}) + yt.fail_playlists = {"PLtest"} + code, data = run(repo, yt) + assert code == 1 + assert "All 2 channels and playlists failed" in data["fatal"] + assert not (repo / "data/videos.toml").exists() + + def test_videos_api_failure_is_fatal(self, repo): + run(repo, fake_api()) + before = (repo / "data/videos.toml").read_bytes() + yt = fake_api() + yt.fail_videos = True + code, data = run(repo, yt) + assert code == 1 + assert "quotaExceeded" in data["fatal"] + assert (repo / "data/videos.toml").read_bytes() == before + + def test_corrupt_videos_toml_is_fatal(self, repo): + (repo / "data/videos.toml").write_text("this is [not toml") + code, data = run(repo, fake_api()) + assert code == 1 + assert "Could not parse" in data["fatal"] + assert (repo / "data/videos.toml").read_text() == "this is [not toml" + + def test_missing_config_is_fatal(self, repo): + (repo / "data/youtube.toml").unlink() + code, data = run(repo, fake_api()) + assert code == 1 and "Config file not found" in data["fatal"] + + def test_missing_api_key_is_fatal(self, repo, monkeypatch): + monkeypatch.delenv("YOUTUBE_API_KEY", raising=False) + summary = repo / "s.json" + code = u.main(["--project-root", str(repo), "--summary-json", str(summary)]) + assert code == 1 + assert "YOUTUBE_API_KEY" in json.loads(summary.read_text())["fatal"] + + def test_dry_run_writes_nothing(self, repo): + code, data = run(repo, fake_api(), "--dry-run") + assert code == 0 + assert not (repo / "data/videos.toml").exists() + assert data["counts"] == {"new": 4} + + def test_duplicate_slug_is_stable(self, repo): + yt = fake_api() + yt.playlists_data["UUposit"].append("video000004") + yt.videos_data["video000004"] = yt_item("video000004", "First talk", published="2026-10-01T23:00:00Z") + run(repo, yt) + first = read_videos(repo) + code, data = run(repo, yt) + assert read_videos(repo) == first + dupes = [i for i in data["items"] if i["status"] == "duplicate_slug"] + assert len(dupes) == 1 and dupes[0]["slug"] == "2026-10-01_first-talk" + + def test_logs_changes(self, repo, capsys, monkeypatch): + monkeypatch.setenv("CI", "true") + run(repo, fake_api()) + err = capsys.readouterr().err + assert "New: 2026-10-01_first-talk (video000001) 'First talk'" in err + assert "API quota used" in err + assert "Summary: new=4" in err + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/test-validate-blog-posts.py b/scripts/test-validate-blog-posts.py old mode 100644 new mode 100755 index 8c8b51bfc..820498e22 --- a/scripts/test-validate-blog-posts.py +++ b/scripts/test-validate-blog-posts.py @@ -566,3 +566,7 @@ def test_subdirectory_image(self, tmp_path): (p.parent / "assets" / "hero.png").touch() fm = {"image": "assets/hero.png"} assert v.check_image_exists(p, fm, ctx) == [] + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/test-video-pipeline.py b/scripts/test-video-pipeline.py new file mode 100755 index 000000000..7fa592172 --- /dev/null +++ b/scripts/test-video-pipeline.py @@ -0,0 +1,214 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "google-api-python-client>=2.100.0", +# "tomli-w>=1.0.0", +# "python-dotenv>=1.0.0", +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "requests>=2.28.0", +# "openai>=2.0.0", +# "anthropic>=0.40.0", +# "yt-dlp>=2025.0", +# "pytest>=8.0", +# ] +# /// + +""" +End-to-end test of the video pipeline: the four scripts and the report, run +in order against a fixture repository, with every external service faked. + +The key property: a second run without changes on YouTube changes nothing, +which is what keeps the nightly pull requests small. +""" + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from videotesting import ( # noqa: E402 + FakeAnthropic, + FakeHttp, + FakeOpenAI, + FakeYouTube, + FakeYtDlp, + load_script, + make_repo, + no_sleep, + yt_item, +) + +update = load_script("update-youtube-videos.py") +sync = load_script("sync-videos.py") +transcribe = load_script("transcribe.py") +fmt = load_script("format-transcriptions.py") +report = load_script("video-sync-report.py") + + +def youtube(**titles): + titles = {"video000001": "Quarto with Hadley Wickham", "video000002": "Shiny tips", **titles} + return FakeYouTube( + channels={"PositPBC": ("UUposit", "Posit PBC")}, + playlists={"UUposit": list(titles), "PLtest": []}, + videos={ + vid: yt_item(vid, title, published=f"2026-11-0{i + 1}T10:00:00Z", duration="PT26S", views=10) + for i, (vid, title) in enumerate(titles.items()) + }, + ) + + +class Pipeline: + def __init__(self, root: Path): + self.root = root + self.ytdlp = FakeYtDlp() + self.openai = FakeOpenAI() + self.anthropic = FakeAnthropic() + self.http = FakeHttp() + + def run(self, yt: FakeYouTube, *update_args: str) -> dict[str, int]: + summaries = self.root.parent / "summaries" + common = ["--project-root", str(self.root)] + codes = { + "update": update.main( + [*common, *update_args, "--summary-json", str(summaries / "1.json")], youtube=yt + ), + "sync": sync.main([*common, "--summary-json", str(summaries / "2.json")], http_get=self.http), + "transcribe": transcribe.main( + [*common, "--summary-json", str(summaries / "3.json")], + client=self.openai, + run=self.ytdlp, + sleep=no_sleep, + min_request_interval=0, + ), + "format": fmt.main( + [*common, "--summary-json", str(summaries / "4.json")], + client=self.anthropic, + sleep=no_sleep, + min_request_interval=0, + ), + } + self.report, self.annotations = report.build_report( + report.load_summaries(summaries), expected=list(report.STEPS) + ) + return codes + + def snapshot(self) -> dict[str, tuple[bytes, int]]: + return { + str(p.relative_to(self.root)): (p.read_bytes(), p.stat().st_mtime_ns) + for p in sorted(self.root.rglob("*")) + if p.is_file() + } + + +@pytest.fixture +def pipeline(tmp_path): + return Pipeline(make_repo(tmp_path / "repo")) + + +def changed(before, after): + return sorted( + path + for path in set(before) | set(after) + if before.get(path, (None,))[0] != after.get(path, (None,))[0] + ) + + +def test_first_run_creates_everything(pipeline): + codes = pipeline.run(youtube()) + assert codes == {"update": 0, "sync": 0, "transcribe": 0, "format": 0} + d = pipeline.root / "content/resources/videos/2026-11-01_quarto-with-hadley-wickham" + assert sorted(p.name for p in d.iterdir()) == ["_index.md", "thumbnail.jpg", "transcription.txt", "transcription.vtt"] + html = (d / "transcription.txt").read_text() + assert 'Hadley Wickham' in html + fm = sync.parse_frontmatter((d / "_index.md").read_text())[0] + assert fm["people"] == ["Hadley Wickham"] and fm["software"] == ["quarto"] + # The transcription prompt used the synced frontmatter. + assert pipeline.openai.calls[-1]["prompt"].startswith("Quarto with Hadley Wickham") + assert "### New videos (2)" in pipeline.report + assert "| Transcription | 2 transcribed" in pipeline.report + assert "| Formatting | 2 formatted" in pipeline.report + assert pipeline.annotations == [] + + +def test_second_run_changes_nothing(pipeline): + pipeline.run(youtube()) + before = pipeline.snapshot() + calls = (len(pipeline.ytdlp.calls), len(pipeline.openai.calls), len(pipeline.anthropic.calls)) + codes = pipeline.run(youtube()) + after = pipeline.snapshot() + assert codes == {"update": 0, "sync": 0, "transcribe": 0, "format": 0} + assert before == after # contents and modification times + assert (len(pipeline.ytdlp.calls), len(pipeline.openai.calls), len(pipeline.anthropic.calls)) == calls + assert "| YouTube metadata | 2 unchanged |" in pipeline.report + + +def test_title_change_renames_without_new_transcripts(pipeline): + pipeline.run(youtube()) + before = pipeline.snapshot() + calls = (len(pipeline.ytdlp.calls), len(pipeline.openai.calls), len(pipeline.anthropic.calls)) + pipeline.run(youtube(video000002="Shiny tips and tricks")) + after = pipeline.snapshot() + old, new = "content/resources/videos/2026-11-02_shiny-tips", "content/resources/videos/2026-11-02_shiny-tips-and-tricks" + assert changed(before, after) == sorted([ + f"{old}/_index.md", + f"{old}/thumbnail.jpg", + f"{old}/transcription.txt", + f"{old}/transcription.vtt", + f"{new}/_index.md", + f"{new}/thumbnail.jpg", + f"{new}/transcription.txt", + f"{new}/transcription.vtt", + "data/videos.toml", + ]) + # The transcripts moved along; nothing was transcribed or formatted again. + assert after[f"{new}/transcription.vtt"][0] == before[f"{old}/transcription.vtt"][0] + assert (len(pipeline.ytdlp.calls), len(pipeline.openai.calls), len(pipeline.anthropic.calls)) == calls + assert "`2026-11-02_shiny-tips` → `2026-11-02_shiny-tips-and-tricks`" in pipeline.report + + +def test_stats_refresh_only_touches_metadata(pipeline): + pipeline.run(youtube()) + before = pipeline.snapshot() + yt = youtube() + for item in yt.videos_data.values(): + item["statistics"]["viewCount"] = "5000" + pipeline.run(yt) + assert changed(before, pipeline.snapshot()) == [] # not a stats day + pipeline.run(yt, "--refresh-stats") + assert changed(before, pipeline.snapshot()) == [ + "content/resources/videos/2026-11-01_quarto-with-hadley-wickham/_index.md", + "content/resources/videos/2026-11-02_shiny-tips/_index.md", + "data/videos.toml", + ] + assert "stats refreshed" in pipeline.report + + +def test_blocked_downloads_are_retried_next_run(pipeline): + blocked = "ERROR: [youtube] video000001: Sign in to confirm you’re not a bot." + pipeline.ytdlp = FakeYtDlp({"https://www.youtube.com/watch?v=video000001": [blocked]}) + codes = pipeline.run(youtube()) + assert codes["transcribe"] == 2 and codes["format"] == 0 + assert "**Audio downloads failed**" in pipeline.report + assert "::warning title=Audio downloads failed::" in pipeline.annotations[0] + d = pipeline.root / "content/resources/videos/2026-11-01_quarto-with-hadley-wickham" + assert not (d / "transcription.vtt").exists() + # Next night the download works and the video catches up. + codes = pipeline.run(youtube()) + assert codes == {"update": 0, "sync": 0, "transcribe": 0, "format": 0} + assert (d / "transcription.txt").exists() + + +def test_summaries_are_valid_json(pipeline): + pipeline.run(youtube()) + for path in sorted((pipeline.root.parent / "summaries").glob("*.json")): + data = json.loads(path.read_text()) + assert data["exit_code"] == 0 and data["finished_at"] + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/test-video-sync-report.py b/scripts/test-video-sync-report.py new file mode 100755 index 000000000..d20196080 --- /dev/null +++ b/scripts/test-video-sync-report.py @@ -0,0 +1,272 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for video-sync-report.py.""" + +import json +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 +from videotesting import load_script # noqa: E402 + +r = load_script("video-sync-report.py") + +ALL = list(r.STEPS) +RUN_URL = "https://github.com/posit-dev/open-source-website/actions/runs/1" + + +def summary(script, items=(), tallies=None, extra=None, fatal=None): + s = videolib.RunSummary(script) + for item in items: + item = dict(item) + s.add(item.pop("status"), **item) + for status, n in (tallies or {}).items(): + s.tally(status, n) + s.extra.update(extra or {}) + s.fatal = fatal + return s.to_dict() + + +def quiet_night(): + return { + "update-youtube-videos": summary("update-youtube-videos", tallies={"unchanged": 1750}, extra={"refresh_stats": False}), + "sync-videos": summary("sync-videos", tallies={"unchanged": 1750}), + "transcribe": summary("transcribe", tallies={"already_transcribed": 1750}), + "format-transcriptions": summary("format-transcriptions", tallies={"already_formatted": 1750}), + } + + +def typical_night(): + s = quiet_night() + s["update-youtube-videos"] = summary( + "update-youtube-videos", + items=[ + {"status": "new", "url": "https://www.youtube.com/watch?v=aaaaaaaaaaa", "slug": "2026-11-11_new-talk", "title": "New | talk"}, + {"status": "changed", "slug": "2026-10-01_renamed", "changed_fields": ["title", "slug"], "old_slug": "2026-10-01_old"}, + {"status": "missing", "slug": "2025-01-01_gone", "url": "https://www.youtube.com/watch?v=ggggggggggg"}, + ], + tallies={"unchanged": 1748}, + extra={"refresh_stats": False}, + ) + s["sync-videos"] = summary( + "sync-videos", + items=[ + {"status": "created", "slug": "2026-11-11_new-talk"}, + { + "status": "renamed", + "action": "renamed", + "old_slug": "2026-10-01_old", + "slug": "2026-10-01_renamed", + "updated_references": ["content/blog/post/index.md"], + }, + ], + tallies={"unchanged": 1749}, + ) + s["transcribe"] = summary( + "transcribe", + items=[{"status": "transcribed", "slug": "2026-11-11_new-talk", "estimated_cost_usd": 0.12, "duration_s": 1200}], + tallies={"already_transcribed": 1750}, + extra={"estimated_cost_usd": 0.12, "breaker_tripped": False}, + ) + s["format-transcriptions"] = summary( + "format-transcriptions", + items=[{"status": "formatted", "slug": "2026-11-11_new-talk", "estimated_cost_usd": 0.3, "warnings": None}], + tallies={"already_formatted": 1750}, + extra={"estimated_cost_usd": 0.3}, + ) + return s + + +def blocked_night(): + s = typical_night() + s["transcribe"] = summary( + "transcribe", + items=[ + {"status": "error", "slug": "2026-11-11_new-talk", "stage": "download", "reason": "bot_check", "detail": "ERROR: Sign in to confirm you’re not a bot"}, + {"status": "error", "slug": "2026-11-10_b", "stage": "download", "reason": "bot_check", "detail": "ERROR: Sign in"}, + {"status": "error", "slug": "2026-11-09_c", "stage": "download", "reason": "private", "detail": "ERROR: Private video"}, + {"status": "skipped", "slug": "2026-11-08_d", "reason": "download_blocked"}, + ], + extra={"breaker_tripped": True}, + ) + s["format-transcriptions"] = summary("format-transcriptions", tallies={"already_formatted": 1750}) + return s + + +def build(summaries, expected=ALL): + return r.build_report(summaries, expected=expected, run_url=RUN_URL) + + +def test_quiet_night(): + report, annotations = build(quiet_night()) + assert report == ( + "Nightly sync of [Posit's YouTube videos](https://www.youtube.com/@PositPBC) into " + f"`content/resources/videos/` ([workflow run]({RUN_URL})).\n" + "\n" + "Review the changes below, then approve and merge to publish them.\n" + "\n" + "| Step | Result |\n" + "| --- | --- |\n" + "| YouTube metadata | 1,750 unchanged |\n" + "| Video pages | 1,750 unchanged |\n" + "| Transcription | nothing to do |\n" + "| Formatting | nothing to do |\n" + "\n" + "---\n" + "\n" + "Adjust a video with `include`, `exclude`, or `override` blocks in its `_index.md`, hide it with " + "`publish = false` in `data/videos.toml`, or skip its transcript with `transcribe: false`. " + f"Logs and run summaries are attached to the [workflow run]({RUN_URL}).\n" + ) + assert annotations == [] + + +def test_typical_night(): + report, annotations = build(typical_night()) + assert "| YouTube metadata | 1 new, 1 changed, 1,748 unchanged |" in report + assert "| Video pages | 1 created, 1 renamed, 1,749 unchanged |" in report + assert "| Transcription | 1 transcribed (~$0.12) |" in report + assert "| Formatting | 1 formatted (~$0.30) |" in report + assert "### New videos (1)" in report + assert "- [New \\| talk](https://www.youtube.com/watch?v=aaaaaaaaaaa) `2026-11-11_new-talk`" in report + assert "- `2026-10-01_old` → `2026-10-01_renamed`; references updated in `content/blog/post/index.md`" in report + assert "- `2026-10-01_renamed`: title, slug" in report + assert "- `2026-11-11_new-talk`: transcribed and formatted" in report + assert "### ⚠️ Problems" not in report + assert "### Notes" in report and "no longer on the channel" in report + assert annotations == [] + + +def test_blocked_night(): + report, annotations = build(blocked_night()) + assert "| Transcription | ⚠️ 3 failed, downloads blocked |" in report + assert "**YouTube blocked the downloads**" in report + assert "1 more video skipped after repeated bot_check errors" in report + assert "`just update-videos 15 chrome`" in report + assert "`2026-11-10_b`: **bot_check**" in report + assert "Fix: `just transcribe --video 2026-11-10_b --cookies-from-browser chrome`\n" in report + assert "`2026-11-09_c`: **private**" in report and "or set `transcribe: false`" in report + assert annotations == [ + "::warning title=YouTube blocked the downloads::1 item(s); see the step summary or PR description", + "::warning title=Audio downloads failed::3 item(s); see the step summary or PR description", + ] + + +def test_waiting_for_transcripts(): + s = quiet_night() + del s["transcribe"] + s["format-transcriptions"] = summary( + "format-transcriptions", + tallies={"already_formatted": 1700, "no_vtt": 2}, + extra={"needs_transcript": ["2026-11-11_b", "2026-11-10_a"]}, + ) + report, annotations = build(s, expected=["update-youtube-videos", "sync-videos", "format-transcriptions"]) + assert "Transcription" not in report # the step didn't run and wasn't expected + assert "**2 videos waiting for a transcript.**" in report + assert "git switch -C automation/sync-videos origin/automation/sync-videos" in report + assert "just transcribe --limit 15" in report + assert "- `2026-11-11_b`\n- `2026-11-10_a`" in report + assert annotations == [] + + +def test_stats_refresh(): + s = quiet_night() + s["update-youtube-videos"] = summary( + "update-youtube-videos", tallies={"stats_refreshed": 1700, "unchanged": 50}, extra={"refresh_stats": True} + ) + report, _ = build(s) + assert "| YouTube metadata | 1,750 unchanged, stats refreshed |" in report + + +def test_formatting_errors_and_warnings(): + s = typical_night() + s["format-transcriptions"] = summary( + "format-transcriptions", + items=[ + {"status": "error", "slug": "2026-11-11_new-talk", "reason": "low_coverage", "detail": "Only 80% of the words"}, + {"status": "formatted", "slug": "2026-11-01_x", "warnings": ["timestamp_overshoot: 44s"]}, + ], + ) + report, annotations = build(s) + assert "`2026-11-11_new-talk`: transcribed (not formatted yet)" in report + assert "`2026-11-01_x`: ⚠️ timestamp_overshoot: 44s" in report + assert "**Formatting failed**" in report + assert "`just format-transcriptions --video 2026-11-11_new-talk`" in report + assert annotations == ["::warning title=Formatting failed::1 item(s); see the step summary or PR description"] + + +def test_fatal_and_missing_summaries(): + s = quiet_night() + s["update-youtube-videos"] = summary("update-youtube-videos", fatal="YOUTUBE_API_KEY not found") + del s["format-transcriptions"] + report, annotations = build(s) + assert "| YouTube metadata | ❌ failed: YOUTUBE_API_KEY not found |" in report + assert "| Formatting | ⚠️ no summary (crashed?) |" in report + assert "**YouTube metadata failed**" in report + assert annotations[0].startswith("::error title=YouTube metadata failed::") + assert annotations[1].startswith("::error title=Formatting produced no summary::") + + +def test_skipped_steps_not_expected(): + s = quiet_night() + del s["transcribe"], s["format-transcriptions"] + report, annotations = build(s, expected=["update-youtube-videos", "sync-videos"]) + assert "Transcription" not in report and annotations == [] + + +def test_long_lists_are_truncated(): + s = quiet_night() + s["update-youtube-videos"] = summary( + "update-youtube-videos", + items=[ + {"status": "new", "url": f"https://www.youtube.com/watch?v={i:011d}", "slug": f"s{i}", "title": f"T{i}"} + for i in range(100) + ], + ) + report, _ = build(s) + assert "…and 60 more" in report + assert len(report) < 65_536 + + +def test_main_reads_directory(tmp_path, capsys): + for script, data in typical_night().items(): + (tmp_path / f"{script}.json").write_text(json.dumps(data)) + (tmp_path / "junk.json").write_text("{not json") + out = tmp_path / "body.md" + assert r.main([str(tmp_path), "--output", str(out), "--annotations", "--run-url", RUN_URL]) == 0 + assert "### New videos (1)" in out.read_text() + captured = capsys.readouterr() + assert captured.out == "" # no annotations on a typical night + assert "could not read" in captured.err + + +def test_main_missing_directory(tmp_path, capsys): + assert r.main([str(tmp_path / "nope"), "--annotations"]) == 0 + out = capsys.readouterr().out + assert "⚠️ no summary (crashed?)" in out + assert "::error title=YouTube metadata produced no summary::" in out + + +def test_default_run_url(monkeypatch): + monkeypatch.setenv("GITHUB_SERVER_URL", "https://github.com") + monkeypatch.setenv("GITHUB_REPOSITORY", "posit-dev/open-source-website") + monkeypatch.setenv("GITHUB_RUN_ID", "1") + assert r.default_run_url() == RUN_URL + monkeypatch.delenv("GITHUB_RUN_ID") + assert r.default_run_url() is None + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/test-videolib.py b/scripts/test-videolib.py new file mode 100755 index 000000000..e14c8ad48 --- /dev/null +++ b/scripts/test-videolib.py @@ -0,0 +1,412 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pyyaml>=6.0", +# "rich>=13.0.0", +# "pytest>=8.0", +# ] +# /// + +"""Tests for videolib.py, the shared helpers of the video pipeline.""" + +import asyncio +import json +import logging +import sys +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 +from videotesting import RateLimitError, StatusError, no_sleep # noqa: E402 + +# --- VTT --- + + +class TestVttTimestamps: + @pytest.mark.parametrize( + "ts, seconds", + [ + ("00:00:00.000", 0.0), + ("00:01:30.500", 90.5), + ("01:02:03.100", 3723.1), + ("02:05.250", 125.25), # no hours + ], + ) + def test_parse(self, ts, seconds): + assert videolib.parse_vtt_timestamp(ts) == pytest.approx(seconds) + + @pytest.mark.parametrize( + "seconds, ts", + [ + (0, "00:00:00.000"), + (90.5, "00:01:30.500"), + (3723.1, "01:02:03.100"), + (59.9996, "00:01:00.000"), # rounds up instead of "00:00:60.000" + ], + ) + def test_format(self, seconds, ts): + assert videolib.format_vtt_timestamp(seconds) == ts + + +class TestParseVtt: + def test_basic(self): + vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:02.500\nHello\n\n00:00:03.000 --> 00:00:04.000\nWorld\n" + assert videolib.parse_vtt(vtt) == [(1.0, 2.5, "Hello"), (3.0, 4.0, "World")] + + def test_multiline_cue_and_identifier(self): + vtt = "WEBVTT\n\ncue-1\n00:00:01.000 --> 00:00:02.000\nline one\nline two\n" + assert videolib.parse_vtt(vtt) == [(1.0, 2.0, "line one\nline two")] + + def test_empty_cues_dropped(self): + vtt = "WEBVTT\n\n00:00:01.000 --> 00:00:02.000\n\n\n00:00:03.000 --> 00:00:04.000\nkept\n" + assert videolib.parse_vtt(vtt) == [(3.0, 4.0, "kept")] + + def test_crlf(self): + vtt = "WEBVTT\r\n\r\n00:00:01.000 --> 00:00:02.000\r\nHi\r\n" + assert videolib.parse_vtt(vtt) == [(1.0, 2.0, "Hi")] + + def test_not_vtt(self): + assert videolib.parse_vtt("") == [] + assert videolib.parse_vtt("just some text") == [] + + def test_round_trip(self): + cues = [(0.0, 4.0, "a"), (4.0, 9.5, "b c"), (3600.25, 3605.0, "d")] + assert videolib.parse_vtt(videolib.render_vtt(cues)) == cues + + +# --- Frontmatter --- + + +class TestFrontmatter: + def test_split(self): + fm, section, rest = videolib.split_frontmatter("---\ntitle: X\n---\nbody\n") + assert fm == {"title": "X"} + assert section == "title: X" + assert rest == "body\n" + + @pytest.mark.parametrize( + "content", + [ + "no frontmatter", + "---\ntitle: X\nnever closed\n", + "---\ntitle: [unclosed\n---\n", + "---\n- a list\n---\n", + ], + ) + def test_invalid(self, content): + fm, section, rest = videolib.split_frontmatter(content) + assert fm == {} and section == "" and rest == content + + def test_read_missing_file(self, tmp_path): + assert videolib.read_frontmatter(tmp_path / "nope.md") == {} + + +# --- Files --- + + +class TestAtomicWrite: + def test_writes(self, tmp_path): + p = tmp_path / "sub" / "f.txt" + videolib.atomic_write_text(p, "héllo") + assert p.read_text(encoding="utf-8") == "héllo" + + def test_failure_leaves_original_and_no_temp_files(self, tmp_path, monkeypatch): + p = tmp_path / "f.txt" + p.write_text("original") + + def boom(*_): + raise OSError("disk full") + + monkeypatch.setattr(videolib.os, "replace", boom) + with pytest.raises(OSError): + videolib.atomic_write_text(p, "new") + assert p.read_text() == "original" + assert list(tmp_path.iterdir()) == [p] + + +# --- Video directories --- + + +class TestVideoDirs: + def test_newest_first(self, tmp_path): + for name in ["2024-01-01_a", "2026-05-01_b", "2025-03-03_c"]: + (tmp_path / name).mkdir() + (tmp_path / "file.txt").write_text("x") + assert [d.name for d in videolib.video_dirs(tmp_path)] == [ + "2026-05-01_b", + "2025-03-03_c", + "2024-01-01_a", + ] + + def test_resolve(self, tmp_path): + (tmp_path / "2026-01-01_talk-one").mkdir() + (tmp_path / "2026-01-01_talk-two").mkdir() + (tmp_path / "2025-01-01_other").mkdir() + assert videolib.resolve_video_dir("2025", tmp_path).name == "2025-01-01_other" + assert videolib.resolve_video_dir("2026-01-01_talk-one", tmp_path).name == "2026-01-01_talk-one" + with pytest.raises(ValueError, match="Ambiguous"): + videolib.resolve_video_dir("2026-01-01_talk", tmp_path) + with pytest.raises(ValueError, match="not found"): + videolib.resolve_video_dir("1999", tmp_path) + + @pytest.mark.parametrize( + "url, vid", + [ + ("https://www.youtube.com/watch?v=ohUZNk5WCGQ", "ohUZNk5WCGQ"), + ("https://youtu.be/ohUZNk5WCGQ", "ohUZNk5WCGQ"), + ("https://www.youtube.com/shorts/ohUZNk5WCGQ", "ohUZNk5WCGQ"), + ("https://example.com", None), + (None, None), + ], + ) + def test_youtube_id(self, url, vid): + assert videolib.youtube_id(url) == vid + + +# --- Run summary --- + + +class TestRunSummary: + def test_exit_codes(self): + s = videolib.RunSummary("x") + s.add("created", slug="a") + assert s.exit_code == videolib.EXIT_OK + s.add("error", slug="b", reason="boom") + assert s.exit_code == videolib.EXIT_PARTIAL + s.fatal = "no API key" + assert s.exit_code == videolib.EXIT_FATAL + + def test_counts_merge_items_and_tallies(self): + s = videolib.RunSummary("x") + s.add("created", slug="a") + s.add("created", slug="b") + s.tally("unchanged", 1700) + s.tally("created") + assert s.counts() == {"created": 3, "unchanged": 1700} + + def test_none_fields_dropped(self): + s = videolib.RunSummary("x") + item = s.add("error", slug="a", reason=None, detail="d") + assert item == {"status": "error", "slug": "a", "detail": "d"} + + def test_json_schema(self, tmp_path): + s = videolib.RunSummary("sync-videos") + s.add("error", slug="a", reason="boom") + s.extra["dry_run"] = False + path = tmp_path / "out" / "summary.json" + s.write(path) + data = json.loads(path.read_text()) + assert set(data) == { + "script", + "started_at", + "finished_at", + "exit_code", + "fatal", + "counts", + "extra", + "environment", + "items", + } + assert data["script"] == "sync-videos" + assert data["exit_code"] == 2 + assert data["counts"] == {"error": 1} + assert data["finished_at"] is not None + + def test_run_main_turns_exceptions_into_fatal(self, tmp_path, caplog): + s = videolib.RunSummary("x") + path = tmp_path / "s.json" + + def body(): + raise RuntimeError("kaput") + + with caplog.at_level(logging.ERROR): + code = videolib.run_main(s, logging.getLogger("t"), path, body) + assert code == 1 + assert json.loads(path.read_text())["fatal"] == "RuntimeError: kaput" + assert "Unhandled error" in caplog.text + + def test_run_main_ok(self, tmp_path): + s = videolib.RunSummary("x") + assert videolib.run_main(s, logging.getLogger("t"), None, lambda: None) == 0 + + +# --- Retries --- + + +class TestRetryableErrors: + @pytest.mark.parametrize( + "error, retryable", + [ + (RateLimitError(), True), + (StatusError(500), True), + (StatusError(529), True), # Anthropic "overloaded" + (StatusError(408), True), + (StatusError(400), False), + (StatusError(401), False), + (TimeoutError(), True), + (ValueError("bad"), False), + ], + ) + def test_is_retryable(self, error, retryable): + assert videolib.is_retryable_api_error(error) is retryable + + def test_connection_error_by_class_name(self): + class APIConnectionError(Exception): + pass + + class APITimeoutError(APIConnectionError): + pass + + assert videolib.is_retryable_api_error(APITimeoutError()) + + def test_describe(self): + assert videolib.describe_api_error(StatusError(429, "slow down")) == ( + "StatusError status=429 request_id=req_429: slow down" + ) + assert videolib.describe_api_error(ValueError()) == "ValueError" + + +class TestRetryAsync: + def run(self, fn, **kwargs): + sleeps = [] + + async def fake_sleep(s): + sleeps.append(s) + + result = asyncio.run( + videolib.retry_async( + fn, label="video-x", logger=logging.getLogger("t"), sleep=fake_sleep, **kwargs + ) + ) + return result, sleeps + + def test_retries_then_succeeds(self, caplog): + outcomes = [RateLimitError(), StatusError(503), "ok"] + + async def fn(): + r = outcomes.pop(0) + if isinstance(r, Exception): + raise r + return r + + with caplog.at_level(logging.WARNING): + result, sleeps = self.run(fn, base_delay=1.0) + assert result == "ok" + assert len(sleeps) == 2 + assert 1.0 <= sleeps[0] < 2.0 and 2.0 <= sleeps[1] < 3.0 # exponential + jitter + assert "video-x: attempt 1/6 failed: RateLimitError status=429" in caplog.text + assert "attempt 2/6 failed: StatusError status=503" in caplog.text + + def test_default_backoff_rides_out_a_minute(self): + async def fn(): + raise TimeoutError() + + sleeps = [] + + async def fake_sleep(s): + sleeps.append(s) + + with pytest.raises(TimeoutError): + asyncio.run(videolib.retry_async(fn, label="x", logger=logging.getLogger("t"), sleep=fake_sleep)) + assert len(sleeps) == 5 + assert 60 <= sum(sleeps) <= 75 + assert max(sleeps) <= 62 + + def test_delay_capped(self): + sleeps = [] + + async def fake_sleep(s): + sleeps.append(s) + + async def fn(): + raise TimeoutError() + + with pytest.raises(TimeoutError): + asyncio.run( + videolib.retry_async( + fn, label="x", logger=logging.getLogger("t"), sleep=fake_sleep, attempts=10, max_delay=5 + ) + ) + assert max(sleeps) <= 5 + 2.0 + + def test_non_retryable_raises_immediately(self, caplog): + calls = [] + + async def fn(): + calls.append(1) + raise StatusError(400, "bad request") + + with caplog.at_level(logging.ERROR), pytest.raises(StatusError): + self.run(fn) + assert calls == [1] + assert "not retryable" in caplog.text + + def test_gives_up(self, caplog): + calls = [] + + async def fn(): + calls.append(1) + raise StatusError(500) + + with caplog.at_level(logging.ERROR), pytest.raises(StatusError): + self.run(fn, attempts=3) + assert len(calls) == 3 + assert "giving up after 3 attempts" in caplog.text + + +class TestRateLimiter: + def test_spacing(self): + async def go(): + limiter = videolib.RateLimiter(min_interval=0.05) + loop = asyncio.get_running_loop() + times = [] + for _ in range(3): + await limiter.wait() + times.append(loop.time()) + return times + + times = asyncio.run(go()) + assert times[1] - times[0] >= 0.045 + assert times[2] - times[1] >= 0.045 + + +# --- Logging --- + + +class TestLogging: + def test_plain_format_in_ci(self, monkeypatch, capsys): + monkeypatch.setenv("CI", "true") + logger = videolib.setup_logging("test-logger") + logger.info("hello %s", "world") + err = capsys.readouterr().err + assert "INFO test-logger: hello world" in err + assert err[:4].isdigit() and "Z " in err # UTC timestamp first + + def test_verbose_and_env_level(self, monkeypatch): + monkeypatch.setenv("CI", "true") + assert videolib.setup_logging("a", verbose=True).getEffectiveLevel() == logging.DEBUG + monkeypatch.setenv("LOG_LEVEL", "warning") + assert videolib.setup_logging("a").getEffectiveLevel() == logging.WARNING + + def test_noisy_loggers_quieted(self, monkeypatch): + monkeypatch.setenv("CI", "true") + videolib.setup_logging("a") + assert logging.getLogger("httpx").level == logging.WARNING + + def test_log_environment(self, caplog): + with caplog.at_level(logging.INFO): + env = videolib.log_environment( + logging.getLogger("t"), packages=["pyyaml", "no-such-package-xyz"], tools=["no-such-tool-xyz"] + ) + assert env["packages"]["pyyaml"] + assert env["packages"]["no-such-package-xyz"] is None + assert env["tools"]["no-such-tool-xyz"] is None + assert "tool no-such-tool-xyz: NOT FOUND" in caplog.text + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-p", "no:cacheprovider", *sys.argv[1:]])) diff --git a/scripts/transcribe.py b/scripts/transcribe.py index c5d7d83d2..6201f4b02 100755 --- a/scripts/transcribe.py +++ b/scripts/transcribe.py @@ -1,5 +1,6 @@ -#!/usr/bin/env -S uv run +#!/usr/bin/env -S uv run --script # /// script +# requires-python = ">=3.11" # dependencies = [ # "pyyaml>=6.0", # "rich>=13.0.0", @@ -9,86 +10,255 @@ # ] # /// - """ -Download audio from YouTube and transcribe it for each video under -content/resources/videos/. - -For each subdirectory: -- If transcription.vtt already exists, skip. -- If no audio file exists, download it from external.url in _index.md using - yt-dlp (pass --cookies-from-browser BROWSER if authentication is needed). -- If the audio exceeds the 25 MB API limit, split into chunks at silence - boundaries, transcribe each chunk, and merge the VTT output with corrected - timestamps. -- Delete the audio file after a successful transcription. - -Requires OPENAI_API_KEY in .env. -Runs up to --jobs transcriptions in parallel (default 5) with at least 1 second -between API requests and exponential backoff on rate-limit / server errors. +Download audio from YouTube and transcribe it with OpenAI Whisper. + +Processes video directories under content/resources/videos/, newest first, +that don't have a transcription.vtt yet. Skips videos with +`transcribe: false` in their frontmatter and videos without a duration +(live streams and premieres that haven't happened yet). + +For each video: +- Download the audio with yt-dlp into a temporary directory (an audio.* + file already in the video directory is used instead, and deleted after a + successful transcription). +- If the audio exceeds the 25 MB API limit, split it at silences, transcribe + each chunk, and merge the VTT output with corrected timestamps. +- Check the result (at least one cue; a warning if it ends well before the + video does) and write transcription.vtt. + +YouTube may block downloads from datacenter IPs (such as GitHub Actions +runners). Download errors are classified (bot_check, po_token, http_403, ...) +and logged with yt-dlp's full output. After three blocked downloads in a row +the remaining downloads are skipped, so we don't make matters worse. + +Requires OPENAI_API_KEY in the environment or .env, ffmpeg/ffprobe, and a +JavaScript runtime for yt-dlp (Deno). + +Exit codes: 0 = success, 1 = fatal, 2 = partial (some videos failed). """ +from __future__ import annotations + import argparse import asyncio +import logging import os -import random import re +import shutil import subprocess import sys import tempfile +import time +from dataclasses import dataclass, field from pathlib import Path -from typing import Any - -import openai -import yaml -from dotenv import load_dotenv -from rich.console import Console -from rich.progress import ( - BarColumn, - MofNCompleteColumn, - Progress, - SpinnerColumn, - TaskID, - TextColumn, - TimeElapsedColumn, -) - -console = Console(stderr=True) +from typing import Any, Awaitable, Callable -MAX_BYTES = 25 * 1024 * 1024 -MAX_RETRIES = 5 -BASE_BACKOFF = 1.0 - - -def parse_frontmatter(content: str) -> dict[str, Any]: - lines = content.split("\n") - if not lines or lines[0].strip() != "---": - return {} - end_idx = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) - if end_idx is None: - return {} - try: - return yaml.safe_load("\n".join(lines[1:end_idx])) or {} - except yaml.YAMLError: - return {} - - -def find_audio_file(video_dir: Path) -> Path | None: - for f in video_dir.glob("audio.*"): - if f.suffix != ".part": - return f - return None +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import videolib # noqa: E402 + +log = logging.getLogger("transcribe") + +MAX_BYTES = 25 * 1024 * 1024 +WHISPER_MODEL = "whisper-1" +WHISPER_USD_PER_MINUTE = 0.006 + +# Download failures that mean YouTube is blocking us, not this one video. +BLOCKING_REASONS = {"bot_check", "po_token", "http_429"} +BREAKER_THRESHOLD = 3 +# Download failures worth one more try. +RETRY_REASONS = {"network", "http_403"} +DOWNLOAD_RETRY_DELAY = 5.0 +# Warn when the transcript ends before this fraction of the video. +TRUNCATION_RATIO = 0.9 +# Whisper sometimes returns a long run of cues without words (just ".") +# for part of the audio. A chunk with a stretch this long is retried once; +# a transcript that still has one is rejected. +MAX_WORDLESS_S = 120 +# Aim chunks at this share of the API limit: bitrates vary, and a chunk +# that ends up just over the limit gets split again into a tiny piece. +CHUNK_TARGET_RATIO = 0.9 +MAX_STDERR_CHARS = 4000 + + +# --------------------------------------------------------------------------- +# Selecting videos +# --------------------------------------------------------------------------- + + +@dataclass +class Candidate: + dir: Path + url: str + duration: int + frontmatter: dict[str, Any] = field(default_factory=dict) + + @property + def slug(self) -> str: + return self.dir.name + + @property + def video_id(self) -> str | None: + return videolib.youtube_id(self.url) + + +def select_videos( + dirs: list[Path], + summary: videolib.RunSummary, + *, + limit: int | None = None, + max_audio_minutes: float | None = None, +) -> list[Candidate]: + """Pick the videos to transcribe this run, in the given order. + + ``limit`` caps the number of videos and ``max_audio_minutes`` the total + duration (a single video longer than the budget is still taken if it is + the first one, so it doesn't block the queue forever). + """ + selected: list[Candidate] = [] + minutes = 0.0 + deferred: list[str] = [] + + for d in dirs: + if (d / "transcription.vtt").exists(): + summary.tally("already_transcribed") + continue + index = d / "_index.md" + if not index.exists(): + summary.add("skipped", slug=d.name, reason="no_index") + continue + fm = videolib.read_frontmatter(index) + if fm.get("transcribe", True) is False: + summary.tally("excluded") + continue + external = fm.get("external") or {} + url = external.get("url") + if not url: + summary.add("skipped", slug=d.name, reason="no_url") + log.warning("Skipping %s: no external.url", d.name) + continue + duration = int(external.get("duration") or 0) + if duration <= 0: + summary.add("skipped", slug=d.name, video_id=videolib.youtube_id(url), reason="no_duration") + log.info("Skipping %s: no duration (live stream or premiere?)", d.name) + continue -def get_video_url(video_dir: Path) -> str | None: - index_file = video_dir / "_index.md" - if not index_file.exists(): - return None - frontmatter = parse_frontmatter(index_file.read_text(encoding="utf-8")) - return frontmatter.get("external", {}).get("url") + over_limit = limit is not None and len(selected) >= limit + over_budget = ( + max_audio_minutes is not None + and selected + and minutes + duration / 60 > max_audio_minutes + ) + if over_limit or over_budget: + deferred.append(d.name) + continue + selected.append(Candidate(d, url, duration, fm)) + minutes += duration / 60 + + if deferred: + summary.tally("deferred", len(deferred)) + summary.extra["deferred"] = deferred + log.info("Deferred %d videos to a later run (--limit / --max-audio-minutes)", len(deferred)) + summary.extra["selected_audio_minutes"] = round(minutes, 1) + return selected + + +# --------------------------------------------------------------------------- +# Downloading +# --------------------------------------------------------------------------- + +# Ordered: the first matching pattern wins. +YTDLP_ERROR_PATTERNS: list[tuple[str, str]] = [ + ("age_restricted", r"confirm your age|age[- ]restricted|inappropriate for some users"), + ("bot_check", r"confirm you(?:'|’)?re not a bot|Sign in to confirm you|not a bot"), + ("po_token", r"PO Token|po_token|GVS PO"), + ("members_only", r"members[- ]only|Join this channel"), + ("private", r"Private video|video is private"), + ("live_or_upcoming", r"live event will begin|Premieres in|This live event|is not available until|is_upcoming"), + ("js_runtime", r"JavaScript runtime|JS runtime|challenge solv|n challenge|Signature extraction failed"), + ("unavailable", r"Video unavailable|video is not available|has been removed|This video is no longer available|account associated with this video has been terminated"), + ("http_403", r"HTTP Error 403|403: Forbidden|status code 403"), + ("http_429", r"HTTP Error 429|Too Many Requests"), + ("format_unavailable", r"Requested format is not available|No video formats found"), + ( + "network", + r"timed out|Connection reset|Temporary failure in name resolution|Network is unreachable" + r"|Remote end closed|IncompleteRead|Connection refused|getaddrinfo failed|SSL", + ), +] + + +# Warnings that explain a vague ERROR line ("Requested format is not +# available", "HTTP Error 403") better than the error itself. +HINT_REASONS = ("bot_check", "po_token", "js_runtime") +VAGUE_REASONS = ("unknown", "format_unavailable", "http_403") + + +def _match(text: str, reasons: tuple[str, ...] | None = None) -> str: + for reason, pattern in YTDLP_ERROR_PATTERNS: + if (reasons is None or reason in reasons) and re.search(pattern, text, flags=re.IGNORECASE): + return reason + return "unknown" + + +def classify_ytdlp_error(output: str) -> str: + """Classify a failed yt-dlp run from its output. + + The ERROR lines decide; when they're vague, warnings such as "GVS PO + Token which was not provided" or "No supported JavaScript runtime" are + used instead. + """ + error_lines = "\n".join(line for line in output.splitlines() if "ERROR" in line) + reason = _match(error_lines or output) + if reason in VAGUE_REASONS: + hint = _match(output, HINT_REASONS) + if hint != "unknown": + return hint + return reason + + +def last_error_line(output: str) -> str: + """The most useful single line of yt-dlp output: the last ERROR line.""" + lines = [line.strip() for line in output.splitlines() if line.strip()] + errors = [line for line in lines if "ERROR" in line] + return (errors or lines or ["(no output)"])[-1] + + +class DownloadError(Exception): + def __init__(self, reason: str, stderr: str, attempts: int = 1): + super().__init__(last_error_line(stderr)) + self.reason = reason + self.stderr = stderr + self.attempts = attempts + + +class CircuitBreaker: + """Trips after ``threshold`` blocked downloads in a row.""" + + def __init__(self, threshold: int = BREAKER_THRESHOLD): + self.threshold = threshold + self.consecutive = 0 + self.tripped = False + + def record(self, reason: str | None) -> None: + if reason in BLOCKING_REASONS: + self.consecutive += 1 + if self.consecutive >= self.threshold and not self.tripped: + self.tripped = True + log.error( + "%d downloads in a row were blocked by YouTube (%s); " + "skipping the remaining downloads this run", + self.consecutive, + reason, + ) + else: + self.consecutive = 0 -def download_audio(url: str, dest_dir: Path, browser: str | None) -> None: +def build_ytdlp_cmd( + url: str, dest_dir: Path, browser: str | None, cookies: Path | None, verbose: bool = False +) -> list[str]: cmd = [ sys.executable, "-m", @@ -100,39 +270,85 @@ def download_audio(url: str, dest_dir: Path, browser: str | None) -> None: "--output", str(dest_dir / "audio.%(ext)s"), "--no-playlist", - "--quiet", - "--no-warnings", + "--no-progress", + # --verbose shows the JS runtime, PO token providers, and clients used. + "--verbose" if verbose else "--quiet", ] if browser: cmd += ["--cookies-from-browser", browser] + if cookies: + cmd += ["--cookies", str(cookies)] cmd.append(url) - result = subprocess.run(cmd, capture_output=True, text=True) + return cmd + + +def find_audio_file(directory: Path) -> Path | None: + for f in sorted(directory.glob("audio.*")): + if f.suffix not in (".part", ".ytdl"): + return f + return None + + +def download_audio( + url: str, + dest_dir: Path, + *, + browser: str | None, + cookies: Path | None, + run: Callable[..., subprocess.CompletedProcess], + label: str, +) -> Path: + cmd = build_ytdlp_cmd(url, dest_dir, browser, cookies, verbose=log.isEnabledFor(logging.DEBUG)) + log.debug("%s: running %s", label, " ".join(cmd)) + result = run(cmd, capture_output=True, text=True) + output = "\n".join(s for s in (result.stderr, result.stdout) if s and s.strip()) if result.returncode != 0: - raise RuntimeError(result.stderr.strip() or result.stdout.strip()) + raise DownloadError(classify_ytdlp_error(output), output) + for line in output.splitlines(): + log.debug("%s: yt-dlp: %s", label, line) + audio = find_audio_file(dest_dir) + if audio is None: + raise DownloadError("no_file", output or "yt-dlp succeeded but wrote no audio file") + return audio + + +async def download_with_retry( + candidate: Candidate, + dest_dir: Path, + args: argparse.Namespace, + run: Callable[..., subprocess.CompletedProcess], + sleep: Callable[[float], Awaitable[Any]], +) -> Path: + label = f"{candidate.slug} ({candidate.video_id})" + for attempt in (1, 2): + try: + return await asyncio.to_thread( + download_audio, + candidate.url, + dest_dir, + browser=args.cookies_from_browser, + cookies=args.cookies, + run=run, + label=label, + ) + except DownloadError as e: + e.attempts = attempt + if attempt == 1 and e.reason in RETRY_REASONS: + log.warning("%s: download failed (%s: %s); retrying in %.0fs", label, e.reason, e, DOWNLOAD_RETRY_DELAY) + await sleep(DOWNLOAD_RETRY_DELAY) + continue + raise + raise AssertionError("unreachable") -def build_prompt(frontmatter: dict[str, Any]) -> str: - parts = [] - title = frontmatter.get("title", "") - if title: - parts.append(title) - software = frontmatter.get("software", []) or [] - parts.extend(software) - return ", ".join(parts) +# --------------------------------------------------------------------------- +# Splitting large audio files +# --------------------------------------------------------------------------- def get_audio_duration(src: Path) -> float: result = subprocess.run( - [ - "ffprobe", - "-v", - "error", - "-show_entries", - "format=duration", - "-of", - "default=noprint_wrappers=1:nokey=1", - str(src), - ], + ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", str(src)], capture_output=True, text=True, check=True, @@ -140,20 +356,9 @@ def get_audio_duration(src: Path) -> float: return float(result.stdout.strip()) -def detect_silence( - src: Path, noise_db: float = -30, min_duration: float = 0.5 -) -> list[float]: +def detect_silence(src: Path, noise_db: float = -30, min_duration: float = 0.5) -> list[float]: result = subprocess.run( - [ - "ffmpeg", - "-i", - str(src), - "-af", - f"silencedetect=noise={noise_db}dB:d={min_duration}", - "-f", - "null", - "-", - ], + ["ffmpeg", "-i", str(src), "-af", f"silencedetect=noise={noise_db}dB:d={min_duration}", "-f", "null", "-"], capture_output=True, text=True, ) @@ -171,8 +376,9 @@ def compute_chunk_boundaries( duration: float, file_size: int, silence_points: list[float], + max_bytes: int = MAX_BYTES, ) -> list[tuple[float, float]]: - max_chunk_secs = MAX_BYTES / (file_size / duration) + max_chunk_secs = CHUNK_TARGET_RATIO * max_bytes / (file_size / duration) boundaries: list[tuple[float, float]] = [] start = 0.0 while start < duration: @@ -187,299 +393,372 @@ def compute_chunk_boundaries( return boundaries -def split_at_silence(src: Path, _offset: float = 0.0) -> list[tuple[Path, float]]: +def split_at_silence( + src: Path, work_dir: Path, _offset: float = 0.0, max_bytes: int = MAX_BYTES +) -> list[tuple[Path, float]]: + """Split ``src`` into chunks under ``max_bytes``; return (path, offset) pairs.""" duration = get_audio_duration(src) silence = detect_silence(src) - boundaries = compute_chunk_boundaries(duration, src.stat().st_size, silence) + boundaries = compute_chunk_boundaries(duration, src.stat().st_size, silence, max_bytes) + log.debug("Splitting %s (%.0fs) at %s", src.name, duration, [round(e) for _, e in boundaries]) chunks: list[tuple[Path, float]] = [] for start, end in boundaries: - tmp = Path(tempfile.mktemp(suffix=src.suffix)) + fd, name = tempfile.mkstemp(suffix=src.suffix, prefix="chunk-", dir=work_dir) + os.close(fd) + tmp = Path(name) subprocess.run( - [ - "ffmpeg", - "-y", - "-ss", - str(start), - "-i", - str(src), - "-t", - str(end - start), - "-c", - "copy", - str(tmp), - ], + ["ffmpeg", "-y", "-ss", str(start), "-i", str(src), "-t", str(end - start), "-c", "copy", str(tmp)], check=True, capture_output=True, ) - if tmp.stat().st_size > MAX_BYTES: - sub_chunks = split_at_silence(tmp, _offset=_offset + start) + if tmp.stat().st_size > max_bytes and end - start > 1: + chunks.extend(split_at_silence(tmp, work_dir, _offset=_offset + start, max_bytes=max_bytes)) tmp.unlink(missing_ok=True) - chunks.extend(sub_chunks) else: chunks.append((tmp, _offset + start)) return chunks -def parse_vtt_timestamp(ts: str) -> float: - parts = ts.strip().split(":") - if len(parts) == 3: - return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2]) - return int(parts[0]) * 60 + float(parts[1]) - +# --------------------------------------------------------------------------- +# Transcribing +# --------------------------------------------------------------------------- -def format_vtt_timestamp(seconds: float) -> str: - h = int(seconds // 3600) - m = int((seconds % 3600) // 60) - s = seconds % 60 - return f"{h:02d}:{m:02d}:{s:06.3f}" - -def parse_vtt(content: str) -> list[tuple[float, float, str]]: - cues: list[tuple[float, float, str]] = [] - for block in re.split(r"\n{2,}", content.strip()): - lines = block.strip().splitlines() - arrow = next((i for i, line in enumerate(lines) if "-->" in line), None) - if arrow is None: - continue - m = re.match(r"([\d:.]+)\s+-->\s+([\d:.]+)", lines[arrow]) - if not m: - continue - start = parse_vtt_timestamp(m.group(1)) - end = parse_vtt_timestamp(m.group(2)) - text = "\n".join(lines[arrow + 1 :]).strip() - if text: - cues.append((start, end, text)) - return cues +def merge_vtt(parts: list[tuple[str, float]]) -> str: + cues = [ + (start + offset, end + offset, text) + for vtt_content, offset in parts + for start, end, text in videolib.parse_vtt(vtt_content) + ] + return videolib.render_vtt(cues) -def merge_vtt(parts: list[tuple[str, float]]) -> str: - lines = ["WEBVTT", ""] - for vtt_content, offset in parts: - for start, end, text in parse_vtt(vtt_content): - lines.append( - f"{format_vtt_timestamp(start + offset)} --> " - f"{format_vtt_timestamp(end + offset)}" - ) - lines.append(text) - lines.append("") - return "\n".join(lines) +def build_prompt(frontmatter: dict[str, Any]) -> str: + parts = [] + title = frontmatter.get("title", "") + if title: + parts.append(str(title)) + parts.extend(str(s) for s in frontmatter.get("software", []) or []) + return ", ".join(parts) -class RateLimiter: - """Ensures at least `min_interval` seconds between API requests.""" +class TranscriptError(Exception): + def __init__(self, reason: str, detail: str): + super().__init__(detail) + self.reason = reason - def __init__(self, min_interval: float = 1.0) -> None: - self._lock = asyncio.Lock() - self._last: float = 0.0 - self._interval = min_interval - async def wait(self) -> None: - async with self._lock: - loop = asyncio.get_running_loop() - elapsed = loop.time() - self._last - if elapsed < self._interval: - await asyncio.sleep(self._interval - elapsed) - self._last = asyncio.get_running_loop().time() +def longest_wordless_stretch(cues: list[videolib.Cue]) -> tuple[float, float]: + """``(seconds, start)`` of the longest run of consecutive cues without words.""" + longest, longest_start = 0.0, 0.0 + run_start = None + for start, end, text in cues: + if re.search(r"\w", text): + run_start = None + continue + if run_start is None: + run_start = start + if end - run_start > longest: + longest, longest_start = end - run_start, run_start + return longest, longest_start + + +def check_transcript(vtt: str, duration: int) -> tuple[list[videolib.Cue], list[str]]: + """Return ``(cues, warnings)``; raise TranscriptError if unusable.""" + cues = videolib.parse_vtt(vtt) + if not cues: + raise TranscriptError("empty_transcript", f"Whisper returned no cues ({len(vtt)} characters)") + stretch, at = longest_wordless_stretch(cues) + if stretch > MAX_WORDLESS_S: + raise TranscriptError( + "degenerate_transcript", + f"{stretch:.0f}s of cues without words from {videolib.format_vtt_timestamp(at)[:8]}", + ) + warnings = [] + last_end = cues[-1][1] + if duration > 0 and last_end < TRUNCATION_RATIO * duration: + warnings.append( + f"suspect_truncated: last cue ends at {videolib.format_vtt_timestamp(last_end)} " + f"of {videolib.format_vtt_timestamp(duration)}" + ) + return cues, warnings -async def transcribe_audio( +async def transcribe_file( + client: Any, audio_file: Path, prompt: str, - name: str, - client: openai.AsyncOpenAI, - rate_limiter: RateLimiter, + *, + label: str, + rate_limiter: videolib.RateLimiter, + sleep: Callable[[float], Awaitable[Any]], ) -> str: - kwargs: dict[str, Any] = { - "model": "whisper-1", - "response_format": "vtt", - } + kwargs: dict[str, Any] = {"model": WHISPER_MODEL, "response_format": "vtt"} if prompt: kwargs["prompt"] = prompt - for attempt in range(MAX_RETRIES): - if attempt > 0: - delay = BASE_BACKOFF * (2 ** (attempt - 1)) + random.uniform(0, 1) - console.print( - f" [yellow]↻[/] {name} retry {attempt + 1}/{MAX_RETRIES} in {delay:.1f}s" - ) - await asyncio.sleep(delay) - + async def call() -> str: await rate_limiter.wait() + with open(audio_file, "rb") as f: + return await client.audio.transcriptions.create(file=f, **kwargs) + + return await videolib.retry_async(call, label=label, logger=log, sleep=sleep) + + +@dataclass +class Context: + client: Any + args: argparse.Namespace + semaphore: asyncio.Semaphore + rate_limiter: videolib.RateLimiter + breaker: CircuitBreaker + run: Callable[..., subprocess.CompletedProcess] + sleep: Callable[[float], Awaitable[Any]] + max_bytes: int = MAX_BYTES + + +async def process_video(c: Candidate, ctx: Context) -> dict[str, Any]: + async with ctx.semaphore: + label = f"{c.slug} ({c.video_id})" + item: dict[str, Any] = {"slug": c.slug, "video_id": c.video_id, "duration_s": c.duration} + if ctx.breaker.tripped: + log.info("%s: skipped, downloads are blocked this run", label) + return {**item, "status": "skipped", "reason": "download_blocked"} + + timings: dict[str, float] = {} + item["timings"] = timings + legacy_audio = find_audio_file(c.dir) try: - with open(audio_file, "rb") as f: - return await client.audio.transcriptions.create(file=f, **kwargs) - except openai.RateLimitError: - if attempt == MAX_RETRIES - 1: - raise - except openai.APIStatusError as e: - if e.status_code < 500 or attempt == MAX_RETRIES - 1: - raise - - raise RuntimeError("Exceeded max retries") - - -async def process_video_async( - video_dir: Path, - client: openai.AsyncOpenAI, - semaphore: asyncio.Semaphore, - rate_limiter: RateLimiter, - progress: Progress, - overall_task: TaskID, - browser: str | None, -) -> str: - async with semaphore: - name = video_dir.name - job_task = progress.add_task(f"[dim]{name}", total=None) - try: - transcription_file = video_dir / "transcription.vtt" - if transcription_file.exists(): - return "skipped" - - audio_file = find_audio_file(video_dir) - - if audio_file is None: - url = get_video_url(video_dir) - if not url: - return "no_url" - progress.update(job_task, description=f"[cyan]{name} downloading") - await asyncio.to_thread(download_audio, url, video_dir, browser) - audio_file = find_audio_file(video_dir) - if audio_file is None: - raise RuntimeError("Download succeeded but audio file not found") - - index_file = video_dir / "_index.md" - frontmatter = ( - parse_frontmatter(index_file.read_text(encoding="utf-8")) - if index_file.exists() - else {} - ) - prompt = build_prompt(frontmatter) - - if audio_file.stat().st_size > MAX_BYTES: - size_mb = audio_file.stat().st_size / 1024 / 1024 - progress.update( - job_task, - description=f"[yellow]{name} chunking audio ({size_mb:.1f} MB)", + with tempfile.TemporaryDirectory(prefix="transcribe-") as tmp: + work_dir = Path(tmp) + if legacy_audio: + audio = legacy_audio + log.info("%s: using existing %s", label, audio.name) + else: + log.info("%s: downloading audio (%s)", label, videolib.format_vtt_timestamp(c.duration)[:8]) + t0 = time.monotonic() + try: + audio = await download_with_retry(c, work_dir, ctx.args, ctx.run, ctx.sleep) + except DownloadError as e: + ctx.breaker.record(e.reason) + log.error("%s: download failed: %s: %s", label, e.reason, e) + for line in e.stderr.splitlines(): + log.error("%s: yt-dlp: %s", label, line) + return { + **item, + "status": "error", + "reason": e.reason, + "stage": "download", + "detail": str(e), + "attempts": e.attempts, + "stderr": e.stderr[-MAX_STDERR_CHARS:], + } + ctx.breaker.record(None) + timings["download_s"] = round(time.monotonic() - t0, 1) + + size = audio.stat().st_size + item["audio_bytes"] = size + item["audio_format"] = audio.suffix.lstrip(".") + log.info( + "%s: audio %s, %.1f MB%s", + label, + item["audio_format"], + size / 1024 / 1024, + f" in {timings['download_s']}s" if "download_s" in timings else "", ) - chunks: list[tuple[Path, float]] = await asyncio.to_thread( - split_at_silence, audio_file - ) - is_temp = True - else: - chunks = [(audio_file, 0.0)] - is_temp = False - - try: - vtt_parts: list[tuple[str, float]] = [] - total_chunks = len(chunks) - for i, (chunk_path, offset) in enumerate(chunks): - label = f" {i + 1}/{total_chunks}" if total_chunks > 1 else "" - progress.update( - job_task, description=f"[cyan]{name} transcribing{label}" - ) - vtt = await transcribe_audio( - chunk_path, prompt, name, client, rate_limiter + + if size > ctx.max_bytes: + t0 = time.monotonic() + chunks = await asyncio.to_thread(split_at_silence, audio, work_dir, 0.0, ctx.max_bytes) + timings["split_s"] = round(time.monotonic() - t0, 1) + log.info("%s: split into %d chunks at %s", label, len(chunks), ", ".join(f"{o:.0f}s" for _, o in chunks)) + else: + chunks = [(audio, 0.0)] + item["chunks"] = len(chunks) + + prompt = build_prompt(c.frontmatter) + parts: list[tuple[str, float]] = [] + t0 = time.monotonic() + for i, (chunk, offset) in enumerate(chunks, start=1): + chunk_label = f"{label} chunk {i}/{len(chunks)}" if len(chunks) > 1 else label + tc = time.monotonic() + for attempt in (1, 2): + vtt = await transcribe_file( + ctx.client, chunk, prompt, label=chunk_label, rate_limiter=ctx.rate_limiter, sleep=ctx.sleep + ) + stretch, at = longest_wordless_stretch(videolib.parse_vtt(vtt)) + if stretch <= MAX_WORDLESS_S: + break + log.warning( + "%s: Whisper returned %.0fs of cues without words from %s (attempt %d/2)%s", + chunk_label, + stretch, + videolib.format_vtt_timestamp(at + offset)[:8], + attempt, + "; retrying" if attempt == 1 else "", + ) + item["whisper_retries"] = item.get("whisper_retries", 0) + (attempt == 1) + log.info( + "%s: transcribed %.1f MB (offset %.0fs) in %.1fs", + chunk_label, + chunk.stat().st_size / 1024 / 1024, + offset, + time.monotonic() - tc, ) - vtt_parts.append((vtt, offset)) - finally: - if is_temp: - for chunk_path, _ in chunks: - chunk_path.unlink(missing_ok=True) - - if len(vtt_parts) == 1: - data = vtt_parts[0][0] - else: - progress.update( - job_task, description=f"[cyan]{name} merging transcript" - ) - data = merge_vtt(vtt_parts) - transcription_file.write_text(data, encoding="utf-8") - audio_file.unlink() - return "transcribed" + parts.append((vtt, offset)) + timings["transcribe_s"] = round(time.monotonic() - t0, 1) + + data = parts[0][0] if len(parts) == 1 else merge_vtt(parts) + cues, warnings = check_transcript(data, c.duration) + for w in warnings: + log.warning("%s: %s", label, w) + videolib.atomic_write_text(c.dir / "transcription.vtt", data) + if legacy_audio: + legacy_audio.unlink(missing_ok=True) + except TranscriptError as e: + log.error("%s: %s: %s", label, e.reason, e) + return {**item, "status": "error", "reason": e.reason, "stage": "check", "detail": str(e)} + except Exception as e: # noqa: BLE001 - every failure goes in the summary + log.exception("%s: transcription failed", label) + return { + **item, + "status": "error", + "reason": "transcription_failed", + "stage": "transcribe", + "detail": videolib.describe_api_error(e), + } + + cost = round(c.duration / 60 * WHISPER_USD_PER_MINUTE, 4) + log.info( + "%s: wrote transcription.vtt (%d cues, last at %s, ~$%.3f)", + label, + len(cues), + videolib.format_vtt_timestamp(cues[-1][1])[:8], + cost, + ) + return { + **item, + "status": "transcribed", + "cues": len(cues), + "estimated_cost_usd": cost, + "warnings": warnings or None, + } - except Exception as e: - console.print(f" [red]✗[/] {name}: {e}") - return "error" - finally: - progress.remove_task(job_task) - progress.advance(overall_task) +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- -async def main_async(jobs: int, browser: str | None) -> None: - console.print("\n[bold blue]Video Transcriber[/]\n") - load_dotenv() +def build_client(project_root: Path) -> Any: + import openai + from dotenv import load_dotenv + + load_dotenv(project_root / ".env") api_key = os.getenv("OPENAI_API_KEY") if not api_key: - console.print("[bold red]Error:[/] OPENAI_API_KEY not set in .env") - sys.exit(1) - - client = openai.AsyncOpenAI(api_key=api_key) - - videos_dir = Path(__file__).parent.parent / "content" / "resources" / "videos" - if not videos_dir.exists(): - console.print(f"[bold red]Error:[/] Videos directory not found: {videos_dir}") - sys.exit(1) - - dirs = sorted(d for d in videos_dir.iterdir() if d.is_dir()) - console.print(f"[dim]Found {len(dirs)} video directories (--jobs {jobs})[/]\n") - - semaphore = asyncio.Semaphore(jobs) - rate_limiter = RateLimiter(min_interval=1.0) - - with Progress( - SpinnerColumn(), - TextColumn("{task.description}"), - BarColumn(), - MofNCompleteColumn(), - TimeElapsedColumn(), - console=console, - ) as progress: - overall_task = progress.add_task("[bold cyan]Transcribing", total=len(dirs)) - - results: list[str] = await asyncio.gather( - *[ - process_video_async( - d, client, semaphore, rate_limiter, progress, overall_task, browser - ) - for d in dirs - ] - ) - - transcribed = results.count("transcribed") - skipped = results.count("skipped") - no_url = results.count("no_url") - errors = results.count("error") + raise RuntimeError("OPENAI_API_KEY not set in .env or environment") + return openai.AsyncOpenAI(api_key=api_key, max_retries=0) # we retry ourselves, with logging - console.print("\n[bold]Summary:[/]") - console.print(f" [green]✓[/] Transcribed: {transcribed}") - console.print(f" [dim]○[/] Skipped: {skipped}") - console.print(f" [dim]○[/] No URL: {no_url}") - if errors: - console.print(f" [red]✗[/] Errors: {errors}") - console.print("\n[bold green]✓ Done![/]\n") - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Download and transcribe video audio files using OpenAI Whisper" - ) +def parse_args(argv: list[str] | None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Download and transcribe video audio with OpenAI Whisper") + parser.add_argument("--jobs", type=int, default=5, metavar="N", help="parallel jobs (default: 5)") + parser.add_argument("--limit", type=int, metavar="N", help="transcribe at most N videos (newest first)") parser.add_argument( - "--jobs", - type=int, - default=5, - metavar="N", - help="number of parallel transcription jobs (default: 5)", + "--max-audio-minutes", type=float, metavar="M", help="stop selecting videos after M minutes of audio" ) + parser.add_argument("--video", metavar="NAME", help="only this video directory (name, prefix, or path)") parser.add_argument( "--cookies-from-browser", metavar="BROWSER", - help="browser to read cookies from for yt-dlp (e.g. chrome, firefox, safari)", + help="browser to read YouTube cookies from for yt-dlp (e.g. chrome, firefox, safari)", ) - args = parser.parse_args() - asyncio.run(main_async(args.jobs, args.cookies_from_browser)) + parser.add_argument("--cookies", type=Path, metavar="FILE", help="cookies.txt file for yt-dlp") + parser.add_argument("--dry-run", action="store_true", help="list what would be transcribed") + videolib.add_common_args(parser) + return parser.parse_args(argv) + + +def main( + argv: list[str] | None = None, + client: Any = None, + run: Callable[..., subprocess.CompletedProcess] = subprocess.run, + sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, + max_bytes: int = MAX_BYTES, + min_request_interval: float = 1.0, +) -> int: + args = parse_args(argv) + videolib.setup_logging("transcribe", args.verbose) + summary = videolib.RunSummary("transcribe") + summary.extra.update(dry_run=args.dry_run, limit=args.limit, max_audio_minutes=args.max_audio_minutes) + videos_dir = args.project_root / "content" / "resources" / "videos" + + def body() -> None: + nonlocal client + summary.environment = videolib.log_environment( + log, packages=["yt-dlp", "yt-dlp-ejs", "openai", "bgutil-ytdlp-pot-provider"], tools=["deno", "node", "ffmpeg", "ffprobe"] + ) + if not videos_dir.exists(): + summary.fatal = f"Videos directory not found: {videos_dir}" + return + if not args.dry_run: + missing = [t for t in ("ffmpeg", "ffprobe") if not shutil.which(t)] + if missing: + summary.fatal = f"Required tools not found: {', '.join(missing)}" + return + if not (shutil.which("deno") or shutil.which("node")): + log.warning("No JavaScript runtime (deno or node) found; YouTube downloads will probably fail") + + dirs = [videolib.resolve_video_dir(args.video, videos_dir)] if args.video else videolib.video_dirs(videos_dir) + candidates = select_videos(dirs, summary, limit=args.limit, max_audio_minutes=args.max_audio_minutes) + log.info( + "Selected %d of %d video directories (%.0f minutes of audio, --jobs %d)", + len(candidates), + len(dirs), + summary.extra["selected_audio_minutes"], + args.jobs, + ) + if args.dry_run: + for c in candidates: + log.info("Would transcribe %s (%s, %ds)", c.slug, c.video_id, c.duration) + summary.add("would_transcribe", slug=c.slug, video_id=c.video_id, duration_s=c.duration) + return + if not candidates: + return + if client is None: + client = build_client(args.project_root) + + ctx = Context( + client=client, + args=args, + semaphore=asyncio.Semaphore(args.jobs), + rate_limiter=videolib.RateLimiter(min_interval=min_request_interval), + breaker=CircuitBreaker(), + run=run, + sleep=sleep, + max_bytes=max_bytes, + ) + + async def go() -> list[dict[str, Any]]: + return await asyncio.gather(*(process_video(c, ctx) for c in candidates)) + + for result in asyncio.run(go()): + summary.add(result.pop("status"), **result) + + transcribed = summary.by_status("transcribed") + summary.extra["breaker_tripped"] = ctx.breaker.tripped + summary.extra["estimated_cost_usd"] = round(sum(i["estimated_cost_usd"] for i in transcribed), 3) + summary.extra["transcribed_audio_minutes"] = round(sum(i["duration_s"] for i in transcribed) / 60, 1) + log.info( + "Transcribed %d videos (%.0f minutes, ~$%.2f)", + len(transcribed), + summary.extra["transcribed_audio_minutes"], + summary.extra["estimated_cost_usd"], + ) + + return videolib.run_main(summary, log, args.summary_json, body) if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/scripts/update-youtube-videos.py b/scripts/update-youtube-videos.py index 29ebe4459..087f411cd 100755 --- a/scripts/update-youtube-videos.py +++ b/scripts/update-youtube-videos.py @@ -1,9 +1,11 @@ -#!/usr/bin/env -S uv run +#!/usr/bin/env -S uv run --script # /// script +# requires-python = ">=3.11" # dependencies = [ # "google-api-python-client>=2.100.0", # "tomli-w>=1.0.0", # "python-dotenv>=1.0.0", +# "pyyaml>=6.0", # "rich>=13.0.0", # ] # /// @@ -11,38 +13,63 @@ """ Fetch video metadata from YouTube channels and playlists. -Reads channel names and playlist IDs from data/youtube.toml, -fetches metadata via the YouTube Data API v3, and writes -video information to data/videos.toml. +Reads channel names, playlist IDs, and individual video IDs from +data/youtube.toml, fetches metadata via the YouTube Data API v3, and writes +it to data/videos.toml. + +Every run fetches metadata for all tracked videos (~40 quota units), but a +video's entry, including its `last_updated` timestamp, only changes when one +of its fields actually changed. View, like, and comment counts change all +the time, so they're only refreshed with --refresh-stats (the nightly +workflow does this once a week); new videos always get them. + +Videos that disappear from YouTube are reported but never removed. + +Requires YOUTUBE_API_KEY in the environment or .env. + +Exit codes: 0 = success, 1 = fatal (nothing written), 2 = partial (a channel +or playlist could not be fetched; everything else was written). """ +from __future__ import annotations + import argparse +import logging import os import re import sys import tomllib import unicodedata -from datetime import datetime, timedelta, timezone +from datetime import datetime, timezone from pathlib import Path from typing import Any import tomli_w -from dotenv import load_dotenv -from googleapiclient.discovery import build -from rich.console import Console -from rich.progress import ( - BarColumn, - Progress, - SpinnerColumn, - TaskProgressColumn, - TextColumn, - TimeRemainingColumn, -) - -console = Console(stderr=True) + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import videolib # noqa: E402 + +log = logging.getLogger("update-youtube-videos") THUMBNAIL_PREFERENCE = ["maxres", "high", "standard", "medium", "default"] +# Fields that change constantly; only refreshed with --refresh-stats. +STATS_FIELDS = ("view_count", "like_count", "comment_count") +# Bookkeeping fields that never count as a change by themselves. +VOLATILE_FIELDS = ("last_updated",) + +API_RETRIES = 3 + + +class FatalError(Exception): + """Something is wrong enough that videos.toml must not be written.""" + + +# --------------------------------------------------------------------------- +# Pure helpers +# --------------------------------------------------------------------------- + def slugify(text: str) -> str: text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode("ascii") @@ -53,30 +80,132 @@ def slugify(text: str) -> str: def parse_iso8601_duration(duration: str) -> int: - match = re.match(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", duration) + match = re.fullmatch(r"P(?:(\d+)D)?(?:T(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?)?", duration or "") if not match: return 0 - hours = int(match.group(1) or 0) - minutes = int(match.group(2) or 0) - seconds = int(match.group(3) or 0) - return hours * 3600 + minutes * 60 + seconds + days, hours, minutes, seconds = (int(g or 0) for g in match.groups()) + return days * 86400 + hours * 3600 + minutes * 60 + seconds -def get_api_key() -> str: - load_dotenv() - key = os.getenv("YOUTUBE_API_KEY") - if not key: - console.print( - "[bold red]Error:[/] YOUTUBE_API_KEY not found in .env or environment" - ) - sys.exit(1) - return key +def best_thumbnail(thumbnails: dict) -> str: + for key in THUMBNAIL_PREFERENCE: + if key in thumbnails: + return thumbnails[key]["url"] + return "" + + +def transform_video(item: dict, channel: str, playlist: str, now: str) -> dict[str, Any]: + snippet = item.get("snippet", {}) + content = item.get("contentDetails", {}) + stats = item.get("statistics", {}) + title = snippet.get("title", "") + date = snippet.get("publishedAt", "") + date_prefix = date[:10] if len(date) >= 10 else "" + slug = f"{date_prefix}_{slugify(title)}" if date_prefix else slugify(title) + return { + "url": f"https://www.youtube.com/watch?v={item['id']}", + "title": title, + "slug": slug, + "date": date, + "description": snippet.get("description", ""), + "duration": parse_iso8601_duration(content.get("duration", "")), + "channel": channel or snippet.get("channelTitle", ""), + "playlist": playlist, + "tags": snippet.get("tags", []), + "view_count": int(stats.get("viewCount", 0)), + "like_count": int(stats.get("likeCount", 0)), + "comment_count": int(stats.get("commentCount", 0)), + "thumbnail": best_thumbnail(snippet.get("thumbnails", {})), + "language": snippet.get("defaultAudioLanguage", ""), + "has_captions": content.get("caption") == "true", + "definition": content.get("definition", ""), + "publish": True, + "last_updated": now, + } + + +def merge_video( + existing: dict[str, Any] | None, + fetched: dict[str, Any], + *, + refresh_stats: bool, + now: str, +) -> tuple[dict[str, Any], list[str], list[str]]: + """Merge freshly fetched metadata into the stored entry. + + Returns ``(video, changed_fields, changed_stats)``. ``last_updated`` only + moves when a field changed. Without ``refresh_stats`` the stored counts + are kept. ``publish = false`` and any keys this script doesn't produce + (added by hand) are preserved. + """ + if existing is None: + return {**fetched, "last_updated": now}, [], [] + + merged = dict(fetched) + for key, value in existing.items(): + merged.setdefault(key, value) + if existing.get("publish") is False: + merged["publish"] = False + if not refresh_stats: + for f in STATS_FIELDS: + if f in existing: + merged[f] = existing[f] + + keys = [k for k in merged if k not in VOLATILE_FIELDS] + changed = [k for k in keys if k not in STATS_FIELDS and merged.get(k) != existing.get(k)] + changed_stats = [k for k in keys if k in STATS_FIELDS and merged.get(k) != existing.get(k)] + + if changed or changed_stats: + merged["last_updated"] = now + else: + merged["last_updated"] = existing.get("last_updated", now) + return merged, changed, changed_stats + + +def deduplicate_videos( + videos: dict[str, dict[str, Any]], + existing_urls: set[str], +) -> tuple[dict[str, dict[str, Any]], list[dict[str, str]]]: + """Resolve slug collisions deterministically. + + Two videos with the same title on the same day get the same slug. Prefer + the entry already in videos.toml (so the winner doesn't flip from night + to night), then the later publish time, then the URL. + """ + + def rank(url: str) -> tuple: + v = videos[url] + return (url in existing_urls, v.get("date", ""), url) + + winners: dict[str, str] = {} + for url, video in videos.items(): + slug = video.get("slug", "") + if slug not in winners or rank(url) > rank(winners[slug]): + winners[slug] = url + + keep = set(winners.values()) + result = {url: v for url, v in videos.items() if url in keep} + discarded = [ + {"url": url, "slug": v.get("slug", ""), "kept": winners[v.get("slug", "")]} + for url, v in videos.items() + if url not in keep + ] + return result, discarded + + +def render_videos(videos: dict[str, dict[str, Any]]) -> str: + videos_list = [{k: v for k, v in video.items() if v is not None} for video in videos.values()] + return tomli_w.dumps({"videos": videos_list}) + + +# --------------------------------------------------------------------------- +# Files +# --------------------------------------------------------------------------- def load_youtube_config(config_path: Path) -> dict[str, list[str]]: if not config_path.exists(): - console.print(f"[bold red]Error:[/] Config file not found: {config_path}") - sys.exit(1) + raise FatalError(f"Config file not found: {config_path}") with open(config_path, "rb") as f: data = tomllib.load(f) return { @@ -92,315 +221,299 @@ def load_existing_videos(output_path: Path) -> dict[str, dict[str, Any]]: try: with open(output_path, "rb") as f: data = tomllib.load(f) - return {v["url"]: v for v in data.get("videos", [])} - except Exception as e: - console.print(f"[yellow]Warning:[/] Could not load existing videos: {e}") - return {} + except tomllib.TOMLDecodeError as e: + # Never continue with an empty list: that would wipe the file. + raise FatalError(f"Could not parse {output_path}: {e}") from e + return {v["url"]: v for v in data.get("videos", [])} -def deduplicate_videos( - videos: dict[str, dict[str, Any]], -) -> dict[str, dict[str, Any]]: - """Remove slug collisions, keeping the entry with the more recent last_updated.""" - seen: dict[str, str] = {} # slug -> url - result: dict[str, dict[str, Any]] = {} - for url, video in videos.items(): - slug = video.get("slug", "") - if slug not in seen: - seen[slug] = url - result[url] = video - else: - prev_url = seen[slug] - prev = result[prev_url] - if video.get("last_updated", "") > prev.get("last_updated", ""): - del result[prev_url] - seen[slug] = url - result[url] = video - console.print( - f" [yellow]Warning:[/] Duplicate slug '{slug}': " - f"replaced {prev_url} with {url}" - ) - else: - console.print( - f" [yellow]Warning:[/] Duplicate slug '{slug}': " - f"kept {prev_url}, discarded {url}" - ) - return result +# --------------------------------------------------------------------------- +# YouTube API +# --------------------------------------------------------------------------- -def write_videos(videos: dict[str, dict[str, Any]], output_path: Path) -> None: - videos_list = [ - {k: v for k, v in video.items() if v is not None} for video in videos.values() - ] - with open(output_path, "wb") as f: - tomli_w.dump({"videos": videos_list}, f) +class Quota: + """Rough quota bookkeeping: list calls cost 1 unit each.""" + def __init__(self) -> None: + self.calls: dict[str, int] = {} -def should_update_video(existing: dict[str, Any] | None, force: bool) -> bool: - if force or not existing: - return True - last_updated_str = existing.get("last_updated") - if not last_updated_str: - return True - try: - last_updated = datetime.fromisoformat(last_updated_str) - return (datetime.now(timezone.utc) - last_updated) >= timedelta(hours=12) - except Exception: - return True + def add(self, call: str, units: int = 1) -> None: + self.calls[call] = self.calls.get(call, 0) + units + @property + def total(self) -> int: + return sum(self.calls.values()) -def resolve_channel_uploads(youtube: Any, channel_name: str) -> tuple[str, str] | None: - for params in [ - {"forUsername": channel_name}, - {"forHandle": f"@{channel_name}"}, - ]: - try: - resp = ( - youtube.channels() - .list(part="contentDetails,snippet", **params) - .execute() - ) - if resp.get("items"): - item = resp["items"][0] - uploads_id = item["contentDetails"]["relatedPlaylists"]["uploads"] - title = item["snippet"]["title"] - console.print( - f" [green]\u2713[/] Resolved uploads playlist: {uploads_id}" - ) - return uploads_id, title - except Exception as e: - console.print(f" [yellow]Warning:[/] API error for {params}: {e}") - console.print( - f" [bold red]Error:[/] Could not resolve channel '{channel_name}', skipping" - ) + +def resolve_channel_uploads(youtube: Any, channel_name: str, quota: Quota) -> tuple[str, str] | None: + for params in [{"forUsername": channel_name}, {"forHandle": f"@{channel_name}"}]: + quota.add("channels.list") + resp = ( + youtube.channels() + .list(part="contentDetails,snippet", **params) + .execute(num_retries=API_RETRIES) + ) + if resp.get("items"): + item = resp["items"][0] + uploads_id = item["contentDetails"]["relatedPlaylists"]["uploads"] + title = item["snippet"]["title"] + log.info(" Resolved channel %r (%s) to uploads playlist %s", channel_name, title, uploads_id) + return uploads_id, title + log.debug(" No channel found for %s", params) return None -def fetch_playlist_video_ids(youtube: Any, playlist_id: str) -> list[str]: +def fetch_playlist_video_ids(youtube: Any, playlist_id: str, quota: Quota) -> list[str]: video_ids: list[str] = [] request = youtube.playlistItems().list( part="contentDetails", playlistId=playlist_id, maxResults=50 ) while request: - response = request.execute() + quota.add("playlistItems.list") + response = request.execute(num_retries=API_RETRIES) for item in response.get("items", []): - vid = item["contentDetails"]["videoId"] - video_ids.append(vid) + video_ids.append(item["contentDetails"]["videoId"]) request = youtube.playlistItems().list_next(request, response) return video_ids -def fetch_video_metadata(youtube: Any, video_ids: list[str]) -> list[dict]: +def fetch_video_metadata(youtube: Any, video_ids: list[str], quota: Quota) -> list[dict]: items: list[dict] = [] for i in range(0, len(video_ids), 50): batch = video_ids[i : i + 50] + quota.add("videos.list") resp = ( youtube.videos() .list(part="snippet,contentDetails,statistics", id=",".join(batch)) - .execute() + .execute(num_retries=API_RETRIES) ) items.extend(resp.get("items", [])) + log.debug(" Fetched metadata batch %d (%d videos)", i // 50 + 1, len(batch)) return items -def best_thumbnail(thumbnails: dict) -> str: - for key in THUMBNAIL_PREFERENCE: - if key in thumbnails: - return thumbnails[key]["url"] - return "" +def collect_video_ids( + youtube: Any, + config: dict[str, list[str]], + quota: Quota, + summary: videolib.RunSummary, +) -> dict[str, tuple[str, str]]: + """Map video id -> (channel title, playlist id) over all sources. + A source that fails is recorded as an error; if every channel and + playlist fails, something is wrong globally (API key, quota, network). + """ + video_context: dict[str, tuple[str, str]] = {} + sources = len(config["channels"]) + len(config["playlists"]) + failed = 0 -def transform_video(item: dict, channel: str, playlist: str) -> dict[str, Any]: - snippet = item.get("snippet", {}) - content = item.get("contentDetails", {}) - stats = item.get("statistics", {}) - title = snippet.get("title", "") - date = snippet.get("publishedAt", "") - date_prefix = date[:10] if len(date) >= 10 else "" - slug = f"{date_prefix}_{slugify(title)}" if date_prefix else slugify(title) - return { - "url": f"https://www.youtube.com/watch?v={item['id']}", - "title": title, - "slug": slug, - "date": date, - "description": snippet.get("description", ""), - "duration": parse_iso8601_duration(content.get("duration", "")), - "channel": channel or snippet.get("channelTitle", ""), - "playlist": playlist, - "tags": snippet.get("tags", []), - "view_count": int(stats.get("viewCount", 0)), - "like_count": int(stats.get("likeCount", 0)), - "comment_count": int(stats.get("commentCount", 0)), - "thumbnail": best_thumbnail(snippet.get("thumbnails", {})), - "language": snippet.get("defaultAudioLanguage", ""), - "has_captions": content.get("caption") == "true", - "definition": content.get("definition", ""), - "publish": True, - "last_updated": datetime.now(timezone.utc).isoformat(), - } + for channel_name in config["channels"]: + log.info("Channel %s", channel_name) + try: + resolved = resolve_channel_uploads(youtube, channel_name, quota) + if not resolved: + raise LookupError(f"channel {channel_name!r} not found by username or handle") + uploads_id, channel_title = resolved + ids = fetch_playlist_video_ids(youtube, uploads_id, quota) + except Exception as e: # noqa: BLE001 - recorded and reported + failed += 1 + log.error(" Could not fetch channel %s: %s", channel_name, e) + summary.add("error", reason="source_failed", source=f"channel:{channel_name}", detail=str(e)) + continue + log.info(" Found %d videos", len(ids)) + for vid in ids: + video_context.setdefault(vid, (channel_title, "")) + for playlist_id in config["playlists"]: + log.info("Playlist %s", playlist_id) + try: + ids = fetch_playlist_video_ids(youtube, playlist_id, quota) + except Exception as e: # noqa: BLE001 - recorded and reported + failed += 1 + log.error(" Could not fetch playlist %s: %s", playlist_id, e) + summary.add("error", reason="source_failed", source=f"playlist:{playlist_id}", detail=str(e)) + continue + log.info(" Found %d videos", len(ids)) + for vid in ids: + video_context[vid] = (video_context.get(vid, ("", ""))[0], playlist_id) + + if sources and failed == sources: + raise FatalError(f"All {sources} channels and playlists failed; check the API key and quota") + + if config["videos"]: + log.info("Individual videos: %d", len(config["videos"])) + for vid in config["videos"]: + if vid in video_context: + log.debug(" Already tracked: %s", vid) + else: + video_context[vid] = ("", "") + log.info(" Added %s", vid) -def main() -> None: - parser = argparse.ArgumentParser( - description="Fetch video metadata from YouTube channels and playlists." - ) - parser.add_argument( - "--force", - "-f", - action="store_true", - help="Force update all videos, ignoring last_updated timestamps.", - ) - args = parser.parse_args() + return video_context - console.print("\n[bold blue]YouTube Video Metadata Fetcher[/]\n") - script_dir = Path(__file__).parent - project_root = script_dir.parent - config_path = project_root / "data" / "youtube.toml" - output_path = project_root / "data" / "videos.toml" +# --------------------------------------------------------------------------- +# Core +# --------------------------------------------------------------------------- - console.print(f"[dim]Config: {config_path}[/]") - console.print(f"[dim]Output: {output_path}[/]\n") - config = load_youtube_config(config_path) - channels = config["channels"] - playlists = config["playlists"] - individual_videos = config["videos"] +def update_videos( + youtube: Any, + config: dict[str, list[str]], + existing: dict[str, dict[str, Any]], + *, + refresh_stats: bool, + summary: videolib.RunSummary, + now: str | None = None, +) -> dict[str, dict[str, Any]]: + """Return the new contents of videos.toml, recording results in ``summary``.""" + now = now or datetime.now(timezone.utc).isoformat() + quota = Quota() + + video_context = collect_video_ids(youtube, config, quota, summary) + log.info("Total unique videos in sources: %d", len(video_context)) + + log.info("Fetching metadata for %d videos", len(video_context)) + items = fetch_video_metadata(youtube, list(video_context), quota) + fetched_ids = {item["id"] for item in items} + + # Start from the stored order so the file diff stays minimal; new videos + # are appended. + videos: dict[str, dict[str, Any]] = dict(existing) + results: dict[str, dict[str, Any]] = {} # url -> summary fields + + for item in items: + vid = item["id"] + channel, playlist = video_context[vid] + fetched = transform_video(item, channel, playlist, now) + url = fetched["url"] + old = existing.get(url) + video, changed, changed_stats = merge_video(old, fetched, refresh_stats=refresh_stats, now=now) + videos[url] = video + + if old is None: + results[url] = {"status": "new", "slug": video["slug"], "title": video["title"]} + log.info("New: %s (%s) %r", video["slug"], vid, video["title"]) + elif changed: + result = {"status": "changed", "slug": video["slug"], "changed_fields": changed} + if "slug" in changed: + result["old_slug"] = old.get("slug") + results[url] = result + log.info("Changed: %s (%s): %s", video["slug"], vid, ", ".join(changed)) + for f in changed: + log.debug(" %s: %r -> %r", f, old.get(f), video.get(f)) + elif changed_stats: + results[url] = {"status": "stats_refreshed"} + else: + results[url] = {"status": "unchanged"} + + for vid in video_context: + if vid not in fetched_ids: + url = f"https://www.youtube.com/watch?v={vid}" + stored = existing.get(url, {}) + results[url] = {"status": "unavailable", "slug": stored.get("slug")} + log.warning("Unavailable: %s is listed but not returned by the API (private or deleted?)", vid) + + source_urls = {f"https://www.youtube.com/watch?v={vid}" for vid in video_context} + for url, stored in existing.items(): + if url not in source_urls: + results[url] = {"status": "missing", "slug": stored.get("slug"), "title": stored.get("title")} + log.warning( + "Missing: %s (%s) is in videos.toml but no longer in any channel or playlist", + stored.get("slug"), + videolib.youtube_id(url), + ) - console.print(f"[cyan]Channels:[/] {', '.join(channels) or '(none)'}") - console.print(f"[cyan]Playlists:[/] {', '.join(playlists) or '(none)'}") - console.print(f"[cyan]Videos:[/] {', '.join(individual_videos) or '(none)'}") + videos, discarded = deduplicate_videos(videos, set(existing)) + for d in discarded: + results[d["url"]] = {"status": "duplicate_slug", "slug": d["slug"], "kept": d["kept"]} + log.warning("Duplicate slug %s: kept %s, skipped %s", d["slug"], d["kept"], d["url"]) - if args.force: - console.print("[yellow]Force mode:[/] Ignoring last_updated timestamps") + for url, result in results.items(): + status = result.pop("status") + if status in ("unchanged", "stats_refreshed"): + summary.tally(status) + else: + summary.add(status, url=url, video_id=videolib.youtube_id(url), **result) - existing_videos = load_existing_videos(output_path) - if existing_videos: - console.print(f"[dim]Loaded {len(existing_videos)} existing videos[/]") + summary.extra["quota_used"] = quota.total + summary.extra["quota_calls"] = quota.calls + summary.extra["videos_in_sources"] = len(video_context) + log.info("API quota used: ~%d units (%s)", quota.total, quota.calls) + return videos - api_key = get_api_key() - youtube = build("youtube", "v3", developerKey=api_key) - # video_id -> (channel, playlist) - video_context: dict[str, tuple[str, str]] = {} - quota_used = 0 - - # Phase 2 + 3: Resolve channels and collect video IDs - for channel_name in channels: - console.print(f"\n[bold]Channel: {channel_name}[/]") - result = resolve_channel_uploads(youtube, channel_name) - quota_used += 1 - if not result: - continue - uploads_id, channel_title = result - video_ids = fetch_playlist_video_ids(youtube, uploads_id) - quota_used += (len(video_ids) // 50) + 1 - console.print(f" [green]\u2713[/] Found {len(video_ids)} videos") - for vid in video_ids: - if vid not in video_context: - video_context[vid] = (channel_title, "") - - for playlist_id in playlists: - console.print(f"\n[bold]Playlist: {playlist_id}[/]") +def build_youtube_client(project_root: Path) -> Any: + from dotenv import load_dotenv + from googleapiclient.discovery import build + + load_dotenv(project_root / ".env") + key = os.getenv("YOUTUBE_API_KEY") + if not key: + raise FatalError("YOUTUBE_API_KEY not found in .env or environment") + return build("youtube", "v3", developerKey=key, cache_discovery=False) + + +def main(argv: list[str] | None = None, youtube: Any = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.strip().split("\n\n")[0]) + parser.add_argument( + "--refresh-stats", + action="store_true", + help="also update view, like, and comment counts of existing videos", + ) + parser.add_argument("--force", "-f", action="store_true", help="alias for --refresh-stats") + parser.add_argument("--dry-run", action="store_true", help="show what would change; write nothing") + videolib.add_common_args(parser) + args = parser.parse_args(argv) + refresh_stats = args.refresh_stats or args.force + + videolib.setup_logging("update-youtube-videos", args.verbose) + summary = videolib.RunSummary("update-youtube-videos") + summary.extra.update(refresh_stats=refresh_stats, dry_run=args.dry_run) + + config_path = args.project_root / "data" / "youtube.toml" + output_path = args.project_root / "data" / "videos.toml" + + def body() -> None: + nonlocal youtube + summary.environment = videolib.log_environment( + log, packages=["google-api-python-client", "tomli-w"] + ) try: - video_ids = fetch_playlist_video_ids(youtube, playlist_id) - quota_used += (len(video_ids) // 50) + 1 - console.print(f" [green]\u2713[/] Found {len(video_ids)} videos") - for vid in video_ids: - video_context[vid] = (video_context.get(vid, ("", ""))[0], playlist_id) - except Exception as e: - console.print(f" [bold red]Error:[/] Could not fetch playlist: {e}") - - if individual_videos: - console.print(f"\n[bold]Individual videos: {len(individual_videos)}[/]") - for vid in individual_videos: - if vid not in video_context: - video_context[vid] = ("", "") - console.print(f" [green]\u2713[/] Added {vid}") - else: - console.print(f" [dim]○[/] Already tracked: {vid}") + config = load_youtube_config(config_path) + log.info("Config: %s", config_path) + for kind in ("channels", "playlists", "videos"): + log.info(" %s: %s", kind, ", ".join(config[kind]) or "(none)") + log.info("Stats: %s", "refreshing" if refresh_stats else "keeping stored counts") - console.print(f"\n[cyan]Total unique videos:[/] {len(video_context)}") + existing = load_existing_videos(output_path) + log.info("Loaded %d existing videos from %s", len(existing), output_path) - # Phase 4 + 5: Fetch metadata, deduplicate, merge - ids_to_fetch = [] - counters = {"new": 0, "updated": 0, "skipped": 0, "errors": 0} + if youtube is None: + youtube = build_youtube_client(args.project_root) - for vid, (channel, playlist) in video_context.items(): - url = f"https://www.youtube.com/watch?v={vid}" - existing = existing_videos.get(url) - if should_update_video(existing, args.force): - ids_to_fetch.append(vid) + videos = update_videos( + youtube, config, existing, refresh_stats=refresh_stats, summary=summary + ) + except FatalError as e: + summary.fatal = str(e) + log.error("%s", e) + return + + new_text = render_videos(videos) + old_text = output_path.read_text(encoding="utf-8") if output_path.exists() else None + if new_text == old_text: + log.info("%s is unchanged (%d videos)", output_path, len(videos)) + elif args.dry_run: + log.info("Dry run: would write %d videos to %s", len(videos), output_path) else: - counters["skipped"] += 1 + videolib.atomic_write_text(output_path, new_text) + log.info("Wrote %d videos to %s", len(videos), output_path) + summary.extra["written"] = new_text != old_text and not args.dry_run - if not ids_to_fetch: - console.print("\n[dim]All videos are up to date, nothing to fetch.[/]") - else: - console.print( - f"\n[bold]Fetching metadata for {len(ids_to_fetch)} videos[/]" - f" [dim]({counters['skipped']} skipped as fresh)[/]" - ) - - all_videos = dict(existing_videos) - - with Progress( - SpinnerColumn(), - TextColumn("[progress.description]{task.description}"), - BarColumn(), - TaskProgressColumn(), - TimeRemainingColumn(), - console=console, - ) as progress: - task = progress.add_task("[cyan]Fetching...", total=len(ids_to_fetch)) - - for i in range(0, len(ids_to_fetch), 50): - batch = ids_to_fetch[i : i + 50] - items = fetch_video_metadata(youtube, batch) - quota_used += 1 - - fetched_ids = {item["id"] for item in items} - for vid in batch: - if vid not in fetched_ids: - console.print( - f" [yellow]Warning:[/] Video {vid} not returned by API (private/deleted?)" - ) - counters["errors"] += 1 - - for item in items: - vid = item["id"] - channel, playlist = video_context[vid] - url = f"https://www.youtube.com/watch?v={vid}" - existing = all_videos.get(url) - video = transform_video(item, channel, playlist) - if existing and existing.get("publish") is False: - video["publish"] = False - all_videos[url] = video - if existing: - counters["updated"] += 1 - else: - counters["new"] += 1 - - write_videos(all_videos, output_path) - progress.advance(task, len(batch)) - - existing_videos = all_videos - - # Write final state (includes skipped videos preserved from existing data) - existing_videos = deduplicate_videos(existing_videos) - write_videos(existing_videos, output_path) - - # Phase 6: Summary report - total = len(existing_videos) - console.print( - f"\n[bold green]\u2713 Done![/] {total} videos written to {output_path}" - ) - console.print( - f" New: {counters['new']} | Updated: {counters['updated']}" - f" | Skipped: {counters['skipped']} | Errors: {counters['errors']}" - ) - console.print(f" API quota used: ~{quota_used} units\n") + return videolib.run_main(summary, log, args.summary_json, body) if __name__ == "__main__": - main() + sys.exit(main()) diff --git a/scripts/video-sync-report.py b/scripts/video-sync-report.py new file mode 100755 index 000000000..41a4ed529 --- /dev/null +++ b/scripts/video-sync-report.py @@ -0,0 +1,447 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// + +""" +Turn the run summaries of the video pipeline into a Markdown report. + +Reads the --summary-json files written by update-youtube-videos.py, +sync-videos.py, transcribe.py, and format-transcriptions.py, and writes the +pull request description of the nightly sync (also used for the GitHub +Actions step summary). With --annotations, also prints ::warning:: and +::error:: workflow commands so problems show up on the run page. + +Usage: + ./scripts/video-sync-report.py SUMMARY_DIR [--output FILE] [--annotations] + [--expect update-youtube-videos,sync-videos,...] [--run-url URL] +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from pathlib import Path +from typing import Any + +STEPS = { + "update-youtube-videos": "YouTube metadata", + "sync-videos": "Video pages", + "transcribe": "Transcription", + "format-transcriptions": "Formatting", +} + +# Download failures that won't go away by trying again. +PERMANENT_REASONS = {"private", "members_only", "unavailable", "age_restricted"} +# Download failures that mean YouTube is blocking the runner (see transcribe.py). +BLOCKING_REASONS = {"bot_check", "po_token", "http_429"} + +MAX_LIST = 40 # items per list, to stay well under GitHub's 65,536-character limit + + +def load_summaries(directory: Path) -> dict[str, dict[str, Any]]: + summaries = {} + for path in sorted(directory.glob("*.json")): + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + print(f"warning: could not read {path}: {e}", file=sys.stderr) + continue + if isinstance(data, dict) and data.get("script") in STEPS: + summaries[data["script"]] = data + return summaries + + +def items(summary: dict[str, Any] | None, *statuses: str) -> list[dict[str, Any]]: + if not summary: + return [] + return [i for i in summary.get("items", []) if i.get("status") in statuses] + + +def count(summary: dict[str, Any] | None, status: str) -> int: + return (summary or {}).get("counts", {}).get(status, 0) + + +def plural(n: int, word: str, plural_word: str | None = None) -> str: + return f"{n:,} {word if n == 1 else (plural_word or word + 's')}" + + +def bullet_list(lines: list[str]) -> list[str]: + out = [f"- {line}" for line in lines[:MAX_LIST]] + if len(lines) > MAX_LIST: + out.append(f"- …and {len(lines) - MAX_LIST:,} more (see the run summaries)") + return out + + +def details(summary_text: str, lines: list[str]) -> list[str]: + return ["
" + summary_text + "", "", *bullet_list(lines), "", "
", ""] + + +def money(value: float | None) -> str: + return f" (~${value:.2f})" if value else "" + + +def code(text: str | None) -> str: + return f"`{text}`" if text else "(unknown)" + + +def one_line(text: str | None, limit: int = 200) -> str: + text = " ".join((text or "").split()) + text = text.replace("|", "\\|") + return text if len(text) <= limit else text[: limit - 1] + "…" + + +# --------------------------------------------------------------------------- +# Report sections +# --------------------------------------------------------------------------- + + +def results_table(summaries: dict[str, dict[str, Any]], expected: list[str]) -> list[str]: + update = summaries.get("update-youtube-videos") + sync = summaries.get("sync-videos") + transcribe = summaries.get("transcribe") + fmt = summaries.get("format-transcriptions") + + rows: list[tuple[str, str]] = [] + + def describe(script: str, parts: list[str]) -> str: + s = summaries.get(script) + if s is None: + return "⚠️ no summary (crashed?)" if script in expected else "skipped" + if s.get("fatal"): + return f"❌ failed: {one_line(s['fatal'], 120)}" + text = ", ".join(p for p in parts if p) or "nothing to do" + return ("⚠️ " if s.get("exit_code") == 2 else "") + text + + if "update-youtube-videos" in expected or update: + stats = (update or {}).get("extra", {}).get("refresh_stats") + rows.append( + ( + STEPS["update-youtube-videos"], + describe( + "update-youtube-videos", + [ + f"{count(update, 'new'):,} new" if count(update, "new") else "", + f"{count(update, 'changed'):,} changed" if count(update, "changed") else "", + f"{count(update, 'unchanged') + count(update, 'stats_refreshed'):,} unchanged", + "stats refreshed" if stats else "", + ], + ), + ) + ) + if "sync-videos" in expected or sync: + rows.append( + ( + STEPS["sync-videos"], + describe( + "sync-videos", + [ + f"{count(sync, 'created'):,} created" if count(sync, "created") else "", + f"{count(sync, 'updated'):,} updated" if count(sync, "updated") else "", + f"{count(sync, 'renamed'):,} renamed" if count(sync, "renamed") else "", + f"{count(sync, 'error'):,} failed" if count(sync, "error") else "", + f"{count(sync, 'unchanged'):,} unchanged" if count(sync, "unchanged") else "", + ], + ), + ) + ) + if "transcribe" in expected or transcribe: + extra = (transcribe or {}).get("extra", {}) + rows.append( + ( + STEPS["transcribe"], + describe( + "transcribe", + [ + f"{count(transcribe, 'transcribed'):,} transcribed{money(extra.get('estimated_cost_usd'))}" + if count(transcribe, "transcribed") + else "", + f"{count(transcribe, 'error'):,} failed" if count(transcribe, "error") else "", + "downloads blocked" if extra.get("breaker_tripped") else "", + f"{count(transcribe, 'deferred'):,} deferred" if count(transcribe, "deferred") else "", + ], + ), + ) + ) + if "format-transcriptions" in expected or fmt: + extra = (fmt or {}).get("extra", {}) + rows.append( + ( + STEPS["format-transcriptions"], + describe( + "format-transcriptions", + [ + f"{count(fmt, 'formatted'):,} formatted{money(extra.get('estimated_cost_usd'))}" + if count(fmt, "formatted") + else "", + f"{count(fmt, 'error'):,} failed" if count(fmt, "error") else "", + f"{count(fmt, 'deferred'):,} deferred" if count(fmt, "deferred") else "", + ], + ), + ) + ) + if not rows: + return [] + return ["| Step | Result |", "| --- | --- |", *(f"| {step} | {result} |" for step, result in rows), ""] + + +def changes_sections(summaries: dict[str, dict[str, Any]]) -> list[str]: + update = summaries.get("update-youtube-videos") + sync = summaries.get("sync-videos") + transcribe = summaries.get("transcribe") + fmt = summaries.get("format-transcriptions") + out: list[str] = [] + + new = items(update, "new") + if new: + out += [f"### New videos ({len(new)})", ""] + out += bullet_list([f"[{one_line(i.get('title'), 120)}]({i['url']}) {code(i.get('slug'))}" for i in new]) + out.append("") + + renamed = items(sync, "renamed") + if renamed: + out += [f"### Renamed ({len(renamed)})", "", "The URLs of these pages change; links in the repo were updated.", ""] + lines = [] + for i in renamed: + line = f"{code(i.get('old_slug'))} → {code(i.get('slug'))}" + if i.get("action") == "merged": + line += " (merged into an existing directory)" + refs = i.get("updated_references") or [] + if refs: + line += f"; references updated in {', '.join(code(r) for r in refs)}" + lines.append(line) + out += bullet_list(lines) + [""] + + changed = items(update, "changed") + if changed: + out += ["### Changed metadata", ""] + out += details( + plural(len(changed), "video"), + [f"{code(i.get('slug'))}: {', '.join(i.get('changed_fields', []))}" for i in changed], + ) + + transcribed = items(transcribe, "transcribed") + formatted = items(fmt, "formatted") + if transcribed or formatted: + out += ["### Transcripts", ""] + lines = [] + formatted_slugs = {i["slug"] for i in formatted} + for i in transcribed: + note = " and formatted" if i["slug"] in formatted_slugs else " (not formatted yet)" + warn = f" ⚠️ {'; '.join(i['warnings'])}" if i.get("warnings") else "" + lines.append(f"{code(i['slug'])}: transcribed{note}{warn}") + transcribed_slugs = {i["slug"] for i in transcribed} + for i in formatted: + if i["slug"] not in transcribed_slugs: + lines.append(f"{code(i['slug'])}: formatted") + for i in formatted: + if i.get("warnings"): + lines.append(f"{code(i['slug'])}: ⚠️ {'; '.join(i['warnings'])}") + out += details(plural(len(lines), "video"), lines) + return out + + +def problems_section(summaries: dict[str, dict[str, Any]], expected: list[str]) -> tuple[list[str], list[str]]: + """Return (markdown lines, annotations).""" + update = summaries.get("update-youtube-videos") + sync = summaries.get("sync-videos") + transcribe = summaries.get("transcribe") + fmt = summaries.get("format-transcriptions") + out: list[str] = [] + annotations: list[str] = [] + + def block(title: str, lines: list[str], hint: str | None = None, level: str = "warning") -> None: + if not lines: + return + nonlocal out + out += [f"**{title}**", ""] + if hint: + out += [hint, ""] + out += bullet_list(lines) + [""] + annotations.append(f"::{level} title={title}::{len(lines)} item(s); see the step summary or PR description") + + for script in STEPS: + s = summaries.get(script) + if s and s.get("fatal"): + block(f"{STEPS[script]} failed", [one_line(s["fatal"], 500)], level="error") + elif s is None and script in expected: + block(f"{STEPS[script]} produced no summary", ["The step crashed before writing it; check the job log."], level="error") + + block( + "Could not fetch some YouTube sources", + [f"{code(i.get('source'))}: {one_line(i.get('detail'))}" for i in items(update, "error")], + ) + + extra = (transcribe or {}).get("extra", {}) + download_errors = [i for i in items(transcribe, "error") if i.get("stage") == "download"] + blocked = items(transcribe, "skipped") + blocked = [i for i in blocked if i.get("reason") == "download_blocked"] + if extra.get("breaker_tripped"): + block( + "YouTube blocked the downloads", + [ + f"{plural(len(blocked), 'more video')} skipped after repeated " + f"{', '.join(sorted({i['reason'] for i in download_errors} & BLOCKING_REASONS)) or 'blocking'} errors" + ], + hint="Transcribe them locally with `just update-videos 15 chrome` on this branch and push, " + "or wait for the next run.", + ) + if download_errors: + lines = [] + for i in download_errors: + fix = f"`just transcribe --video {i['slug']} --cookies-from-browser chrome`" + if i.get("reason") in PERMANENT_REASONS: + fix += " or set `transcribe: false` in its `_index.md`" + lines.append(f"{code(i['slug'])}: **{i.get('reason')}**: {one_line(i.get('detail'))}. Fix: {fix}") + block("Audio downloads failed", lines) + + block( + "Transcription failed", + [ + f"{code(i['slug'])}: **{i.get('reason')}**: {one_line(i.get('detail'))}. " + f"Fix: `just transcribe --video {i['slug']}`" + for i in items(transcribe, "error") + if i.get("stage") != "download" + ], + ) + block( + "Formatting failed", + [ + f"{code(i['slug'])}: **{i.get('reason')}**: {one_line(i.get('detail'))}. " + f"It's retried in the next run, or run `just format-transcriptions --video {i['slug']}`" + for i in items(fmt, "error") + ], + ) + block( + "Video pages failed", + [f"{code(i.get('slug'))}: **{i.get('reason')}**: {one_line(i.get('detail'))}" for i in items(sync, "error")], + ) + + # Informational: not errors, but worth a look. + info: list[str] = [] + missing = items(update, "missing") + if missing: + info += details( + f"{plural(len(missing), 'video')} in data/videos.toml no longer on the channel or playlists " + "(set publish = false to unpublish)", + [f"{code(i.get('slug'))} ({i.get('url')})" for i in missing], + ) + unavailable = items(update, "unavailable") + if unavailable: + info += details( + f"{plural(len(unavailable), 'video')} listed but not returned by the API (private or deleted?)", + [f"{code(i.get('slug') or i.get('video_id'))} ({i.get('url')})" for i in unavailable], + ) + duplicates = items(update, "duplicate_slug") + if duplicates: + info += details( + f"{plural(len(duplicates), 'video')} skipped because another video has the same slug", + [f"{code(i.get('slug'))}: skipped {i.get('url')}, kept {i.get('kept')}" for i in duplicates], + ) + orphans = items(sync, "orphan") + if orphans: + info += details( + f"{plural(len(orphans), 'directory', 'directories')} without an entry in data/videos.toml", + [code(i.get("slug")) for i in orphans], + ) + skipped = [i for i in items(transcribe, "skipped") if i.get("reason") != "download_blocked"] + if skipped: + info += details( + f"{plural(len(skipped), 'video')} not transcribed", + [f"{code(i.get('slug'))}: {i.get('reason')}" for i in skipped], + ) + + waiting = (fmt or {}).get("extra", {}).get("needs_transcript") or [] + if waiting: + info += [ + f"**{plural(len(waiting), 'video')} waiting for a transcript.** YouTube blocks audio downloads from " + "GitHub's runners, so transcribe them locally on this branch and push; the next run formats them:", + "", + "```bash", + "git fetch origin && git switch -C automation/sync-videos origin/automation/sync-videos", + "just transcribe --limit 15 # add --cookies-from-browser chrome if YouTube asks you to sign in", + 'git add content/resources/videos && git commit -m "Add transcripts" && git push', + "```", + "", + ] + info += details("Videos without a transcript (newest first)", [code(slug) for slug in waiting]) + + if out: + out = ["### ⚠️ Problems", ""] + out + if info: + out += ["### Notes", ""] + info + return out, annotations + + +def build_report( + summaries: dict[str, dict[str, Any]], + *, + expected: list[str], + run_url: str | None = None, +) -> tuple[str, list[str]]: + out = [ + "Nightly sync of [Posit's YouTube videos](https://www.youtube.com/@PositPBC) into " + "`content/resources/videos/`" + + (f" ([workflow run]({run_url}))" if run_url else "") + + ".", + "", + "Review the changes below, then approve and merge to publish them.", + "", + ] + out += results_table(summaries, expected) + out += changes_sections(summaries) + problems, annotations = problems_section(summaries, expected) + out += problems + + out += [ + "---", + "", + "Adjust a video with `include`, `exclude`, or `override` blocks in its `_index.md`, " + "hide it with `publish = false` in `data/videos.toml`, or skip its transcript with " + "`transcribe: false`." + + (f" Logs and run summaries are attached to the [workflow run]({run_url})." if run_url else ""), + ] + return "\n".join(out).rstrip() + "\n", annotations + + +def default_run_url() -> str | None: + server = os.getenv("GITHUB_SERVER_URL") + repo = os.getenv("GITHUB_REPOSITORY") + run_id = os.getenv("GITHUB_RUN_ID") + if server and repo and run_id: + return f"{server}/{repo}/actions/runs/{run_id}" + return None + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.strip().split("\n\n")[0]) + parser.add_argument("summary_dir", type=Path, help="directory with the *.json run summaries") + parser.add_argument("--output", "-o", type=Path, help="write the report here instead of stdout") + parser.add_argument("--annotations", action="store_true", help="print GitHub Actions annotations to stdout") + parser.add_argument( + "--expect", + default=",".join(STEPS), + help="comma-separated steps that should have a summary (default: all four)", + ) + parser.add_argument("--run-url", default=default_run_url(), help="link to the workflow run") + args = parser.parse_args(argv) + + expected = [s for s in args.expect.split(",") if s] + summaries = load_summaries(args.summary_dir) if args.summary_dir.exists() else {} + report, annotations = build_report(summaries, expected=expected, run_url=args.run_url) + + if args.output: + args.output.write_text(report, encoding="utf-8") + else: + sys.stdout.write(report) + if args.annotations: + for line in annotations: + print(line) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/videolib.py b/scripts/videolib.py new file mode 100644 index 000000000..f1e252582 --- /dev/null +++ b/scripts/videolib.py @@ -0,0 +1,581 @@ +""" +Shared helpers for the video pipeline scripts: + + update-youtube-videos.py -> sync-videos.py -> transcribe.py -> format-transcriptions.py + +The scripts are PEP 723 uv scripts that import this module from their own +directory. Only pyyaml and rich are required here (imported lazily where +possible), so every script must declare both in its inline dependencies. + +Conventions shared by the scripts: + +- Exit codes: 0 = everything succeeded, 1 = fatal error (nothing useful was + written), 2 = partial success (some videos failed, the rest was written). +- ``--summary-json PATH`` writes a machine-readable run summary (see + ``RunSummary``), which ``video-sync-report.py`` turns into the PR body. +- Logging goes to stderr. In a terminal it is colored by Rich; elsewhere (CI) + it is plain, timestamped text in UTC so that logs can be correlated. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import os +import platform +import random +import re +import shutil +import subprocess +import sys +import tempfile +import time +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Awaitable, Callable, TypeVar + +T = TypeVar("T") + +EXIT_OK = 0 +EXIT_FATAL = 1 +EXIT_PARTIAL = 2 + +DEFAULT_PROJECT_ROOT = Path(__file__).resolve().parent.parent + +log = logging.getLogger("videolib") + + +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- + +_console = None + + +def is_interactive() -> bool: + """True when stderr is a terminal and we're not running in CI.""" + return sys.stderr.isatty() and not os.getenv("CI") + + +def get_console(): + """The Rich console shared by the log handler and progress bars.""" + global _console + if _console is None: + from rich.console import Console + + _console = Console(stderr=True) + return _console + + +class _UTCFormatter(logging.Formatter): + converter = time.gmtime + + def formatTime(self, record, datefmt=None): # noqa: N802 (logging API) + t = time.gmtime(record.created) + return time.strftime("%Y-%m-%dT%H:%M:%S", t) + f".{int(record.msecs):03d}Z" + + +NOISY_LOGGERS = [ + "httpx", + "httpx2", + "httpcore", + "openai", + "anthropic", + "googleapiclient.discovery", + "googleapiclient.discovery_cache", + "urllib3", +] + + +def setup_logging(name: str, verbose: bool = False) -> logging.Logger: + """Configure the root logger and return the logger for ``name``. + + The level is DEBUG with ``verbose``, otherwise ``$LOG_LEVEL`` or INFO. + """ + level_name = "DEBUG" if verbose else os.getenv("LOG_LEVEL", "INFO").upper() + level = getattr(logging, level_name, logging.INFO) + + root = logging.getLogger() + for handler in list(root.handlers): + root.removeHandler(handler) + + if is_interactive(): + from rich.logging import RichHandler + + handler: logging.Handler = RichHandler( + console=get_console(), + show_path=False, + markup=False, + rich_tracebacks=True, + log_time_format="[%X]", + ) + else: + handler = logging.StreamHandler(sys.stderr) + handler.setFormatter( + _UTCFormatter("%(asctime)s %(levelname)-7s %(name)s: %(message)s") + ) + root.addHandler(handler) + root.setLevel(level) + + # Third-party HTTP clients are chatty at INFO/DEBUG; only show their + # warnings unless we're debugging. + for noisy in NOISY_LOGGERS: + logging.getLogger(noisy).setLevel(level if verbose else logging.WARNING) + + return logging.getLogger(name) + + +def make_progress(*columns_extra): + """A Rich progress bar that is disabled outside interactive terminals.""" + from rich.progress import ( + BarColumn, + MofNCompleteColumn, + Progress, + SpinnerColumn, + TextColumn, + TimeElapsedColumn, + ) + + return Progress( + SpinnerColumn(), + TextColumn("{task.description}"), + BarColumn(), + MofNCompleteColumn(), + TimeElapsedColumn(), + *columns_extra, + console=get_console(), + disable=not is_interactive(), + transient=True, + ) + + +def add_common_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--summary-json", + type=Path, + metavar="PATH", + help="write a machine-readable run summary to PATH", + ) + parser.add_argument( + "--verbose", "-v", action="store_true", help="enable debug logging" + ) + # Hidden: lets the tests run a script against a temporary fixture repo. + parser.add_argument( + "--project-root", + type=Path, + default=DEFAULT_PROJECT_ROOT, + help=argparse.SUPPRESS, + ) + + +def log_environment( + logger: logging.Logger, + packages: list[str] | None = None, + tools: list[str] | None = None, +) -> dict[str, Any]: + """Log interpreter, package, and external tool versions; return them.""" + from importlib import metadata + + env: dict[str, Any] = { + "python": platform.python_version(), + "platform": platform.platform(), + "ci": bool(os.getenv("CI")), + "packages": {}, + "tools": {}, + } + for package in packages or []: + try: + env["packages"][package] = metadata.version(package) + except metadata.PackageNotFoundError: + env["packages"][package] = None + for tool in tools or []: + env["tools"][tool] = tool_version(tool) + + logger.info( + "Environment: python %s on %s%s", + env["python"], + env["platform"], + " (CI)" if env["ci"] else "", + ) + for package, version in env["packages"].items(): + logger.info(" package %s: %s", package, version or "not installed") + for tool, version in env["tools"].items(): + logger.info(" tool %s: %s", tool, version or "NOT FOUND") + return env + + +def tool_version(tool: str) -> str | None: + """First line of ``tool --version`` (``-version`` for ffmpeg), or None.""" + path = shutil.which(tool) + if not path: + return None + flag = "-version" if tool in ("ffmpeg", "ffprobe") else "--version" + try: + result = subprocess.run( + [path, flag], capture_output=True, text=True, timeout=15 + ) + except (OSError, subprocess.SubprocessError) as e: + return f"error: {e}" + out = (result.stdout or result.stderr).strip().splitlines() + return out[0] if out else "unknown" + + +# --------------------------------------------------------------------------- +# Run summary +# --------------------------------------------------------------------------- + + +def utcnow_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +@dataclass +class RunSummary: + """Collects per-video results of one script run. + + ``items`` hold the interesting results (new, changed, failed, ...), each + a dict with at least ``status``. ``tallies`` count bulk outcomes that + aren't worth listing one by one (e.g. 1,700 "already transcribed"). + An item with status ``error`` makes the run partial (exit code 2); + ``fatal`` makes it fatal (exit code 1). + """ + + script: str + started_at: str = field(default_factory=utcnow_iso) + finished_at: str | None = None + items: list[dict[str, Any]] = field(default_factory=list) + tallies: dict[str, int] = field(default_factory=dict) + fatal: str | None = None + extra: dict[str, Any] = field(default_factory=dict) + environment: dict[str, Any] = field(default_factory=dict) + + def add(self, status: str, **fields: Any) -> dict[str, Any]: + item = {"status": status, **{k: v for k, v in fields.items() if v is not None}} + self.items.append(item) + return item + + def tally(self, status: str, n: int = 1) -> None: + self.tallies[status] = self.tallies.get(status, 0) + n + + def counts(self) -> dict[str, int]: + counts = dict(self.tallies) + for item in self.items: + counts[item["status"]] = counts.get(item["status"], 0) + 1 + return dict(sorted(counts.items())) + + def by_status(self, *statuses: str) -> list[dict[str, Any]]: + return [i for i in self.items if i["status"] in statuses] + + @property + def errors(self) -> list[dict[str, Any]]: + return self.by_status("error") + + @property + def exit_code(self) -> int: + if self.fatal: + return EXIT_FATAL + if self.errors: + return EXIT_PARTIAL + return EXIT_OK + + def to_dict(self) -> dict[str, Any]: + return { + "script": self.script, + "started_at": self.started_at, + "finished_at": self.finished_at, + "exit_code": self.exit_code, + "fatal": self.fatal, + "counts": self.counts(), + "extra": self.extra, + "environment": self.environment, + "items": self.items, + } + + def write(self, path: Path | None) -> None: + self.finished_at = self.finished_at or utcnow_iso() + if path is None: + return + path.parent.mkdir(parents=True, exist_ok=True) + atomic_write_text(path, json.dumps(self.to_dict(), indent=2, default=str) + "\n") + + def log_counts(self, logger: logging.Logger) -> None: + counts = self.counts() + if counts: + logger.info( + "Summary: %s", + ", ".join(f"{status}={n}" for status, n in counts.items()), + ) + else: + logger.info("Summary: nothing to do") + if self.fatal: + logger.error("Fatal: %s", self.fatal) + elif self.errors: + logger.warning("%d error(s); exit code %d", len(self.errors), self.exit_code) + + +def run_main( + summary: RunSummary, + logger: logging.Logger, + summary_path: Path | None, + body: Callable[[], None], +) -> int: + """Run ``body``, turning uncaught exceptions into a fatal summary. + + The summary is always written, so the report can explain a crash. + """ + try: + body() + except KeyboardInterrupt: + summary.fatal = "interrupted" + raise + except Exception as e: # noqa: BLE001 - we want every failure in the summary + logger.exception("Unhandled error") + summary.fatal = f"{type(e).__name__}: {e}" + finally: + summary.write(summary_path) + summary.log_counts(logger) + return summary.exit_code + + +# --------------------------------------------------------------------------- +# Files +# --------------------------------------------------------------------------- + + +def atomic_write_bytes(path: Path, data: bytes) -> None: + """Write ``data`` to a temp file next to ``path``, then rename it.""" + path.parent.mkdir(parents=True, exist_ok=True) + fd, tmp_name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent) + tmp = Path(tmp_name) + try: + with os.fdopen(fd, "wb") as f: + f.write(data) + os.replace(tmp, path) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + +def atomic_write_text(path: Path, text: str) -> None: + atomic_write_bytes(path, text.encode("utf-8")) + + +# --------------------------------------------------------------------------- +# Frontmatter +# --------------------------------------------------------------------------- + + +def split_frontmatter(content: str) -> tuple[dict[str, Any], str, str]: + """Return ``(frontmatter, yaml_section, remaining_content)``. + + Without (valid) frontmatter, returns ``({}, "", content)``. + """ + import yaml + + lines = content.split("\n") + if not lines or lines[0].strip() != "---": + return {}, "", content + end_idx = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) + if end_idx is None: + return {}, "", content + yaml_section = "\n".join(lines[1:end_idx]) + try: + frontmatter = yaml.safe_load(yaml_section) or {} + except yaml.YAMLError as e: + log.warning("Failed to parse YAML frontmatter: %s", e) + return {}, "", content + if not isinstance(frontmatter, dict): + return {}, "", content + return frontmatter, yaml_section, "\n".join(lines[end_idx + 1 :]) + + +def parse_frontmatter(content: str) -> dict[str, Any]: + return split_frontmatter(content)[0] + + +def read_frontmatter(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + return parse_frontmatter(path.read_text(encoding="utf-8")) + + +# --------------------------------------------------------------------------- +# Video directories +# --------------------------------------------------------------------------- + + +def video_dirs(videos_dir: Path) -> list[Path]: + """Video directories, newest first (names start with the YYYY-MM-DD date).""" + return sorted((d for d in videos_dir.iterdir() if d.is_dir()), reverse=True) + + +def resolve_video_dir(name_or_path: str, videos_dir: Path) -> Path: + """Find a video directory by path, exact name, or unique name prefix.""" + p = Path(name_or_path) + if p.is_absolute() and p.is_dir(): + return p + candidate = videos_dir / name_or_path + if candidate.is_dir(): + return candidate + matches = [ + d for d in videos_dir.iterdir() if d.is_dir() and d.name.startswith(name_or_path) + ] + if len(matches) == 1: + return matches[0] + if len(matches) > 1: + listed = "\n ".join(m.name for m in sorted(matches)) + raise ValueError(f"Ambiguous video name, matches:\n {listed}") + raise ValueError(f"Video directory not found: {name_or_path}") + + +def youtube_id(url: str | None) -> str | None: + if not url: + return None + m = re.search(r"(?:v=|youtu\.be/|/shorts/)([\w-]{11})", url) + return m.group(1) if m else None + + +# --------------------------------------------------------------------------- +# WebVTT +# --------------------------------------------------------------------------- + +Cue = tuple[float, float, str] + + +def parse_vtt_timestamp(ts: str) -> float: + parts = ts.strip().split(":") + if len(parts) == 3: + return int(parts[0]) * 3600 + int(parts[1]) * 60 + float(parts[2]) + return int(parts[0]) * 60 + float(parts[1]) + + +def format_vtt_timestamp(seconds: float) -> str: + # Work in whole milliseconds so 59.9999 doesn't render as "00:00:60.000". + ms = round(seconds * 1000) + h, ms = divmod(ms, 3_600_000) + m, ms = divmod(ms, 60_000) + s, ms = divmod(ms, 1000) + return f"{h:02d}:{m:02d}:{s:02d}.{ms:03d}" + + +def parse_vtt(content: str) -> list[Cue]: + cues: list[Cue] = [] + for block in re.split(r"\n{2,}", content.replace("\r\n", "\n").strip()): + lines = block.strip().splitlines() + arrow = next((i for i, line in enumerate(lines) if "-->" in line), None) + if arrow is None: + continue + m = re.match(r"([\d:.]+)\s+-->\s+([\d:.]+)", lines[arrow].strip()) + if not m: + continue + start = parse_vtt_timestamp(m.group(1)) + end = parse_vtt_timestamp(m.group(2)) + text = "\n".join(lines[arrow + 1 :]).strip() + if text: + cues.append((start, end, text)) + return cues + + +def render_vtt(cues: list[Cue]) -> str: + lines = ["WEBVTT", ""] + for start, end, text in cues: + lines.append(f"{format_vtt_timestamp(start)} --> {format_vtt_timestamp(end)}") + lines.append(text) + lines.append("") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# API calls: rate limiting and retries +# --------------------------------------------------------------------------- + + +class RateLimiter: + """Ensures at least ``min_interval`` seconds between API requests.""" + + def __init__(self, min_interval: float = 1.0) -> None: + self._lock = asyncio.Lock() + self._last: float = 0.0 + self._interval = min_interval + + async def wait(self) -> None: + async with self._lock: + loop = asyncio.get_running_loop() + elapsed = loop.time() - self._last + if elapsed < self._interval: + await asyncio.sleep(self._interval - elapsed) + self._last = asyncio.get_running_loop().time() + + +RETRYABLE_ERROR_NAMES = {"APIConnectionError", "APITimeoutError", "RateLimitError"} + + +def is_retryable_api_error(e: BaseException) -> bool: + """Rate limits, timeouts, connection errors, and 5xx responses. + + Works for both the OpenAI and Anthropic SDKs without importing either. + """ + if any(cls.__name__ in RETRYABLE_ERROR_NAMES for cls in type(e).__mro__): + return True + status = getattr(e, "status_code", None) + if isinstance(status, int): + return status in (408, 409, 429) or status >= 500 + return isinstance(e, (TimeoutError, ConnectionError)) + + +def describe_api_error(e: BaseException) -> str: + """``RateLimitError status=429 request_id=req_123: message``""" + parts = [type(e).__name__] + status = getattr(e, "status_code", None) + if status is not None: + parts.append(f"status={status}") + request_id = getattr(e, "request_id", None) + if request_id: + parts.append(f"request_id={request_id}") + message = str(e).strip().splitlines()[0][:300] if str(e).strip() else "" + return " ".join(parts) + (f": {message}" if message else "") + + +async def retry_async( + fn: Callable[[], Awaitable[T]], + *, + label: str, + logger: logging.Logger, + attempts: int = 6, + base_delay: float = 2.0, + max_delay: float = 60.0, + is_retryable: Callable[[BaseException], bool] = is_retryable_api_error, + describe: Callable[[BaseException], str] = describe_api_error, + sleep: Callable[[float], Awaitable[Any]] = asyncio.sleep, +) -> T: + """Call ``fn`` with exponential backoff (plus jitter) on retryable errors. + + Every failed attempt is logged with the error description, so a flaky + API is visible in the logs even when the call eventually succeeds. The + defaults wait about a minute in total, enough to ride out a short + network outage (with 1 s steps, a ten-second outage failed 15 videos). + """ + for attempt in range(1, attempts + 1): + try: + return await fn() + except Exception as e: + if not is_retryable(e): + logger.error("%s: %s (not retryable)", label, describe(e)) + raise + if attempt == attempts: + logger.error("%s: %s (giving up after %d attempts)", label, describe(e), attempts) + raise + delay = min(max_delay, base_delay * (2 ** (attempt - 1))) + random.uniform(0, base_delay) + logger.warning( + "%s: attempt %d/%d failed: %s; retrying in %.1fs", + label, + attempt, + attempts, + describe(e), + delay, + ) + await sleep(delay) + raise AssertionError("unreachable") diff --git a/scripts/videotesting.py b/scripts/videotesting.py new file mode 100644 index 000000000..12d839f1c --- /dev/null +++ b/scripts/videotesting.py @@ -0,0 +1,373 @@ +""" +Test helpers for the video pipeline tests (scripts/test-*.py). + +Provides a loader for the hyphenated scripts, a tiny fixture repository, and +fakes for every external service the pipeline talks to: the YouTube Data +API, thumbnail downloads, yt-dlp, OpenAI Whisper, and the Anthropic API. +Nothing in here touches the network. +""" + +from __future__ import annotations + +import importlib.util +import re +import subprocess +import sys +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +SCRIPTS_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(SCRIPTS_DIR)) + +import videolib # noqa: E402 + + +def load_script(filename: str): + """Import a hyphenated script (e.g. ``sync-videos.py``) as a module.""" + name = filename.removesuffix(".py").replace("-", "_") + if name in sys.modules: + return sys.modules[name] + spec = importlib.util.spec_from_file_location(name, SCRIPTS_DIR / filename) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +# --------------------------------------------------------------------------- +# Fixture repository +# --------------------------------------------------------------------------- + + +def write(path: Path, text: str) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return path + + +def make_repo(root: Path) -> Path: + """A minimal project tree with people, software, and YouTube config.""" + write(root / "content/people/hadley-wickham/_index.md", "---\ntitle: Hadley Wickham\n---\n") + write(root / "content/people/mine-cetinkaya-rundel/_index.md", "---\ntitle: Mine Çetinkaya-Rundel\n---\n") + write(root / "content/software/quarto/_index.md", "---\ntitle: Quarto\n---\n") + write(root / "content/software/shiny/_index.md", "---\ntitle: Shiny\n---\n") + write(root / "content/software/commons/_index.md", "---\ntitle: commons\n---\n") + write(root / "data/software-match-ignore.toml", 'names = ["commons"]\n') + write( + root / "data/youtube.toml", + 'channels = ["PositPBC"]\nplaylists = ["PLtest"]\nvideos = ["indiv000001"]\n', + ) + (root / "content/resources/videos").mkdir(parents=True, exist_ok=True) + return root + + +def write_video_dir( + root: Path, + slug: str, + *, + url: str = "https://www.youtube.com/watch?v=abcdefghijk", + title: str = "A talk", + duration: int = 600, + extra: str = "", + vtt: str | None = None, + html: str | None = None, +) -> Path: + """Write a video directory as sync-videos.py would have produced it.""" + d = root / "content/resources/videos" / slug + write( + d / "_index.md", + f"---\ntitle: {title}\nresource_type: video\nsoftware:\n- quarto\n{extra}" + f"external:\n url: {url}\n duration: {duration}\n---\n", + ) + if vtt is not None: + write(d / "transcription.vtt", vtt) + if html is not None: + write(d / "transcription.txt", html) + return d + + +SAMPLE_VTT = """WEBVTT + +00:00:00.000 --> 00:00:04.000 +Hello everyone, today we talk about Quarto. + +00:00:04.000 --> 00:00:09.500 +It was built by the team that also made Shiny. + +00:00:20.000 --> 00:00:25.000 +Hadley Wickham joins us later. +""" + + +def vtt_to_html(vtt: str) -> str: + """What a well-behaved model returns: one

with one per cue.""" + cues = videolib.parse_vtt(vtt) + anchors = " ".join(f'{text}' for start, _, text in cues) + return f"

{anchors}

" + + +# --------------------------------------------------------------------------- +# YouTube Data API fake +# --------------------------------------------------------------------------- + + +def yt_item( + video_id: str, + title: str = "A talk", + *, + published: str = "2026-10-01T12:00:00Z", + description: str = "", + duration: str = "PT10M", + views: int = 100, + likes: int = 10, + comments: int = 1, + tags: list[str] | None = None, + channel_title: str = "Posit PBC", +) -> dict[str, Any]: + return { + "id": video_id, + "snippet": { + "title": title, + "publishedAt": published, + "description": description, + "channelTitle": channel_title, + "tags": tags or [], + "thumbnails": {"high": {"url": f"https://i.ytimg.com/vi/{video_id}/hqdefault.jpg"}}, + "defaultAudioLanguage": "en", + }, + "contentDetails": {"duration": duration, "caption": "false", "definition": "hd"}, + "statistics": {"viewCount": str(views), "likeCount": str(likes), "commentCount": str(comments)}, + } + + +class HttpError(Exception): + """Stand-in for googleapiclient.errors.HttpError.""" + + def __init__(self, status: int, message: str = "error"): + super().__init__(f"") + self.status_code = status + + +class _Request: + def __init__(self, fn, kwargs): + self._fn = fn + self.kwargs = kwargs + + def execute(self, num_retries: int = 0): + return self._fn(**self.kwargs) + + +class FakeYouTube: + """Mimics the parts of ``googleapiclient`` the metadata script uses. + + ``channels``: handle -> (uploads playlist id, channel title) + ``playlists``: playlist id -> list of video ids + ``videos``: video id -> API item (missing ids are "private/deleted") + ``page_size``: items per playlistItems page, to exercise pagination + ``fail_playlists`` / ``fail_videos``: raise HttpError for these + """ + + def __init__(self, channels=None, playlists=None, videos=None, page_size=50): + self.channels_data = channels or {} + self.playlists_data = playlists or {} + self.videos_data = videos or {} + self.page_size = page_size + self.fail_playlists: set[str] = set() + self.fail_videos = False + self.calls: list[str] = [] + + # channels().list(...) + def channels(self): + def list_(part, forUsername=None, forHandle=None): # noqa: N803 - API names + def run(): + self.calls.append("channels.list") + key = forUsername or (forHandle or "").lstrip("@") + if forUsername and key not in self.channels_data: + return {} + if key in self.channels_data: + uploads, title = self.channels_data[key] + return { + "items": [ + { + "contentDetails": {"relatedPlaylists": {"uploads": uploads}}, + "snippet": {"title": title}, + } + ] + } + return {} + + return _Request(lambda: run(), {}) + + return SimpleNamespace(list=list_) + + # playlistItems().list(...) / list_next(...) + def playlistItems(self): # noqa: N802 - API name + def page(playlistId, start): # noqa: N803 + self.calls.append("playlistItems.list") + if playlistId in self.fail_playlists or playlistId not in self.playlists_data: + raise HttpError(404, f"playlist {playlistId} not found") + ids = self.playlists_data[playlistId] + chunk = ids[start : start + self.page_size] + response = {"items": [{"contentDetails": {"videoId": v}} for v in chunk]} + if start + self.page_size < len(ids): + response["nextPageToken"] = str(start + self.page_size) + return response + + def list_(part, playlistId, maxResults=50, pageToken=None): # noqa: N803 + start = int(pageToken or 0) + req = _Request(lambda: page(playlistId, start), {}) + req.playlist_id = playlistId + return req + + def list_next(request, response): + token = response.get("nextPageToken") + if not token: + return None + return list_("contentDetails", request.playlist_id, pageToken=token) + + return SimpleNamespace(list=list_, list_next=list_next) + + # videos().list(...) + def videos(self): + def list_(part, id): # noqa: A002 - API name + def run(): + self.calls.append("videos.list") + if self.fail_videos: + raise HttpError(403, "quotaExceeded") + ids = id.split(",") + return {"items": [self.videos_data[v] for v in ids if v in self.videos_data]} + + return _Request(lambda: run(), {}) + + return SimpleNamespace(list=list_) + + +# --------------------------------------------------------------------------- +# HTTP (thumbnails) +# --------------------------------------------------------------------------- + + +class FakeHttp: + """``get(url, timeout)`` returning a response with ``content``.""" + + def __init__(self, fail_urls: set[str] | None = None): + self.fail_urls = fail_urls or set() + self.requested: list[str] = [] + + def __call__(self, url: str, timeout: float = 15): + self.requested.append(url) + if url in self.fail_urls: + raise OSError(f"connection refused: {url}") + return SimpleNamespace(content=b"JPEG:" + url.encode(), raise_for_status=lambda: None) + + +# --------------------------------------------------------------------------- +# yt-dlp fake +# --------------------------------------------------------------------------- + + +class FakeYtDlp: + """Stands in for ``subprocess.run`` when transcribe.py calls yt-dlp. + + ``outcomes`` maps a video URL to a list of results, consumed in order; + each result is either ``"ok"`` (an audio file is written) or an stderr + string (exit code 1). URLs without outcomes always succeed. + """ + + def __init__(self, outcomes: dict[str, list[str]] | None = None, size: int = 1024): + self.outcomes = {k: list(v) for k, v in (outcomes or {}).items()} + self.size = size + self.calls: list[list[str]] = [] + + def __call__(self, cmd, capture_output=True, text=True, **kwargs): + self.calls.append(cmd) + url = cmd[-1] + queue = self.outcomes.get(url) + result = queue.pop(0) if queue else "ok" + if result != "ok": + return subprocess.CompletedProcess(cmd, 1, stdout="", stderr=result) + template = cmd[cmd.index("--output") + 1] + out = Path(template.replace("%(ext)s", "webm")) + out.write_bytes(b"\0" * self.size) + return subprocess.CompletedProcess(cmd, 0, stdout="", stderr="") + + +# --------------------------------------------------------------------------- +# OpenAI / Anthropic fakes +# --------------------------------------------------------------------------- + + +class StatusError(Exception): + """Mimics an SDK ``APIStatusError`` (has ``status_code`` and ``request_id``).""" + + def __init__(self, status: int, message: str = "boom"): + super().__init__(message) + self.status_code = status + self.request_id = f"req_{status}" + + +class RateLimitError(StatusError): + def __init__(self): + super().__init__(429, "rate limited") + + +class FakeOpenAI: + """``client.audio.transcriptions.create(file=..., **kwargs)`` returning VTT. + + ``responses`` is a list consumed in order; an Exception instance is + raised instead of returned. When exhausted, ``default`` is returned. + """ + + def __init__(self, responses: list | None = None, default: str = SAMPLE_VTT): + self.responses = list(responses or []) + self.default = default + self.calls: list[dict[str, Any]] = [] + self.audio = SimpleNamespace(transcriptions=SimpleNamespace(create=self._create)) + + async def _create(self, file, **kwargs): + self.calls.append({"file": getattr(file, "name", None), **kwargs}) + result = self.responses.pop(0) if self.responses else self.default + if isinstance(result, BaseException): + raise result + return result + + +class FakeAnthropic: + """``client.messages.create(...)`` returning a message-like object. + + ``responses`` entries may be a string (the text), a ``(text, + stop_reason)`` tuple, or an Exception to raise. When exhausted, the + response is generated from the VTT in the user message. + """ + + def __init__(self, responses: list | None = None): + self.responses = list(responses or []) + self.calls: list[dict[str, Any]] = [] + self.messages = SimpleNamespace(create=self._create) + + async def _create(self, **kwargs): + self.calls.append(kwargs) + if self.responses: + result = self.responses.pop(0) + else: + user = kwargs["messages"][0]["content"] + vtt = user.split("\n\n---\n\n", 1)[-1] + result = vtt_to_html(vtt) + if isinstance(result, BaseException): + raise result + text, stop_reason = result if isinstance(result, tuple) else (result, "end_turn") + return SimpleNamespace( + content=[SimpleNamespace(type="text", text=text)], + stop_reason=stop_reason, + usage=SimpleNamespace(input_tokens=1000, output_tokens=500), + _request_id="req_fake", + ) + + +async def no_sleep(_seconds: float) -> None: + return None + + +def strip_ansi(text: str) -> str: + return re.sub(r"\x1b\[[0-9;]*m", "", text)